diff --git a/Documentation/devicetree/bindings/clock/eswin,eic7700-hspcrg.yaml b/Documentation/devicetree/bindings/clock/eswin,eic7700-hspcrg.yaml index 43df689ae647..ee4e88e518b6 100644 --- a/Documentation/devicetree/bindings/clock/eswin,eic7700-hspcrg.yaml +++ b/Documentation/devicetree/bindings/clock/eswin,eic7700-hspcrg.yaml @@ -14,7 +14,11 @@ description: properties: compatible: - const: eswin,eic7700-hspcrg + oneOf: + - const: eswin,eic7700-hspcrg + - items: + - const: eswin,eic7700-hspcrg + - const: syscon reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/i2c/snps,designware-i2c.yaml b/Documentation/devicetree/bindings/i2c/snps,designware-i2c.yaml index 467bdcbb8538..ea3d4d4c299f 100644 --- a/Documentation/devicetree/bindings/i2c/snps,designware-i2c.yaml +++ b/Documentation/devicetree/bindings/i2c/snps,designware-i2c.yaml @@ -40,6 +40,7 @@ properties: - const: snps,designware-i2c - items: - enum: + - eswin,eic7700-i2c - mobileye,eyeq6lplus-i2c - mscc,ocelot-i2c - sophgo,sg2044-i2c diff --git a/Documentation/devicetree/bindings/mmc/snps,dwcmshc-sdhci.yaml b/Documentation/devicetree/bindings/mmc/snps,dwcmshc-sdhci.yaml index cd823a3ef213..552bff11021c 100644 --- a/Documentation/devicetree/bindings/mmc/snps,dwcmshc-sdhci.yaml +++ b/Documentation/devicetree/bindings/mmc/snps,dwcmshc-sdhci.yaml @@ -168,23 +168,45 @@ allOf: - const: bus - const: timer else: - properties: - clocks: - minItems: 1 - items: - - description: core clock - - description: bus clock for optional - - description: axi clock for rockchip specified - - description: block clock for rockchip specified - - description: timer clock for rockchip specified - clock-names: - minItems: 1 - items: - - const: core - - const: bus - - const: axi - - const: block - - const: timer + if: + properties: + compatible: + contains: + const: eswin,eic7700-dwcmshc + then: + properties: + clocks: + minItems: 1 + items: + - description: core clock + - description: bus clock + - description: AXI clock + - description: timer clock + clock-names: + minItems: 1 + items: + - const: core + - const: bus + - const: axi + - const: timer + else: + properties: + clocks: + minItems: 1 + items: + - description: core clock + - description: bus clock for optional + - description: axi clock for rockchip specified + - description: block clock for rockchip specified + - description: timer clock for rockchip specified + clock-names: + minItems: 1 + items: + - const: core + - const: bus + - const: axi + - const: block + - const: timer - if: properties: diff --git a/Documentation/devicetree/bindings/pwm/snps,dw-apb-timers-pwm2.yaml b/Documentation/devicetree/bindings/pwm/snps,dw-apb-timers-pwm2.yaml index 7523a89a1773..87a56ffed09d 100644 --- a/Documentation/devicetree/bindings/pwm/snps,dw-apb-timers-pwm2.yaml +++ b/Documentation/devicetree/bindings/pwm/snps,dw-apb-timers-pwm2.yaml @@ -20,12 +20,11 @@ description: instead of having to encode the IP version number in the device tree compatible. -allOf: - - $ref: pwm.yaml# - properties: compatible: - const: snps,dw-apb-timers-pwm2 + enum: + - snps,dw-apb-timers-pwm2 + - eswin,eic7700-pwm reg: maxItems: 1 @@ -43,6 +42,17 @@ properties: - const: bus - const: timer + resets: + minItems: 1 + items: + - description: Interface bus (presetn) reset + - description: PWM timer logic (timer_N_resetn) reset + + reset-names: + items: + - const: bus + - const: timer + snps,pwm-number: $ref: /schemas/types.yaml#/definitions/uint32 description: The number of PWM channels configured for this instance @@ -54,6 +64,29 @@ required: - clocks - clock-names +allOf: + - $ref: pwm.yaml# + + - if: + properties: + compatible: + contains: + const: eswin,eic7700-pwm + then: + properties: + resets: + items: + - description: Combined APB interface and timer logic reset + reset-names: false + required: + - resets + else: + properties: + resets: + minItems: 2 + reset-names: + minItems: 2 + additionalProperties: false examples: @@ -65,3 +98,12 @@ examples: clocks = <&bus>, <&timer>; clock-names = "bus", "timer"; }; + - | + pwm@50818000 { + compatible = "eswin,eic7700-pwm"; + reg = <0x50818000 0x4000>; + #pwm-cells = <3>; + clocks = <&bus>, <&timer>; + clock-names = "bus", "timer"; + resets = <&reset>; + }; diff --git a/Documentation/devicetree/bindings/riscv/physical-memory.yaml b/Documentation/devicetree/bindings/riscv/physical-memory.yaml new file mode 100644 index 000000000000..8beaa588c71c --- /dev/null +++ b/Documentation/devicetree/bindings/riscv/physical-memory.yaml @@ -0,0 +1,92 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/riscv/physical-memory.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: RISC-V Physical Memory Regions + +maintainers: + - Samuel Holland + +description: + The RISC-V Privileged Architecture defines a number of Physical Memory + Attributes (PMAs) which apply to a given region of memory. These include the + types of accesses (read, write, execute, LR/SC, and/or AMO) allowed within + a region, the supported access widths and alignments, the cacheability and + coherence of the region, and whether or not accesses to the region may have + side effects. + + Some RISC-V platforms provide multiple physical address mappings for main + memory or certain peripherals. Each alias of a region generally has different + PMAs (e.g. cacheable vs non-cacheable), which allows software to dynamically + select the PMAs for an access by referencing the corresponding alias. + + On DT-based RISC-V platforms, this information is provided by the + riscv,physical-memory-regions property of the root node. + +properties: + $nodename: + const: '/' + + riscv,physical-memory-regions: + $ref: /schemas/types.yaml#/definitions/uint32-matrix + description: + Each table entry provides PMAs for a specific physical memory region, + which must not overlap with any other table entry. + minItems: 1 + maxItems: 256 + items: + minItems: 4 + maxItems: 6 + additionalItems: true + items: + - description: CPU physical address (#address-cells) + - description: > + Size (#size-cells). For entry 0, if the size is zero, the size is + assumed to be 2^(32 * #size-cells). + - description: > + Flags describing the most restrictive PMAs for any address within + the region. + + The least significant byte indicates the types of accesses allowed + for this region. Note that a memory region may support a type of + access (e.g. AMOs) even if the CPU does not. + + The next byte describes the cacheability, coherence, idempotency, + and ordering PMAs for this region. It also includes a flag to + indicate that accesses to a region are unsafe and must be + prohibited by software (for example using PMPs or Smmpt). + + The third byte is reserved for future PMAs. + + The most significant byte is the index of the lowest-numbered entry + which this entry is an alias of, if any. Aliases need not be the + same size, for example if a smaller memory region repeats within a + larger alias. + - description: Reserved for describing future PMAs + +additionalProperties: true + +examples: + - | + #include + + / { + compatible = "beagle,beaglev-starlight-jh7100-r0", "starfive,jh7100"; + model = "BeagleV Starlight Beta"; + #address-cells = <2>; + #size-cells = <2>; + + riscv,physical-memory-regions = + <0x00 0x18000000 0x00 0x00020000 (PMA_RWX | PMA_NONCACHEABLE_MEMORY) 0x0>, + <0x00 0x18080000 0x00 0x00020000 (PMA_RWX | PMA_NONCACHEABLE_MEMORY) 0x0>, + <0x00 0x41000000 0x00 0x1f000000 (PMA_RWX | PMA_NONCACHEABLE_MEMORY) 0x0>, + <0x00 0x61000000 0x00 0x1f000000 (PMA_RWXA | PMA_NONCOHERENT_MEMORY | PMR_ALIAS(3)) 0x0>, + <0x00 0x80000000 0x08 0x00000000 (PMA_RWXA | PMA_NONCOHERENT_MEMORY) 0x0>, + <0x10 0x00000000 0x08 0x00000000 (PMA_RWX | PMA_NONCACHEABLE_MEMORY | PMR_ALIAS(5)) 0x0>, + <0x20 0x00000000 0x10 0x00000000 (PMA_RWX | PMA_NONCACHEABLE_MEMORY) 0x0>, + <0x30 0x00000000 0x10 0x00000000 (PMA_RWXA | PMA_NONCOHERENT_MEMORY | PMR_ALIAS(7)) 0x0>; + }; + +... diff --git a/Documentation/devicetree/bindings/timer/snps,dw-apb-timer.yaml b/Documentation/devicetree/bindings/timer/snps,dw-apb-timer.yaml index d33c9205a909..4d30cd01ac65 100644 --- a/Documentation/devicetree/bindings/timer/snps,dw-apb-timer.yaml +++ b/Documentation/devicetree/bindings/timer/snps,dw-apb-timer.yaml @@ -25,7 +25,10 @@ properties: maxItems: 1 resets: - maxItems: 1 + minItems: 1 + items: + - description: Timer ticks reference (timer_N_resetn) reset + - description: APB interface (presetn) reset clocks: minItems: 1 diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 5874ca30f4f6..e6e1c3f35a26 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -618,6 +618,7 @@ config RISCV_ISA_SVPBMT depends on 64BIT && MMU depends on RISCV_ALTERNATIVE default y + select DMA_DIRECT_REMAP help Add support for the Svpbmt ISA-extension (Supervisor-mode: page-based memory types) in the kernel when it is detected at boot. @@ -831,7 +832,6 @@ config RISCV_ISA_ZICBOM depends on RISCV_ALTERNATIVE default y select RISCV_DMA_NONCOHERENT - select DMA_DIRECT_REMAP help Add support for the Zicbom extension (Cache Block Management Operations) and enable its use in the kernel when it is detected @@ -910,6 +910,23 @@ config TOOLCHAIN_NEEDS_OLD_ISA_SPEC versions of GCC to be passed to GAS, which has the same result as passing zicsr and zifencei to -march. +config RISCV_ISA_XLINUXMEMALIAS + bool "Use physical memory aliases to emulate page-based memory types" + depends on 64BIT && MMU + depends on RISCV_ALTERNATIVE + default y + select DMA_DIRECT_REMAP + help + Add support for the kernel to alter the Physical Memory Attributes + (PMAs) of a page at runtime by selecting from among the aliases of + that page in the physical address space. + + On systems where physical memory aliases are present, this option + is required in order to mark pages as non-cacheable for use with + non-coherent DMA devices. + + If you don't know what to do here, say Y. + config FPU bool "FPU support" default y diff --git a/arch/riscv/Kconfig.errata b/arch/riscv/Kconfig.errata index 622ade116a2d..46827d01d098 100644 --- a/arch/riscv/Kconfig.errata +++ b/arch/riscv/Kconfig.errata @@ -77,13 +77,11 @@ config ERRATA_SIFIVE_CIP_1200 If you don't know what to do here, say "Y". config ERRATA_STARFIVE_JH7100 - bool "StarFive JH7100 support" + bool "StarFive JH7100 support for old devicetrees" depends on ARCH_STARFIVE depends on !DMA_DIRECT_REMAP depends on NONPORTABLE select DMA_GLOBAL_POOL - select RISCV_DMA_NONCOHERENT - select RISCV_NONSTANDARD_CACHE_OPS select CACHEMAINT_FOR_DMA select SIFIVE_CCACHE default n @@ -94,7 +92,10 @@ config ERRATA_STARFIVE_JH7100 cache operations through the SiFive cache controller. Say "Y" if you want to support the BeagleV Starlight and/or - StarFive VisionFive V1 boards. + StarFive VisionFive V1 boards with older devicetrees that reserve + memory for DMA using a "shared-dma-pool". If your devicetree has + the "riscv,physical-memory-regions" property, you should instead + enable RISCV_ISA_XLINUXMEMALIAS and use a portable kernel. config ERRATA_SIFIVE_MAL_9092 bool "Apply SiFive HLV/HLVX/HSV fence errata" @@ -123,6 +124,7 @@ config ERRATA_THEAD config ERRATA_THEAD_MAE bool "Apply T-Head's memory attribute extension (XTheadMae) errata" depends on ERRATA_THEAD && 64BIT && MMU + select DMA_DIRECT_REMAP select RISCV_ALTERNATIVE_EARLY default y help @@ -134,7 +136,6 @@ config ERRATA_THEAD_MAE config ERRATA_THEAD_CMO bool "Apply T-Head cache management errata" depends on ERRATA_THEAD && MMU - select DMA_DIRECT_REMAP select RISCV_DMA_NONCOHERENT select RISCV_NONSTANDARD_CACHE_OPS default y diff --git a/arch/riscv/Kconfig.socs b/arch/riscv/Kconfig.socs index cecbce0c4983..b85174e62ffa 100644 --- a/arch/riscv/Kconfig.socs +++ b/arch/riscv/Kconfig.socs @@ -14,6 +14,10 @@ config ARCH_ANLOGIC config ARCH_ESWIN bool "ESWIN SoCs" + select CACHEMAINT_FOR_DMA + select SIFIVE_CCACHE + select RISCV_DMA_NONCOHERENT + select RISCV_NONSTANDARD_CACHE_OPS help This enables support for ESWIN SoC platform hardware, including the ESWIN EIC7700 SoC. @@ -56,6 +60,8 @@ config SOC_STARFIVE bool "StarFive SoCs" select PINCTRL select RESET_CONTROLLER + select RISCV_DMA_NONCOHERENT + select RISCV_NONSTANDARD_CACHE_OPS select ARM_AMBA help This enables support for StarFive SoC platform hardware. diff --git a/arch/riscv/boot/dts/eswin/eic7700-hifive-premier-p550.dts b/arch/riscv/boot/dts/eswin/eic7700-hifive-premier-p550.dts index 131ed1fc6b2e..30562627955f 100644 --- a/arch/riscv/boot/dts/eswin/eic7700-hifive-premier-p550.dts +++ b/arch/riscv/boot/dts/eswin/eic7700-hifive-premier-p550.dts @@ -6,24 +6,206 @@ /dts-v1/; #include "eic7700.dtsi" +#include "eic7700-pinctrl.dtsi" +#include / { compatible = "sifive,hifive-premier-p550", "eswin,eic7700"; model = "SiFive HiFive Premier P550"; aliases { + ethernet0 = &gmac0; + ethernet1 = &gmac1; + mmc0 = &emmc; serial0 = &uart0; }; chosen { stdout-path = "serial0:115200n8"; }; + + leds { + compatible = "gpio-leds"; + pinctrl-names = "default"; + pinctrl-0 = <&gpio84_pins>; + + led-0 { + gpios = <&gpioC 20 GPIO_ACTIVE_HIGH>; + }; + }; + + vcc_1v8: regulator-vcc1v8 { + compatible = "regulator-fixed"; + regulator-name = "vcc1v8"; + regulator-always-on; + regulator-boot-on; + regulator-min-microvolt = <1800000>; + regulator-max-microvolt = <1800000>; + }; +}; + +&aon_i2c0 { + status = "okay"; + + eeprom@50 { + compatible = "atmel,24c02"; + reg = <0x50>; + }; +}; + +&aon_i2c1 { + status = "okay"; + + adc@10 { + compatible = "microchip,pac1934"; + reg = <0x10>; + #address-cells = <1>; + #size-cells = <0>; + + channel@1 { + reg = <0x1>; + shunt-resistor-micro-ohms = <1000>; + label = "VDD_SOM"; + }; + + channel@2 { + reg = <0x2>; + shunt-resistor-micro-ohms = <1000>; + label = "VDD_SOC"; + }; + + channel@3 { + reg = <0x3>; + shunt-resistor-micro-ohms = <1000>; + label = "VDD_CPU"; + }; + + channel@4 { + reg = <0x4>; + shunt-resistor-micro-ohms = <1000>; + label = "VDD_LPDDR"; + }; + }; + + power-sensor@44 { + compatible = "ti,ina226"; + reg = <0x44>; + #io-channel-cells = <1>; + label = "sys_power"; + shunt-resistor = <1000>; + }; +}; + +&emmc { + assigned-clocks = <&clk EIC7700_CLK_GATE_HSP_MSHC0_CORE_CLK>; + assigned-clock-rates = <200000000>; + bus-width = <8>; + max-frequency = <200000000>; + mmc-hs400-1_8v; + non-removable; + no-sd; + no-sdio; + status = "okay"; +}; + +&gmac0 { + phy-handle = <&gmac0_phy0>; + phy-mode = "rgmii-id"; + pinctrl-names = "default"; + pinctrl-0 = <&gpio106_pins>; + rx-internal-delay-ps = <20>; + tx-internal-delay-ps = <100>; + status = "okay"; +}; + +&gmac0_mdio { + gmac0_phy0: ethernet-phy@0 { + compatible = "ethernet-phy-id001c.c916"; + reg = <0>; + reset-gpios = <&gpioD 10 GPIO_ACTIVE_LOW>; + reset-assert-us = <10000>; + reset-deassert-us = <80000>; + }; +}; + +&gmac1 { + phy-handle = <&gmac1_phy0>; + phy-mode = "rgmii-id"; + pinctrl-names = "default"; + pinctrl-0 = <&gpio111_pins>; + rx-internal-delay-ps = <200>; + tx-internal-delay-ps = <2200>; + status = "okay"; +}; + +&gmac1_mdio { + gmac1_phy0: ethernet-phy@0 { + compatible = "ethernet-phy-id001c.c916"; + reg = <0>; + reset-gpios = <&gpioD 15 GPIO_ACTIVE_LOW>; + reset-assert-us = <10000>; + reset-deassert-us = <80000>; + }; +}; + +&gpio { + pinctrl-names = "default"; + pinctrl-0 = <&gpio43_pins>; +}; + +&gpio43_pins { + bias-disable; + input-disable; +}; + +&gpio84_pins { + bias-disable; + input-disable; +}; + +&gpioB { + usb-hub-reset-hog { + gpio-hog; + gpios = <11 GPIO_ACTIVE_LOW>; + /* Deassert the active-low reset of the on-board USB hub. */ + output-low; + line-name = "usb-hub-reset"; + }; +}; + +&osc { + clock-frequency = <24000000>; + clock-output-names = "osc_24m"; +}; + +&pcie { + pinctrl-names = "default"; + pinctrl-0 = <&pcie_pins>; + status = "okay"; +}; + +&pinctrl { + vrgmii-supply = <&vcc_1v8>; }; &uart0 { status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&uart0_pins>; }; &uart2 { status = "okay"; + pinctrl-names = "default"; + pinctrl-0 = <&uart2_pins>; +}; + +&usb0 { + dr_mode = "peripheral"; + status = "okay"; +}; + +&usb1 { + dr_mode = "host"; + status = "okay"; }; diff --git a/arch/riscv/boot/dts/eswin/eic7700-pinctrl.dtsi b/arch/riscv/boot/dts/eswin/eic7700-pinctrl.dtsi new file mode 100644 index 000000000000..c11f42e79c6e --- /dev/null +++ b/arch/riscv/boot/dts/eswin/eic7700-pinctrl.dtsi @@ -0,0 +1,1065 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) +/* + * Copyright (c) 2025 Beijing ESWIN Computing Technology Co., Ltd. + * + * ESWIN's EIC7700 SoC pin-mux and pin-config options are listed as + * device tree nodes in this file. + * + * Authors: Yulin Lu + */ + +&pinctrl { + default-pins-grp { + /omit-if-no-ref/ + sata_act_led_pins: sata-act-led-pins { + pins = "gpio27"; + function = "sata"; + }; + + /omit-if-no-ref/ + emmc_led_ctrl_pins: emmc-led-ctrl-pins { + pins = "gpio29"; + function = "emmc"; + }; + + /omit-if-no-ref/ + sd0_led_ctrl_pins: sd0-led-ctrl-pins { + pins = "gpio34"; + function = "sdio"; + }; + + /omit-if-no-ref/ + sd1_led_ctrl_pins: sd1-led-ctrl-pins { + pins = "spi1_d2"; + function = "sdio"; + }; + + /omit-if-no-ref/ + i2c6_pins: i2c6-pins { + pins = "uart1_cts", "uart1_rts"; + function = "i2c"; + }; + + /omit-if-no-ref/ + i2c7_pins: i2c7-pins { + pins = "uart2_tx", "uart2_rx"; + function = "i2c"; + }; + + /omit-if-no-ref/ + i2c9_pins: i2c9-pins { + pins = "spi1_d0", "spi1_d1"; + function = "i2c"; + }; + + /omit-if-no-ref/ + pwm1_pins: pwm1-pins { + pins = "spi1_d3"; + function = "pwm"; + }; + + /omit-if-no-ref/ + pwm2_pins: pwm2-pins { + pins = "spi1_cs1_n"; + function = "pwm"; + }; + + /omit-if-no-ref/ + mipi_csi_xtrig0_pins: mipi-csi-xtrig0-pins { + pins = "gpio92"; + function = "mipi_csi"; + }; + + /omit-if-no-ref/ + mipi_csi_xtrig1_pins: mipi-csi-xtrig1-pins { + pins = "gpio93"; + function = "mipi_csi"; + }; + + /omit-if-no-ref/ + dual_spi2_pins: dual-spi2-pins { + pins = "spi2_cs0_n", "jtag0_tck", "jtag0_tms", "jtag0_tdi"; + function = "spi"; + }; + + /omit-if-no-ref/ + quad_spi2_pins: quad-spi2-pins { + pins = "spi2_cs1_n", "jtag0_tdo", "gpio5"; + function = "spi"; + }; + + /omit-if-no-ref/ + gpio1_pins: gpio1-pins { + pins = "jtag0_tck"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio2_pins: gpio2-pins { + pins = "jtag0_tms"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio3_pins: gpio3-pins { + pins = "jtag0_tdi"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio4_pins: gpio4-pins { + pins = "jtag0_tdo"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio6_pins: gpio6-pins { + pins = "spi2_cs0_n"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio7_pins: gpio7-pins { + pins = "jtag1_tck"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio8_pins: gpio8-pins { + pins = "jtag1_tms"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio9_pins: gpio9-pins { + pins = "jtag1_tdi"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio10_pins: gpio10-pins { + pins = "jtag1_tdo"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio12_pins: gpio12-pins { + pins = "spi2_cs1_n"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio13_pins: gpio13-pins { + pins = "mode_set0"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio14_pins: gpio14-pins { + pins = "mode_set1"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio15_pins: gpio15-pins { + pins = "mode_set2"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio16_pins: gpio16-pins { + pins = "mode_set3"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio17_pins: gpio17-pins { + pins = "jtag2_trst"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio18_pins: gpio18-pins { + pins = "i2s0_bclk"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio19_pins: gpio19-pins { + pins = "i2s0_wclk"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio20_pins: gpio20-pins { + pins = "i2s0_sdi"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio21_pins: gpio21-pins { + pins = "i2s0_sdo"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio22_pins: gpio22-pins { + pins = "i2s_mclk"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio23_pins: gpio23-pins { + pins = "i2s2_bclk"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio24_pins: gpio24-pins { + pins = "i2s2_wclk"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio25_pins: gpio25-pins { + pins = "i2s2_sdi"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio26_pins: gpio26-pins { + pins = "i2s2_sdo"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio29_pins: gpio29-pins { + pins = "gpio29"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio30_pins: gpio30-pins { + pins = "i2s1_bclk"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio31_pins: gpio31-pins { + pins = "i2s1_wclk"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio32_pins: gpio32-pins { + pins = "i2s1_sdi"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio33_pins: gpio33-pins { + pins = "i2s1_sdo"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio34_pins: gpio34-pins { + pins = "gpio34"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio35_pins: gpio35-pins { + pins = "spi1_cs0_n"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio36_pins: gpio36-pins { + pins = "spi1_clk"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio37_pins: gpio37-pins { + pins = "spi1_d0"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio38_pins: gpio38-pins { + pins = "spi1_d1"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio39_pins: gpio39-pins { + pins = "spi1_d2"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio40_pins: gpio40-pins { + pins = "spi1_d3"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio41_pins: gpio41-pins { + pins = "spi1_cs1_n"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio42_pins: gpio42-pins { + pins = "usb0_pwren"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio43_pins: gpio43-pins { + pins = "usb1_pwren"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio44_pins: gpio44-pins { + pins = "i2c0_scl"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio45_pins: gpio45-pins { + pins = "i2c0_sda"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio46_pins: gpio46-pins { + pins = "i2c1_scl"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio47_pins: gpio47-pins { + pins = "i2c1_sda"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio48_pins: gpio48-pins { + pins = "i2c2_scl"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio49_pins: gpio49-pins { + pins = "i2c2_sda"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio50_pins: gpio50-pins { + pins = "i2c3_scl"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio51_pins: gpio51-pins { + pins = "i2c3_sda"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio52_pins: gpio52-pins { + pins = "i2c4_scl"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio53_pins: gpio53-pins { + pins = "i2c4_sda"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio54_pins: gpio54-pins { + pins = "i2c5_scl"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio55_pins: gpio55-pins { + pins = "i2c5_sda"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio56_pins: gpio56-pins { + pins = "uart0_tx"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio57_pins: gpio57-pins { + pins = "uart0_rx"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio58_pins: gpio58-pins { + pins = "uart1_tx"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio59_pins: gpio59-pins { + pins = "uart1_rx"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio60_pins: gpio60-pins { + pins = "uart1_cts"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio61_pins: gpio61-pins { + pins = "uart1_rts"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio62_pins: gpio62-pins { + pins = "uart2_tx"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio63_pins: gpio63-pins { + pins = "uart2_rx"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio64_pins: gpio64-pins { + pins = "jtag2_tck"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio65_pins: gpio65-pins { + pins = "jtag2_tms"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio66_pins: gpio66-pins { + pins = "jtag2_tdi"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio67_pins: gpio67-pins { + pins = "jtag2_tdo"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio68_pins: gpio68-pins { + pins = "fan_pwm"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio69_pins: gpio69-pins { + pins = "fan_tach"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio70_pins: gpio70-pins { + pins = "mipi_csi0_xvs"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio71_pins: gpio71-pins { + pins = "mipi_csi0_xhs"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio72_pins: gpio72-pins { + pins = "mipi_csi0_mclk"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio73_pins: gpio73-pins { + pins = "mipi_csi1_xvs"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio74_pins: gpio74-pins { + pins = "mipi_csi1_xhs"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio75_pins: gpio75-pins { + pins = "mipi_csi1_mclk"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio76_pins: gpio76-pins { + pins = "mipi_csi2_xvs"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio77_pins: gpio77-pins { + pins = "mipi_csi2_xhs"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio78_pins: gpio78-pins { + pins = "mipi_csi2_mclk"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio79_pins: gpio79-pins { + pins = "mipi_csi3_xvs"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio80_pins: gpio80-pins { + pins = "mipi_csi3_xhs"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio81_pins: gpio81-pins { + pins = "mipi_csi3_mclk"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio82_pins: gpio82-pins { + pins = "mipi_csi4_xvs"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio83_pins: gpio83-pins { + pins = "mipi_csi4_xhs"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio84_pins: gpio84-pins { + pins = "mipi_csi4_mclk"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio85_pins: gpio85-pins { + pins = "mipi_csi5_xvs"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio86_pins: gpio86-pins { + pins = "mipi_csi5_xhs"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio87_pins: gpio87-pins { + pins = "mipi_csi5_mclk"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio88_pins: gpio88-pins { + pins = "spi3_cs_n"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio89_pins: gpio89-pins { + pins = "spi3_clk"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio90_pins: gpio90-pins { + pins = "spi3_di"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio91_pins: gpio91-pins { + pins = "spi3_do"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio92_pins: gpio92-pins { + pins = "gpio92"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio93_pins: gpio93-pins { + pins = "gpio93"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio94_pins: gpio94-pins { + pins = "s_mode"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio95_pins: gpio95-pins { + pins = "gpio95"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio96_pins: gpio96-pins { + pins = "spi0_cs_n"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio97_pins: gpio97-pins { + pins = "spi0_clk"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio98_pins: gpio98-pins { + pins = "spi0_d0"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio99_pins: gpio99-pins { + pins = "spi0_d1"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio100_pins: gpio100-pins { + pins = "spi0_d2"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio101_pins: gpio101-pins { + pins = "spi0_d3"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio102_pins: gpio102-pins { + pins = "i2c10_scl"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio103_pins: gpio103-pins { + pins = "i2c10_sda"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio104_pins: gpio104-pins { + pins = "i2c11_scl"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio105_pins: gpio105-pins { + pins = "i2c11_sda"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio107_pins: gpio107-pins { + pins = "boot_sel0"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio108_pins: gpio108-pins { + pins = "boot_sel1"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio109_pins: gpio109-pins { + pins = "boot_sel2"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio110_pins: gpio110-pins { + pins = "boot_sel3"; + function = "gpio"; + }; + + /omit-if-no-ref/ + uart3_pins: uart3-pins { + pins = "gpio92", "gpio93"; + function = "uart"; + }; + + /omit-if-no-ref/ + uart4_pins: uart4-pins { + pins = "spi1_d0", "spi1_d1"; + function = "uart"; + }; + + /omit-if-no-ref/ + gpio0_pins: gpio0-pins { + pins = "gpio0"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio5_pins: gpio5-pins { + pins = "gpio5"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio11_pins: gpio11-pins { + pins = "gpio11"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio27_pins: gpio27-pins { + pins = "gpio27"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio28_pins: gpio28-pins { + pins = "gpio28"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio106_pins: gpio106-pins { + pins = "gpio106"; + function = "gpio"; + }; + + /omit-if-no-ref/ + gpio111_pins: gpio111-pins { + pins = "gpio111"; + function = "gpio"; + }; + + /omit-if-no-ref/ + chip_mode_pins: chip-mode-pins { + pins = "chip_mode"; + function = "chip_mode"; + }; + + /omit-if-no-ref/ + sdio0_pins: sdio0-pins { + pins = "mode_set0", "mode_set1"; + function = "sdio"; + }; + + /omit-if-no-ref/ + sdio1_pins: sdio1-pins { + pins = "mode_set2", "mode_set3"; + function = "sdio"; + }; + + /omit-if-no-ref/ + jtag0_pins: jtag0-pins { + pins = "jtag0_tck", "jtag0_tms", "jtag0_tdi", "jtag0_tdo"; + function = "jtag"; + }; + + /omit-if-no-ref/ + jtag1_pins: jtag1-pins { + pins = "jtag1_tck", "jtag1_tms", "jtag1_tdi", "jtag1_tdo"; + function = "jtag"; + }; + + /omit-if-no-ref/ + jtag2_pins: jtag2-pins { + pins = "jtag2_trst", "jtag2_tck", "jtag2_tms", "jtag2_tdi", + "jtag2_tdo"; + function = "jtag"; + }; + + /omit-if-no-ref/ + pcie_pins: pcie-pins { + pins = "pcie_clkreq_n", "pcie_wake_n", "pcie_perst_n"; + function = "pcie"; + }; + + /omit-if-no-ref/ + hdmi_pins: hdmi-pins { + pins = "hdmi_scl", "hdmi_sda", "hdmi_cec"; + function = "hdmi"; + }; + + /omit-if-no-ref/ + rgmii0_pins: rgmii0-pins { + pins = "rgmii0_clk_125", "rgmii0_txen", "rgmii0_txclk", "rgmii0_txd0", + "rgmii0_txd1", "rgmii0_txd2", "rgmii0_txd3", "rgmii0_rxclk", + "rgmii0_rxdv", "rgmii0_rxd0", "rgmii0_rxd1", "rgmii0_rxd2", + "rgmii0_rxd3", "rgmii0_mdc", "rgmii0_mdio", "rgmii0_intb"; + function = "rgmii"; + }; + + /omit-if-no-ref/ + rgmii1_pins: rgmii1-pins { + pins = "rgmii1_clk_125", "rgmii1_txen", "rgmii1_txclk", "rgmii1_txd0", + "rgmii1_txd1", "rgmii1_txd2", "rgmii1_txd3", "rgmii1_rxclk", + "rgmii1_rxdv", "rgmii1_rxd0", "rgmii1_rxd1", "rgmii1_rxd2", + "rgmii1_rxd3", "rgmii1_mdc", "rgmii1_mdio", "rgmii1_intb"; + function = "rgmii"; + }; + + /omit-if-no-ref/ + i2s0_pins: i2s0-pins { + pins = "i2s0_bclk", "i2s0_wclk", "i2s0_sdi", "i2s0_sdo", "i2s_mclk"; + function = "i2s"; + }; + + /omit-if-no-ref/ + i2s1_pins: i2s1-pins { + pins = "i2s1_bclk", "i2s1_wclk", "i2s1_sdi", "i2s1_sdo", "i2s_mclk"; + function = "i2s"; + }; + + /omit-if-no-ref/ + i2s2_pins: i2s2-pins { + pins = "i2s2_bclk", "i2s2_wclk", "i2s2_sdi", "i2s2_sdo", "i2s_mclk"; + function = "i2s"; + }; + + /omit-if-no-ref/ + usb0_pwren_pins: usb0-pwren-pins { + pins = "usb0_pwren"; + function = "usb"; + }; + + /omit-if-no-ref/ + usb1_pwren_pins: usb1-pwren-pins { + pins = "usb1_pwren"; + function = "usb"; + }; + + /omit-if-no-ref/ + i2c0_pins: i2c0-pins { + pins = "i2c0_scl", "i2c0_sda"; + function = "i2c"; + }; + + /omit-if-no-ref/ + i2c1_pins: i2c1-pins { + pins = "i2c1_scl", "i2c1_sda"; + function = "i2c"; + }; + + /omit-if-no-ref/ + i2c2_pins: i2c2-pins { + pins = "i2c2_scl", "i2c2_sda"; + function = "i2c"; + }; + + /omit-if-no-ref/ + i2c3_pins: i2c3-pins { + pins = "i2c3_scl", "i2c3_sda"; + function = "i2c"; + }; + + /omit-if-no-ref/ + i2c4_pins: i2c4-pins { + pins = "i2c4_scl", "i2c4_sda"; + function = "i2c"; + }; + + /omit-if-no-ref/ + i2c5_pins: i2c5-pins { + pins = "i2c5_scl", "i2c5_sda"; + function = "i2c"; + }; + + /omit-if-no-ref/ + i2c8_pins: i2c8-pins { + pins = "gpio92", "gpio93"; + function = "i2c"; + }; + + /omit-if-no-ref/ + i2c10_pins: i2c10-pins { + pins = "i2c10_scl", "i2c10_sda"; + function = "i2c"; + }; + + /omit-if-no-ref/ + i2c11_pins: i2c11-pins { + pins = "i2c11_scl", "i2c11_sda"; + function = "i2c"; + }; + + /omit-if-no-ref/ + uart0_pins: uart0-pins { + pins = "uart0_tx", "uart0_rx"; + function = "uart"; + }; + + /omit-if-no-ref/ + uart1_pins: uart1-pins { + pins = "uart1_tx", "uart1_rx"; + function = "uart"; + }; + + /omit-if-no-ref/ + uart1_cts_rts_pins: uart1-cts-rts-pins { + pins = "uart1_cts", "uart1_rts"; + function = "uart"; + }; + + /omit-if-no-ref/ + uart2_pins: uart2-pins { + pins = "uart2_tx", "uart2_rx"; + function = "uart"; + }; + + /omit-if-no-ref/ + fan_pwm_pins: fan-pwm-pins { + pins = "fan_pwm"; + function = "pwm"; + }; + + /omit-if-no-ref/ + fan_tach_pins: fan-tach-pins { + pins = "fan_tach"; + function = "fan_tach"; + }; + + /omit-if-no-ref/ + mipi_csi0_pins: mipi-csi0-pins { + pins = "mipi_csi0_xvs", "mipi_csi0_xhs", "mipi_csi0_mclk"; + function = "mipi_csi"; + }; + + /omit-if-no-ref/ + mipi_csi1_pins: mipi-csi1-pins { + pins = "mipi_csi1_xvs", "mipi_csi1_xhs", "mipi_csi1_mclk"; + function = "mipi_csi"; + }; + + /omit-if-no-ref/ + mipi_csi2_pins: mipi-csi2-pins { + pins = "mipi_csi2_xvs", "mipi_csi2_xhs", "mipi_csi2_mclk"; + function = "mipi_csi"; + }; + + /omit-if-no-ref/ + mipi_csi3_pins: mipi-csi3-pins { + pins = "mipi_csi3_xvs", "mipi_csi3_xhs", "mipi_csi3_mclk"; + function = "mipi_csi"; + }; + + /omit-if-no-ref/ + mipi_csi4_pins: mipi-csi4-pins { + pins = "mipi_csi4_xvs", "mipi_csi4_xhs", "mipi_csi4_mclk"; + function = "mipi_csi"; + }; + + /omit-if-no-ref/ + mipi_csi5_pins: mipi-csi5-pins { + pins = "mipi_csi5_xvs", "mipi_csi5_xhs", "mipi_csi5_mclk"; + function = "mipi_csi"; + }; + + /omit-if-no-ref/ + s_mode_pins: s-mode-pins { + pins = "s_mode"; + function = "s_mode"; + }; + + /omit-if-no-ref/ + boot_sel0_pins: boot-sel0-pins { + pins = "boot_sel0"; + function = "boot_sel"; + }; + + /omit-if-no-ref/ + boot_sel1_pins: boot-sel1-pins { + pins = "boot_sel1"; + function = "boot_sel"; + }; + + /omit-if-no-ref/ + boot_sel2_pins: boot-sel2-pins { + pins = "boot_sel2"; + function = "boot_sel"; + }; + + /omit-if-no-ref/ + boot_sel3_pins: boot-sel3-pins { + pins = "boot_sel3"; + function = "boot_sel"; + }; + + /omit-if-no-ref/ + ddr_ref_clk_sel_pins: ddr-ref-clk-sel-pins { + pins = "gpio95"; + function = "ddr_ref_clk_sel"; + }; + + /omit-if-no-ref/ + lpddr_ref_clk_pins: lpddr-ref-clk-pins { + pins = "lpddr_ref_clk"; + function = "lpddr_ref_clk"; + }; + + /omit-if-no-ref/ + spi0_pins: spi0-pins { + pins = "spi0_cs_n", "spi0_clk", "spi0_d0", "spi0_d1", "spi0_d2", + "spi0_d3"; + function = "spi"; + }; + + /omit-if-no-ref/ + dual_spi1_pins: dual-spi1-pins { + pins = "spi1_cs0_n", "spi1_clk", "spi1_d0", "spi1_d1"; + function = "spi"; + }; + + /omit-if-no-ref/ + quad_spi1_pins: quad-spi1-pins { + pins = "spi1_cs1_n", "spi1_d2", "spi1_d3"; + function = "spi"; + }; + + /omit-if-no-ref/ + spi3_pins: spi3-pins { + pins = "spi3_cs_n", "spi3_clk", "spi3_di", "spi3_do"; + function = "spi"; + }; + }; +}; diff --git a/arch/riscv/boot/dts/eswin/eic7700.dtsi b/arch/riscv/boot/dts/eswin/eic7700.dtsi index c3ed93008bca..0aee4041d8c7 100644 --- a/arch/riscv/boot/dts/eswin/eic7700.dtsi +++ b/arch/riscv/boot/dts/eswin/eic7700.dtsi @@ -4,10 +4,18 @@ */ /dts-v1/; +#include +#include +#include +#include +#include / { #address-cells = <2>; #size-cells = <2>; + riscv,physical-memory-regions = + <0x000 0x80000000 0x00f 0x80000000 (PMA_RWXA | PMA_NONCOHERENT_MEMORY) 0x0>, + <0x0c0 0x00000000 0x010 0x00000000 (PMA_RWX | PMA_NONCACHEABLE_MEMORY | PMR_ALIAS(0)) 0x0>; cpus { #address-cells = <1>; @@ -202,6 +210,11 @@ pmu { <0x00000000 0x0000000f 0xfffffffc 0x000000ff 0x00000078>; }; + osc: oscillator { + compatible = "fixed-clock"; + #clock-cells = <0>; + }; + soc { compatible = "simple-bus"; ranges; @@ -245,6 +258,184 @@ plic: interrupt-controller@c000000 { #interrupt-cells = <1>; }; + hsp: bus@50400000 { + compatible = "simple-pm-bus"; + ranges = <0x0 0x50400000 0x0 0x50400000 0x0 0xa0000>; + clocks = <&clk EIC7700_CLK_GATE_HSP_CFG_CLK>; + #address-cells = <2>; + #size-cells = <2>; + + gmac0: ethernet@50400000 { + compatible = "eswin,eic7700-qos-eth", + "snps,dwmac-5.20"; + reg = <0x0 0x50400000 0x0 0x10000>; + interrupts = <61>; + interrupt-names = "macirq"; + clocks = <&clk EIC7700_CLK_GATE_HSP_ACLK>, + <&clk EIC7700_CLK_GATE_HSP_CFG_CLK>, + <&clk EIC7700_CLK_MUX_ETH_CORE_2MUX1>, + <&clk EIC7700_CLK_GATE_HSP_ETH0_CORE_CLK>; + clock-names = "axi", "cfg", "stmmaceth", "tx"; + resets = <&reset EIC7700_RESET_HSP_ETH0_ARST>; + reset-names = "stmmaceth"; + eswin,hsp-sp-csr = <&hspcrg 0x100 0x108 0x118 0x114 0x11c>; + snps,aal; + snps,fixed-burst; + snps,tso; + snps,axi-config = <&stmmac_axi_setup_gmac0>; + status = "disabled"; + + gmac0_mdio: mdio { + compatible = "snps,dwmac-mdio"; + #address-cells = <1>; + #size-cells = <0>; + }; + + stmmac_axi_setup_gmac0: stmmac-axi-config { + snps,blen = <0 0 0 0 16 8 4>; + snps,rd_osr_lmt = <2>; + snps,wr_osr_lmt = <2>; + }; + }; + + gmac1: ethernet@50410000 { + compatible = "eswin,eic7700-qos-eth-clk-inversion", + "snps,dwmac-5.20"; + reg = <0x0 0x50410000 0x0 0x10000>; + interrupts = <70>; + interrupt-names = "macirq"; + clocks = <&clk EIC7700_CLK_GATE_HSP_ACLK>, + <&clk EIC7700_CLK_GATE_HSP_CFG_CLK>, + <&clk EIC7700_CLK_MUX_ETH_CORE_2MUX1>, + <&clk EIC7700_CLK_GATE_HSP_ETH1_CORE_CLK>; + clock-names = "axi", "cfg", "stmmaceth", "tx"; + resets = <&reset EIC7700_RESET_HSP_ETH1_ARST>; + reset-names = "stmmaceth"; + eswin,hsp-sp-csr = <&hspcrg 0x200 0x208 0x218 0x214 0x21c>; + snps,aal; + snps,fixed-burst; + snps,tso; + snps,axi-config = <&stmmac_axi_setup_gmac1>; + status = "disabled"; + + gmac1_mdio: mdio { + compatible = "snps,dwmac-mdio"; + #address-cells = <1>; + #size-cells = <0>; + }; + + stmmac_axi_setup_gmac1: stmmac-axi-config { + snps,blen = <0 0 0 0 16 8 4>; + snps,rd_osr_lmt = <2>; + snps,wr_osr_lmt = <2>; + }; + }; + + hspcrg: clock-controller@50440000 { + compatible = "eswin,eic7700-hspcrg", "syscon"; + reg = <0x0 0x50440000 0x0 0x2000>; + clocks = <&clk EIC7700_CLK_GATE_HSP_CFG_CLK>, + <&clk EIC7700_CLK_FIXED_FACTOR_CLK_1M_DIV24>, + <&clk EIC7700_CLK_GATE_HSP_SATA_OOB_CLK>; + clock-names = "cfg", "mmc", "sata"; + #clock-cells = <1>; + #reset-cells = <1>; + }; + + emmc: mmc@50450000 { + compatible = "eswin,eic7700-dwcmshc"; + reg = <0x0 0x50450000 0x0 0x10000>; + interrupts = <79>; + clocks = <&clk EIC7700_CLK_GATE_HSP_MSHC0_CORE_CLK>, + <&hspcrg EIC7700_HSP_CLK_GATE_EMMC>, + <&clk EIC7700_CLK_GATE_HSP_ACLK>, + <&hspcrg EIC7700_HSP_CLK_GATE_MSHC0_TMR>; + clock-names = "core", "bus", "axi", "timer"; + resets = <&reset EIC7700_RESET_HSP_EMMC_ARST>, + <&reset EIC7700_RESET_MSHC0_PHY>, + <&reset EIC7700_RESET_HSP_EMMC_PRST>, + <&reset EIC7700_RESET_MSHC0_TXRX>; + reset-names = "axi", "phy", "prstn", "txrx"; + eswin,hsp-sp-csr = <&hspcrg 0x508 0x50c>; + eswin,drive-impedance-ohms = <50>; + status = "disabled"; + }; + + usb0: usb@50480000 { + compatible = "eswin,eic7700-dwc3"; + reg = <0x0 0x50480000 0x0 0x10000>; + interrupts = <85>; + interrupt-names = "peripheral"; + clocks = <&clk EIC7700_CLK_GATE_HSP_ACLK>, + <&clk EIC7700_CLK_GATE_HSP_CFG_CLK>, + <&hspcrg EIC7700_HSP_CLK_GATE_USB0>; + clock-names = "aclk", "cfg", "usb_en"; + resets = <&reset EIC7700_RESET_USB0_VAUX>, + <&hspcrg EIC7700_HSP_RST_USB0>, + <&hspcrg EIC7700_HSP_RST_USB0_PHY>; + reset-names = "vaux", "usb_rst", "usb_phy"; + eswin,hsp-sp-csr = <&hspcrg 0x800 0x818>; + phy_type = "utmi"; + maximum-speed = "high-speed"; + status = "disabled"; + }; + + usb1: usb@50490000 { + compatible = "eswin,eic7700-dwc3"; + reg = <0x0 0x50490000 0x0 0x10000>; + interrupts = <86>; + interrupt-names = "peripheral"; + clocks = <&clk EIC7700_CLK_GATE_HSP_ACLK>, + <&clk EIC7700_CLK_GATE_HSP_CFG_CLK>, + <&hspcrg EIC7700_HSP_CLK_GATE_USB1>; + clock-names = "aclk", "cfg", "usb_en"; + resets = <&reset EIC7700_RESET_USB1_VAUX>, + <&hspcrg EIC7700_HSP_RST_USB1>, + <&hspcrg EIC7700_HSP_RST_USB1_PHY>; + reset-names = "vaux", "usb_rst", "usb_phy"; + eswin,hsp-sp-csr = <&hspcrg 0x900 0x918>; + phy_type = "utmi"; + maximum-speed = "high-speed"; + status = "disabled"; + }; + }; + + wdt0: watchdog@50800000 { + compatible = "snps,dw-wdt"; + reg = <0x0 0x50800000 0x0 0x4000>; + interrupts = <87>; + clocks = <&clk EIC7700_CLK_GATE_LSP_WDT0_PCLK>; + clock-names = "tclk"; + resets = <&reset EIC7700_RESET_WDT0>; + }; + + wdt1: watchdog@50804000 { + compatible = "snps,dw-wdt"; + reg = <0x0 0x50804000 0x0 0x4000>; + interrupts = <88>; + clocks = <&clk EIC7700_CLK_GATE_LSP_WDT1_PCLK>; + clock-names = "tclk"; + resets = <&reset EIC7700_RESET_WDT1>; + }; + + wdt2: watchdog@50808000 { + compatible = "snps,dw-wdt"; + reg = <0x0 0x50808000 0x0 0x4000>; + interrupts = <89>; + clocks = <&clk EIC7700_CLK_GATE_LSP_WDT2_PCLK>; + clock-names = "tclk"; + resets = <&reset EIC7700_RESET_WDT2>; + }; + + wdt3: watchdog@5080c000 { + compatible = "snps,dw-wdt"; + reg = <0x0 0x5080c000 0x0 0x4000>; + interrupts = <90>; + clocks = <&clk EIC7700_CLK_GATE_LSP_WDT3_PCLK>; + clock-names = "tclk"; + resets = <&reset EIC7700_RESET_WDT3>; + }; + uart0: serial@50900000 { compatible = "snps,dw-apb-uart"; reg = <0x0 0x50900000 0x0 0x10000>; @@ -295,7 +486,157 @@ uart4: serial@50940000 { status = "disabled"; }; - gpio@51600000 { + i2c0: i2c@50950000 { + compatible = "eswin,eic7700-i2c", "snps,designware-i2c"; + reg = <0x0 0x50950000 0x0 0x8000>; + interrupts = <105>; + clocks = <&clk EIC7700_CLK_GATE_LSP_I2C0_PCLK>; + clock-names = "ref"; + clock-frequency = <100000>; + pinctrl-names = "default"; + pinctrl-0 = <&i2c0_pins>; + resets = <&reset EIC7700_RESET_I2C0>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c1: i2c@50960000 { + compatible = "eswin,eic7700-i2c", "snps,designware-i2c"; + reg = <0x0 0x50960000 0x0 0x8000>; + interrupts = <106>; + clocks = <&clk EIC7700_CLK_GATE_LSP_I2C1_PCLK>; + clock-names = "ref"; + clock-frequency = <100000>; + pinctrl-names = "default"; + pinctrl-0 = <&i2c1_pins>; + resets = <&reset EIC7700_RESET_I2C1>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c2: i2c@50970000 { + compatible = "eswin,eic7700-i2c", "snps,designware-i2c"; + reg = <0x0 0x50970000 0x0 0x8000>; + interrupts = <107>; + clocks = <&clk EIC7700_CLK_GATE_LSP_I2C2_PCLK>; + clock-names = "ref"; + clock-frequency = <100000>; + pinctrl-names = "default"; + pinctrl-0 = <&i2c2_pins>; + resets = <&reset EIC7700_RESET_I2C2>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c3: i2c@50980000 { + compatible = "eswin,eic7700-i2c", "snps,designware-i2c"; + reg = <0x0 0x50980000 0x0 0x8000>; + interrupts = <108>; + clocks = <&clk EIC7700_CLK_GATE_LSP_I2C3_PCLK>; + clock-names = "ref"; + clock-frequency = <100000>; + pinctrl-names = "default"; + pinctrl-0 = <&i2c3_pins>; + resets = <&reset EIC7700_RESET_I2C3>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c4: i2c@50990000 { + compatible = "eswin,eic7700-i2c", "snps,designware-i2c"; + reg = <0x0 0x50990000 0x0 0x8000>; + interrupts = <109>; + clocks = <&clk EIC7700_CLK_GATE_LSP_I2C4_PCLK>; + clock-names = "ref"; + clock-frequency = <100000>; + pinctrl-names = "default"; + pinctrl-0 = <&i2c4_pins>; + resets = <&reset EIC7700_RESET_I2C4>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c5: i2c@509a0000 { + compatible = "eswin,eic7700-i2c", "snps,designware-i2c"; + reg = <0x0 0x509a0000 0x0 0x8000>; + interrupts = <110>; + clocks = <&clk EIC7700_CLK_GATE_LSP_I2C5_PCLK>; + clock-names = "ref"; + clock-frequency = <100000>; + pinctrl-names = "default"; + pinctrl-0 = <&i2c5_pins>; + resets = <&reset EIC7700_RESET_I2C5>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c6: i2c@509b0000 { + compatible = "eswin,eic7700-i2c", "snps,designware-i2c"; + reg = <0x0 0x509b0000 0x0 0x8000>; + interrupts = <111>; + clocks = <&clk EIC7700_CLK_GATE_LSP_I2C6_PCLK>; + clock-names = "ref"; + clock-frequency = <100000>; + pinctrl-names = "default"; + pinctrl-0 = <&i2c6_pins>; + resets = <&reset EIC7700_RESET_I2C6>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c7: i2c@509c0000 { + compatible = "eswin,eic7700-i2c", "snps,designware-i2c"; + reg = <0x0 0x509c0000 0x0 0x8000>; + interrupts = <112>; + clocks = <&clk EIC7700_CLK_GATE_LSP_I2C7_PCLK>; + clock-names = "ref"; + clock-frequency = <100000>; + pinctrl-names = "default"; + pinctrl-0 = <&i2c7_pins>; + resets = <&reset EIC7700_RESET_I2C7>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c8: i2c@509d0000 { + compatible = "eswin,eic7700-i2c", "snps,designware-i2c"; + reg = <0x0 0x509d0000 0x0 0x8000>; + interrupts = <113>; + clocks = <&clk EIC7700_CLK_GATE_LSP_I2C8_PCLK>; + clock-names = "ref"; + clock-frequency = <100000>; + pinctrl-names = "default"; + pinctrl-0 = <&i2c8_pins>; + resets = <&reset EIC7700_RESET_I2C8>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + i2c9: i2c@509e0000 { + compatible = "eswin,eic7700-i2c", "snps,designware-i2c"; + reg = <0x0 0x509e0000 0x0 0x8000>; + interrupts = <114>; + clocks = <&clk EIC7700_CLK_GATE_LSP_I2C9_PCLK>; + clock-names = "ref"; + clock-frequency = <100000>; + pinctrl-names = "default"; + pinctrl-0 = <&i2c9_pins>; + resets = <&reset EIC7700_RESET_I2C9>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + gpio: gpio@51600000 { compatible = "snps,dw-apb-gpio"; reg = <0x0 0x51600000 0x0 0x80>; #address-cells = <1>; @@ -341,5 +682,96 @@ gpioD: gpio-port@3 { #gpio-cells = <2>; }; }; + + pinctrl: pinctrl@51600080 { + compatible = "eswin,eic7700-pinctrl"; + reg = <0x0 0x51600080 0x0 0x1fff80>; + }; + + clk: clock-controller@51828000 { + compatible = "eswin,eic7700-clock"; + reg = <0x0 0x51828000 0x0 0x300>; + clocks = <&osc>; + #clock-cells = <1>; + }; + + reset: reset-controller@51828300 { + compatible = "eswin,eic7700-reset"; + reg = <0x0 0x51828300 0x0 0x200>; + #reset-cells = <1>; + }; + + aon_i2c0: i2c@51830000 { + compatible = "eswin,eic7700-i2c", "snps,designware-i2c"; + reg = <0x0 0x51830000 0x0 0x8000>; + interrupts = <290>; + clocks = <&clk EIC7700_CLK_GATE_AON_I2C0_PCLK>; + clock-names = "ref"; + clock-frequency = <100000>; + pinctrl-names = "default"; + pinctrl-0 = <&i2c10_pins>; + resets = <&reset EIC7700_RESET_ANO0>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + aon_i2c1: i2c@51838000 { + compatible = "eswin,eic7700-i2c", "snps,designware-i2c"; + reg = <0x0 0x51838000 0x0 0x8000>; + interrupts = <291>; + clocks = <&clk EIC7700_CLK_GATE_AON_I2C1_PCLK>; + clock-names = "ref"; + clock-frequency = <100000>; + pinctrl-names = "default"; + pinctrl-0 = <&i2c11_pins>; + resets = <&reset EIC7700_RESET_ANO1>; + #address-cells = <1>; + #size-cells = <0>; + status = "disabled"; + }; + + pcie: pcie@54000000 { + compatible = "eswin,eic7700-pcie"; + reg = <0x0 0x54000000 0x0 0x4000000>, + <0x0 0x40000000 0x0 0x800000>, + <0x0 0x50000000 0x0 0x100000>; + reg-names = "dbi", "config", "elbi"; + #address-cells = <3>; + #size-cells = <2>; + #interrupt-cells = <1>; + ranges = <0x01000000 0x0 0x40800000 0x0 0x40800000 0x0 0x800000>, + <0x02000000 0x0 0x41000000 0x0 0x41000000 0x0 0xf000000>, + <0x43000000 0x80 0x00000000 0x80 0x00000000 0x2 0x00000000>; + bus-range = <0x0 0xff>; + clocks = <&clk EIC7700_CLK_GATE_PCIET_ACLK>, + <&clk EIC7700_CLK_GATE_PCIET_CFG_CLK>, + <&clk EIC7700_CLK_GATE_PCIET_CR_CLK>, + <&clk EIC7700_CLK_GATE_PCIET_AUX_CLK>; + clock-names = "mstr", "dbi", "phy_reg", "aux"; + resets = <&reset EIC7700_RESET_PCIE_CFG>, + <&reset EIC7700_RESET_PCIE_POWEUP>; + reset-names = "dbi", "pwr"; + interrupts = <220>, <179>, <180>, <181>, <182>; + interrupt-names = "msi", "inta", "intb", "intc", "intd"; + interrupt-map-mask = <0x0 0x0 0x0 0x7>; + interrupt-map = <0x0 0x0 0x0 0x1 &plic 179>, + <0x0 0x0 0x0 0x2 &plic 180>, + <0x0 0x0 0x0 0x3 &plic 181>, + <0x0 0x0 0x0 0x4 &plic 182>; + device_type = "pci"; + status = "disabled"; + + pcie_port: pcie@0 { + reg = <0x0 0x0 0x0 0x0 0x0>; + #address-cells = <3>; + #size-cells = <2>; + ranges; + device_type = "pci"; + num-lanes = <4>; + resets = <&reset EIC7700_RESET_PCIE_PERST>; + reset-names = "perst"; + }; + }; }; }; diff --git a/arch/riscv/boot/dts/starfive/jh7100-common.dtsi b/arch/riscv/boot/dts/starfive/jh7100-common.dtsi index ae1a6aeb0aea..47d0cf55bfc0 100644 --- a/arch/riscv/boot/dts/starfive/jh7100-common.dtsi +++ b/arch/riscv/boot/dts/starfive/jh7100-common.dtsi @@ -42,30 +42,6 @@ led-ack { }; }; - reserved-memory { - #address-cells = <2>; - #size-cells = <2>; - ranges; - - dma-reserved@fa000000 { - reg = <0x0 0xfa000000 0x0 0x1000000>; - no-map; - }; - - linux,dma@107a000000 { - compatible = "shared-dma-pool"; - reg = <0x10 0x7a000000 0x0 0x1000000>; - no-map; - linux,dma-default; - }; - }; - - soc { - dma-ranges = <0x00 0x80000000 0x00 0x80000000 0x00 0x7a000000>, - <0x00 0xfa000000 0x10 0x7a000000 0x00 0x01000000>, - <0x00 0xfb000000 0x00 0xfb000000 0x07 0x85000000>; - }; - wifi_pwrseq: wifi-pwrseq { compatible = "mmc-pwrseq-simple"; reset-gpios = <&gpio 37 GPIO_ACTIVE_LOW>; diff --git a/arch/riscv/boot/dts/starfive/jh7100.dtsi b/arch/riscv/boot/dts/starfive/jh7100.dtsi index 7de0732b8eab..c7d7ec9ed8c9 100644 --- a/arch/riscv/boot/dts/starfive/jh7100.dtsi +++ b/arch/riscv/boot/dts/starfive/jh7100.dtsi @@ -7,11 +7,15 @@ /dts-v1/; #include #include +#include / { compatible = "starfive,jh7100"; #address-cells = <2>; #size-cells = <2>; + riscv,physical-memory-regions = + <0x00 0x80000000 0x08 0x00000000 (PMA_RWXA | PMA_NONCOHERENT_MEMORY) 0x0>, + <0x10 0x00000000 0x08 0x00000000 (PMA_RWX | PMA_NONCACHEABLE_MEMORY | PMR_ALIAS(0)) 0x0>; cpus: cpus { #address-cells = <1>; diff --git a/arch/riscv/include/asm/alternative-macros.h b/arch/riscv/include/asm/alternative-macros.h index 9619bd5c8eba..e8bf384da5c2 100644 --- a/arch/riscv/include/asm/alternative-macros.h +++ b/arch/riscv/include/asm/alternative-macros.h @@ -50,8 +50,17 @@ ALT_NEW_CONTENT \vendor_id_2, \patch_id_2, \enable_2, "\new_c_2" .endm +.macro ALTERNATIVE_CFG_3 old_c, new_c_1, vendor_id_1, patch_id_1, enable_1, \ + new_c_2, vendor_id_2, patch_id_2, enable_2, \ + new_c_3, vendor_id_3, patch_id_3, enable_3 + ALTERNATIVE_CFG_2 "\old_c", "\new_c_1", \vendor_id_1, \patch_id_1, \enable_1 \ + "\new_c_2", \vendor_id_2, \patch_id_2, \enable_2 + ALT_NEW_CONTENT \vendor_id_3, \patch_id_3, \enable_3, "\new_c_3" +.endm + #define __ALTERNATIVE_CFG(...) ALTERNATIVE_CFG __VA_ARGS__ #define __ALTERNATIVE_CFG_2(...) ALTERNATIVE_CFG_2 __VA_ARGS__ +#define __ALTERNATIVE_CFG_3(...) ALTERNATIVE_CFG_3 __VA_ARGS__ #else /* !__ASSEMBLER__ */ @@ -98,6 +107,13 @@ __ALTERNATIVE_CFG(old_c, new_c_1, vendor_id_1, patch_id_1, enable_1) \ ALT_NEW_CONTENT(vendor_id_2, patch_id_2, enable_2, new_c_2) +#define __ALTERNATIVE_CFG_3(old_c, new_c_1, vendor_id_1, patch_id_1, enable_1, \ + new_c_2, vendor_id_2, patch_id_2, enable_2, \ + new_c_3, vendor_id_3, patch_id_3, enable_3) \ + __ALTERNATIVE_CFG_2(old_c, new_c_1, vendor_id_1, patch_id_1, enable_1, \ + new_c_2, vendor_id_2, patch_id_2, enable_2) \ + ALT_NEW_CONTENT(vendor_id_3, patch_id_3, enable_3, new_c_3) + #endif /* __ASSEMBLER__ */ #define _ALTERNATIVE_CFG(old_c, new_c, vendor_id, patch_id, CONFIG_k) \ @@ -108,6 +124,13 @@ __ALTERNATIVE_CFG_2(old_c, new_c_1, vendor_id_1, patch_id_1, IS_ENABLED(CONFIG_k_1), \ new_c_2, vendor_id_2, patch_id_2, IS_ENABLED(CONFIG_k_2)) +#define _ALTERNATIVE_CFG_3(old_c, new_c_1, vendor_id_1, patch_id_1, CONFIG_k_1, \ + new_c_2, vendor_id_2, patch_id_2, CONFIG_k_2, \ + new_c_3, vendor_id_3, patch_id_3, CONFIG_k_3) \ + __ALTERNATIVE_CFG_3(old_c, new_c_1, vendor_id_1, patch_id_1, IS_ENABLED(CONFIG_k_1), \ + new_c_2, vendor_id_2, patch_id_2, IS_ENABLED(CONFIG_k_2), \ + new_c_3, vendor_id_3, patch_id_3, IS_ENABLED(CONFIG_k_3)) + #else /* CONFIG_RISCV_ALTERNATIVE */ #ifdef __ASSEMBLER__ @@ -118,11 +141,17 @@ #define __ALTERNATIVE_CFG(old_c, ...) ALTERNATIVE_CFG old_c #define __ALTERNATIVE_CFG_2(old_c, ...) ALTERNATIVE_CFG old_c +#define _ALTERNATIVE_CFG_3(old_c, ...) \ + ALTERNATIVE_CFG old_c + #else /* !__ASSEMBLER__ */ #define __ALTERNATIVE_CFG(old_c, ...) old_c "\n" #define __ALTERNATIVE_CFG_2(old_c, ...) old_c "\n" +#define _ALTERNATIVE_CFG_3(old_c, ...) \ + __ALTERNATIVE_CFG(old_c) + #endif /* __ASSEMBLER__ */ #define _ALTERNATIVE_CFG(old_c, ...) __ALTERNATIVE_CFG(old_c) @@ -147,15 +176,21 @@ _ALTERNATIVE_CFG(old_content, new_content, vendor_id, patch_id, CONFIG_k) /* - * A vendor wants to replace an old_content, but another vendor has used - * ALTERNATIVE() to patch its customized content at the same location. In - * this case, this vendor can create a new macro ALTERNATIVE_2() based - * on the following sample code and then replace ALTERNATIVE() with - * ALTERNATIVE_2() to append its customized content. + * Variant of ALTERNATIVE() that supports two sets of replacement content. */ #define ALTERNATIVE_2(old_content, new_content_1, vendor_id_1, patch_id_1, CONFIG_k_1, \ new_content_2, vendor_id_2, patch_id_2, CONFIG_k_2) \ _ALTERNATIVE_CFG_2(old_content, new_content_1, vendor_id_1, patch_id_1, CONFIG_k_1, \ new_content_2, vendor_id_2, patch_id_2, CONFIG_k_2) +/* + * Variant of ALTERNATIVE() that supports three sets of replacement content. + */ +#define ALTERNATIVE_3(old_content, new_content_1, vendor_id_1, patch_id_1, CONFIG_k_1, \ + new_content_2, vendor_id_2, patch_id_2, CONFIG_k_2, \ + new_content_3, vendor_id_3, patch_id_3, CONFIG_k_3) \ + _ALTERNATIVE_CFG_3(old_content, new_content_1, vendor_id_1, patch_id_1, CONFIG_k_1, \ + new_content_2, vendor_id_2, patch_id_2, CONFIG_k_2, \ + new_content_3, vendor_id_3, patch_id_3, CONFIG_k_3) + #endif diff --git a/arch/riscv/include/asm/errata_list.h b/arch/riscv/include/asm/errata_list.h index 6694b5ccdcf8..fa03021b7074 100644 --- a/arch/riscv/include/asm/errata_list.h +++ b/arch/riscv/include/asm/errata_list.h @@ -53,51 +53,6 @@ asm(ALTERNATIVE( \ : /* no inputs */ \ : "memory") -/* - * _val is marked as "will be overwritten", so need to set it to 0 - * in the default case. - */ -#define ALT_SVPBMT_SHIFT 61 -#define ALT_THEAD_MAE_SHIFT 59 -#define ALT_SVPBMT(_val, prot) \ -asm(ALTERNATIVE_2("li %0, 0\t\nnop", \ - "li %0, %1\t\nslli %0,%0,%3", 0, \ - RISCV_ISA_EXT_SVPBMT, CONFIG_RISCV_ISA_SVPBMT, \ - "li %0, %2\t\nslli %0,%0,%4", THEAD_VENDOR_ID, \ - ERRATA_THEAD_MAE, CONFIG_ERRATA_THEAD_MAE) \ - : "=r"(_val) \ - : "I"(prot##_SVPBMT >> ALT_SVPBMT_SHIFT), \ - "I"(prot##_THEAD >> ALT_THEAD_MAE_SHIFT), \ - "I"(ALT_SVPBMT_SHIFT), \ - "I"(ALT_THEAD_MAE_SHIFT)) - -#ifdef CONFIG_ERRATA_THEAD_MAE -/* - * IO/NOCACHE memory types are handled together with svpbmt, - * so on T-Head chips, check if no other memory type is set, - * and set the non-0 PMA type if applicable. - */ -#define ALT_THEAD_PMA(_val) \ -asm volatile(ALTERNATIVE( \ - __nops(7), \ - "li t3, %1\n\t" \ - "slli t3, t3, %3\n\t" \ - "and t3, %0, t3\n\t" \ - "bne t3, zero, 2f\n\t" \ - "li t3, %2\n\t" \ - "slli t3, t3, %3\n\t" \ - "or %0, %0, t3\n\t" \ - "2:", THEAD_VENDOR_ID, \ - ERRATA_THEAD_MAE, CONFIG_ERRATA_THEAD_MAE) \ - : "+r"(_val) \ - : "I"(_PAGE_MTMASK_THEAD >> ALT_THEAD_MAE_SHIFT), \ - "I"(_PAGE_PMA_THEAD >> ALT_THEAD_MAE_SHIFT), \ - "I"(ALT_THEAD_MAE_SHIFT) \ - : "t3") -#else -#define ALT_THEAD_PMA(_val) -#endif - #define ALT_CMO_OP(_op, _start, _size, _cachesize) \ asm volatile(ALTERNATIVE( \ __nops(5), \ diff --git a/arch/riscv/include/asm/hwcap.h b/arch/riscv/include/asm/hwcap.h index f8db798b2654..f57be99ca162 100644 --- a/arch/riscv/include/asm/hwcap.h +++ b/arch/riscv/include/asm/hwcap.h @@ -123,6 +123,7 @@ #define RISCV_ISA_EXT_ZICCIF 114 #define RISCV_ISA_EXT_ZA64RS 115 +#define RISCV_ISA_EXT_XLINUXMEMALIAS 126 #define RISCV_ISA_EXT_XLINUXENVCFG 127 #define RISCV_ISA_EXT_MAX 128 diff --git a/arch/riscv/include/asm/pgtable-32.h b/arch/riscv/include/asm/pgtable-32.h index 00f3369570a8..eb556ab95732 100644 --- a/arch/riscv/include/asm/pgtable-32.h +++ b/arch/riscv/include/asm/pgtable-32.h @@ -17,21 +17,8 @@ #define MAX_POSSIBLE_PHYSMEM_BITS 34 -/* - * rv32 PTE format: - * | XLEN-1 10 | 9 8 | 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0 - * PFN reserved for SW D A G U X W R V - */ -#define _PAGE_PFN_MASK GENMASK(31, 10) - -#define _PAGE_NOCACHE 0 -#define _PAGE_IO 0 -#define _PAGE_MTMASK 0 - -/* Set of bits to preserve across pte_modify() */ -#define _PAGE_CHG_MASK (~(unsigned long)(_PAGE_PRESENT | _PAGE_READ | \ - _PAGE_WRITE | _PAGE_EXEC | \ - _PAGE_USER | _PAGE_GLOBAL)) +#define ALT_FIXUP_MT(_val) +#define ALT_UNFIX_MT(_val) static const __maybe_unused int pgtable_l4_enabled; static const __maybe_unused int pgtable_l5_enabled; diff --git a/arch/riscv/include/asm/pgtable-64.h b/arch/riscv/include/asm/pgtable-64.h index 6e789fa58514..f1ecd022e3ee 100644 --- a/arch/riscv/include/asm/pgtable-64.h +++ b/arch/riscv/include/asm/pgtable-64.h @@ -8,7 +8,7 @@ #include #include -#include +#include extern bool pgtable_l4_enabled; extern bool pgtable_l5_enabled; @@ -66,25 +66,10 @@ typedef struct { #define pmd_val(x) ((x).pmd) #define __pmd(x) ((pmd_t) { (x) }) - #define PTRS_PER_PMD (PAGE_SIZE / sizeof(pmd_t)) #define MAX_POSSIBLE_PHYSMEM_BITS 56 -/* - * rv64 PTE format: - * | 63 | 62 61 | 60 54 | 53 10 | 9 8 | 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0 - * N MT RSV PFN reserved for SW D A G U X W R V - */ -#define _PAGE_PFN_MASK GENMASK(53, 10) - -/* - * [63] Svnapot definitions: - * 0 Svnapot disabled - * 1 Svnapot enabled - */ -#define _PAGE_NAPOT_SHIFT 63 -#define _PAGE_NAPOT BIT(_PAGE_NAPOT_SHIFT) /* * Only 64KB (order 4) napot ptes supported. */ @@ -112,67 +97,174 @@ enum napot_cont_order { #define HUGE_MAX_HSTATE 2 #endif -/* - * [62:61] Svpbmt Memory Type definitions: - * - * 00 - PMA Normal Cacheable, No change to implied PMA memory type - * 01 - NC Non-cacheable, idempotent, weakly-ordered Main Memory - * 10 - IO Non-cacheable, non-idempotent, strongly-ordered I/O memory - * 11 - Rsvd Reserved for future standard use - */ -#define _PAGE_NOCACHE_SVPBMT (1UL << 61) -#define _PAGE_IO_SVPBMT (1UL << 62) -#define _PAGE_MTMASK_SVPBMT (_PAGE_NOCACHE_SVPBMT | _PAGE_IO_SVPBMT) +#if defined(CONFIG_RISCV_ISA_SVPBMT) || defined(CONFIG_RISCV_ISA_XLINUXMEMALIAS) || \ + defined(CONFIG_ERRATA_THEAD_MAE) /* + * ALT_FIXUP_MT + * + * On systems that do not support any form of page-based memory type + * configuration, this code sequence clears the memory type bits in the PTE. + * + * On systems that support Svpbmt, the memory type bits are left alone. + * + * On systems that support XLinuxMemalias, PTEs with a nonzero memory type have + * the memory type bits cleared and the PFN replaced with the matching alias. + * + * On systems that support XTheadMae, a Svpbmt memory type is transformed + * into the corresponding XTheadMae memory type. + * * [63:59] T-Head Memory Type definitions: * bit[63] SO - Strong Order * bit[62] C - Cacheable * bit[61] B - Bufferable * bit[60] SH - Shareable * bit[59] Sec - Trustable - * 00110 - NC Weakly-ordered, Non-cacheable, Bufferable, Shareable, Non-trustable * 01110 - PMA Weakly-ordered, Cacheable, Bufferable, Shareable, Non-trustable + * 00110 - NC Weakly-ordered, Non-cacheable, Bufferable, Shareable, Non-trustable * 10010 - IO Strongly-ordered, Non-cacheable, Non-bufferable, Shareable, Non-trustable + * + * Pseudocode operating on bits [63:60]: + * t0 = mt << 1 + * if (t0 == 0) + * t0 |= 2 + * t0 ^= 0x5 + * mt ^= t0 */ -#define _PAGE_PMA_THEAD ((1UL << 62) | (1UL << 61) | (1UL << 60)) -#define _PAGE_NOCACHE_THEAD ((1UL << 61) | (1UL << 60)) -#define _PAGE_IO_THEAD ((1UL << 63) | (1UL << 60)) -#define _PAGE_MTMASK_THEAD (_PAGE_PMA_THEAD | _PAGE_IO_THEAD | (1UL << 59)) -static inline u64 riscv_page_mtmask(void) +#define ALT_FIXUP_MT(_val) \ + asm(ALTERNATIVE_3("addi t0, zero, 0x3\n\t" \ + "slli t0, t0, 61\n\t" \ + "not t0, t0\n\t" \ + "and %0, %0, t0\n\t" \ + "nop\n\t" \ + "nop\n\t" \ + "nop\n\t" \ + "nop", \ + __nops(8), \ + 0, RISCV_ISA_EXT_SVPBMT, CONFIG_RISCV_ISA_SVPBMT, \ + "addi t0, zero, 0x3\n\t" \ + "slli t0, t0, 61\n\t" \ + "and t0, %0, t0\n\t" \ + "beqz t0, 2f\n\t" \ + "xor t1, %0, t0\n\t" \ + "1: auipc t0, %%pcrel_hi(riscv_fixup_memory_alias)\n\t" \ + "jalr t0, t0, %%pcrel_lo(1b)\n\t" \ + "mv %0, t1\n" \ + "2:", \ + 0, RISCV_ISA_EXT_XLINUXMEMALIAS, \ + CONFIG_RISCV_ISA_XLINUXMEMALIAS, \ + "srli t0, %0, 59\n\t" \ + "seqz t1, t0\n\t" \ + "slli t1, t1, 1\n\t" \ + "or t0, t0, t1\n\t" \ + "xori t0, t0, 0x5\n\t" \ + "slli t0, t0, 60\n\t" \ + "xor %0, %0, t0\n\t" \ + "nop", \ + THEAD_VENDOR_ID, ERRATA_THEAD_MAE, CONFIG_ERRATA_THEAD_MAE) \ + : "+r" (_val) :: "t0", "t1") + +#else + +#define ALT_FIXUP_MT(_val) + +#endif /* CONFIG_RISCV_ISA_SVPBMT || CONFIG_RISCV_ISA_XLINUXMEMALIAS || CONFIG_ERRATA_THEAD_MAE */ + +#if defined(CONFIG_RISCV_ISA_XLINUXMEMALIAS) || defined(CONFIG_ERRATA_THEAD_MAE) + +/* + * ALT_UNFIX_MT + * + * On systems that support Svpbmt, or do not support any form of page-based + * memory type configuration, the memory type bits are left alone. + * + * On systems that support XLinuxMemalias, PTEs with an aliased PFN have the + * matching memory type set and the PFN replaced with the normal memory alias. + * + * On systems that support XTheadMae, the XTheadMae memory type (or zero) is + * transformed back into the corresponding Svpbmt memory type. + * + * Pseudocode operating on bits [63:60]: + * t0 = mt & 0xd + * t0 ^= t0 >> 1 + * mt ^= t0 + */ + +#define ALT_UNFIX_MT(_val) \ + asm(ALTERNATIVE_2(__nops(6), \ + "mv t1, %0\n\t" \ + "1: auipc t0, %%pcrel_hi(riscv_unfix_memory_alias)\n\t" \ + "jalr t0, t0, %%pcrel_lo(1b)\n\t" \ + "mv %0, t1\n\t" \ + "nop\n\t" \ + "nop", \ + 0, RISCV_ISA_EXT_XLINUXMEMALIAS, \ + CONFIG_RISCV_ISA_XLINUXMEMALIAS, \ + "srli t0, %0, 60\n\t" \ + "andi t0, t0, 0xd\n\t" \ + "srli t1, t0, 1\n\t" \ + "xor t0, t0, t1\n\t" \ + "slli t0, t0, 60\n\t" \ + "xor %0, %0, t0", \ + THEAD_VENDOR_ID, ERRATA_THEAD_MAE, CONFIG_ERRATA_THEAD_MAE) \ + : "+r" (_val) :: "t0", "t1") + +#define ptep_get ptep_get +static inline pte_t ptep_get(pte_t *ptep) +{ + pte_t pte = READ_ONCE(*ptep); + + ALT_UNFIX_MT(pte); + + return pte; +} + +#define pmdp_get pmdp_get +static inline pmd_t pmdp_get(pmd_t *pmdp) +{ + pmd_t pmd = READ_ONCE(*pmdp); + + ALT_UNFIX_MT(pmd); + + return pmd; +} + +#define pudp_get pudp_get +static inline pud_t pudp_get(pud_t *pudp) { - u64 val; + pud_t pud = READ_ONCE(*pudp); + + ALT_UNFIX_MT(pud); - ALT_SVPBMT(val, _PAGE_MTMASK); - return val; + return pud; } -static inline u64 riscv_page_nocache(void) +#define p4dp_get p4dp_get +static inline p4d_t p4dp_get(p4d_t *p4dp) { - u64 val; + p4d_t p4d = READ_ONCE(*p4dp); - ALT_SVPBMT(val, _PAGE_NOCACHE); - return val; + ALT_UNFIX_MT(p4d); + + return p4d; } -static inline u64 riscv_page_io(void) +#define pgdp_get pgdp_get +static inline pgd_t pgdp_get(pgd_t *pgdp) { - u64 val; + pgd_t pgd = READ_ONCE(*pgdp); + + ALT_UNFIX_MT(pgd); - ALT_SVPBMT(val, _PAGE_IO); - return val; + return pgd; } -#define _PAGE_NOCACHE riscv_page_nocache() -#define _PAGE_IO riscv_page_io() -#define _PAGE_MTMASK riscv_page_mtmask() +#else + +#define ALT_UNFIX_MT(_val) -/* Set of bits to preserve across pte_modify() */ -#define _PAGE_CHG_MASK (~(unsigned long)(_PAGE_PRESENT | _PAGE_READ | \ - _PAGE_WRITE | _PAGE_EXEC | \ - _PAGE_USER | _PAGE_GLOBAL | \ - _PAGE_MTMASK)) +#endif /* CONFIG_RISCV_ISA_XLINUXMEMALIAS || CONFIG_ERRATA_THEAD_MAE */ static inline int pud_present(pud_t pud) { @@ -202,6 +294,7 @@ static inline int pud_user(pud_t pud) static inline void set_pud(pud_t *pudp, pud_t pud) { + ALT_FIXUP_MT(pud); WRITE_ONCE(*pudp, pud); } @@ -252,11 +345,7 @@ static inline bool mm_pud_folded(struct mm_struct *mm) static inline pmd_t pfn_pmd(unsigned long pfn, pgprot_t prot) { - unsigned long prot_val = pgprot_val(prot); - - ALT_THEAD_PMA(prot_val); - - return __pmd((pfn << _PAGE_PFN_SHIFT) | prot_val); + return __pmd((pfn << _PAGE_PFN_SHIFT) | pgprot_val(prot)); } static inline unsigned long _pmd_pfn(pmd_t pmd) @@ -264,6 +353,9 @@ static inline unsigned long _pmd_pfn(pmd_t pmd) return __page_val_to_pfn(pmd_val(pmd)); } +#define pmd_offset_lockless(pudp, pud, address) \ + (pud_pgtable(pud) + pmd_index(address)) + #define pmd_ERROR(e) \ pr_err("%s:%d: bad pmd %016lx.\n", __FILE__, __LINE__, pmd_val(e)) @@ -275,10 +367,8 @@ static inline unsigned long _pmd_pfn(pmd_t pmd) static inline void set_p4d(p4d_t *p4dp, p4d_t p4d) { - if (pgtable_l4_enabled) - WRITE_ONCE(*p4dp, p4d); - else - set_pud((pud_t *)p4dp, (pud_t){ p4d_val(p4d) }); + ALT_FIXUP_MT(p4d); + WRITE_ONCE(*p4dp, p4d); } static inline int p4d_none(p4d_t p4d) @@ -337,15 +427,16 @@ static inline struct page *p4d_page(p4d_t p4d) #define pud_index(addr) (((addr) >> PUD_SHIFT) & (PTRS_PER_PUD - 1)) +#define pud_offset_lockless(p4dp, p4d, address) \ + (pgtable_l4_enabled ? p4d_pgtable(p4d) + pud_index(address) : (pud_t *)(p4dp)) + #define pud_offset pud_offset -pud_t *pud_offset(p4d_t *p4d, unsigned long address); +pud_t *pud_offset(p4d_t *p4dp, unsigned long address); static inline void set_pgd(pgd_t *pgdp, pgd_t pgd) { - if (pgtable_l5_enabled) - WRITE_ONCE(*pgdp, pgd); - else - set_p4d((p4d_t *)pgdp, (p4d_t){ pgd_val(pgd) }); + ALT_FIXUP_MT(pgd); + WRITE_ONCE(*pgdp, pgd); } static inline int pgd_none(pgd_t pgd) @@ -395,8 +486,11 @@ static inline struct page *pgd_page(pgd_t pgd) #define p4d_index(addr) (((addr) >> P4D_SHIFT) & (PTRS_PER_P4D - 1)) +#define p4d_offset_lockless(pgdp, pgd, address) \ + (pgtable_l5_enabled ? pgd_pgtable(pgd) + p4d_index(address) : (p4d_t *)(pgdp)) + #define p4d_offset p4d_offset -p4d_t *p4d_offset(pgd_t *pgd, unsigned long address); +p4d_t *p4d_offset(pgd_t *pgdp, unsigned long address); #ifdef CONFIG_TRANSPARENT_HUGEPAGE static inline pte_t pmd_pte(pmd_t pmd); diff --git a/arch/riscv/include/asm/pgtable-bits.h b/arch/riscv/include/asm/pgtable-bits.h index d5a86b4df3ce..1b72372d4f91 100644 --- a/arch/riscv/include/asm/pgtable-bits.h +++ b/arch/riscv/include/asm/pgtable-bits.h @@ -6,6 +6,16 @@ #ifndef _ASM_RISCV_PGTABLE_BITS_H #define _ASM_RISCV_PGTABLE_BITS_H +/* + * rv32 PTE format: + * | XLEN-1 10 | 9 8 | 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0 + * PFN reserved for SW D A G U X W R V + * + * rv64 PTE format: + * | 63 | 62 61 | 60 54 | 53 10 | 9 8 | 7 | 6 | 5 | 4 | 3 | 2 | 1 | 0 + * N MT RSV PFN reserved for SW D A G U X W R V + */ + #define _PAGE_ACCESSED_OFFSET 6 #define _PAGE_PRESENT (1 << 0) @@ -58,6 +68,37 @@ #define _PAGE_TABLE _PAGE_PRESENT +#define _PAGE_PFN_SHIFT 10 +#ifdef CONFIG_64BIT +#define _PAGE_PFN_MASK GENMASK(53, 10) +#else +#define _PAGE_PFN_MASK GENMASK(31, 10) +#endif /* CONFIG_64BIT */ + +#if defined(CONFIG_RISCV_ISA_SVPBMT) || defined(CONFIG_RISCV_ISA_XLINUXMEMALIAS) || \ + defined(CONFIG_ERRATA_THEAD_MAE) +/* + * [62:61] Svpbmt Memory Type definitions: + * + * 00 - PMA Normal Cacheable, No change to implied PMA memory type + * 01 - NC Non-cacheable, idempotent, weakly-ordered Main Memory + * 10 - IO Non-cacheable, non-idempotent, strongly-ordered I/O memory + * 11 - Rsvd Reserved for future standard use + */ +#define _PAGE_NOCACHE (UL(1) << 61) +#define _PAGE_IO (UL(2) << 61) +#define _PAGE_MTMASK (UL(3) << 61) +#else +#define _PAGE_NOCACHE 0 +#define _PAGE_IO 0 +#define _PAGE_MTMASK 0 +#endif /* CONFIG_RISCV_ISA_SVPBMT || CONFIG_RISCV_ISA_XLINUXMEMALIAS || CONFIG_ERRATA_THEAD_MAE */ + +#ifdef CONFIG_RISCV_ISA_SVNAPOT +#define _PAGE_NAPOT_SHIFT 63 +#define _PAGE_NAPOT BIT(_PAGE_NAPOT_SHIFT) +#endif /* CONFIG_RISCV_ISA_SVNAPOT */ + /* * _PAGE_PROT_NONE is set on not-present pages (and ignored by the hardware) to * distinguish them from swapped out pages @@ -67,8 +108,6 @@ /* Used for swap PTEs only. */ #define _PAGE_SWP_EXCLUSIVE _PAGE_ACCESSED -#define _PAGE_PFN_SHIFT 10 - /* * when all of R/W/X are zero, the PTE is a pointer to the next level * of the page table; otherwise, it is a leaf PTE. diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index 40b1ed4f3ea8..e3b2f9324ced 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -199,6 +199,12 @@ extern struct pt_alloc_ops pt_ops __meminitdata; #define _PAGE_IOREMAP ((_PAGE_KERNEL & ~_PAGE_MTMASK) | _PAGE_IO) #define PAGE_KERNEL_IO __pgprot(_PAGE_IOREMAP) +/* Set of bits to preserve across pte_modify() */ +#define _PAGE_CHG_MASK (~(unsigned long)(_PAGE_PRESENT | _PAGE_READ | \ + _PAGE_WRITE | _PAGE_EXEC | \ + _PAGE_USER | _PAGE_GLOBAL | \ + _PAGE_MTMASK)) + extern pgd_t swapper_pg_dir[]; extern pgd_t trampoline_pg_dir[]; extern pgd_t early_pg_dir[]; @@ -239,6 +245,7 @@ static inline bool pmd_leaf(pmd_t pmd) static inline void set_pmd(pmd_t *pmdp, pmd_t pmd) { + ALT_FIXUP_MT(pmd); WRITE_ONCE(*pmdp, pmd); } @@ -249,11 +256,7 @@ static inline void pmd_clear(pmd_t *pmdp) static inline pgd_t pfn_pgd(unsigned long pfn, pgprot_t prot) { - unsigned long prot_val = pgprot_val(prot); - - ALT_THEAD_PMA(prot_val); - - return __pgd((pfn << _PAGE_PFN_SHIFT) | prot_val); + return __pgd((pfn << _PAGE_PFN_SHIFT) | pgprot_val(prot)); } static inline unsigned long _pgd_pfn(pgd_t pgd) @@ -329,11 +332,7 @@ static inline unsigned long pte_pfn(pte_t pte) /* Constructs a page table entry */ static inline pte_t pfn_pte(unsigned long pfn, pgprot_t prot) { - unsigned long prot_val = pgprot_val(prot); - - ALT_THEAD_PMA(prot_val); - - return __pte((pfn << _PAGE_PFN_SHIFT) | prot_val); + return __pte((pfn << _PAGE_PFN_SHIFT) | pgprot_val(prot)); } #define pte_pgprot pte_pgprot @@ -549,11 +548,7 @@ static inline int pmd_protnone(pmd_t pmd) /* Modify page protection bits */ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) { - unsigned long newprot_val = pgprot_val(newprot); - - ALT_THEAD_PMA(newprot_val); - - return __pte((pte_val(pte) & _PAGE_CHG_MASK) | newprot_val); + return __pte((pte_val(pte) & _PAGE_CHG_MASK) | pgprot_val(newprot)); } #define pgd_ERROR(e) \ @@ -619,9 +614,10 @@ static inline int pte_same(pte_t pte_a, pte_t pte_b) * a page table are directly modified. Thus, the following hook is * made available. */ -static inline void set_pte(pte_t *ptep, pte_t pteval) +static inline void set_pte(pte_t *ptep, pte_t pte) { - WRITE_ONCE(*ptep, pteval); + ALT_FIXUP_MT(pte); + WRITE_ONCE(*ptep, pte); } void flush_icache_pte(struct mm_struct *mm, pte_t pte); @@ -668,13 +664,9 @@ bool ptep_test_and_clear_young(struct vm_area_struct *vma, static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long address, pte_t *ptep) { -#ifdef CONFIG_SMP - pte_t pte = __pte(xchg(&ptep->pte, 0)); -#else - pte_t pte = *ptep; + pte_t pte = __pte(atomic_long_xchg((atomic_long_t *)ptep, 0)); - set_pte(ptep, __pte(0)); -#endif + ALT_UNFIX_MT(pte); page_table_check_pte_clear(mm, address, pte); @@ -1027,13 +1019,9 @@ static inline bool pmdp_test_and_clear_young(struct vm_area_struct *vma, static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, unsigned long address, pmd_t *pmdp) { -#ifdef CONFIG_SMP - pmd_t pmd = __pmd(xchg(&pmdp->pmd, 0)); -#else - pmd_t pmd = *pmdp; + pmd_t pmd = __pmd(atomic_long_xchg((atomic_long_t *)pmdp, 0)); - pmd_clear(pmdp); -#endif + ALT_UNFIX_MT(pmd); page_table_check_pmd_clear(mm, address, pmd); @@ -1052,7 +1040,11 @@ static inline pmd_t pmdp_establish(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp, pmd_t pmd) { page_table_check_pmd_set(vma->vm_mm, address, pmdp, pmd); - return __pmd(atomic_long_xchg((atomic_long_t *)pmdp, pmd_val(pmd))); + ALT_FIXUP_MT(pmd); + pmd = __pmd(atomic_long_xchg((atomic_long_t *)pmdp, pmd_val(pmd))); + ALT_UNFIX_MT(pmd); + + return pmd; } #define pmdp_collapse_flush pmdp_collapse_flush @@ -1121,13 +1113,9 @@ static inline bool pudp_test_and_clear_young(struct vm_area_struct *vma, static inline pud_t pudp_huge_get_and_clear(struct mm_struct *mm, unsigned long address, pud_t *pudp) { -#ifdef CONFIG_SMP - pud_t pud = __pud(xchg(&pudp->pud, 0)); -#else - pud_t pud = *pudp; + pud_t pud = __pud(atomic_long_xchg((atomic_long_t *)pudp, 0)); - pud_clear(pudp); -#endif + ALT_UNFIX_MT(pud); page_table_check_pud_clear(mm, address, pud); @@ -1151,7 +1139,11 @@ static inline pud_t pudp_establish(struct vm_area_struct *vma, unsigned long address, pud_t *pudp, pud_t pud) { page_table_check_pud_set(vma->vm_mm, address, pudp, pud); - return __pud(atomic_long_xchg((atomic_long_t *)pudp, pud_val(pud))); + ALT_FIXUP_MT(pud); + pud = __pud(atomic_long_xchg((atomic_long_t *)pudp, pud_val(pud))); + ALT_UNFIX_MT(pud); + + return pud; } static inline pud_t pud_mkinvalid(pud_t pud) @@ -1281,6 +1273,15 @@ extern u64 satp_mode; void paging_init(void); void misc_mem_init(void); +#ifdef CONFIG_RISCV_ISA_XLINUXMEMALIAS +bool __init riscv_have_memory_alias(void); +void __init riscv_init_memory_alias(void); +#else +static inline bool riscv_have_memory_alias(void) { return false; } +static inline void riscv_init_memory_alias(void) {} +#endif /* CONFIG_RISCV_ISA_XLINUXMEMALIAS */ + + /* * Use set_p*_safe(), and elide TLB flushing, when confident that *no* * TLB flush will be required as a result of the "set". For example, use @@ -1291,13 +1292,15 @@ void misc_mem_init(void); */ #define set_p4d_safe(p4dp, p4d) \ ({ \ - WARN_ON_ONCE(p4d_present(*p4dp) && !p4d_same(*p4dp, p4d)); \ + p4d_t old = p4dp_get(p4dp); \ + WARN_ON_ONCE(p4d_present(old) && !p4d_same(old, p4d)); \ set_p4d(p4dp, p4d); \ }) #define set_pgd_safe(pgdp, pgd) \ ({ \ - WARN_ON_ONCE(pgd_present(*pgdp) && !pgd_same(*pgdp, pgd)); \ + pgd_t old = pgdp_get(pgdp); \ + WARN_ON_ONCE(pgd_present(old) && !pgd_same(old, pgd)); \ set_pgd(pgdp, pgd); \ }) #endif /* !__ASSEMBLER__ */ diff --git a/arch/riscv/kernel/alternative.c b/arch/riscv/kernel/alternative.c index c0c9306022c5..386aae59f9be 100644 --- a/arch/riscv/kernel/alternative.c +++ b/arch/riscv/kernel/alternative.c @@ -126,8 +126,8 @@ void riscv_alternative_fix_offsets(void *alt_ptr, unsigned int len, if (!riscv_insn_is_jalr(insn2)) continue; - /* if instruction pair is a call, it will use the ra register */ - if (RV_EXTRACT_RD_REG(insn) != 1) + /* if instruction pair is a call, it will save a link register */ + if (RV_EXTRACT_RD_REG(insn) == 0) continue; riscv_alternative_fix_auipc_jalr(alt_ptr + i * sizeof(u32), diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c index 7329792978cc..ca478381d949 100644 --- a/arch/riscv/kernel/cpufeature.c +++ b/arch/riscv/kernel/cpufeature.c @@ -1191,6 +1191,12 @@ void __init riscv_fill_hwcap(void) riscv_v_setup_vsize(); } + /* Vendor-independent alternatives require a bit in the ISA bitmap. */ + if (riscv_have_memory_alias()) { + set_bit(RISCV_ISA_EXT_XLINUXMEMALIAS, riscv_isa); + pr_info("Using physical memory alias for noncached mappings\n"); + } + memset(print_str, 0, sizeof(print_str)); for (i = 0, j = 0; i < NUM_ALPHA_EXTS; i++) if (riscv_isa[0] & BIT_MASK(i)) diff --git a/arch/riscv/kernel/hibernate.c b/arch/riscv/kernel/hibernate.c index 982843828adb..c3a7261c38cd 100644 --- a/arch/riscv/kernel/hibernate.c +++ b/arch/riscv/kernel/hibernate.c @@ -171,7 +171,7 @@ static int temp_pgtable_map_pte(pmd_t *dst_pmdp, pmd_t *src_pmdp, unsigned long pte_t *src_ptep; pte_t *dst_ptep; - if (pmd_none(READ_ONCE(*dst_pmdp))) { + if (pmd_none(pmdp_get(dst_pmdp))) { dst_ptep = (pte_t *)get_safe_page(GFP_ATOMIC); if (!dst_ptep) return -ENOMEM; @@ -183,7 +183,7 @@ static int temp_pgtable_map_pte(pmd_t *dst_pmdp, pmd_t *src_pmdp, unsigned long src_ptep = pte_offset_kernel(src_pmdp, start); do { - pte_t pte = READ_ONCE(*src_ptep); + pte_t pte = ptep_get(src_ptep); if (pte_present(pte)) set_pte(dst_ptep, __pte(pte_val(pte) | pgprot_val(prot))); @@ -200,7 +200,7 @@ static int temp_pgtable_map_pmd(pud_t *dst_pudp, pud_t *src_pudp, unsigned long pmd_t *src_pmdp; pmd_t *dst_pmdp; - if (pud_none(READ_ONCE(*dst_pudp))) { + if (pud_none(pudp_get(dst_pudp))) { dst_pmdp = (pmd_t *)get_safe_page(GFP_ATOMIC); if (!dst_pmdp) return -ENOMEM; @@ -212,7 +212,7 @@ static int temp_pgtable_map_pmd(pud_t *dst_pudp, pud_t *src_pudp, unsigned long src_pmdp = pmd_offset(src_pudp, start); do { - pmd_t pmd = READ_ONCE(*src_pmdp); + pmd_t pmd = pmdp_get(src_pmdp); next = pmd_addr_end(start, end); @@ -239,7 +239,7 @@ static int temp_pgtable_map_pud(p4d_t *dst_p4dp, p4d_t *src_p4dp, unsigned long pud_t *dst_pudp; pud_t *src_pudp; - if (p4d_none(READ_ONCE(*dst_p4dp))) { + if (p4d_none(p4dp_get(dst_p4dp))) { dst_pudp = (pud_t *)get_safe_page(GFP_ATOMIC); if (!dst_pudp) return -ENOMEM; @@ -251,7 +251,7 @@ static int temp_pgtable_map_pud(p4d_t *dst_p4dp, p4d_t *src_p4dp, unsigned long src_pudp = pud_offset(src_p4dp, start); do { - pud_t pud = READ_ONCE(*src_pudp); + pud_t pud = pudp_get(src_pudp); next = pud_addr_end(start, end); @@ -278,7 +278,7 @@ static int temp_pgtable_map_p4d(pgd_t *dst_pgdp, pgd_t *src_pgdp, unsigned long p4d_t *dst_p4dp; p4d_t *src_p4dp; - if (pgd_none(READ_ONCE(*dst_pgdp))) { + if (pgd_none(pgdp_get(dst_pgdp))) { dst_p4dp = (p4d_t *)get_safe_page(GFP_ATOMIC); if (!dst_p4dp) return -ENOMEM; @@ -290,7 +290,7 @@ static int temp_pgtable_map_p4d(pgd_t *dst_pgdp, pgd_t *src_pgdp, unsigned long src_p4dp = p4d_offset(src_pgdp, start); do { - p4d_t p4d = READ_ONCE(*src_p4dp); + p4d_t p4d = p4dp_get(src_p4dp); next = p4d_addr_end(start, end); @@ -317,7 +317,7 @@ static int temp_pgtable_mapping(pgd_t *pgdp, unsigned long start, unsigned long unsigned long ret; do { - pgd_t pgd = READ_ONCE(*src_pgdp); + pgd_t pgd = pgdp_get(src_pgdp); next = pgd_addr_end(start, end); diff --git a/arch/riscv/kernel/setup.c b/arch/riscv/kernel/setup.c index d79f99b86033..c88480c8a4db 100644 --- a/arch/riscv/kernel/setup.c +++ b/arch/riscv/kernel/setup.c @@ -352,6 +352,7 @@ void __init setup_arch(char **cmdline_p) } riscv_init_cbo_blocksizes(); + riscv_init_memory_alias(); riscv_fill_hwcap(); apply_boot_alternatives(); init_rt_signal_env(); diff --git a/arch/riscv/kvm/gstage.c b/arch/riscv/kvm/gstage.c index e5002cb9cbef..c422308da8d9 100644 --- a/arch/riscv/kvm/gstage.c +++ b/arch/riscv/kvm/gstage.c @@ -129,6 +129,9 @@ bool kvm_riscv_gstage_try_update_pte(struct kvm_gstage *gstage, u32 level, gpa_t addr, pte_t *ptep, pte_t old_pte, pte_t new_pte) { + /* The atomic operation compares and stores hardware-format entries. */ + ALT_FIXUP_MT(old_pte); + ALT_FIXUP_MT(new_pte); if (cmpxchg(&ptep->pte, pte_val(old_pte), pte_val(new_pte)) != pte_val(old_pte)) return false; @@ -172,8 +175,8 @@ int kvm_riscv_gstage_set_pte(struct kvm_gstage *gstage, ptep = &next_ptep[gstage_pte_index(gstage, map->addr, current_level)]; } - if (pte_val(*ptep) != pte_val(map->pte)) { - bool was_invalid = !pte_val(*ptep); + if (pte_val(ptep_get(ptep)) != pte_val(map->pte)) { + bool was_invalid = !pte_val(ptep_get(ptep)); set_pte(ptep, map->pte); if (gstage_pte_leaf(ptep) && !(was_invalid && riscv_has_extension_unlikely(RISCV_ISA_EXT_SVVPTC))) @@ -402,12 +405,12 @@ bool kvm_riscv_gstage_op_pte(struct kvm_gstage *gstage, gpa_t addr, if (op == GSTAGE_OP_CLEAR) put_page(virt_to_page(next_ptep)); } else { - old_pte = *ptep; + old_pte = ptep_get(ptep); if (op == GSTAGE_OP_CLEAR) set_pte(ptep, __pte(0)); else if (op == GSTAGE_OP_WP) set_pte(ptep, __pte(pte_val(ptep_get(ptep)) & ~_PAGE_WRITE)); - if (pte_val(*ptep) != pte_val(old_pte)) + if (pte_val(ptep_get(ptep)) != pte_val(old_pte)) flush = true; } diff --git a/arch/riscv/mm/Makefile b/arch/riscv/mm/Makefile index b916a68d324a..b4d757226efb 100644 --- a/arch/riscv/mm/Makefile +++ b/arch/riscv/mm/Makefile @@ -33,3 +33,4 @@ endif obj-$(CONFIG_DEBUG_VIRTUAL) += physaddr.o obj-$(CONFIG_RISCV_DMA_NONCOHERENT) += dma-noncoherent.o obj-$(CONFIG_RISCV_NONSTANDARD_CACHE_OPS) += cache-ops.o +obj-$(CONFIG_RISCV_ISA_XLINUXMEMALIAS) += memory-alias.o diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c index fb37b0b67efe..15cc1a03f7dc 100644 --- a/arch/riscv/mm/init.c +++ b/arch/riscv/mm/init.c @@ -440,8 +440,8 @@ static void __meminit create_pte_mapping(pte_t *ptep, uintptr_t va, phys_addr_t BUG_ON(sz != PAGE_SIZE); - if (pte_none(ptep[pte_idx])) - ptep[pte_idx] = pfn_pte(PFN_DOWN(pa), prot); + if (pte_none(ptep_get(ptep + pte_idx))) + set_pte(ptep + pte_idx, pfn_pte(PFN_DOWN(pa), prot)); } #ifndef __PAGETABLE_PMD_FOLDED @@ -505,18 +505,19 @@ static void __meminit create_pmd_mapping(pmd_t *pmdp, uintptr_t pmd_idx = pmd_index(va); if (sz == PMD_SIZE) { - if (pmd_none(pmdp[pmd_idx])) - pmdp[pmd_idx] = pfn_pmd(PFN_DOWN(pa), prot); + if (pmd_none(pmdp_get(pmdp + pmd_idx))) + set_pmd(pmdp + pmd_idx, pfn_pmd(PFN_DOWN(pa), prot)); return; } - if (pmd_none(pmdp[pmd_idx])) { + if (pmd_none(pmdp_get(pmdp + pmd_idx))) { pte_phys = pt_ops.alloc_pte(va); - pmdp[pmd_idx] = pfn_pmd(PFN_DOWN(pte_phys), PAGE_TABLE); + set_pmd(pmdp + pmd_idx, + pfn_pmd(PFN_DOWN(pte_phys), PAGE_TABLE)); ptep = pt_ops.get_pte_virt(pte_phys); memset(ptep, 0, PAGE_SIZE); } else { - pte_phys = PFN_PHYS(_pmd_pfn(pmdp[pmd_idx])); + pte_phys = PFN_PHYS(_pmd_pfn(pmdp_get(pmdp + pmd_idx))); ptep = pt_ops.get_pte_virt(pte_phys); } @@ -607,18 +608,19 @@ static void __meminit create_pud_mapping(pud_t *pudp, uintptr_t va, phys_addr_t uintptr_t pud_index = pud_index(va); if (sz == PUD_SIZE) { - if (pud_val(pudp[pud_index]) == 0) - pudp[pud_index] = pfn_pud(PFN_DOWN(pa), prot); + if (pud_val(pudp_get(pudp + pud_index)) == 0) + set_pud(pudp + pud_index, pfn_pud(PFN_DOWN(pa), prot)); return; } - if (pud_val(pudp[pud_index]) == 0) { + if (pud_val(pudp_get(pudp + pud_index)) == 0) { next_phys = pt_ops.alloc_pmd(va); - pudp[pud_index] = pfn_pud(PFN_DOWN(next_phys), PAGE_TABLE); + set_pud(pudp + pud_index, + pfn_pud(PFN_DOWN(next_phys), PAGE_TABLE)); nextp = pt_ops.get_pmd_virt(next_phys); memset(nextp, 0, PAGE_SIZE); } else { - next_phys = PFN_PHYS(_pud_pfn(pudp[pud_index])); + next_phys = PFN_PHYS(_pud_pfn(pudp_get(pudp + pud_index))); nextp = pt_ops.get_pmd_virt(next_phys); } @@ -633,18 +635,19 @@ static void __meminit create_p4d_mapping(p4d_t *p4dp, uintptr_t va, phys_addr_t uintptr_t p4d_index = p4d_index(va); if (sz == P4D_SIZE) { - if (p4d_val(p4dp[p4d_index]) == 0) - p4dp[p4d_index] = pfn_p4d(PFN_DOWN(pa), prot); + if (p4d_val(p4dp_get(p4dp + p4d_index)) == 0) + set_p4d(p4dp + p4d_index, pfn_p4d(PFN_DOWN(pa), prot)); return; } - if (p4d_val(p4dp[p4d_index]) == 0) { + if (p4d_val(p4dp_get(p4dp + p4d_index)) == 0) { next_phys = pt_ops.alloc_pud(va); - p4dp[p4d_index] = pfn_p4d(PFN_DOWN(next_phys), PAGE_TABLE); + set_p4d(p4dp + p4d_index, + pfn_p4d(PFN_DOWN(next_phys), PAGE_TABLE)); nextp = pt_ops.get_pud_virt(next_phys); memset(nextp, 0, PAGE_SIZE); } else { - next_phys = PFN_PHYS(_p4d_pfn(p4dp[p4d_index])); + next_phys = PFN_PHYS(_p4d_pfn(p4dp_get(p4dp + p4d_index))); nextp = pt_ops.get_pud_virt(next_phys); } @@ -690,18 +693,19 @@ void __meminit create_pgd_mapping(pgd_t *pgdp, uintptr_t va, phys_addr_t pa, phy uintptr_t pgd_idx = pgd_index(va); if (sz == PGDIR_SIZE) { - if (pgd_val(pgdp[pgd_idx]) == 0) - pgdp[pgd_idx] = pfn_pgd(PFN_DOWN(pa), prot); + if (pgd_val(pgdp_get(pgdp + pgd_idx)) == 0) + set_pgd(pgdp + pgd_idx, pfn_pgd(PFN_DOWN(pa), prot)); return; } - if (pgd_val(pgdp[pgd_idx]) == 0) { + if (pgd_val(pgdp_get(pgdp + pgd_idx)) == 0) { next_phys = alloc_pgd_next(va); - pgdp[pgd_idx] = pfn_pgd(PFN_DOWN(next_phys), PAGE_TABLE); + set_pgd(pgdp + pgd_idx, + pfn_pgd(PFN_DOWN(next_phys), PAGE_TABLE)); nextp = get_pgd_next_virt(next_phys); memset(nextp, 0, PAGE_SIZE); } else { - next_phys = PFN_PHYS(_pgd_pfn(pgdp[pgd_idx])); + next_phys = PFN_PHYS(_pgd_pfn(pgdp_get(pgdp + pgd_idx))); nextp = get_pgd_next_virt(next_phys); } @@ -1487,14 +1491,14 @@ struct execmem_info __init *execmem_arch_setup(void) #ifdef CONFIG_MEMORY_HOTPLUG static void __meminit free_pte_table(pte_t *pte_start, pmd_t *pmd) { - struct page *page = pmd_page(*pmd); + struct page *page = pmd_page(pmdp_get(pmd)); struct ptdesc *ptdesc = page_ptdesc(page); pte_t *pte; int i; for (i = 0; i < PTRS_PER_PTE; i++) { pte = pte_start + i; - if (!pte_none(*pte)) + if (!pte_none(ptep_get(pte))) return; } @@ -1508,14 +1512,14 @@ static void __meminit free_pte_table(pte_t *pte_start, pmd_t *pmd) static void __meminit free_pmd_table(pmd_t *pmd_start, pud_t *pud, bool is_vmemmap) { - struct page *page = pud_page(*pud); + struct page *page = pud_page(pudp_get(pud)); struct ptdesc *ptdesc = page_ptdesc(page); pmd_t *pmd; int i; for (i = 0; i < PTRS_PER_PMD; i++) { pmd = pmd_start + i; - if (!pmd_none(*pmd)) + if (!pmd_none(pmdp_get(pmd))) return; } @@ -1530,13 +1534,13 @@ static void __meminit free_pmd_table(pmd_t *pmd_start, pud_t *pud, bool is_vmemm static void __meminit free_pud_table(pud_t *pud_start, p4d_t *p4d) { - struct page *page = p4d_page(*p4d); + struct page *page = p4d_page(p4dp_get(p4d)); pud_t *pud; int i; for (i = 0; i < PTRS_PER_PUD; i++) { pud = pud_start + i; - if (!pud_none(*pud)) + if (!pud_none(pudp_get(pud))) return; } @@ -1581,7 +1585,7 @@ static void __meminit remove_pte_mapping(pte_t *pte_base, unsigned long addr, un ptep = pte_base + pte_index(addr); pte = ptep_get(ptep); - if (!pte_present(*ptep)) + if (!pte_present(ptep_get(ptep))) continue; pte_clear(&init_mm, addr, ptep); @@ -1611,7 +1615,7 @@ static void __meminit remove_pmd_mapping(pmd_t *pmd_base, unsigned long addr, un continue; } - pte_base = (pte_t *)pmd_page_vaddr(*pmdp); + pte_base = (pte_t *)pmd_page_vaddr(pmdp_get(pmdp)); remove_pte_mapping(pte_base, addr, next, is_vmemmap, altmap); free_pte_table(pte_base, pmdp); } @@ -1690,10 +1694,10 @@ static void __meminit remove_pgd_mapping(unsigned long va, unsigned long end, bo next = pgd_addr_end(addr, end); pgd = pgd_offset_k(addr); - if (!pgd_present(*pgd)) + if (!pgd_present(pgdp_get(pgd))) continue; - if (pgd_leaf(*pgd)) + if (pgd_leaf(pgdp_get(pgd))) continue; p4d_base = p4d_offset(pgd, 0); diff --git a/arch/riscv/mm/memory-alias.S b/arch/riscv/mm/memory-alias.S new file mode 100644 index 000000000000..e37b83d11591 --- /dev/null +++ b/arch/riscv/mm/memory-alias.S @@ -0,0 +1,123 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2024 SiFive + */ + +#include +#include +#include +#include + +#define CACHED_BASE_OFFSET (0 * RISCV_SZPTR) +#define NONCACHED_BASE_OFFSET (1 * RISCV_SZPTR) +#define SIZE_OFFSET (2 * RISCV_SZPTR) + +#define SIZEOF_PAIR (4 * RISCV_SZPTR) + +/* + * Called from ALT_FIXUP_MT with a non-standard calling convention: + * t0 => return address + * t1 => page table entry + * all other registers are callee-saved + */ +SYM_CODE_START(riscv_fixup_memory_alias) + addi sp, sp, -4 * SZREG + REG_S t2, (0 * SZREG)(sp) + REG_S t3, (1 * SZREG)(sp) + REG_S t4, (2 * SZREG)(sp) +#ifdef CONFIG_RISCV_ISA_SVNAPOT + REG_S t5, (3 * SZREG)(sp) + + /* Save and mask off _PAGE_NAPOT if present. */ + li t5, _PAGE_NAPOT + and t5, t1, t5 + xor t1, t1, t5 +#endif + + /* Ignore !pte_present() PTEs, including swap PTEs. */ + andi t2, t1, (_PAGE_PRESENT | _PAGE_PROT_NONE) + beqz t2, .Lfixup_end + + lla t2, memory_alias_pairs +.Lfixup_loop: + REG_L t3, SIZE_OFFSET(t2) + beqz t3, .Lfixup_end + REG_L t4, CACHED_BASE_OFFSET(t2) + sub t4, t1, t4 + bltu t4, t3, .Lfixup_found + addi t2, t2, SIZEOF_PAIR + j .Lfixup_loop + +.Lfixup_found: + REG_L t3, NONCACHED_BASE_OFFSET(t2) + add t1, t3, t4 + +.Lfixup_end: +#ifdef CONFIG_RISCV_ISA_SVNAPOT + xor t1, t1, t5 + + REG_L t5, (3 * SZREG)(sp) +#endif + REG_L t4, (2 * SZREG)(sp) + REG_L t3, (1 * SZREG)(sp) + REG_L t2, (0 * SZREG)(sp) + addi sp, sp, 4 * SZREG + jr t0 +SYM_CODE_END(riscv_fixup_memory_alias) +EXPORT_SYMBOL(riscv_fixup_memory_alias) + +/* + * Called from ALT_UNFIX_MT with a non-standard calling convention: + * t0 => return address + * t1 => page table entry + * all other registers are callee-saved + */ +SYM_CODE_START(riscv_unfix_memory_alias) + addi sp, sp, -4 * SZREG + REG_S t2, (0 * SZREG)(sp) + REG_S t3, (1 * SZREG)(sp) + REG_S t4, (2 * SZREG)(sp) +#ifdef CONFIG_RISCV_ISA_SVNAPOT + REG_S t5, (3 * SZREG)(sp) + + /* Save and mask off _PAGE_NAPOT if present. */ + li t5, _PAGE_NAPOT + and t5, t1, t5 + xor t1, t1, t5 +#endif + + /* Ignore !pte_present() PTEs, including swap PTEs. */ + andi t2, t1, (_PAGE_PRESENT | _PAGE_PROT_NONE) + beqz t2, .Lunfix_end + + lla t2, memory_alias_pairs +.Lunfix_loop: + REG_L t3, SIZE_OFFSET(t2) + beqz t3, .Lunfix_end + REG_L t4, NONCACHED_BASE_OFFSET(t2) + sub t4, t1, t4 + bltu t4, t3, .Lunfix_found + addi t2, t2, SIZEOF_PAIR + j .Lunfix_loop + +.Lunfix_found: + REG_L t3, CACHED_BASE_OFFSET(t2) + add t1, t3, t4 + + /* PFN was in the noncached alias, so mark it as such. */ + li t2, _PAGE_NOCACHE + or t1, t1, t2 + +.Lunfix_end: +#ifdef CONFIG_RISCV_ISA_SVNAPOT + xor t1, t1, t5 + + REG_L t5, (3 * SZREG)(sp) +#endif + REG_L t4, (2 * SZREG)(sp) + REG_L t3, (1 * SZREG)(sp) + REG_L t2, (0 * SZREG)(sp) + addi sp, sp, 4 * SZREG + jr t0 +SYM_CODE_END(riscv_unfix_memory_alias) +EXPORT_SYMBOL(riscv_unfix_memory_alias) diff --git a/arch/riscv/mm/pgtable.c b/arch/riscv/mm/pgtable.c index 9c4427d0b187..40897b049c9b 100644 --- a/arch/riscv/mm/pgtable.c +++ b/arch/riscv/mm/pgtable.c @@ -1,8 +1,12 @@ // SPDX-License-Identifier: GPL-2.0 #include +#include +#include #include #include +#include +#include #include int ptep_set_access_flags(struct vm_area_struct *vma, @@ -39,21 +43,15 @@ bool ptep_test_and_clear_young(struct vm_area_struct *vma, EXPORT_SYMBOL_GPL(ptep_test_and_clear_young); #ifdef CONFIG_64BIT -pud_t *pud_offset(p4d_t *p4d, unsigned long address) +pud_t *pud_offset(p4d_t *p4dp, unsigned long address) { - if (pgtable_l4_enabled) - return p4d_pgtable(p4dp_get(p4d)) + pud_index(address); - - return (pud_t *)p4d; + return pud_offset_lockless(p4dp, p4dp_get(p4dp), address); } EXPORT_SYMBOL_GPL(pud_offset); -p4d_t *p4d_offset(pgd_t *pgd, unsigned long address) +p4d_t *p4d_offset(pgd_t *pgdp, unsigned long address) { - if (pgtable_l5_enabled) - return pgd_pgtable(pgdp_get(pgd)) + p4d_index(address); - - return (p4d_t *)pgd; + return p4d_offset_lockless(pgdp, pgdp_get(pgdp), address); } EXPORT_SYMBOL_GPL(p4d_offset); #endif @@ -94,8 +92,8 @@ int pud_free_pmd_page(pud_t *pud, unsigned long addr) flush_tlb_kernel_range(addr, addr + PUD_SIZE); for (i = 0; i < PTRS_PER_PMD; i++) { - if (!pmd_none(pmd[i])) { - pte_t *pte = (pte_t *)pmd_page_vaddr(pmd[i]); + if (!pmd_none(pmdp_get(pmd + i))) { + pte_t *pte = (pte_t *)pmd_page_vaddr(pmdp_get(pmd + i)); pte_free_kernel(NULL, pte); } @@ -157,8 +155,9 @@ pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, pud_t pudp_invalidate(struct vm_area_struct *vma, unsigned long address, pud_t *pudp) { - VM_WARN_ON_ONCE(!pud_present(*pudp)); - pud_t old = pudp_establish(vma, address, pudp, pud_mkinvalid(*pudp)); + VM_WARN_ON_ONCE(!pud_present(pudp_get(pudp))); + pud_t old = pudp_establish(vma, address, pudp, + pud_mkinvalid(pudp_get(pudp))); flush_pud_tlb_range(vma, address, address + HPAGE_PUD_SIZE); return old; @@ -180,3 +179,90 @@ pmd_t pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma) return pmd_mkwrite_novma(pmd); } + +#ifdef CONFIG_RISCV_ISA_XLINUXMEMALIAS +struct memory_alias_pair { + unsigned long cached_base; + unsigned long noncached_base; + unsigned long size; + int index; +} memory_alias_pairs[5]; + +bool __init riscv_have_memory_alias(void) +{ + return memory_alias_pairs[0].size; +} + +void __init riscv_init_memory_alias(void) +{ + int na = of_n_addr_cells(of_root); + int ns = of_n_size_cells(of_root); + int nc = na + ns + 2; + const __be32 *prop; + int pairs = 0; + int len; + + prop = of_get_property(of_root, "riscv,physical-memory-regions", &len); + if (!prop) + return; + + len /= sizeof(__be32); + for (int i = 0; len >= nc; i++, prop += nc, len -= nc) { + unsigned long base = of_read_ulong(prop, na); + unsigned long size = of_read_ulong(prop + na, ns); + unsigned long flags = be32_to_cpup(prop + na + ns); + struct memory_alias_pair *pair; + + /* We only care about non-coherent memory. */ + if ((flags & PMA_ORDER_MASK) != PMA_ORDER_MEMORY || (flags & PMA_COHERENT)) + continue; + + /* The cacheable alias must be usable memory. */ + if ((flags & PMA_CACHEABLE) && + !memblock_overlaps_region(&memblock.memory, base, size)) + continue; + + if (flags & PMR_IS_ALIAS) { + int alias = FIELD_GET(PMR_ALIAS_MASK, flags); + + pair = NULL; + for (int j = 0; j < pairs; j++) { + if (alias == memory_alias_pairs[j].index) { + pair = &memory_alias_pairs[j]; + break; + } + } + if (!pair) + continue; + } else { + /* Leave room for the null sentinel. */ + if (pairs == ARRAY_SIZE(memory_alias_pairs) - 1) + continue; + pair = &memory_alias_pairs[pairs++]; + pair->index = i; + } + + /* Align the address and size with the page table PFN field. */ + base >>= PAGE_SHIFT - _PAGE_PFN_SHIFT; + size >>= PAGE_SHIFT - _PAGE_PFN_SHIFT; + + if (flags & PMA_CACHEABLE) + pair->cached_base = base; + else + pair->noncached_base = base; + pair->size = min_not_zero(pair->size, size); + } + + /* Remove any unmatched pairs. */ + for (int i = 0; i < pairs; i++) { + struct memory_alias_pair *pair = &memory_alias_pairs[i]; + + if (pair->cached_base && pair->noncached_base && pair->size) + continue; + + for (int j = i + 1; j < pairs; j++) + memory_alias_pairs[j - 1] = memory_alias_pairs[j]; + memory_alias_pairs[--pairs].size = 0; + } +} +#endif /* CONFIG_RISCV_ISA_XLINUXMEMALIAS */ diff --git a/arch/riscv/mm/ptdump.c b/arch/riscv/mm/ptdump.c index f4b4a9fcbbd8..583743d61ff2 100644 --- a/arch/riscv/mm/ptdump.c +++ b/arch/riscv/mm/ptdump.c @@ -135,12 +135,15 @@ struct prot_bits { static const struct prot_bits pte_bits[] = { { -#ifdef CONFIG_64BIT +#ifdef CONFIG_RISCV_ISA_SVNAPOT .mask = _PAGE_NAPOT, .set = "N", .clear = ".", }, { - .mask = _PAGE_MTMASK_SVPBMT, +#endif +#if defined(CONFIG_RISCV_ISA_SVPBMT) || defined(CONFIG_RISCV_ISA_XLINUXMEMALIAS) || \ + defined(CONFIG_ERRATA_THEAD_MAE) + .mask = _PAGE_MTMASK, .set = "MT(%s)", .clear = " .. ", }, { @@ -215,11 +218,12 @@ static void dump_prot(struct pg_state *st) if (val) { if (pte_bits[i].mask == _PAGE_SOFT) snprintf(s, sizeof(s), pte_bits[i].set, val >> 8); -#ifdef CONFIG_64BIT - else if (pte_bits[i].mask == _PAGE_MTMASK_SVPBMT) { - if (val == _PAGE_NOCACHE_SVPBMT) +#if defined(CONFIG_RISCV_ISA_SVPBMT) || defined(CONFIG_RISCV_ISA_XLINUXMEMALIAS) || \ + defined(CONFIG_ERRATA_THEAD_MAE) + else if (pte_bits[i].mask == _PAGE_MTMASK) { + if (val == _PAGE_NOCACHE) snprintf(s, sizeof(s), pte_bits[i].set, "NC"); - else if (val == _PAGE_IO_SVPBMT) + else if (val == _PAGE_IO) snprintf(s, sizeof(s), pte_bits[i].set, "IO"); else snprintf(s, sizeof(s), pte_bits[i].set, "??"); diff --git a/drivers/cache/Kconfig b/drivers/cache/Kconfig index 1518449d47b5..b53b3d63e309 100644 --- a/drivers/cache/Kconfig +++ b/drivers/cache/Kconfig @@ -18,7 +18,7 @@ config AX45MP_L2_CACHE config SIFIVE_CCACHE bool "Sifive Composable Cache controller" - depends on ARCH_SIFIVE || ARCH_STARFIVE + depends on ARCH_ESWIN || ARCH_SIFIVE || ARCH_STARFIVE help Support for the composable cache controller on SiFive platforms. diff --git a/drivers/clk/eswin/Kconfig b/drivers/clk/eswin/Kconfig index 37857bd77335..1d0b5a03fb85 100644 --- a/drivers/clk/eswin/Kconfig +++ b/drivers/clk/eswin/Kconfig @@ -18,6 +18,7 @@ config COMMON_CLK_EIC7700_HSP tristate "EIC7700 HSP Clock Driver" depends on ARCH_ESWIN || COMPILE_TEST select AUXILIARY_BUS + select MFD_SYSCON select COMMON_CLK_EIC7700 select RESET_EIC7700_HSP if RESET_CONTROLLER select REGMAP_MMIO diff --git a/drivers/clk/eswin/clk-eic7700-hsp.c b/drivers/clk/eswin/clk-eic7700-hsp.c index 6ddb8fbd605e..8bae611df538 100644 --- a/drivers/clk/eswin/clk-eic7700-hsp.c +++ b/drivers/clk/eswin/clk-eic7700-hsp.c @@ -10,6 +10,8 @@ #include #include +#include +#include #include #include @@ -264,8 +266,20 @@ static int eic7700_hsp_clk_probe(struct platform_device *pdev) return dev_err_probe(dev, PTR_ERR(data), "failed to get clk data!\n"); - regmap = devm_regmap_init_mmio(dev, data->base, - &eic7700_hsp_regmap_config); + if (of_device_is_compatible(dev->of_node, "syscon")) { + regmap = syscon_node_to_regmap(dev->of_node); + if (IS_ERR(regmap)) + return dev_err_probe(dev, PTR_ERR(regmap), + "failed to get HSP syscon regmap\n"); + + /* Make the same map available to the auxiliary reset driver. */ + ret = regmap_attach_dev(dev, regmap, &eic7700_hsp_regmap_config); + if (ret) + return dev_err_probe(dev, ret, "failed to attach HSP regmap\n"); + } else { + regmap = devm_regmap_init_mmio(dev, data->base, + &eic7700_hsp_regmap_config); + } if (IS_ERR(regmap)) return dev_err_probe(dev, PTR_ERR(regmap), "failed to get regmap!\n"); diff --git a/drivers/clocksource/dw_apb_timer_of.c b/drivers/clocksource/dw_apb_timer_of.c index 3245eb0c602d..3788b46e49ed 100644 --- a/drivers/clocksource/dw_apb_timer_of.c +++ b/drivers/clocksource/dw_apb_timer_of.c @@ -29,9 +29,14 @@ static int __init timer_get_base_and_rate(struct device_node *np, /* * Reset the timer if the reset control is available, wiping - * out the state the firmware may have left it + * out the state the firmware may have left it. + * Use the array variant to fetch and trigger all reset lines + * (e.g., timer and APB interface resets) together, without + * ordering requirements. + * The exclusive references are intentionally held to keep the + * resets deasserted while the timer is running. */ - rstc = of_reset_control_get(np, NULL); + rstc = of_reset_control_array_get_optional_exclusive(np); if (!IS_ERR(rstc)) { reset_control_assert(rstc); reset_control_deassert(rstc); diff --git a/drivers/mmc/host/sdhci-of-dwcmshc.c b/drivers/mmc/host/sdhci-of-dwcmshc.c index 1baa2379c9c3..95eef24bd19c 100644 --- a/drivers/mmc/host/sdhci-of-dwcmshc.c +++ b/drivers/mmc/host/sdhci-of-dwcmshc.c @@ -1764,7 +1764,7 @@ static int eic7700_init(struct device *dev, struct sdhci_host *host, struct dwcm u32 emmc_caps = MMC_CAP2_NO_SD | MMC_CAP2_NO_SDIO; u32 sd_caps = MMC_CAP2_NO_MMC | MMC_CAP2_NO_SDIO; unsigned int val, hsp_int_status, hsp_pwr_ctrl; - static const char * const clk_ids[] = {"axi"}; + static const char * const clk_ids[] = {"axi", "timer"}; struct of_phandle_args args; struct eic7700_priv *priv; struct regmap *hsp_regmap; diff --git a/drivers/pci/controller/dwc/pcie-eswin.c b/drivers/pci/controller/dwc/pcie-eswin.c index ce8d64f8a395..1bb44696057e 100644 --- a/drivers/pci/controller/dwc/pcie-eswin.c +++ b/drivers/pci/controller/dwc/pcie-eswin.c @@ -388,6 +388,7 @@ static const struct of_device_id eswin_pcie_of_match[] = { { .compatible = "eswin,eic7700-pcie", .data = &eswin_eic7700_data }, {} }; +MODULE_DEVICE_TABLE(of, eswin_pcie_of_match); static struct platform_driver eswin_pcie_driver = { .probe = eswin_pcie_probe, diff --git a/drivers/pwm/Kconfig b/drivers/pwm/Kconfig index 729776086879..b8035a686f0d 100644 --- a/drivers/pwm/Kconfig +++ b/drivers/pwm/Kconfig @@ -249,6 +249,16 @@ config PWM_DWC To compile this driver as a module, choose M here: the module will be called pwm-dwc. +config PWM_DWC_OF + tristate "DesignWare PWM Controller (OF bus)" + depends on HAS_IOMEM && (OF || COMPILE_TEST) + select PWM_DWC_CORE + help + PWM driver for Synopsys DWC PWM Controller on an OF bus or + a platform bus. + To compile this driver as a module, choose M here: the module + will be called pwm-dwc-of. + config PWM_EP93XX tristate "Cirrus Logic EP93xx PWM support" depends on ARCH_EP93XX || COMPILE_TEST diff --git a/drivers/pwm/Makefile b/drivers/pwm/Makefile index 5630a521a7cf..acd7dfe98dff 100644 --- a/drivers/pwm/Makefile +++ b/drivers/pwm/Makefile @@ -20,6 +20,7 @@ obj-$(CONFIG_PWM_CRC) += pwm-crc.o obj-$(CONFIG_PWM_CROS_EC) += pwm-cros-ec.o obj-$(CONFIG_PWM_DWC_CORE) += pwm-dwc-core.o obj-$(CONFIG_PWM_DWC) += pwm-dwc.o +obj-$(CONFIG_PWM_DWC_OF) += pwm-dwc-of.o obj-$(CONFIG_PWM_EP93XX) += pwm-ep93xx.o obj-$(CONFIG_PWM_FSL_FTM) += pwm-fsl-ftm.o obj-$(CONFIG_PWM_GPIO) += pwm-gpio.o diff --git a/drivers/pwm/pwm-dwc-core.c b/drivers/pwm/pwm-dwc-core.c index 6dabec93a3c6..f612226618a5 100644 --- a/drivers/pwm/pwm-dwc-core.c +++ b/drivers/pwm/pwm-dwc-core.c @@ -12,8 +12,10 @@ #define DEFAULT_SYMBOL_NAMESPACE "dwc_pwm" #include +#include #include #include +#include #include #include #include @@ -39,26 +41,82 @@ static int __dwc_pwm_configure_timer(struct dwc_pwm *dwc, struct pwm_device *pwm, const struct pwm_state *state) { - u64 tmp; + u64 tmp, period_cyc; u32 ctrl; u32 high; u32 low; - /* - * Calculate width of low and high period in terms of input clock - * periods and check are the result within HW limits between 1 and - * 2^32 periods. - */ - tmp = DIV_ROUND_CLOSEST_ULL(state->duty_cycle, dwc->clk_ns); - if (tmp < 1 || tmp > (1ULL << 32)) - return -ERANGE; - low = tmp - 1; - - tmp = DIV_ROUND_CLOSEST_ULL(state->period - state->duty_cycle, - dwc->clk_ns); - if (tmp < 1 || tmp > (1ULL << 32)) - return -ERANGE; - high = tmp - 1; + if (dwc->clk) + dwc->clk_rate = clk_get_rate(dwc->clk); + + if (dwc->features & DWC_TIM_CTRL_0N100PWM_EN) { + /* + * Calculate the total period in clock cycles first, then the + * duty cycle. Derive the complementary half as the remainder to + * avoid compounding two independent floor-truncation errors: + * floor(duty) + floor(period - duty) can be one cycle short of + * floor(period). The PWM core requires the maximal achievable + * period not exceeding the requested value. + */ + period_cyc = mul_u64_u64_div_u64(state->period, dwc->clk_rate, + NSEC_PER_SEC); + if (!period_cyc) + return -ERANGE; + + tmp = mul_u64_u64_div_u64(state->duty_cycle, dwc->clk_rate, + NSEC_PER_SEC); + /* + * Calculate the width of the low and high periods in terms of + * input clock periods, and check whether the results are within + * the hardware limits of 0 to 2^32 periods. + */ + if (tmp >= (1ULL << 32) || period_cyc - tmp >= (1ULL << 32)) + return -ERANGE; + + /* + * The hardware has no polarity register. Polarity inversion is + * achieved by swapping the low and high load-count registers: + * NORMAL (active-high): duty_cycle -> + * HIGH period (DWC_TIM_LD_CNT2) + * INVERSED (active-low): duty_cycle -> + * LOW period (DWC_TIM_LD_CNT) + */ + if (state->polarity == PWM_POLARITY_NORMAL) { + high = tmp; + low = period_cyc - tmp; + } else { + low = tmp; + high = period_cyc - tmp; + } + } else { + /* + * Calculate width of low and high period in terms of input + * clock periods and check are the result within HW limits + * between 1 and 2^32 periods. + * Polarity inversion uses the same register-swap technique as + * the 0N100 path above. + * Derive the complementary half from the total period to avoid + * compounding two independent floor-truncation errors. + */ + tmp = mul_u64_u64_div_u64(state->duty_cycle, dwc->clk_rate, + NSEC_PER_SEC); + if (tmp < 1 || tmp > (1ULL << 32)) + return -ERANGE; + + period_cyc = mul_u64_u64_div_u64(state->period, dwc->clk_rate, + NSEC_PER_SEC); + /* period_cyc - tmp: complementary half; tmp <= period_cyc */ + if (period_cyc - tmp < 1 || period_cyc - tmp > (1ULL << 32)) + return -ERANGE; + + if (state->polarity == PWM_POLARITY_NORMAL) { + high = tmp - 1; + low = period_cyc - tmp - 1; + } else { + low = tmp - 1; + high = period_cyc - tmp - 1; + } + } /* * Specification says timer usage flow is to disable timer, then @@ -74,6 +132,7 @@ static int __dwc_pwm_configure_timer(struct dwc_pwm *dwc, * width of low period and latter the width of high period in terms * multiple of input clock periods: * Width = ((Count + 1) * input clock period). + * Width = (Count * input clock period) : supported 0% and 100%. */ dwc_pwm_writel(dwc, low, DWC_TIM_LD_CNT(pwm->hwpwm)); dwc_pwm_writel(dwc, high, DWC_TIM_LD_CNT2(pwm->hwpwm)); @@ -85,6 +144,14 @@ static int __dwc_pwm_configure_timer(struct dwc_pwm *dwc, * periods are set by Load Count registers. */ ctrl = DWC_TIM_CTRL_MODE_USER | DWC_TIM_CTRL_PWM; + /* + * Mask interrupts to prevent unmasked timer interrupts on shared IRQ + * systems where no IRQ handler is installed. + */ + ctrl |= DWC_TIM_CTRL_INT_MASK; + if (dwc->features & DWC_TIM_CTRL_0N100PWM_EN) + ctrl |= DWC_TIM_CTRL_0N100PWM_EN; + dwc_pwm_writel(dwc, ctrl, DWC_TIM_CTRL(pwm->hwpwm)); /* @@ -99,14 +166,18 @@ static int dwc_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, const struct pwm_state *state) { struct dwc_pwm *dwc = to_dwc_pwm(chip); - - if (state->polarity != PWM_POLARITY_INVERSED) - return -EINVAL; + int ret; if (state->enabled) { - if (!pwm->state.enabled) - pm_runtime_get_sync(pwmchip_parent(chip)); - return __dwc_pwm_configure_timer(dwc, pwm, state); + if (!pwm->state.enabled) { + ret = pm_runtime_resume_and_get(pwmchip_parent(chip)); + if (ret < 0) + return ret; + } + ret = __dwc_pwm_configure_timer(dwc, pwm, state); + if (ret && !pwm->state.enabled) + pm_runtime_put_sync(pwmchip_parent(chip)); + return ret; } else { if (pwm->state.enabled) { __dwc_pwm_set_enable(dwc, pwm->hwpwm, false); @@ -121,10 +192,23 @@ static int dwc_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm, struct pwm_state *state) { struct dwc_pwm *dwc = to_dwc_pwm(chip); - u64 duty, period; + unsigned long clk_rate; u32 ctrl, ld, ld2; + u64 duty, period; + int ret; + + ret = pm_runtime_resume_and_get(pwmchip_parent(chip)); + if (ret < 0) + return ret; + + if (dwc->clk) + dwc->clk_rate = clk_get_rate(dwc->clk); - pm_runtime_get_sync(pwmchip_parent(chip)); + clk_rate = dwc->clk_rate; + if (!clk_rate) { + pm_runtime_put_sync(pwmchip_parent(chip)); + return -EINVAL; + } ctrl = dwc_pwm_readl(dwc, DWC_TIM_CTRL(pwm->hwpwm)); ld = dwc_pwm_readl(dwc, DWC_TIM_LD_CNT(pwm->hwpwm)); @@ -132,22 +216,46 @@ static int dwc_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm, state->enabled = !!(ctrl & DWC_TIM_CTRL_EN); + /* + * The hardware has no polarity status register; polarity is encoded + * implicitly by which of DWC_TIM_LD_CNT / DWC_TIM_LD_CNT2 holds the + * duty-cycle period (see __dwc_pwm_configure_timer). Report the + * polarity that was last programmed by apply(). On the initial read + * (before any apply call), pwm->state.polarity defaults to + * PWM_POLARITY_NORMAL, which is the natural zero-initialised value. + */ + state->polarity = pwm->state.polarity; + /* * If we're not in PWM, technically the output is a 50-50 * based on the timer load-count only. */ if (ctrl & DWC_TIM_CTRL_PWM) { - duty = (ld + 1) * dwc->clk_ns; - period = (ld2 + 1) * dwc->clk_ns; - period += duty; + if (ctrl & DWC_TIM_CTRL_0N100PWM_EN) { + /* + * NORMAL: duty_cycle was written to DWC_TIM_LD_CNT2. + * INVERSED: duty_cycle was written to DWC_TIM_LD_CNT. + */ + if (state->polarity == PWM_POLARITY_NORMAL) + duty = ld2; + else + duty = ld; + period = (u64)ld + ld2; + } else { + if (state->polarity == PWM_POLARITY_NORMAL) + duty = ld2 + 1; + else + duty = ld + 1; + period = (u64)ld + ld2 + 2; + } } else { - duty = (ld + 1) * dwc->clk_ns; + duty = ld + 1; period = duty * 2; + state->polarity = PWM_POLARITY_INVERSED; } - state->polarity = PWM_POLARITY_INVERSED; - state->period = period; - state->duty_cycle = duty; + state->period = mul_u64_u64_div_u64(period, NSEC_PER_SEC, clk_rate); + state->duty_cycle = mul_u64_u64_div_u64(duty, NSEC_PER_SEC, clk_rate); pm_runtime_put_sync(pwmchip_parent(chip)); @@ -169,7 +277,7 @@ struct pwm_chip *dwc_pwm_alloc(struct device *dev) return chip; dwc = to_dwc_pwm(chip); - dwc->clk_ns = 10; + dwc->clk_rate = NSEC_PER_SEC / 10; chip->ops = &dwc_pwm_ops; return chip; diff --git a/drivers/pwm/pwm-dwc-of.c b/drivers/pwm/pwm-dwc-of.c new file mode 100644 index 000000000000..e96e5569eb81 --- /dev/null +++ b/drivers/pwm/pwm-dwc-of.c @@ -0,0 +1,298 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * DesignWare PWM Controller driver OF + * + * Copyright (C) 2026 SiFive, Inc. + */ + +#define DEFAULT_SYMBOL_NAMESPACE "dwc_pwm_of" + +#include +#include +#include +#include +#include + +#include "pwm-dwc.h" + +static int dwc_pwm_plat_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct dwc_pwm_drvdata *data; + u32 ctrl[DWC_TIMERS_TOTAL]; + struct pwm_chip *chip; + struct dwc_pwm *dwc; + bool pwm_en = false; + u32 nr_pwm, tim_id; + unsigned int i; + int ret; + + data = devm_kzalloc(dev, struct_size(data, chips, 1), GFP_KERNEL); + if (!data) + return -ENOMEM; + + chip = dwc_pwm_alloc(dev); + if (IS_ERR(chip)) + return dev_err_probe(dev, PTR_ERR(chip), + "failed to alloc pwm\n"); + + dwc = to_dwc_pwm(chip); + + dwc->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(dwc->base)) + return PTR_ERR(dwc->base); + + if (!device_property_read_u32(dev, "snps,pwm-number", &nr_pwm)) { + if (nr_pwm > DWC_TIMERS_TOTAL) + dev_warn(dev, "too many PWMs (%u), capping at %u\n", + nr_pwm, chip->npwm); + else + chip->npwm = nr_pwm; + } + + dwc->bus_clk = devm_clk_get(dev, "bus"); + if (IS_ERR(dwc->bus_clk)) + return dev_err_probe(dev, PTR_ERR(dwc->bus_clk), + "failed to get bus clock\n"); + + dwc->clk = devm_clk_get(dev, "timer"); + if (IS_ERR(dwc->clk)) + return dev_err_probe(dev, PTR_ERR(dwc->clk), + "failed to get timer clock\n"); + + ret = devm_clk_rate_exclusive_get(dev, dwc->clk); + if (ret) + return dev_err_probe(dev, ret, + "failed to get exclusive rate\n"); + + dwc->clk_rate = clk_get_rate(dwc->clk); + if (!dwc->clk_rate) + return dev_err_probe(dev, -EINVAL, + "failed to get a valid clock rate\n"); + + dwc->rst = devm_reset_control_array_get_optional_exclusive(dev); + + if (IS_ERR(dwc->rst)) + return dev_err_probe(dev, PTR_ERR(dwc->rst), + "failed to get reset control\n"); + + ret = clk_prepare_enable(dwc->bus_clk); + if (ret) + return dev_err_probe(dev, ret, + "failed to enable bus clock\n"); + + ret = clk_prepare_enable(dwc->clk); + if (ret) { + ret = dev_err_probe(dev, ret, "failed to enable timer clock\n"); + goto disable_busclk; + } + + /* Ensure the device is out of reset before accessing MMIO registers. */ + ret = reset_control_deassert(dwc->rst); + if (ret) { + ret = dev_err_probe(dev, ret, "failed to deassert reset\n"); + goto disable_clk; + } + + /* + * Check all channels to see if any channel is enabled. + * Read the control register of each channel and extract the enable bit + */ + for (i = 0; i < chip->npwm; i++) { + ctrl[i] = dwc_pwm_readl(dwc, DWC_TIM_CTRL(i)) & DWC_TIM_CTRL_EN; + if (ctrl[i]) + pwm_en = true; + } + + /* + * Only issue a reset pulse when all channels are disabled, so a PWM + * channel already running (e.g. configured by firmware before Linux + * took over) is left undisturbed. + */ + if (!pwm_en) { + ret = reset_control_reset(dwc->rst); + if (ret) { + ret = dev_err_probe(dev, ret, "failed to reset\n"); + goto disable_clk; + } + } + + /* init PWM feature */ + dwc->features = 0; + /* + * Support for 0% and 100% duty cycle mode was added in version 2.11a + * and later. + */ + tim_id = dwc_pwm_readl(dwc, DWC_TIMERS_COMP_VERSION); + if (tim_id >= DWC_TIM_VERSION_ID_2_11A) + dwc->features |= DWC_TIM_CTRL_0N100PWM_EN; + + data->chips[0] = chip; + dev_set_drvdata(dev, data); + + /* + * Save hardware state to PM context before first resume. + * Prevents zero-initialized context from overwriting registers + * and losing bootloader-preset configuration. + */ + for (i = 0; i < chip->npwm; i++) { + dwc->ctx[i].cnt = dwc_pwm_readl(dwc, DWC_TIM_LD_CNT(i)); + dwc->ctx[i].cnt2 = dwc_pwm_readl(dwc, DWC_TIM_LD_CNT2(i)); + dwc->ctx[i].ctrl = dwc_pwm_readl(dwc, DWC_TIM_CTRL(i)); + } + + /* + * If any PWM channel is enabled, mark device active and hold runtime PM + * references for each enabled channel. Otherwise, gate the clocks. + * + * When CONFIG_PM is disabled, pm_runtime callbacks are stubs that never + * re-enable clocks, so keep clocks on unconditionally in that case. + */ + if (pwm_en) { + pm_runtime_set_active(dev); + for (i = 0; i < chip->npwm; i++) { + if (ctrl[i]) + pm_runtime_get_noresume(dev); + } + } else if (IS_ENABLED(CONFIG_PM)) { + clk_disable_unprepare(dwc->clk); + clk_disable_unprepare(dwc->bus_clk); + } + + pm_runtime_enable(dev); + + ret = pwmchip_add(chip); + if (ret) { + ret = dev_err_probe(dev, ret, "failed to add pwm chip\n"); + goto pm_disable; + } + + return 0; + +pm_disable: + pm_runtime_disable(dev); + if (pwm_en) { + for (i = 0; i < chip->npwm; i++) { + if (ctrl[i]) + pm_runtime_put_noidle(dev); + } + goto disable_clk; + } + /* + * When CONFIG_PM is disabled, clocks were never gated and must be + * disabled here. + */ + if (!IS_ENABLED(CONFIG_PM)) + goto disable_clk; + + return ret; + +disable_clk: + clk_disable_unprepare(dwc->clk); +disable_busclk: + clk_disable_unprepare(dwc->bus_clk); + + return ret; +} + +static void dwc_pwm_plat_remove(struct platform_device *pdev) +{ + struct dwc_pwm_drvdata *data = platform_get_drvdata(pdev); + struct pwm_chip *chip = data->chips[0]; + struct dwc_pwm *dwc = to_dwc_pwm(chip); + unsigned int idx; + u32 ctrl; + + pwmchip_remove(chip); + pm_runtime_disable(&pdev->dev); + + if (!pm_runtime_status_suspended(&pdev->dev)) { + for (idx = 0; idx < chip->npwm; idx++) { + ctrl = dwc_pwm_readl(dwc, DWC_TIM_CTRL(idx)); + if (ctrl & DWC_TIM_CTRL_EN) { + dwc_pwm_writel(dwc, ctrl & ~DWC_TIM_CTRL_EN, + DWC_TIM_CTRL(idx)); + pm_runtime_put_noidle(&pdev->dev); + } + } + clk_disable_unprepare(dwc->clk); + clk_disable_unprepare(dwc->bus_clk); + } +} + +static int dwc_pwm_runtime_suspend(struct device *dev) +{ + struct dwc_pwm_drvdata *data = dev_get_drvdata(dev); + struct pwm_chip *chip = data->chips[0]; + struct dwc_pwm *dwc = to_dwc_pwm(chip); + unsigned int idx; + + for (idx = 0; idx < chip->npwm; idx++) { + dwc->ctx[idx].cnt = dwc_pwm_readl(dwc, DWC_TIM_LD_CNT(idx)); + dwc->ctx[idx].cnt2 = dwc_pwm_readl(dwc, DWC_TIM_LD_CNT2(idx)); + dwc->ctx[idx].ctrl = dwc_pwm_readl(dwc, DWC_TIM_CTRL(idx)); + } + + clk_disable_unprepare(dwc->clk); + clk_disable_unprepare(dwc->bus_clk); + + return 0; +} + +static int dwc_pwm_runtime_resume(struct device *dev) +{ + struct dwc_pwm_drvdata *data = dev_get_drvdata(dev); + struct pwm_chip *chip = data->chips[0]; + struct dwc_pwm *dwc = to_dwc_pwm(chip); + unsigned int idx; + int ret; + + ret = clk_prepare_enable(dwc->bus_clk); + if (ret) { + dev_err(dev, "failed to enable bus clock: %d\n", ret); + return ret; + } + + ret = clk_prepare_enable(dwc->clk); + if (ret) { + dev_err(dev, "failed to enable timer clock: %d\n", ret); + clk_disable_unprepare(dwc->bus_clk); + return ret; + } + + for (idx = 0; idx < chip->npwm; idx++) { + dwc_pwm_writel(dwc, dwc->ctx[idx].cnt, DWC_TIM_LD_CNT(idx)); + dwc_pwm_writel(dwc, dwc->ctx[idx].cnt2, DWC_TIM_LD_CNT2(idx)); + dwc_pwm_writel(dwc, dwc->ctx[idx].ctrl, DWC_TIM_CTRL(idx)); + } + + return 0; +} + +static const struct dev_pm_ops dwc_pwm_pm_ops = { + RUNTIME_PM_OPS(dwc_pwm_runtime_suspend, dwc_pwm_runtime_resume, NULL) + SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume) +}; + +static const struct of_device_id dwc_pwm_dt_ids[] = { + { .compatible = "snps,dw-apb-timers-pwm2" }, + { .compatible = "eswin,eic7700-pwm" }, + { } +}; +MODULE_DEVICE_TABLE(of, dwc_pwm_dt_ids); + +static struct platform_driver dwc_pwm_plat_driver = { + .driver = { + .name = "dwc-pwm", + .pm = pm_ptr(&dwc_pwm_pm_ops), + .of_match_table = dwc_pwm_dt_ids, + }, + .probe = dwc_pwm_plat_probe, + .remove = dwc_pwm_plat_remove, +}; + +module_platform_driver(dwc_pwm_plat_driver); + +MODULE_AUTHOR("Ben Dooks "); +MODULE_DESCRIPTION("DesignWare PWM Controller"); +MODULE_LICENSE("GPL"); diff --git a/drivers/pwm/pwm-dwc.h b/drivers/pwm/pwm-dwc.h index 1562594e7f85..75f7c2d031c4 100644 --- a/drivers/pwm/pwm-dwc.h +++ b/drivers/pwm/pwm-dwc.h @@ -26,12 +26,19 @@ MODULE_IMPORT_NS("dwc_pwm"); #define DWC_TIMERS_TOTAL 8 /* Timer Control Register */ -#define DWC_TIM_CTRL_EN BIT(0) -#define DWC_TIM_CTRL_MODE BIT(1) -#define DWC_TIM_CTRL_MODE_FREE (0 << 1) -#define DWC_TIM_CTRL_MODE_USER (1 << 1) -#define DWC_TIM_CTRL_INT_MASK BIT(2) -#define DWC_TIM_CTRL_PWM BIT(3) +#define DWC_TIM_CTRL_EN BIT(0) +#define DWC_TIM_CTRL_MODE BIT(1) +#define DWC_TIM_CTRL_MODE_FREE (0 << 1) +#define DWC_TIM_CTRL_MODE_USER BIT(1) +#define DWC_TIM_CTRL_INT_MASK BIT(2) +#define DWC_TIM_CTRL_PWM BIT(3) +#define DWC_TIM_CTRL_0N100PWM_EN BIT(4) + +/* + * The version 2.11a and later add "Pulse Width Modulation with + * 0% and 100% Duty Cycle". + */ +#define DWC_TIM_VERSION_ID_2_11A 0x3231312a struct dwc_pwm_info { unsigned int nr; @@ -52,8 +59,12 @@ struct dwc_pwm_ctx { struct dwc_pwm { void __iomem *base; - unsigned int clk_ns; + struct clk *bus_clk; + struct clk *clk; + unsigned long clk_rate; + struct reset_control *rst; struct dwc_pwm_ctx ctx[DWC_TIMERS_TOTAL]; + u32 features; }; static inline struct dwc_pwm *to_dwc_pwm(struct pwm_chip *chip) diff --git a/fs/dax.c b/fs/dax.c index 1fbba0d21c13..692bcd71dc1a 100644 --- a/fs/dax.c +++ b/fs/dax.c @@ -1940,7 +1940,7 @@ static vm_fault_t dax_iomap_pte_fault(struct vm_fault *vmf, unsigned long *pfnp, * the PTE we need to set up. If so just return and the fault will be * retried. */ - if (pmd_trans_huge(*vmf->pmd)) { + if (pmd_trans_huge(pmdp_get(vmf->pmd))) { ret = VM_FAULT_NOPAGE; goto unlock_entry; } @@ -2061,7 +2061,7 @@ static vm_fault_t dax_iomap_pmd_fault(struct vm_fault *vmf, unsigned long *pfnp, * the PMD we need to set up. If so just return and the fault will be * retried. */ - if (!pmd_none(*vmf->pmd) && !pmd_trans_huge(*vmf->pmd)) { + if (!pmd_none(pmdp_get(vmf->pmd)) && !pmd_trans_huge(pmdp_get(vmf->pmd))) { ret = 0; goto unlock_entry; } diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c index 5c54aebe2118..d933d3db3d4c 100644 --- a/fs/proc/task_mmu.c +++ b/fs/proc/task_mmu.c @@ -1103,13 +1103,13 @@ static void smaps_pmd_entry(pmd_t *pmd, unsigned long addr, bool present = false; struct folio *folio; - if (pmd_none(*pmd)) + if (pmd_none(pmdp_get(pmd))) return; - if (pmd_present(*pmd)) { - page = vm_normal_page_pmd(vma, addr, *pmd); + if (pmd_present(pmdp_get(pmd))) { + page = vm_normal_page_pmd(vma, addr, pmdp_get(pmd)); present = true; } else if (unlikely(thp_migration_supported())) { - const softleaf_t entry = softleaf_from_pmd(*pmd); + const softleaf_t entry = softleaf_from_pmd(pmdp_get(pmd)); if (softleaf_has_pfn(entry)) page = softleaf_to_page(entry); @@ -1126,7 +1126,8 @@ static void smaps_pmd_entry(pmd_t *pmd, unsigned long addr, else mss->file_thp += HPAGE_PMD_SIZE; - smaps_account(mss, page, true, pmd_young(*pmd), pmd_dirty(*pmd), + smaps_account(mss, page, true, pmd_young(pmdp_get(pmd)), + pmd_dirty(pmdp_get(pmd)), locked, present); } #else @@ -1738,7 +1739,7 @@ static inline void clear_soft_dirty(struct vm_area_struct *vma, static inline void clear_soft_dirty_pmd(struct vm_area_struct *vma, unsigned long addr, pmd_t *pmdp) { - pmd_t old, pmd = *pmdp; + pmd_t old, pmd = pmdp_get(pmdp); if (!pgtable_supports_soft_dirty()) return; @@ -1783,10 +1784,10 @@ static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr, goto out; } - if (!pmd_present(*pmd)) + if (!pmd_present(pmdp_get(pmd))) goto out; - folio = pmd_folio(*pmd); + folio = pmd_folio(pmdp_get(pmd)); /* Clear accessed and referenced bits. */ pmdp_test_and_clear_young(vma, addr, pmd); @@ -2094,7 +2095,7 @@ static int pagemap_pmd_range_thp(pmd_t *pmdp, unsigned long addr, { unsigned int idx = (addr & ~PMD_MASK) >> PAGE_SHIFT; u64 flags = 0, frame = 0; - pmd_t pmd = *pmdp; + pmd_t pmd = pmdp_get(pmdp); struct page *page = NULL; struct folio *folio = NULL; int err = 0; @@ -2575,7 +2576,7 @@ static unsigned long pagemap_thp_category(struct pagemap_scan_private *p, static void make_uffd_wp_pmd(struct vm_area_struct *vma, unsigned long addr, pmd_t *pmdp) { - pmd_t old, pmd = *pmdp; + pmd_t old, pmd = pmdp_get(pmdp); if (pmd_present(pmd)) { old = pmdp_invalidate_ad(vma, addr, pmdp); @@ -2826,7 +2827,7 @@ static int pagemap_scan_thp_entry(pmd_t *pmd, unsigned long start, return -ENOENT; categories = p->cur_vma_category | - pagemap_thp_category(p, vma, start, *pmd); + pagemap_thp_category(p, vma, start, pmdp_get(pmd)); if (!pagemap_scan_is_interesting_page(categories, p)) goto out_unlock; @@ -3435,9 +3436,9 @@ static int gather_pte_stats(pmd_t *pmd, unsigned long addr, if (ptl) { struct page *page; - page = can_gather_numa_stats_pmd(*pmd, vma, addr); + page = can_gather_numa_stats_pmd(pmdp_get(pmd), vma, addr); if (page) - gather_stats(page, md, pmd_dirty(*pmd), + gather_stats(page, md, pmd_dirty(pmdp_get(pmd)), HPAGE_PMD_SIZE/PAGE_SIZE); spin_unlock(ptl); return 0; diff --git a/include/dt-bindings/riscv/physical-memory.h b/include/dt-bindings/riscv/physical-memory.h new file mode 100644 index 000000000000..d6ed8015c535 --- /dev/null +++ b/include/dt-bindings/riscv/physical-memory.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */ + +#ifndef _DT_BINDINGS_RISCV_PHYSICAL_MEMORY_H +#define _DT_BINDINGS_RISCV_PHYSICAL_MEMORY_H + +#define PMA_READ (1 << 0) +#define PMA_WRITE (1 << 1) +#define PMA_EXECUTE (1 << 2) +#define PMA_AMO_MASK (3 << 4) +#define PMA_AMO_NONE (0 << 4) +#define PMA_AMO_SWAP (1 << 4) +#define PMA_AMO_LOGICAL (2 << 4) +#define PMA_AMO_ARITHMETIC (3 << 4) +#define PMA_RSRV_MASK (3 << 6) +#define PMA_RSRV_NONE (0 << 6) +#define PMA_RSRV_NON_EVENTUAL (1 << 6) +#define PMA_RSRV_EVENTUAL (2 << 6) + +#define PMA_RW (PMA_READ | PMA_WRITE) +#define PMA_RWA (PMA_RW | PMA_AMO_ARITHMETIC | PMA_RSRV_EVENTUAL) +#define PMA_RWX (PMA_RW | PMA_EXECUTE) +#define PMA_RWXA (PMA_RWA | PMA_EXECUTE) + +#define PMA_ORDER_MASK (3 << 8) +#define PMA_ORDER_IO_RELAXED (0 << 8) +#define PMA_ORDER_IO_STRONG (1 << 8) +#define PMA_ORDER_MEMORY (2 << 8) +#define PMA_READ_IDEMPOTENT (1 << 10) +#define PMA_WRITE_IDEMPOTENT (1 << 11) +#define PMA_CACHEABLE (1 << 12) +#define PMA_COHERENT (1 << 13) + +#define PMA_UNSAFE (1 << 15) + +#define PMA_IO (PMA_ORDER_IO_RELAXED) +#define PMA_NONCACHEABLE_MEMORY (PMA_ORDER_MEMORY | PMA_READ_IDEMPOTENT | \ + PMA_WRITE_IDEMPOTENT) +#define PMA_NONCOHERENT_MEMORY (PMA_NONCACHEABLE_MEMORY | PMA_CACHEABLE) +#define PMA_NORMAL_MEMORY (PMA_NONCOHERENT_MEMORY | PMA_COHERENT) + +#define PMR_ALIAS_MASK (0x7f << 24) +#define PMR_IS_ALIAS (0x80 << 24) +#define PMR_ALIAS(n) (PMR_IS_ALIAS | ((n) << 24)) + +#endif /* _DT_BINDINGS_RISCV_PHYSICAL_MEMORY_H */ diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index c745f7ad2298..434bbc5e9c88 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h @@ -461,7 +461,7 @@ static inline bool pmd_is_huge(pmd_t pmd) #define split_huge_pmd(__vma, __pmd, __address) \ do { \ pmd_t *____pmd = (__pmd); \ - if (pmd_is_huge(*____pmd)) \ + if (pmd_is_huge(pmdp_get(____pmd))) \ __split_huge_pmd(__vma, __pmd, __address, \ false); \ } while (0) @@ -504,7 +504,7 @@ change_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma, #define split_huge_pud(__vma, __pud, __address) \ do { \ pud_t *____pud = (__pud); \ - if (pud_trans_huge(*____pud)) \ + if (pud_trans_huge(pudp_get(____pud))) \ __split_huge_pud(__vma, __pud, __address); \ } while (0) @@ -521,7 +521,7 @@ spinlock_t *__pud_trans_huge_lock(pud_t *pud, struct vm_area_struct *vma); static inline spinlock_t *pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma) { - if (pmd_is_huge(*pmd)) + if (pmd_is_huge(pmdp_get(pmd))) return __pmd_trans_huge_lock(pmd, vma); return NULL; @@ -529,7 +529,7 @@ static inline spinlock_t *pmd_trans_huge_lock(pmd_t *pmd, static inline spinlock_t *pud_trans_huge_lock(pud_t *pud, struct vm_area_struct *vma) { - if (pud_trans_huge(*pud)) + if (pud_trans_huge(pudp_get(pud))) return __pud_trans_huge_lock(pud, vma); else return NULL; diff --git a/include/linux/mm.h b/include/linux/mm.h index dd09c438fa23..303ffeec9a37 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -3694,20 +3694,20 @@ int __pte_alloc_kernel(pmd_t *pmd); static inline p4d_t *p4d_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address) { - return (unlikely(pgd_none(*pgd)) && __p4d_alloc(mm, pgd, address)) ? + return (unlikely(pgd_none(pgdp_get(pgd))) && __p4d_alloc(mm, pgd, address)) ? NULL : p4d_offset(pgd, address); } static inline pud_t *pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address) { - return (unlikely(p4d_none(*p4d)) && __pud_alloc(mm, p4d, address)) ? + return (unlikely(p4d_none(p4dp_get(p4d))) && __pud_alloc(mm, p4d, address)) ? NULL : pud_offset(p4d, address); } static inline pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address) { - return (unlikely(pud_none(*pud)) && __pmd_alloc(mm, pud, address))? + return (unlikely(pud_none(pudp_get(pud))) && __pmd_alloc(mm, pud, address)) ? NULL: pmd_offset(pud, address); } #endif /* CONFIG_MMU */ @@ -3859,9 +3859,9 @@ static inline spinlock_t *ptlock_ptr(struct ptdesc *ptdesc) } #endif /* ALLOC_SPLIT_PTLOCKS */ -static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmd) +static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmdvalp) { - return ptlock_ptr(page_ptdesc(pmd_page(*pmd))); + return ptlock_ptr(page_ptdesc(pmd_page(*pmdvalp))); } static inline spinlock_t *ptep_lockptr(struct mm_struct *mm, pte_t *pte) @@ -3957,7 +3957,7 @@ pte_t *pte_offset_map_rw_nolock(struct mm_struct *mm, pmd_t *pmd, pte_unmap(pte); \ } while (0) -#define pte_alloc(mm, pmd) (unlikely(pmd_none(*(pmd))) && __pte_alloc(mm, pmd)) +#define pte_alloc(mm, pmd) (unlikely(pmd_none(pmdp_get(pmd))) && __pte_alloc(mm, pmd)) #define pte_alloc_map(mm, pmd, address) \ (pte_alloc(mm, pmd) ? NULL : pte_offset_map(pmd, address)) @@ -3967,7 +3967,7 @@ pte_t *pte_offset_map_rw_nolock(struct mm_struct *mm, pmd_t *pmd, NULL : pte_offset_map_lock(mm, pmd, address, ptlp)) #define pte_alloc_kernel(pmd, address) \ - ((unlikely(pmd_none(*(pmd))) && __pte_alloc_kernel(pmd))? \ + ((unlikely(pmd_none(pmdp_get(pmd))) && __pte_alloc_kernel(pmd)) ? \ NULL: pte_offset_kernel(pmd, address)) #if defined(CONFIG_SPLIT_PMD_PTLOCKS) diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h index 8c093c119e5a..46ec06544621 100644 --- a/include/linux/pgtable.h +++ b/include/linux/pgtable.h @@ -71,6 +71,41 @@ static inline unsigned long pud_index(unsigned long address) #define pgd_index(a) (((a) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1)) #endif +#ifndef ptep_get +static inline pte_t ptep_get(pte_t *ptep) +{ + return READ_ONCE(*ptep); +} +#endif + +#ifndef pmdp_get +static inline pmd_t pmdp_get(pmd_t *pmdp) +{ + return READ_ONCE(*pmdp); +} +#endif + +#ifndef pudp_get +static inline pud_t pudp_get(pud_t *pudp) +{ + return READ_ONCE(*pudp); +} +#endif + +#ifndef p4dp_get +static inline p4d_t p4dp_get(p4d_t *p4dp) +{ + return READ_ONCE(*p4dp); +} +#endif + +#ifndef pgdp_get +static inline pgd_t pgdp_get(pgd_t *pgdp) +{ + return READ_ONCE(*pgdp); +} +#endif + #ifndef kernel_pte_init static inline void kernel_pte_init(void *addr) { @@ -95,14 +130,14 @@ static inline void pud_init(void *addr) #ifndef pte_offset_kernel static inline pte_t *pte_offset_kernel(pmd_t *pmd, unsigned long address) { - return (pte_t *)pmd_page_vaddr(*pmd) + pte_index(address); + return (pte_t *)pmd_page_vaddr(pmdp_get(pmd)) + pte_index(address); } #define pte_offset_kernel pte_offset_kernel #endif #ifdef CONFIG_HIGHPTE #define __pte_map(pmd, address) \ - ((pte_t *)kmap_local_page(pmd_page(*(pmd))) + pte_index((address))) + ((pte_t *)kmap_local_page(pmd_page(pmdp_get(pmd))) + pte_index((address))) #define pte_unmap(pte) do { \ kunmap_local((pte)); \ rcu_read_unlock(); \ @@ -124,7 +159,7 @@ void pte_free_defer(struct mm_struct *mm, pgtable_t pgtable); #ifndef pmd_offset static inline pmd_t *pmd_offset(pud_t *pud, unsigned long address) { - return pud_pgtable(*pud) + pmd_index(address); + return pud_pgtable(pudp_get(pud)) + pmd_index(address); } #define pmd_offset pmd_offset #endif @@ -132,7 +167,7 @@ static inline pmd_t *pmd_offset(pud_t *pud, unsigned long address) #ifndef pud_offset static inline pud_t *pud_offset(p4d_t *p4d, unsigned long address) { - return p4d_pgtable(*p4d) + pud_index(address); + return p4d_pgtable(p4dp_get(p4d)) + pud_index(address); } #define pud_offset pud_offset #endif @@ -176,7 +211,7 @@ static inline pte_t *virt_to_kpte(unsigned long vaddr) { pmd_t *pmd = pmd_off_k(vaddr); - return pmd_none(*pmd) ? NULL : pte_offset_kernel(pmd, vaddr); + return pmd_none(pmdp_get(pmd)) ? NULL : pte_offset_kernel(pmd, vaddr); } #ifndef pmd_young @@ -489,41 +524,6 @@ static inline int pudp_set_access_flags(struct vm_area_struct *vma, #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ #endif -#ifndef ptep_get -static inline pte_t ptep_get(pte_t *ptep) -{ - return READ_ONCE(*ptep); -} -#endif - -#ifndef pmdp_get -static inline pmd_t pmdp_get(pmd_t *pmdp) -{ - return READ_ONCE(*pmdp); -} -#endif - -#ifndef pudp_get -static inline pud_t pudp_get(pud_t *pudp) -{ - return READ_ONCE(*pudp); -} -#endif - -#ifndef p4dp_get -static inline p4d_t p4dp_get(p4d_t *p4dp) -{ - return READ_ONCE(*p4dp); -} -#endif - -#ifndef pgdp_get -static inline pgd_t pgdp_get(pgd_t *pgdp) -{ - return READ_ONCE(*pgdp); -} -#endif - #ifndef __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG static inline bool ptep_test_and_clear_young(struct vm_area_struct *vma, unsigned long address, pte_t *ptep) @@ -544,7 +544,7 @@ static inline bool ptep_test_and_clear_young(struct vm_area_struct *vma, static inline bool pmdp_test_and_clear_young(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp) { - pmd_t pmd = *pmdp; + pmd_t pmd = pmdp_get(pmdp); bool young = true; if (!pmd_young(pmd)) @@ -799,7 +799,7 @@ static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, unsigned long address, pmd_t *pmdp) { - pmd_t pmd = *pmdp; + pmd_t pmd = pmdp_get(pmdp); pmd_clear(pmdp); page_table_check_pmd_clear(mm, address, pmd); @@ -812,7 +812,7 @@ static inline pud_t pudp_huge_get_and_clear(struct mm_struct *mm, unsigned long address, pud_t *pudp) { - pud_t pud = *pudp; + pud_t pud = pudp_get(pudp); pud_clear(pudp); page_table_check_pud_clear(mm, address, pud); @@ -1218,7 +1218,7 @@ static inline pte_t pte_sw_mkyoung(pte_t pte) static inline void pmdp_set_wrprotect(struct mm_struct *mm, unsigned long address, pmd_t *pmdp) { - pmd_t old_pmd = *pmdp; + pmd_t old_pmd = pmdp_get(pmdp); set_pmd_at(mm, address, pmdp, pmd_wrprotect(old_pmd)); } #else @@ -1235,7 +1235,7 @@ static inline void pmdp_set_wrprotect(struct mm_struct *mm, static inline void pudp_set_wrprotect(struct mm_struct *mm, unsigned long address, pud_t *pudp) { - pud_t old_pud = *pudp; + pud_t old_pud = pudp_get(pudp); set_pud_at(mm, address, pudp, pud_wrprotect(old_pud)); } @@ -1259,7 +1259,7 @@ static inline pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, pmd_t *pmdp) { BUILD_BUG(); - return *pmdp; + return pmdp_get(pmdp); } #define pmdp_collapse_flush pmdp_collapse_flush #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ @@ -1287,7 +1287,7 @@ extern pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp); static inline pmd_t generic_pmdp_establish(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp, pmd_t pmd) { - pmd_t old_pmd = *pmdp; + pmd_t old_pmd = pmdp_get(pmdp); set_pmd_at(vma->vm_mm, address, pmdp, pmd); return old_pmd; } @@ -1541,9 +1541,9 @@ void pmd_clear_bad(pmd_t *); static inline int pgd_none_or_clear_bad(pgd_t *pgd) { - if (pgd_none(*pgd)) + if (pgd_none(pgdp_get(pgd))) return 1; - if (unlikely(pgd_bad(*pgd))) { + if (unlikely(pgd_bad(pgdp_get(pgd)))) { pgd_clear_bad(pgd); return 1; } @@ -1552,9 +1552,9 @@ static inline int pgd_none_or_clear_bad(pgd_t *pgd) static inline int p4d_none_or_clear_bad(p4d_t *p4d) { - if (p4d_none(*p4d)) + if (p4d_none(p4dp_get(p4d))) return 1; - if (unlikely(p4d_bad(*p4d))) { + if (unlikely(p4d_bad(p4dp_get(p4d)))) { p4d_clear_bad(p4d); return 1; } @@ -1563,9 +1563,9 @@ static inline int p4d_none_or_clear_bad(p4d_t *p4d) static inline int pud_none_or_clear_bad(pud_t *pud) { - if (pud_none(*pud)) + if (pud_none(pudp_get(pud))) return 1; - if (unlikely(pud_bad(*pud))) { + if (unlikely(pud_bad(pudp_get(pud)))) { pud_clear_bad(pud); return 1; } @@ -1574,9 +1574,9 @@ static inline int pud_none_or_clear_bad(pud_t *pud) static inline int pmd_none_or_clear_bad(pmd_t *pmd) { - if (pmd_none(*pmd)) + if (pmd_none(pmdp_get(pmd))) return 1; - if (unlikely(pmd_bad(*pmd))) { + if (unlikely(pmd_bad(pmdp_get(pmd)))) { pmd_clear_bad(pmd); return 1; } diff --git a/kernel/events/core.c b/kernel/events/core.c index c4217c996f5a..1bc605a17699 100644 --- a/kernel/events/core.c +++ b/kernel/events/core.c @@ -8570,6 +8570,8 @@ static u64 perf_get_pgtable_size(struct mm_struct *mm, unsigned long addr) if (pmd_leaf(pmd)) return pmd_leaf_size(pmd); + /* transform pmd as if &pmd pointed to a hardware page table */ + set_pmd(&pmd, pmd); ptep = pte_offset_map(&pmd, addr); if (!ptep) goto again; diff --git a/mm/filemap.c b/mm/filemap.c index 00fd89cf6f55..7e84e49d29ee 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -3719,13 +3719,13 @@ static bool filemap_map_pmd(struct vm_fault *vmf, struct folio *folio, struct mm_struct *mm = vmf->vma->vm_mm; /* Huge page is mapped? No need to proceed. */ - if (pmd_trans_huge(*vmf->pmd)) { + if (pmd_trans_huge(pmdp_get(vmf->pmd))) { folio_unlock(folio); folio_put(folio); return true; } - if (pmd_none(*vmf->pmd) && folio_test_pmd_mappable(folio)) { + if (pmd_none(pmdp_get(vmf->pmd)) && folio_test_pmd_mappable(folio)) { struct page *page = folio_file_page(folio, start); vm_fault_t ret = do_set_pmd(vmf, folio, page); if (!ret) { @@ -3735,7 +3735,7 @@ static bool filemap_map_pmd(struct vm_fault *vmf, struct folio *folio, } } - if (pmd_none(*vmf->pmd) && vmf->prealloc_pte) + if (pmd_none(pmdp_get(vmf->pmd)) && vmf->prealloc_pte) pmd_install(mm, vmf->pmd, &vmf->prealloc_pte); return false; diff --git a/mm/gup.c b/mm/gup.c index eb898ea1ee22..b449dbd55182 100644 --- a/mm/gup.c +++ b/mm/gup.c @@ -652,7 +652,7 @@ static struct page *follow_huge_pud(struct vm_area_struct *vma, { struct mm_struct *mm = vma->vm_mm; struct page *page; - pud_t pud = *pudp; + pud_t pud = pudp_get(pudp); unsigned long pfn = pud_pfn(pud); int ret; @@ -704,7 +704,7 @@ static struct page *follow_huge_pmd(struct vm_area_struct *vma, unsigned long *page_mask) { struct mm_struct *mm = vma->vm_mm; - pmd_t pmdval = *pmd; + pmd_t pmdval = pmdp_get(pmd); struct page *page; int ret; @@ -719,7 +719,7 @@ static struct page *follow_huge_pmd(struct vm_area_struct *vma, if ((flags & FOLL_DUMP) && is_huge_zero_pmd(pmdval)) return ERR_PTR(-EFAULT); - if (pmd_protnone(*pmd) && !gup_can_follow_protnone(vma, flags)) + if (pmd_protnone(pmdp_get(pmd)) && !gup_can_follow_protnone(vma, flags)) return NULL; if (!pmd_write(pmdval) && gup_must_unshare(vma, flags, page)) @@ -918,7 +918,7 @@ static struct page *follow_pmd_mask(struct vm_area_struct *vma, return no_page_table(vma, flags, address); ptl = pmd_lock(mm, pmd); - pmdval = *pmd; + pmdval = pmdp_get(pmd); if (unlikely(!pmd_present(pmdval))) { spin_unlock(ptl); return no_page_table(vma, flags, address); @@ -1017,7 +1017,7 @@ static struct page *follow_page_mask(struct vm_area_struct *vma, *page_mask = 0; pgd = pgd_offset(mm, address); - if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd))) + if (pgd_none(pgdp_get(pgd)) || unlikely(pgd_bad(pgdp_get(pgd)))) page = no_page_table(vma, flags, address); else page = follow_p4d_mask(vma, address, pgd, flags, page_mask); @@ -1043,16 +1043,16 @@ static int get_gate_page(struct mm_struct *mm, unsigned long address, if (gup_flags & FOLL_WRITE) return -EFAULT; pgd = pgd_offset(mm, address); - if (pgd_none(*pgd)) + if (pgd_none(pgdp_get(pgd))) return -EFAULT; p4d = p4d_offset(pgd, address); - if (p4d_none(*p4d)) + if (p4d_none(p4dp_get(p4d))) return -EFAULT; pud = pud_offset(p4d, address); - if (pud_none(*pud)) + if (pud_none(pudp_get(pud))) return -EFAULT; pmd = pmd_offset(pud, address); - if (!pmd_present(*pmd)) + if (!pmd_present(pmdp_get(pmd))) return -EFAULT; pte = pte_offset_map(pmd, address); if (!pte) @@ -2833,7 +2833,10 @@ static int gup_fast_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr, int ret = 0; pte_t *ptep, *ptem; + /* transform pmd as if &pmd pointed to a hardware page table */ + set_pmd(&pmd, pmd); ptem = ptep = pte_offset_map(&pmd, addr); + pmd = pmdp_get(&pmd); if (!ptep) return 0; do { diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 1e5d68acf62a..f22a363aa401 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -1440,7 +1440,7 @@ static vm_fault_t __do_huge_pmd_anonymous_page(struct vm_fault *vmf) } vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); - if (unlikely(!pmd_none(*vmf->pmd))) { + if (unlikely(!pmd_none(pmdp_get(vmf->pmd)))) { goto unlock_release; } else { ret = check_stable_address_space(vma->vm_mm); @@ -1488,7 +1488,7 @@ vm_fault_t do_huge_pmd_device_private(struct vm_fault *vmf) } ptl = pmd_lock(vma->vm_mm, vmf->pmd); - if (unlikely(!pmd_same(*vmf->pmd, vmf->orig_pmd))) { + if (unlikely(!pmd_same(pmdp_get(vmf->pmd), vmf->orig_pmd))) { spin_unlock(ptl); return 0; } @@ -1590,7 +1590,7 @@ vm_fault_t do_huge_pmd_anonymous_page(struct vm_fault *vmf) } vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); ret = 0; - if (pmd_none(*vmf->pmd)) { + if (pmd_none(pmdp_get(vmf->pmd))) { ret = check_stable_address_space(vma->vm_mm); if (ret) { spin_unlock(vmf->ptl); @@ -1643,16 +1643,16 @@ static vm_fault_t insert_pmd(struct vm_area_struct *vma, unsigned long addr, } ptl = pmd_lock(mm, pmd); - if (!pmd_none(*pmd)) { + if (!pmd_none(pmdp_get(pmd))) { const unsigned long pfn = fop.is_folio ? folio_pfn(fop.folio) : fop.pfn; if (write) { - if (pmd_pfn(*pmd) != pfn) { - WARN_ON_ONCE(!is_huge_zero_pmd(*pmd)); + if (pmd_pfn(pmdp_get(pmd)) != pfn) { + WARN_ON_ONCE(!is_huge_zero_pmd(pmdp_get(pmd))); goto out_unlock; } - entry = pmd_mkyoung(*pmd); + entry = pmd_mkyoung(pmdp_get(pmd)); entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma); if (pmdp_set_access_flags(vma, addr, pmd, entry, 1)) update_mmu_cache_pmd(vma, addr, pmd); @@ -1767,14 +1767,14 @@ static vm_fault_t insert_pud(struct vm_area_struct *vma, unsigned long addr, return VM_FAULT_SIGBUS; ptl = pud_lock(mm, pud); - if (!pud_none(*pud)) { + if (!pud_none(pudp_get(pud))) { const unsigned long pfn = fop.is_folio ? folio_pfn(fop.folio) : fop.pfn; if (write) { - if (WARN_ON_ONCE(pud_pfn(*pud) != pfn)) + if (WARN_ON_ONCE(pud_pfn(pudp_get(pud)) != pfn)) goto out_unlock; - entry = pud_mkyoung(*pud); + entry = pud_mkyoung(pudp_get(pud)); entry = maybe_pud_mkwrite(pud_mkdirty(entry), vma); if (pudp_set_access_flags(vma, addr, pud, entry, 1)) update_mmu_cache_pud(vma, addr, pud); @@ -1879,7 +1879,7 @@ bool touch_pmd(struct vm_area_struct *vma, unsigned long addr, { pmd_t entry; - entry = pmd_mkyoung(*pmd); + entry = pmd_mkyoung(pmdp_get(pmd)); if (write) entry = pmd_mkdirty(entry); if (pmdp_set_access_flags(vma, addr & HPAGE_PMD_MASK, @@ -1906,9 +1906,9 @@ static void copy_huge_non_present_pmd( softleaf_is_migration_read_exclusive(entry)) { entry = make_readable_migration_entry(swp_offset(entry)); pmd = softleaf_to_pmd(entry); - if (pmd_swp_soft_dirty(*src_pmd)) + if (pmd_swp_soft_dirty(pmdp_get(src_pmd))) pmd = pmd_swp_mksoft_dirty(pmd); - if (pmd_swp_uffd(*src_pmd)) + if (pmd_swp_uffd(pmdp_get(src_pmd))) pmd = pmd_swp_mkuffd(pmd); set_pmd_at(src_mm, addr, src_pmd, pmd); } else if (softleaf_is_device_private(entry)) { @@ -1920,9 +1920,9 @@ static void copy_huge_non_present_pmd( entry = make_readable_device_private_entry(swp_offset(entry)); pmd = softleaf_to_pmd(entry); - if (pmd_swp_soft_dirty(*src_pmd)) + if (pmd_swp_soft_dirty(pmdp_get(src_pmd))) pmd = pmd_swp_mksoft_dirty(pmd); - if (pmd_swp_uffd(*src_pmd)) + if (pmd_swp_uffd(pmdp_get(src_pmd))) pmd = pmd_swp_mkuffd(pmd); set_pmd_at(src_mm, addr, src_pmd, pmd); } @@ -1990,7 +1990,7 @@ int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm, spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); ret = -EAGAIN; - pmd = *src_pmd; + pmd = pmdp_get(src_pmd); if (unlikely(thp_migration_supported() && pmd_is_valid_softleaf(pmd))) { @@ -2065,7 +2065,7 @@ void touch_pud(struct vm_area_struct *vma, unsigned long addr, { pud_t _pud; - _pud = pud_mkyoung(*pud); + _pud = pud_mkyoung(pudp_get(pud)); if (write) _pud = pud_mkdirty(_pud); if (pudp_set_access_flags(vma, addr & HPAGE_PUD_MASK, @@ -2086,7 +2086,7 @@ int copy_huge_pud(struct mm_struct *dst_mm, struct mm_struct *src_mm, spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); ret = -EAGAIN; - pud = *src_pud; + pud = pudp_get(src_pud); if (unlikely(!pud_trans_huge(pud))) goto out_unlock; @@ -2113,7 +2113,7 @@ void huge_pud_set_accessed(struct vm_fault *vmf, pud_t orig_pud) bool write = vmf->flags & FAULT_FLAG_WRITE; vmf->ptl = pud_lock(vmf->vma->vm_mm, vmf->pud); - if (unlikely(!pud_same(*vmf->pud, orig_pud))) + if (unlikely(!pud_same(pudp_get(vmf->pud), orig_pud))) goto unlock; touch_pud(vmf->vma, vmf->address, vmf->pud, write); @@ -2126,7 +2126,7 @@ bool huge_pmd_set_accessed(struct vm_fault *vmf) { bool write = vmf->flags & FAULT_FLAG_WRITE; - if (unlikely(!pmd_same(*vmf->pmd, vmf->orig_pmd))) + if (unlikely(!pmd_same(pmdp_get(vmf->pmd), vmf->orig_pmd))) return false; return touch_pmd(vmf->vma, vmf->address, vmf->pmd, write); @@ -2188,7 +2188,7 @@ vm_fault_t do_huge_pmd_wp_page(struct vm_fault *vmf) spin_lock(vmf->ptl); - if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) { + if (unlikely(!pmd_same(pmdp_get(vmf->pmd), orig_pmd))) { spin_unlock(vmf->ptl); return 0; } @@ -2206,7 +2206,7 @@ vm_fault_t do_huge_pmd_wp_page(struct vm_fault *vmf) spin_unlock(vmf->ptl); folio_lock(folio); spin_lock(vmf->ptl); - if (unlikely(!pmd_same(*vmf->pmd, orig_pmd))) { + if (unlikely(!pmd_same(pmdp_get(vmf->pmd), orig_pmd))) { spin_unlock(vmf->ptl); folio_unlock(folio); folio_put(folio); @@ -2412,7 +2412,7 @@ bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, if (!ptl) goto out_unlocked; - orig_pmd = *pmd; + orig_pmd = pmdp_get(pmd); if (is_huge_zero_pmd(orig_pmd)) goto out; @@ -2636,8 +2636,8 @@ bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr, * should have released it; but move_page_tables() might have already * inserted a page table, if racing against shmem/file collapse. */ - if (!pmd_none(*new_pmd)) { - VM_BUG_ON(pmd_trans_huge(*new_pmd)); + if (!pmd_none(pmdp_get(new_pmd))) { + VM_BUG_ON(pmd_trans_huge(pmdp_get(new_pmd))); return false; } @@ -2653,7 +2653,7 @@ bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr, pmd = pmdp_huge_get_and_clear(mm, old_addr, old_pmd); if (pmd_present(pmd)) force_flush = true; - VM_BUG_ON(!pmd_none(*new_pmd)); + VM_BUG_ON(!pmd_none(pmdp_get(new_pmd))); if (pmd_move_must_withdraw(new_ptl, old_ptl, vma)) { pgtable_t pgtable; @@ -2689,10 +2689,10 @@ static void change_non_present_huge_pmd(struct mm_struct *mm, unsigned long addr, pmd_t *pmd, bool uffd_prot, bool uffd_prot_resolve) { - softleaf_t entry = softleaf_from_pmd(*pmd); + softleaf_t entry = softleaf_from_pmd(pmdp_get(pmd)); pmd_t newpmd; - VM_WARN_ON(!pmd_is_valid_softleaf(*pmd)); + VM_WARN_ON(!pmd_is_valid_softleaf(pmdp_get(pmd))); if (softleaf_is_migration_write(entry)) { const struct folio *folio = softleaf_to_folio(entry); @@ -2705,22 +2705,22 @@ static void change_non_present_huge_pmd(struct mm_struct *mm, else entry = make_readable_migration_entry(swp_offset(entry)); newpmd = softleaf_to_pmd(entry); - if (pmd_swp_soft_dirty(*pmd)) + if (pmd_swp_soft_dirty(pmdp_get(pmd))) newpmd = pmd_swp_mksoft_dirty(newpmd); } else if (softleaf_is_device_private_write(entry)) { entry = make_readable_device_private_entry(swp_offset(entry)); newpmd = softleaf_to_pmd(entry); - if (pmd_swp_uffd(*pmd)) + if (pmd_swp_uffd(pmdp_get(pmd))) newpmd = pmd_swp_mkuffd(newpmd); } else { - newpmd = *pmd; + newpmd = pmdp_get(pmd); } if (uffd_prot) newpmd = pmd_swp_mkuffd(newpmd); else if (uffd_prot_resolve) newpmd = pmd_swp_clear_uffd(newpmd); - if (!pmd_same(*pmd, newpmd)) + if (!pmd_same(pmdp_get(pmd), newpmd)) set_pmd_at(mm, addr, pmd, newpmd); } @@ -2753,16 +2753,16 @@ int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, if (!ptl) return 0; - if (thp_migration_supported() && pmd_is_valid_softleaf(*pmd)) { + if (thp_migration_supported() && pmd_is_valid_softleaf(pmdp_get(pmd))) { change_non_present_huge_pmd(mm, addr, pmd, uffd_prot, uffd_prot_resolve); goto unlock; } /* Already in the desired state */ - if (prot_numa && pmd_protnone(*pmd)) + if (prot_numa && pmd_protnone(pmdp_get(pmd))) goto unlock; - if ((cp_flags & MM_CP_UFFD_RWP) && pmd_protnone(*pmd) && pmd_uffd(*pmd)) + if ((cp_flags & MM_CP_UFFD_RWP) && pmd_protnone(pmdp_get(pmd)) && pmd_uffd(pmdp_get(pmd))) goto unlock; if (prot_numa) { @@ -2772,10 +2772,10 @@ int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, * data is likely to be read-cached on the local CPU and * local/remote hits to the zero page are not interesting. */ - if (is_huge_zero_pmd(*pmd)) + if (is_huge_zero_pmd(pmdp_get(pmd))) goto unlock; - if (!folio_can_map_prot_numa(pmd_folio(*pmd), vma, + if (!folio_can_map_prot_numa(pmd_folio(pmdp_get(pmd)), vma, vma_is_single_threaded_private(vma))) goto unlock; } @@ -2899,7 +2899,7 @@ int move_pages_huge_pmd(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, pm struct mmu_notifier_range range; int err = 0; - src_pmdval = *src_pmd; + src_pmdval = pmdp_get(src_pmd); src_ptl = pmd_lockptr(mm, src_pmd); lockdep_assert_held(src_ptl); @@ -2947,8 +2947,8 @@ int move_pages_huge_pmd(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, pm dst_ptl = pmd_lockptr(mm, dst_pmd); double_pt_lock(src_ptl, dst_ptl); - if (unlikely(!pmd_same(*src_pmd, src_pmdval) || - !pmd_same(*dst_pmd, dst_pmdval))) { + if (unlikely(!pmd_same(pmdp_get(src_pmd), src_pmdval) || + !pmd_same(pmdp_get(dst_pmd), dst_pmdval))) { err = -EAGAIN; goto unlock_ptls; } @@ -3018,7 +3018,7 @@ spinlock_t *__pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma) spinlock_t *ptl; ptl = pmd_lock(vma->vm_mm, pmd); - if (likely(pmd_is_huge(*pmd))) + if (likely(pmd_is_huge(pmdp_get(pmd)))) return ptl; spin_unlock(ptl); return NULL; @@ -3035,7 +3035,7 @@ spinlock_t *__pud_trans_huge_lock(pud_t *pud, struct vm_area_struct *vma) spinlock_t *ptl; ptl = pud_lock(vma->vm_mm, pud); - if (likely(pud_trans_huge(*pud))) + if (likely(pud_trans_huge(pudp_get(pud)))) return ptl; spin_unlock(ptl); return NULL; @@ -3087,7 +3087,7 @@ static void __split_huge_pud_locked(struct vm_area_struct *vma, pud_t *pud, VM_BUG_ON(haddr & ~HPAGE_PUD_MASK); VM_BUG_ON_VMA(vma->vm_start > haddr, vma); VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PUD_SIZE, vma); - VM_BUG_ON(!pud_trans_huge(*pud)); + VM_BUG_ON(!pud_trans_huge(pudp_get(pud))); count_vm_event(THP_SPLIT_PUD); @@ -3120,7 +3120,7 @@ void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud, (address & HPAGE_PUD_MASK) + HPAGE_PUD_SIZE); mmu_notifier_invalidate_range_start(&range); ptl = pud_lock(vma->vm_mm, pud); - if (unlikely(!pud_trans_huge(*pud))) + if (unlikely(!pud_trans_huge(pudp_get(pud)))) goto out; __split_huge_pud_locked(vma, pud, range.start); @@ -3199,7 +3199,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, VM_BUG_ON_VMA(vma->vm_start > haddr, vma); VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PMD_SIZE, vma); - VM_WARN_ON_ONCE(!pmd_is_valid_softleaf(*pmd) && !pmd_trans_huge(*pmd)); + VM_WARN_ON_ONCE(!pmd_is_valid_softleaf(pmdp_get(pmd)) && !pmd_trans_huge(pmdp_get(pmd))); count_vm_event(THP_SPLIT_PMD); @@ -3235,7 +3235,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, return; } - if (is_huge_zero_pmd(*pmd)) { + if (is_huge_zero_pmd(pmdp_get(pmd))) { /* * FIXME: Do we want to invalidate secondary mmu by calling * mmu_notifier_arch_invalidate_secondary_tlbs() see comments below @@ -3248,10 +3248,10 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, return __split_huge_zero_page_pmd(vma, haddr, pmd); } - if (pmd_is_migration_entry(*pmd)) { + if (pmd_is_migration_entry(pmdp_get(pmd))) { softleaf_t entry; - old_pmd = *pmd; + old_pmd = pmdp_get(pmd); entry = softleaf_from_pmd(old_pmd); page = softleaf_to_page(entry); folio = page_folio(page); @@ -3264,10 +3264,10 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, anon_exclusive = softleaf_is_migration_read_exclusive(entry); young = softleaf_is_migration_young(entry); dirty = softleaf_is_migration_dirty(entry); - } else if (pmd_is_device_private_entry(*pmd)) { + } else if (pmd_is_device_private_entry(pmdp_get(pmd))) { softleaf_t entry; - old_pmd = *pmd; + old_pmd = pmdp_get(pmd); entry = softleaf_from_pmd(old_pmd); page = softleaf_to_page(entry); folio = page_folio(page); @@ -3459,7 +3459,7 @@ static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd, } pte_unmap(pte); - if (!pmd_is_migration_entry(*pmd)) + if (!pmd_is_migration_entry(pmdp_get(pmd))) folio_remove_rmap_pmd(folio, page, vma); if (freeze) put_page(page); @@ -3472,7 +3472,7 @@ void split_huge_pmd_locked(struct vm_area_struct *vma, unsigned long address, pmd_t *pmd, bool freeze) { VM_WARN_ON_ONCE(!IS_ALIGNED(address, HPAGE_PMD_SIZE)); - if (pmd_trans_huge(*pmd) || pmd_is_valid_softleaf(*pmd)) + if (pmd_trans_huge(pmdp_get(pmd)) || pmd_is_valid_softleaf(pmdp_get(pmd))) __split_huge_pmd_locked(vma, pmd, address, freeze); } @@ -3560,7 +3560,7 @@ static bool __discard_anon_folio_pmd_locked(struct vm_area_struct *vma, { struct mm_struct *mm = vma->vm_mm; int ref_count, map_count; - pmd_t orig_pmd = *pmdp; + pmd_t orig_pmd = pmdp_get(pmdp); if (pmd_dirty(orig_pmd)) folio_set_dirty(folio); diff --git a/mm/hugetlb.c b/mm/hugetlb.c index cea25773a6c9..f81d373815e8 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -7015,7 +7015,7 @@ pte_t *huge_pmd_share(struct mm_struct *mm, struct vm_area_struct *vma, goto out; spin_lock(&mm->page_table_lock); - if (pud_none(*pud)) { + if (pud_none(pudp_get(pud))) { pud_populate(mm, pud, (pmd_t *)((unsigned long)spte & PAGE_MASK)); mm_inc_nr_pmds(mm); @@ -7154,7 +7154,7 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma, pte = (pte_t *)pud; } else { BUG_ON(sz != PMD_SIZE); - if (want_pmd_share(vma, addr) && pud_none(*pud)) + if (want_pmd_share(vma, addr) && pud_none(pudp_get(pud))) pte = huge_pmd_share(mm, vma, addr, pud); else pte = (pte_t *)pmd_alloc(mm, pud, addr); @@ -7188,17 +7188,17 @@ pte_t *huge_pte_offset(struct mm_struct *mm, pmd_t *pmd; pgd = pgd_offset(mm, addr); - if (!pgd_present(*pgd)) + if (!pgd_present(pgdp_get(pgd))) return NULL; p4d = p4d_offset(pgd, addr); - if (!p4d_present(*p4d)) + if (!p4d_present(p4dp_get(p4d))) return NULL; pud = pud_offset(p4d, addr); if (sz == PUD_SIZE) /* must be pud huge, non-present or none */ return (pte_t *)pud; - if (!pud_present(*pud)) + if (!pud_present(pudp_get(pud))) return NULL; /* must have a valid entry and size to go further */ diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c index 917db0984143..baec0c37b157 100644 --- a/mm/hugetlb_vmemmap.c +++ b/mm/hugetlb_vmemmap.c @@ -73,7 +73,7 @@ static int vmemmap_split_pmd(pmd_t *pmd, struct page *head, unsigned long start, } spin_lock(&init_mm.page_table_lock); - if (likely(pmd_leaf(*pmd))) { + if (likely(pmd_leaf(pmdp_get(pmd)))) { /* * Higher order allocations from buddy allocator must be able to * be treated as independent small pages (as they can be freed @@ -107,7 +107,7 @@ static int vmemmap_pmd_entry(pmd_t *pmd, unsigned long addr, walk->action = ACTION_CONTINUE; spin_lock(&init_mm.page_table_lock); - head = pmd_leaf(*pmd) ? pmd_page(*pmd) : NULL; + head = pmd_leaf(pmdp_get(pmd)) ? pmd_page(pmdp_get(pmd)) : NULL; /* * Due to HugeTLB alignment requirements and the vmemmap * pages being at the start of the hotplugged memory diff --git a/mm/kasan/init.c b/mm/kasan/init.c index 66a883887987..6a8e4fd705f5 100644 --- a/mm/kasan/init.c +++ b/mm/kasan/init.c @@ -121,7 +121,7 @@ static int __ref zero_pmd_populate(pud_t *pud, unsigned long addr, continue; } - if (pmd_none(*pmd)) { + if (pmd_none(pmdp_get(pmd))) { pte_t *p; if (slab_is_available()) @@ -160,7 +160,7 @@ static int __ref zero_pud_populate(p4d_t *p4d, unsigned long addr, continue; } - if (pud_none(*pud)) { + if (pud_none(pudp_get(pud))) { pmd_t *p; if (slab_is_available()) { @@ -202,7 +202,7 @@ static int __ref zero_p4d_populate(pgd_t *pgd, unsigned long addr, continue; } - if (p4d_none(*p4d)) { + if (p4d_none(p4dp_get(p4d))) { pud_t *p; if (slab_is_available()) { @@ -265,7 +265,7 @@ int __ref kasan_populate_early_shadow(const void *shadow_start, continue; } - if (pgd_none(*pgd)) { + if (pgd_none(pgdp_get(pgd))) { if (slab_is_available()) { if (!p4d_alloc(&init_mm, pgd, addr)) @@ -303,7 +303,7 @@ static void kasan_free_pmd(pmd_t *pmd_start, pud_t *pud) for (i = 0; i < PTRS_PER_PMD; i++) { pmd = pmd_start + i; - if (!pmd_none(*pmd)) + if (!pmd_none(pmdp_get(pmd))) return; } @@ -318,7 +318,7 @@ static void kasan_free_pud(pud_t *pud_start, p4d_t *p4d) for (i = 0; i < PTRS_PER_PUD; i++) { pud = pud_start + i; - if (!pud_none(*pud)) + if (!pud_none(pudp_get(pud))) return; } @@ -333,7 +333,7 @@ static void kasan_free_p4d(p4d_t *p4d_start, pgd_t *pgd) for (i = 0; i < PTRS_PER_P4D; i++) { p4d = p4d_start + i; - if (!p4d_none(*p4d)) + if (!p4d_none(p4dp_get(p4d))) return; } @@ -373,10 +373,10 @@ static void kasan_remove_pmd_table(pmd_t *pmd, unsigned long addr, next = pmd_addr_end(addr, end); - if (!pmd_present(*pmd)) + if (!pmd_present(pmdp_get(pmd))) continue; - if (kasan_pte_table(*pmd)) { + if (kasan_pte_table(pmdp_get(pmd))) { if (IS_ALIGNED(addr, PMD_SIZE) && IS_ALIGNED(next, PMD_SIZE)) { pmd_clear(pmd); @@ -399,10 +399,10 @@ static void kasan_remove_pud_table(pud_t *pud, unsigned long addr, next = pud_addr_end(addr, end); - if (!pud_present(*pud)) + if (!pud_present(pudp_get(pud))) continue; - if (kasan_pmd_table(*pud)) { + if (kasan_pmd_table(pudp_get(pud))) { if (IS_ALIGNED(addr, PUD_SIZE) && IS_ALIGNED(next, PUD_SIZE)) { pud_clear(pud); @@ -426,10 +426,10 @@ static void kasan_remove_p4d_table(p4d_t *p4d, unsigned long addr, next = p4d_addr_end(addr, end); - if (!p4d_present(*p4d)) + if (!p4d_present(p4dp_get(p4d))) continue; - if (kasan_pud_table(*p4d)) { + if (kasan_pud_table(p4dp_get(p4d))) { if (IS_ALIGNED(addr, P4D_SIZE) && IS_ALIGNED(next, P4D_SIZE)) { p4d_clear(p4d); @@ -460,10 +460,10 @@ void kasan_remove_zero_shadow(void *start, unsigned long size) next = pgd_addr_end(addr, end); pgd = pgd_offset_k(addr); - if (!pgd_present(*pgd)) + if (!pgd_present(pgdp_get(pgd))) continue; - if (kasan_p4d_table(*pgd)) { + if (kasan_p4d_table(pgdp_get(pgd))) { if (IS_ALIGNED(addr, PGDIR_SIZE) && IS_ALIGNED(next, PGDIR_SIZE)) { pgd_clear(pgd); diff --git a/mm/kasan/shadow.c b/mm/kasan/shadow.c index d286e0a04543..ee1bb6bfe785 100644 --- a/mm/kasan/shadow.c +++ b/mm/kasan/shadow.c @@ -191,20 +191,20 @@ static bool shadow_mapped(unsigned long addr) pmd_t *pmd; pte_t *pte; - if (pgd_none(*pgd)) + if (pgd_none(pgdp_get(pgd))) return false; p4d = p4d_offset(pgd, addr); - if (p4d_none(*p4d)) + if (p4d_none(p4dp_get(p4d))) return false; pud = pud_offset(p4d, addr); - if (pud_none(*pud)) + if (pud_none(pudp_get(pud))) return false; - if (pud_leaf(*pud)) + if (pud_leaf(pudp_get(pud))) return true; pmd = pmd_offset(pud, addr); - if (pmd_none(*pmd)) + if (pmd_none(pmdp_get(pmd))) return false; - if (pmd_leaf(*pmd)) + if (pmd_leaf(pmdp_get(pmd))) return true; pte = pte_offset_kernel(pmd, addr); return !pte_none(ptep_get(pte)); diff --git a/mm/khugepaged.c b/mm/khugepaged.c index 75639298efc2..658c9b2b8cb6 100644 --- a/mm/khugepaged.c +++ b/mm/khugepaged.c @@ -1395,7 +1395,7 @@ static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long s if (unlikely(result != SCAN_SUCCEED)) { spin_lock(pmd_ptl); - VM_WARN_ON_ONCE(!pmd_none(*pmd)); + VM_WARN_ON_ONCE(!pmd_none(pmdp_get(pmd))); /* * We can only use set_pmd_at() when establishing * hugepmds and never for establishing regular pmds that @@ -1431,7 +1431,7 @@ static enum scan_result collapse_huge_page(struct mm_struct *mm, unsigned long s */ __folio_mark_uptodate(folio); spin_lock(pmd_ptl); - VM_WARN_ON_ONCE(!pmd_none(*pmd)); + VM_WARN_ON_ONCE(!pmd_none(pmdp_get(pmd))); if (is_pmd_order(order)) { pgtable = pmd_pgtable(_pmd); pgtable_trans_huge_deposit(mm, pmd, pgtable); @@ -2139,7 +2139,7 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff) struct mmu_notifier_range range; struct mm_struct *mm; unsigned long addr; - pmd_t *pmd, pgt_pmd; + pmd_t *pmd, pgt_pmd, pmdval; spinlock_t *pml; spinlock_t *ptl; bool success = false; @@ -2179,7 +2179,9 @@ static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff) */ if (check_pmd_state(pmd) != SCAN_SUCCEED) goto drop_pml; - ptl = pte_lockptr(mm, pmd); + /* pte_lockptr() needs a value, not a pointer to a page table */ + pmdval = pmdp_get(pmd); + ptl = pte_lockptr(mm, &pmdval); if (ptl != pml) spin_lock_nested(ptl, SINGLE_DEPTH_NESTING); diff --git a/mm/ksm.c b/mm/ksm.c index 49d48d1e0998..416a7ab99ee7 100644 --- a/mm/ksm.c +++ b/mm/ksm.c @@ -1370,7 +1370,7 @@ static int write_protect_page(struct vm_area_struct *vma, struct folio *folio, set_pte_at(mm, pvmw.address, pvmw.pte, entry); } - *orig_pte = entry; + set_pte(orig_pte, entry); err = 0; out_unlock: diff --git a/mm/madvise.c b/mm/madvise.c index eeee82cf2b3f..90dce2c29713 100644 --- a/mm/madvise.c +++ b/mm/madvise.c @@ -383,7 +383,7 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, !can_do_file_pageout(vma); #ifdef CONFIG_TRANSPARENT_HUGEPAGE - if (pmd_trans_huge(*pmd)) { + if (pmd_trans_huge(pmdp_get(pmd))) { pmd_t orig_pmd; unsigned long next = pmd_addr_end(addr, end); @@ -392,7 +392,7 @@ static int madvise_cold_or_pageout_pte_range(pmd_t *pmd, if (!ptl) return 0; - orig_pmd = *pmd; + orig_pmd = pmdp_get(pmd); if (is_huge_zero_pmd(orig_pmd)) goto huge_unlock; @@ -674,7 +674,7 @@ static int madvise_free_pte_range(pmd_t *pmd, unsigned long addr, int nr, max_nr; next = pmd_addr_end(addr, end); - if (pmd_trans_huge(*pmd)) + if (pmd_trans_huge(pmdp_get(pmd))) if (madvise_free_huge_pmd(tlb, vma, pmd, addr, next)) return 0; @@ -1115,7 +1115,7 @@ static int guard_install_set_pte(unsigned long addr, unsigned long next, unsigned long *nr_pages = (unsigned long *)walk->private; /* Simply install a PTE marker, this causes segfault on access. */ - *ptep = make_pte_marker(PTE_MARKER_GUARD); + set_pte(ptep, make_pte_marker(PTE_MARKER_GUARD)); (*nr_pages)++; return 0; diff --git a/mm/memory-failure.c b/mm/memory-failure.c index a8b03e2920ba..f467e3bf6110 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -347,20 +347,20 @@ static unsigned long dev_pagemap_mapping_shift(struct vm_area_struct *vma, VM_BUG_ON_VMA(address == -EFAULT, vma); pgd = pgd_offset(vma->vm_mm, address); - if (!pgd_present(*pgd)) + if (!pgd_present(pgdp_get(pgd))) return 0; p4d = p4d_offset(pgd, address); - if (!p4d_present(*p4d)) + if (!p4d_present(p4dp_get(p4d))) return 0; pud = pud_offset(p4d, address); - if (!pud_present(*pud)) + if (!pud_present(pudp_get(pud))) return 0; - if (pud_trans_huge(*pud)) + if (pud_trans_huge(pudp_get(pud))) return PUD_SHIFT; pmd = pmd_offset(pud, address); - if (!pmd_present(*pmd)) + if (!pmd_present(pmdp_get(pmd))) return 0; - if (pmd_trans_huge(*pmd)) + if (pmd_trans_huge(pmdp_get(pmd))) return PMD_SHIFT; pte = pte_offset_map(pmd, address); if (!pte) @@ -715,7 +715,7 @@ static int check_hwpoisoned_entry(pte_t pte, unsigned long addr, short shift, static int check_hwpoisoned_pmd_entry(pmd_t *pmdp, unsigned long addr, struct hwpoison_walk *hwp) { - pmd_t pmd = *pmdp; + pmd_t pmd = pmdp_get(pmdp); unsigned long pfn; unsigned long hwpoison_vaddr; diff --git a/mm/memory.c b/mm/memory.c index 8b0c2c735d3d..ad79800a8043 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -176,7 +176,7 @@ void mm_trace_rss_stat(struct mm_struct *mm, int member) static void free_pte_range(struct mmu_gather *tlb, pmd_t *pmd, unsigned long addr) { - pgtable_t token = pmd_pgtable(*pmd); + pgtable_t token = pmd_pgtable(pmdp_get(pmd)); pmd_clear(pmd); pte_free_tlb(tlb, token, addr); mm_dec_nr_ptes(tlb->mm); @@ -426,7 +426,7 @@ void pmd_install(struct mm_struct *mm, pmd_t *pmd, pgtable_t *pte) { spinlock_t *ptl = pmd_lock(mm, pmd); - if (likely(pmd_none(*pmd))) { /* Has another populated it ? */ + if (likely(pmd_none(pmdp_get(pmd)))) { /* Has another populated it ? */ mm_inc_nr_ptes(mm); /* * Ensure all pte setup (eg. pte page lock and page clearing) are @@ -467,7 +467,7 @@ int __pte_alloc_kernel(pmd_t *pmd) return -ENOMEM; spin_lock(&init_mm.page_table_lock); - if (likely(pmd_none(*pmd))) { /* Has another populated it ? */ + if (likely(pmd_none(pmdp_get(pmd)))) { /* Has another populated it ? */ smp_wmb(); /* See comment in pmd_install() */ pmd_populate_kernel(&init_mm, pmd, new); new = NULL; @@ -575,9 +575,9 @@ static void __print_bad_page_map_pgtable(struct mm_struct *mm, unsigned long add * see locking requirements for print_bad_page_map(). */ pgdp = pgd_offset(mm, addr); - ptval_to_str(pgd_str, pgd_val(*pgdp)); + ptval_to_str(pgd_str, pgd_val(pgdp_get(pgdp))); - if (!pgd_present(*pgdp) || pgd_leaf(*pgdp)) { + if (!pgd_present(pgdp_get(pgdp)) || pgd_leaf(pgdp_get(pgdp))) { pr_alert("pgd:%s\n", pgd_str); return; } @@ -1473,7 +1473,7 @@ copy_pmd_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, src_pmd = pmd_offset(src_pud, addr); do { next = pmd_addr_end(addr, end); - if (pmd_is_huge(*src_pmd)) { + if (pmd_is_huge(pmdp_get(src_pmd))) { int err; VM_BUG_ON_VMA(next-addr != HPAGE_PMD_SIZE, src_vma); @@ -1510,7 +1510,7 @@ copy_pud_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, src_pud = pud_offset(src_p4d, addr); do { next = pud_addr_end(addr, end); - if (pud_trans_huge(*src_pud)) { + if (pud_trans_huge(pudp_get(src_pud))) { int err; VM_BUG_ON_VMA(next-addr != HPAGE_PUD_SIZE, src_vma); @@ -2131,7 +2131,7 @@ static inline unsigned long zap_pmd_range(struct mmu_gather *tlb, pmd = pmd_offset(pud, addr); do { next = pmd_addr_end(addr, end); - if (pmd_is_huge(*pmd)) { + if (pmd_is_huge(pmdp_get(pmd))) { if (next - addr != HPAGE_PMD_SIZE) __split_huge_pmd(vma, pmd, addr, false); else if (zap_huge_pmd(tlb, vma, pmd, addr)) { @@ -2141,10 +2141,10 @@ static inline unsigned long zap_pmd_range(struct mmu_gather *tlb, /* fall through */ } else if (details && details->single_folio && folio_test_pmd_mappable(details->single_folio) && - next - addr == HPAGE_PMD_SIZE && pmd_none(*pmd)) { + next - addr == HPAGE_PMD_SIZE && pmd_none(pmdp_get(pmd))) { sync_with_folio_pmd_zap(tlb->mm, pmd); } - if (pmd_none(*pmd)) { + if (pmd_none(pmdp_get(pmd))) { addr = next; continue; } @@ -2167,7 +2167,7 @@ static inline unsigned long zap_pud_range(struct mmu_gather *tlb, pud = pud_offset(p4d, addr); do { next = pud_addr_end(addr, end); - if (pud_trans_huge(*pud)) { + if (pud_trans_huge(pudp_get(pud))) { if (next - addr != HPAGE_PUD_SIZE) split_huge_pud(vma, pud, addr); else if (zap_huge_pud(tlb, vma, pud, addr)) @@ -2417,7 +2417,7 @@ static pmd_t *walk_to_pmd(struct mm_struct *mm, unsigned long addr) if (!pmd) return NULL; - VM_BUG_ON(pmd_trans_huge(*pmd)); + VM_BUG_ON(pmd_trans_huge(pmdp_get(pmd))); return pmd; } @@ -3077,7 +3077,7 @@ static inline int remap_pmd_range(struct mm_struct *mm, pud_t *pud, pmd = pmd_alloc(mm, pud, addr); if (!pmd) return -ENOMEM; - VM_BUG_ON(pmd_trans_huge(*pmd)); + VM_BUG_ON(pmd_trans_huge(pmdp_get(pmd))); do { next = pmd_addr_end(addr, end); err = remap_pte_range(mm, pmd, addr, next, @@ -3491,7 +3491,7 @@ static int apply_to_pmd_range(struct mm_struct *mm, pud_t *pud, unsigned long next; int err = 0; - BUG_ON(pud_leaf(*pud)); + BUG_ON(pud_leaf(pudp_get(pud))); if (create) { pmd = pmd_alloc_track(mm, pud, addr, mask); @@ -3502,11 +3502,11 @@ static int apply_to_pmd_range(struct mm_struct *mm, pud_t *pud, } do { next = pmd_addr_end(addr, end); - if (pmd_none(*pmd) && !create) + if (pmd_none(pmdp_get(pmd)) && !create) continue; - if (WARN_ON_ONCE(pmd_leaf(*pmd))) + if (WARN_ON_ONCE(pmd_leaf(pmdp_get(pmd)))) return -EINVAL; - if (!pmd_none(*pmd) && WARN_ON_ONCE(pmd_bad(*pmd))) { + if (!pmd_none(pmdp_get(pmd)) && WARN_ON_ONCE(pmd_bad(pmdp_get(pmd)))) { if (!create) continue; pmd_clear_bad(pmd); @@ -3538,11 +3538,11 @@ static int apply_to_pud_range(struct mm_struct *mm, p4d_t *p4d, } do { next = pud_addr_end(addr, end); - if (pud_none(*pud) && !create) + if (pud_none(pudp_get(pud)) && !create) continue; - if (WARN_ON_ONCE(pud_leaf(*pud))) + if (WARN_ON_ONCE(pud_leaf(pudp_get(pud)))) return -EINVAL; - if (!pud_none(*pud) && WARN_ON_ONCE(pud_bad(*pud))) { + if (!pud_none(pudp_get(pud)) && WARN_ON_ONCE(pud_bad(pudp_get(pud)))) { if (!create) continue; pud_clear_bad(pud); @@ -3574,11 +3574,11 @@ static int apply_to_p4d_range(struct mm_struct *mm, pgd_t *pgd, } do { next = p4d_addr_end(addr, end); - if (p4d_none(*p4d) && !create) + if (p4d_none(p4dp_get(p4d)) && !create) continue; - if (WARN_ON_ONCE(p4d_leaf(*p4d))) + if (WARN_ON_ONCE(p4d_leaf(p4dp_get(p4d)))) return -EINVAL; - if (!p4d_none(*p4d) && WARN_ON_ONCE(p4d_bad(*p4d))) { + if (!p4d_none(p4dp_get(p4d)) && WARN_ON_ONCE(p4d_bad(p4dp_get(p4d)))) { if (!create) continue; p4d_clear_bad(p4d); @@ -3608,13 +3608,13 @@ static int __apply_to_page_range(struct mm_struct *mm, unsigned long addr, pgd = pgd_offset(mm, addr); do { next = pgd_addr_end(addr, end); - if (pgd_none(*pgd) && !create) + if (pgd_none(pgdp_get(pgd)) && !create) continue; - if (WARN_ON_ONCE(pgd_leaf(*pgd))) { + if (WARN_ON_ONCE(pgd_leaf(pgdp_get(pgd)))) { err = -EINVAL; break; } - if (!pgd_none(*pgd) && WARN_ON_ONCE(pgd_bad(*pgd))) { + if (!pgd_none(pgdp_get(pgd)) && WARN_ON_ONCE(pgd_bad(pgdp_get(pgd)))) { if (!create) continue; pgd_clear_bad(pgd); @@ -5556,7 +5556,7 @@ static vm_fault_t __do_fault(struct vm_fault *vmf) * folio_unlock(B) * # flush A, B to clear the writeback */ - if (pmd_none(*vmf->pmd) && !vmf->prealloc_pte) { + if (pmd_none(pmdp_get(vmf->pmd)) && !vmf->prealloc_pte) { vmf->prealloc_pte = pte_alloc_one(vma->vm_mm); if (!vmf->prealloc_pte) return VM_FAULT_OOM; @@ -5651,7 +5651,7 @@ vm_fault_t do_set_pmd(struct vm_fault *vmf, struct folio *folio, struct page *pa } vmf->ptl = pmd_lock(vma->vm_mm, vmf->pmd); - if (unlikely(!pmd_none(*vmf->pmd))) + if (unlikely(!pmd_none(pmdp_get(vmf->pmd)))) goto out; flush_icache_pages(vma, page, HPAGE_PMD_NR); @@ -5803,7 +5803,7 @@ vm_fault_t finish_fault(struct vm_fault *vmf) file_end < folio_next_index(folio); } - if (pmd_none(*vmf->pmd)) { + if (pmd_none(pmdp_get(vmf->pmd))) { if (!needs_fallback && folio_test_pmd_mappable(folio)) { ret = do_set_pmd(vmf, folio, page); if (ret != VM_FAULT_FALLBACK) @@ -5948,7 +5948,7 @@ static vm_fault_t do_fault_around(struct vm_fault *vmf) to_pte = min3(from_pte + nr_pages, (pgoff_t)PTRS_PER_PTE, pte_off + vma_pages(vmf->vma) - vma_off) - 1; - if (pmd_none(*vmf->pmd)) { + if (pmd_none(pmdp_get(vmf->pmd))) { vmf->prealloc_pte = pte_alloc_one(vmf->vma->vm_mm); if (!vmf->prealloc_pte) return VM_FAULT_OOM; @@ -6515,7 +6515,7 @@ static vm_fault_t handle_pte_fault(struct vm_fault *vmf) { pte_t entry; - if (unlikely(pmd_none(*vmf->pmd))) { + if (unlikely(pmd_none(pmdp_get(vmf->pmd)))) { /* * Leave __pte_alloc() until later: because vm_ops->fault may * want to allocate huge page, and if we expose page table @@ -6627,13 +6627,13 @@ static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma, if (!vmf.pud) return VM_FAULT_OOM; retry_pud: - if (pud_none(*vmf.pud) && + if (pud_none(pudp_get(vmf.pud)) && thp_vma_allowable_order(vma, vm_flags, TVA_PAGEFAULT, PUD_ORDER)) { ret = create_huge_pud(&vmf); if (!(ret & VM_FAULT_FALLBACK)) return ret; } else { - pud_t orig_pud = *vmf.pud; + pud_t orig_pud = pudp_get(vmf.pud); barrier(); if (pud_trans_huge(orig_pud)) { @@ -6661,7 +6661,7 @@ static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma, if (pud_trans_unstable(vmf.pud)) goto retry_pud; - if (pmd_none(*vmf.pmd) && + if (pmd_none(pmdp_get(vmf.pmd)) && thp_vma_allowable_order(vma, vm_flags, TVA_PAGEFAULT, PMD_ORDER)) { ret = create_huge_pmd(&vmf); if (ret & VM_FAULT_FALLBACK) @@ -6917,7 +6917,7 @@ int __p4d_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address) return -ENOMEM; spin_lock(&mm->page_table_lock); - if (pgd_present(*pgd)) { /* Another has populated it */ + if (pgd_present(pgdp_get(pgd))) { /* Another has populated it */ p4d_free(mm, new); } else { smp_wmb(); /* See comment in pmd_install() */ @@ -6940,7 +6940,7 @@ int __pud_alloc(struct mm_struct *mm, p4d_t *p4d, unsigned long address) return -ENOMEM; spin_lock(&mm->page_table_lock); - if (!p4d_present(*p4d)) { + if (!p4d_present(p4dp_get(p4d))) { mm_inc_nr_puds(mm); smp_wmb(); /* See comment in pmd_install() */ p4d_populate(mm, p4d, new); @@ -6964,7 +6964,7 @@ int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address) return -ENOMEM; ptl = pud_lock(mm, pud); - if (!pud_present(*pud)) { + if (!pud_present(pudp_get(pud))) { mm_inc_nr_pmds(mm); smp_wmb(); /* See comment in pmd_install() */ pud_populate(mm, pud, new); @@ -7057,7 +7057,7 @@ int follow_pfnmap_start(struct follow_pfnmap_args *args) goto out; retry: pgdp = pgd_offset(mm, address); - if (pgd_none(*pgdp) || unlikely(pgd_bad(*pgdp))) + if (pgd_none(pgdp_get(pgdp)) || unlikely(pgd_bad(pgdp_get(pgdp)))) goto out; p4dp = p4d_offset(pgdp, address); diff --git a/mm/migrate.c b/mm/migrate.c index 15b45832bcfa..2383f64607ae 100644 --- a/mm/migrate.c +++ b/mm/migrate.c @@ -563,9 +563,9 @@ void pmd_migration_entry_wait(struct mm_struct *mm, pmd_t *pmd) spinlock_t *ptl; ptl = pmd_lock(mm, pmd); - if (!pmd_is_migration_entry(*pmd)) + if (!pmd_is_migration_entry(pmdp_get(pmd))) goto unlock; - softleaf_entry_wait_on_locked(softleaf_from_pmd(*pmd), ptl); + softleaf_entry_wait_on_locked(softleaf_from_pmd(pmdp_get(pmd)), ptl); return; unlock: spin_unlock(ptl); diff --git a/mm/migrate_device.c b/mm/migrate_device.c index 009bfa8b212d..1b25bab190b6 100644 --- a/mm/migrate_device.c +++ b/mm/migrate_device.c @@ -146,26 +146,26 @@ static int migrate_vma_collect_huge_pmd(pmd_t *pmdp, unsigned long start, unsigned long write = 0; ptl = pmd_lock(mm, pmdp); - if (pmd_none(*pmdp)) { + if (pmd_none(pmdp_get(pmdp))) { spin_unlock(ptl); return migrate_vma_collect_hole(start, end, -1, walk); } - if (pmd_trans_huge(*pmdp)) { + if (pmd_trans_huge(pmdp_get(pmdp))) { if (!(migrate->flags & MIGRATE_VMA_SELECT_SYSTEM)) { spin_unlock(ptl); return migrate_vma_collect_skip(start, end, walk); } - folio = pmd_folio(*pmdp); + folio = pmd_folio(pmdp_get(pmdp)); if (is_huge_zero_folio(folio)) { spin_unlock(ptl); return migrate_vma_collect_hole(start, end, -1, walk); } - if (pmd_write(*pmdp)) + if (pmd_write(pmdp_get(pmdp))) write = MIGRATE_PFN_WRITE; - } else if (!pmd_present(*pmdp)) { - const softleaf_t entry = softleaf_from_pmd(*pmdp); + } else if (!pmd_present(pmdp_get(pmdp))) { + const softleaf_t entry = softleaf_from_pmd(pmdp_get(pmdp)); if (!softleaf_is_device_private(entry) || !(migrate->flags & MIGRATE_VMA_SELECT_DEVICE_PRIVATE)) { @@ -257,7 +257,7 @@ static int migrate_vma_collect_pmd(pmd_t *pmdp, pte_t *ptep; again: - if (pmd_trans_huge(*pmdp) || !pmd_present(*pmdp)) { + if (pmd_trans_huge(pmdp_get(pmdp)) || !pmd_present(pmdp_get(pmdp))) { int ret = migrate_vma_collect_huge_pmd(pmdp, start, end, walk, fault_folio); if (ret == -EAGAIN) @@ -869,9 +869,9 @@ static int migrate_vma_insert_huge_pmd_page(struct migrate_vma *migrate, if (userfaultfd_missing(vma)) goto unlock_abort; - if (is_huge_zero_pmd(*pmdp)) + if (is_huge_zero_pmd(pmdp_get(pmdp))) flush = true; - else if (!pmd_none(*pmdp)) + else if (!pmd_none(pmdp_get(pmdp))) goto unlock_abort; add_mm_counter(vma->vm_mm, MM_ANONPAGES, HPAGE_PMD_NR); @@ -1015,12 +1015,12 @@ static void migrate_vma_insert_page(struct migrate_vma *migrate, return; } - if (!pmd_none(*pmdp)) { - if (pmd_trans_huge(*pmdp)) { - if (!is_huge_zero_pmd(*pmdp)) + if (!pmd_none(pmdp_get(pmdp))) { + if (pmd_trans_huge(pmdp_get(pmdp))) { + if (!is_huge_zero_pmd(pmdp_get(pmdp))) goto abort; split_huge_pmd(vma, pmdp, addr); - } else if (pmd_leaf(*pmdp)) + } else if (pmd_leaf(pmdp_get(pmdp))) goto abort; } diff --git a/mm/mlock.c b/mm/mlock.c index 39215a3eab1f..255a6ba0e634 100644 --- a/mm/mlock.c +++ b/mm/mlock.c @@ -361,11 +361,11 @@ static int mlock_pte_range(pmd_t *pmd, unsigned long addr, ptl = pmd_trans_huge_lock(pmd, vma); if (ptl) { - if (!pmd_present(*pmd)) + if (!pmd_present(pmdp_get(pmd))) goto out; - if (is_huge_zero_pmd(*pmd)) + if (is_huge_zero_pmd(pmdp_get(pmd))) goto out; - folio = pmd_folio(*pmd); + folio = pmd_folio(pmdp_get(pmd)); if (folio_is_zone_device(folio)) goto out; if (vma_test(vma, VMA_LOCKED_BIT)) diff --git a/mm/mprotect.c b/mm/mprotect.c index 2888ee638d87..d8d5a6cc6bbd 100644 --- a/mm/mprotect.c +++ b/mm/mprotect.c @@ -523,7 +523,7 @@ static inline long change_pmd_range(struct mmu_gather *tlb, break; } - if (pmd_none(*pmd)) + if (pmd_none(pmdp_get(pmd))) goto next; _pmd = pmdp_get_lockless(pmd); diff --git a/mm/mremap.c b/mm/mremap.c index 444f37d7cd28..bf0f3516e71f 100644 --- a/mm/mremap.c +++ b/mm/mremap.c @@ -103,7 +103,7 @@ static pmd_t *get_old_pmd(struct mm_struct *mm, unsigned long addr) return NULL; pmd = pmd_offset(pud, addr); - if (pmd_none(*pmd)) + if (pmd_none(pmdp_get(pmd))) return NULL; return pmd; @@ -135,7 +135,7 @@ static pmd_t *alloc_new_pmd(struct mm_struct *mm, unsigned long addr) if (!pmd) return NULL; - VM_BUG_ON(pmd_trans_huge(*pmd)); + VM_BUG_ON(pmd_trans_huge(pmdp_get(pmd))); return pmd; } @@ -392,7 +392,7 @@ static bool move_normal_pmd(struct pagetable_move_control *pmc, * One alternative might be to just unmap the target pmd at * this point, and verify that it really is empty. We'll see. */ - if (WARN_ON_ONCE(!pmd_none(*new_pmd))) + if (WARN_ON_ONCE(!pmd_none(pmdp_get(new_pmd)))) return false; /* @@ -404,7 +404,7 @@ static bool move_normal_pmd(struct pagetable_move_control *pmc, if (new_ptl != old_ptl) spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING); - pmd = *old_pmd; + pmd = pmdp_get(old_pmd); /* Racing with collapse? */ if (unlikely(!pmd_present(pmd) || pmd_leaf(pmd))) @@ -413,7 +413,7 @@ static bool move_normal_pmd(struct pagetable_move_control *pmc, pmd_clear(old_pmd); res = true; - VM_BUG_ON(!pmd_none(*new_pmd)); + VM_BUG_ON(!pmd_none(pmdp_get(new_pmd))); pmd_populate(mm, new_pmd, pmd_pgtable(pmd)); flush_tlb_range(vma, pmc->old_addr, pmc->old_addr + PMD_SIZE); @@ -449,7 +449,7 @@ static bool move_normal_pud(struct pagetable_move_control *pmc, * The destination pud shouldn't be established, free_pgtables() * should have released it. */ - if (WARN_ON_ONCE(!pud_none(*new_pud))) + if (WARN_ON_ONCE(!pud_none(pudp_get(new_pud)))) return false; /* @@ -462,10 +462,10 @@ static bool move_normal_pud(struct pagetable_move_control *pmc, spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING); /* Clear the pud */ - pud = *old_pud; + pud = pudp_get(old_pud); pud_clear(old_pud); - VM_BUG_ON(!pud_none(*new_pud)); + VM_BUG_ON(!pud_none(pudp_get(new_pud))); pud_populate(mm, new_pud, pud_pgtable(pud)); flush_tlb_range(vma, pmc->old_addr, pmc->old_addr + PUD_SIZE); @@ -496,7 +496,7 @@ static bool move_huge_pud(struct pagetable_move_control *pmc, * The destination pud shouldn't be established, free_pgtables() * should have released it. */ - if (WARN_ON_ONCE(!pud_none(*new_pud))) + if (WARN_ON_ONCE(!pud_none(pudp_get(new_pud)))) return false; /* @@ -509,10 +509,10 @@ static bool move_huge_pud(struct pagetable_move_control *pmc, spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING); /* Clear the pud */ - pud = *old_pud; + pud = pudp_get(old_pud); pud_clear(old_pud); - VM_BUG_ON(!pud_none(*new_pud)); + VM_BUG_ON(!pud_none(pudp_get(new_pud))); /* Set the new pud */ /* mark soft_ditry when we add pud level soft dirty support */ @@ -841,7 +841,7 @@ unsigned long move_page_tables(struct pagetable_move_control *pmc) new_pud = alloc_new_pud(mm, pmc->new_addr); if (!new_pud) break; - if (pud_trans_huge(*old_pud)) { + if (pud_trans_huge(pudp_get(old_pud))) { if (extent == HPAGE_PUD_SIZE) { move_pgt_entry(pmc, HPAGE_PUD, old_pud, new_pud); /* We ignore and continue on error? */ @@ -860,7 +860,7 @@ unsigned long move_page_tables(struct pagetable_move_control *pmc) if (!new_pmd) break; again: - if (pmd_is_huge(*old_pmd)) { + if (pmd_is_huge(pmdp_get(old_pmd))) { if (extent == HPAGE_PMD_SIZE && move_pgt_entry(pmc, HPAGE_PMD, old_pmd, new_pmd)) continue; @@ -874,7 +874,7 @@ unsigned long move_page_tables(struct pagetable_move_control *pmc) if (move_pgt_entry(pmc, NORMAL_PMD, old_pmd, new_pmd)) continue; } - if (pmd_none(*old_pmd)) + if (pmd_none(pmdp_get(old_pmd))) continue; if (pte_alloc(pmc->new->vm_mm, new_pmd)) break; diff --git a/mm/page_table_check.c b/mm/page_table_check.c index 6ffc536359cd..abb6cde0fc1f 100644 --- a/mm/page_table_check.c +++ b/mm/page_table_check.c @@ -248,7 +248,7 @@ void __page_table_check_pmds_set(struct mm_struct *mm, unsigned long addr, page_table_check_pmd_flags(pmd); for (i = 0; i < nr; i++) - __page_table_check_pmd_clear(mm, addr + PMD_SIZE * i, *(pmdp + i)); + __page_table_check_pmd_clear(mm, addr + PMD_SIZE * i, pmdp_get(pmdp + i)); if (pmd_user_accessible_page(mm, addr, pmd) && !page_table_check_huge_zero_pmd(pmd)) page_table_check_set(pmd_pfn(pmd), stride * nr, pmd_write(pmd)); @@ -265,7 +265,7 @@ void __page_table_check_puds_set(struct mm_struct *mm, unsigned long addr, return; for (i = 0; i < nr; i++) - __page_table_check_pud_clear(mm, addr + PUD_SIZE * i, *(pudp + i)); + __page_table_check_pud_clear(mm, addr + PUD_SIZE * i, pudp_get(pudp + i)); if (pud_user_accessible_page(mm, addr, pud)) page_table_check_set(pud_pfn(pud), stride * nr, pud_write(pud)); } @@ -279,7 +279,10 @@ void __page_table_check_pte_clear_range(struct mm_struct *mm, return; if (!pmd_bad(pmd) && !pmd_leaf(pmd)) { + /* transform pmd as if &pmd pointed to a hardware page table */ + set_pmd(&pmd, pmd); pte_t *ptep = pte_offset_map(&pmd, addr); + pmd = pmdp_get(&pmd); unsigned long i; if (WARN_ON(!ptep)) diff --git a/mm/page_vma_mapped.c b/mm/page_vma_mapped.c index 4e964545e5e8..d52c8765d1bf 100644 --- a/mm/page_vma_mapped.c +++ b/mm/page_vma_mapped.c @@ -226,17 +226,17 @@ bool page_vma_mapped_walk(struct page_vma_mapped_walk *pvmw) restart: do { pgd = pgd_offset(mm, pvmw->address); - if (!pgd_present(*pgd)) { + if (!pgd_present(pgdp_get(pgd))) { step_forward(pvmw, PGDIR_SIZE); continue; } p4d = p4d_offset(pgd, pvmw->address); - if (!p4d_present(*p4d)) { + if (!p4d_present(p4dp_get(p4d))) { step_forward(pvmw, P4D_SIZE); continue; } pud = pud_offset(p4d, pvmw->address); - if (!pud_present(*pud)) { + if (!pud_present(pudp_get(pud))) { step_forward(pvmw, PUD_SIZE); continue; } diff --git a/mm/pagewalk.c b/mm/pagewalk.c index cc07fcf50e87..a686872cce0d 100644 --- a/mm/pagewalk.c +++ b/mm/pagewalk.c @@ -128,7 +128,7 @@ static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end, again: walk->action = ACTION_SUBTREE; next = pmd_addr_end(addr, end); - if (pmd_none(*pmd)) { + if (pmd_none(pmdp_get(pmd))) { if (has_install) err = __pte_alloc(walk->mm, pmd); else if (ops->pte_hole) @@ -160,13 +160,13 @@ static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end, * We are ONLY installing, so avoid unnecessarily * splitting a present huge page. */ - if (pmd_present(*pmd) && pmd_trans_huge(*pmd)) + if (pmd_present(pmdp_get(pmd)) && pmd_trans_huge(pmdp_get(pmd))) continue; } if (walk->vma) split_huge_pmd(walk->vma, pmd, addr); - else if (pmd_leaf(*pmd) || !pmd_present(*pmd)) + else if (pmd_leaf(pmdp_get(pmd)) || !pmd_present(pmdp_get(pmd))) continue; /* Nothing to do. */ err = walk_pte_range(pmd, addr, next, walk); @@ -197,7 +197,7 @@ static int walk_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end, again: walk->action = ACTION_SUBTREE; next = pud_addr_end(addr, end); - if (pud_none(*pud)) { + if (pud_none(pudp_get(pud))) { if (has_install) err = __pmd_alloc(walk->mm, pud, addr); else if (ops->pte_hole) @@ -225,13 +225,13 @@ static int walk_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end, * We are ONLY installing, so avoid unnecessarily * splitting a present huge page. */ - if (pud_present(*pud) && pud_trans_huge(*pud)) + if (pud_present(pudp_get(pud)) && pud_trans_huge(pudp_get(pud))) continue; } if (walk->vma) split_huge_pud(walk->vma, pud, addr); - else if (pud_leaf(*pud) || !pud_present(*pud)) + else if (pud_leaf(pudp_get(pud)) || !pud_present(pudp_get(pud))) continue; /* Nothing to do. */ err = walk_pmd_range(pud, addr, next, walk); diff --git a/mm/percpu.c b/mm/percpu.c index a802d72c116f..e116d8decfa7 100644 --- a/mm/percpu.c +++ b/mm/percpu.c @@ -3157,25 +3157,25 @@ void __init __weak pcpu_populate_pte(unsigned long addr) pud_t *pud; pmd_t *pmd; - if (pgd_none(*pgd)) { + if (pgd_none(pgdp_get(pgd))) { p4d = memblock_alloc_or_panic(P4D_TABLE_SIZE, P4D_TABLE_SIZE); pgd_populate_kernel(addr, pgd, p4d); } p4d = p4d_offset(pgd, addr); - if (p4d_none(*p4d)) { + if (p4d_none(p4dp_get(p4d))) { pud = memblock_alloc_or_panic(PUD_TABLE_SIZE, PUD_TABLE_SIZE); p4d_populate_kernel(addr, p4d, pud); } pud = pud_offset(p4d, addr); - if (pud_none(*pud)) { + if (pud_none(pudp_get(pud))) { pmd = memblock_alloc_or_panic(PMD_TABLE_SIZE, PMD_TABLE_SIZE); pud_populate(&init_mm, pud, pmd); } pmd = pmd_offset(pud, addr); - if (!pmd_present(*pmd)) { + if (!pmd_present(pmdp_get(pmd))) { pte_t *new; new = memblock_alloc_or_panic(PTE_TABLE_SIZE, PTE_TABLE_SIZE); diff --git a/mm/pgalloc-track.h b/mm/pgalloc-track.h index 1a6de1358a21..7da26992713c 100644 --- a/mm/pgalloc-track.h +++ b/mm/pgalloc-track.h @@ -10,7 +10,7 @@ static inline p4d_t *p4d_alloc_track(struct mm_struct *mm, pgd_t *pgd, unsigned long address, pgtbl_mod_mask *mod_mask) { - if (unlikely(pgd_none(*pgd))) { + if (unlikely(pgd_none(pgdp_get(pgd)))) { if (__p4d_alloc(mm, pgd, address)) return NULL; *mod_mask |= PGTBL_PGD_MODIFIED; @@ -23,7 +23,7 @@ static inline pud_t *pud_alloc_track(struct mm_struct *mm, p4d_t *p4d, unsigned long address, pgtbl_mod_mask *mod_mask) { - if (unlikely(p4d_none(*p4d))) { + if (unlikely(p4d_none(p4dp_get(p4d)))) { if (__pud_alloc(mm, p4d, address)) return NULL; *mod_mask |= PGTBL_P4D_MODIFIED; @@ -36,7 +36,7 @@ static inline pmd_t *pmd_alloc_track(struct mm_struct *mm, pud_t *pud, unsigned long address, pgtbl_mod_mask *mod_mask) { - if (unlikely(pud_none(*pud))) { + if (unlikely(pud_none(pudp_get(pud)))) { if (__pmd_alloc(mm, pud, address)) return NULL; *mod_mask |= PGTBL_PUD_MODIFIED; @@ -47,7 +47,7 @@ static inline pmd_t *pmd_alloc_track(struct mm_struct *mm, pud_t *pud, #endif /* CONFIG_MMU */ #define pte_alloc_kernel_track(pmd, address, mask) \ - ((unlikely(pmd_none(*(pmd))) && \ + ((unlikely(pmd_none(pmdp_get(pmd))) && \ (__pte_alloc_kernel(pmd) || ({*(mask)|=PGTBL_PMD_MODIFIED;0;})))?\ NULL: pte_offset_kernel(pmd, address)) diff --git a/mm/pgtable-generic.c b/mm/pgtable-generic.c index cd227fc05d2d..32ada4235dde 100644 --- a/mm/pgtable-generic.c +++ b/mm/pgtable-generic.c @@ -26,14 +26,14 @@ void pgd_clear_bad(pgd_t *pgd) { - pgd_ERROR(*pgd); + pgd_ERROR(pgdp_get(pgd)); pgd_clear(pgd); } #ifndef __PAGETABLE_P4D_FOLDED void p4d_clear_bad(p4d_t *p4d) { - p4d_ERROR(*p4d); + p4d_ERROR(p4dp_get(p4d)); p4d_clear(p4d); } #endif @@ -41,7 +41,7 @@ void p4d_clear_bad(p4d_t *p4d) #ifndef __PAGETABLE_PUD_FOLDED void pud_clear_bad(pud_t *pud) { - pud_ERROR(*pud); + pud_ERROR(pudp_get(pud)); pud_clear(pud); } #endif @@ -53,7 +53,7 @@ void pud_clear_bad(pud_t *pud) */ void pmd_clear_bad(pmd_t *pmd) { - pmd_ERROR(*pmd); + pmd_ERROR(pmdp_get(pmd)); pmd_clear(pmd); } @@ -113,7 +113,7 @@ int pmdp_set_access_flags(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp, pmd_t entry, int dirty) { - int changed = !pmd_same(*pmdp, entry); + int changed = !pmd_same(pmdp_get(pmdp), entry); VM_BUG_ON(address & ~HPAGE_PMD_MASK); if (changed) { set_pmd_at(vma->vm_mm, address, pmdp, entry); @@ -143,7 +143,7 @@ pmd_t pmdp_huge_clear_flush(struct vm_area_struct *vma, unsigned long address, { pmd_t pmd; VM_BUG_ON(address & ~HPAGE_PMD_MASK); - VM_BUG_ON(pmd_present(*pmdp) && !pmd_trans_huge(*pmdp)); + VM_BUG_ON(pmd_present(pmdp_get(pmdp)) && !pmd_trans_huge(pmdp_get(pmdp))); pmd = pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp); flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE); return pmd; @@ -156,7 +156,7 @@ pud_t pudp_huge_clear_flush(struct vm_area_struct *vma, unsigned long address, pud_t pud; VM_BUG_ON(address & ~HPAGE_PUD_MASK); - VM_BUG_ON(!pud_trans_huge(*pudp)); + VM_BUG_ON(!pud_trans_huge(pudp_get(pudp))); pud = pudp_huge_get_and_clear(vma->vm_mm, address, pudp); flush_pud_tlb_range(vma, address, address + HPAGE_PUD_SIZE); return pud; @@ -201,8 +201,9 @@ pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp) pmd_t pmdp_invalidate(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp) { - VM_WARN_ON_ONCE(!pmd_present(*pmdp)); - pmd_t old = pmdp_establish(vma, address, pmdp, pmd_mkinvalid(*pmdp)); + VM_WARN_ON_ONCE(!pmd_present(pmdp_get(pmdp))); + pmd_t old = pmdp_establish(vma, address, pmdp, + pmd_mkinvalid(pmdp_get(pmdp))); flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE); return old; } @@ -212,7 +213,7 @@ pmd_t pmdp_invalidate(struct vm_area_struct *vma, unsigned long address, pmd_t pmdp_invalidate_ad(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp) { - VM_WARN_ON_ONCE(!pmd_present(*pmdp)); + VM_WARN_ON_ONCE(!pmd_present(pmdp_get(pmdp))); return pmdp_invalidate(vma, address, pmdp); } #endif @@ -228,7 +229,7 @@ pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long address, pmd_t pmd; VM_BUG_ON(address & ~HPAGE_PMD_MASK); - VM_BUG_ON(pmd_trans_huge(*pmdp)); + VM_BUG_ON(pmd_trans_huge(pmdp_get(pmdp))); pmd = pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp); /* collapse entails shooting down ptes not pmd */ @@ -302,6 +303,8 @@ pte_t *__pte_offset_map(pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp) pmd_clear_bad(pmd); goto nomap; } + /* transform pmdval as if &pmdval pointed to a hardware page table */ + set_pmd(&pmdval, pmdval); return __pte_map(&pmdval, addr); nomap: rcu_read_unlock(); diff --git a/mm/rmap.c b/mm/rmap.c index f3b21aaa34ee..36d4ce8513a3 100644 --- a/mm/rmap.c +++ b/mm/rmap.c @@ -893,15 +893,15 @@ pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address) pmd_t *pmd = NULL; pgd = pgd_offset(mm, address); - if (!pgd_present(*pgd)) + if (!pgd_present(pgdp_get(pgd))) goto out; p4d = p4d_offset(pgd, address); - if (!p4d_present(*p4d)) + if (!p4d_present(p4dp_get(p4d))) goto out; pud = pud_offset(p4d, address); - if (!pud_present(*pud)) + if (!pud_present(pudp_get(pud))) goto out; pmd = pmd_offset(pud, address); diff --git a/mm/sparse-vmemmap.c b/mm/sparse-vmemmap.c index 5a2469fb1838..8f59e9cc8729 100644 --- a/mm/sparse-vmemmap.c +++ b/mm/sparse-vmemmap.c @@ -195,7 +195,7 @@ static void * __meminit vmemmap_alloc_block_zero(unsigned long size, int node) static pmd_t * __meminit vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node) { pmd_t *pmd = pmd_offset(pud, addr); - if (pmd_none(*pmd)) { + if (pmd_none(pmdp_get(pmd))) { void *p = vmemmap_alloc_block_zero(PAGE_SIZE, node); if (!p) return NULL; @@ -208,7 +208,7 @@ static pmd_t * __meminit vmemmap_pmd_populate(pud_t *pud, unsigned long addr, in static pud_t * __meminit vmemmap_pud_populate(p4d_t *p4d, unsigned long addr, int node) { pud_t *pud = pud_offset(p4d, addr); - if (pud_none(*pud)) { + if (pud_none(pudp_get(pud))) { void *p = vmemmap_alloc_block_zero(PAGE_SIZE, node); if (!p) return NULL; @@ -221,7 +221,7 @@ static pud_t * __meminit vmemmap_pud_populate(p4d_t *p4d, unsigned long addr, in static p4d_t * __meminit vmemmap_p4d_populate(pgd_t *pgd, unsigned long addr, int node) { p4d_t *p4d = p4d_offset(pgd, addr); - if (p4d_none(*p4d)) { + if (p4d_none(p4dp_get(p4d))) { void *p = vmemmap_alloc_block_zero(PAGE_SIZE, node); if (!p) return NULL; @@ -234,7 +234,7 @@ static p4d_t * __meminit vmemmap_p4d_populate(pgd_t *pgd, unsigned long addr, in static pgd_t * __meminit vmemmap_pgd_populate(unsigned long addr, int node) { pgd_t *pgd = pgd_offset_k(addr); - if (pgd_none(*pgd)) { + if (pgd_none(pgdp_get(pgd))) { void *p = vmemmap_alloc_block_zero(PAGE_SIZE, node); if (!p) return NULL; diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c index 74f04c323c50..f8cf248384df 100644 --- a/mm/userfaultfd.c +++ b/mm/userfaultfd.c @@ -1555,8 +1555,8 @@ static long move_pages_ptes(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd } /* Sanity checks before the operation */ - if (pmd_none(*dst_pmd) || pmd_none(*src_pmd) || - pmd_trans_huge(*dst_pmd) || pmd_trans_huge(*src_pmd)) { + if (pmd_none(pmdp_get(dst_pmd)) || pmd_none(pmdp_get(src_pmd)) || + pmd_trans_huge(pmdp_get(dst_pmd)) || pmd_trans_huge(pmdp_get(src_pmd))) { ret = -EINVAL; goto out; } @@ -2112,8 +2112,8 @@ static ssize_t move_pages(struct userfaultfd_ctx *ctx, unsigned long dst_start, if (move_splits_huge_pmd(dst_addr, src_addr, src_start + len) || !pmd_none(dst_pmdval)) { /* Can be a migration entry */ - if (pmd_present(*src_pmd)) { - struct folio *folio = pmd_folio(*src_pmd); + if (pmd_present(pmdp_get(src_pmd))) { + struct folio *folio = pmd_folio(pmdp_get(src_pmd)); if (!is_huge_zero_folio(folio) && !PageAnonExclusive(&folio->page)) { @@ -2136,7 +2136,7 @@ static ssize_t move_pages(struct userfaultfd_ctx *ctx, unsigned long dst_start, } else { long ret; - if (pmd_none(*src_pmd)) { + if (pmd_none(pmdp_get(src_pmd))) { if (!(mode & UFFDIO_MOVE_MODE_ALLOW_SRC_HOLES)) { err = -ENOENT; break; @@ -2730,13 +2730,13 @@ static inline bool userfaultfd_must_wait(struct userfaultfd_ctx *ctx, assert_fault_locked(vmf); pgd = pgd_offset(mm, address); - if (!pgd_present(*pgd)) + if (!pgd_present(pgdp_get(pgd))) return true; p4d = p4d_offset(pgd, address); - if (!p4d_present(*p4d)) + if (!p4d_present(p4dp_get(p4d))) return true; pud = pud_offset(p4d, address); - if (!pud_present(*pud)) + if (!pud_present(pudp_get(pud))) return true; pmd = pmd_offset(pud, address); again: diff --git a/mm/vmalloc.c b/mm/vmalloc.c index bea9f76ed7e7..acd1c082402e 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -160,7 +160,7 @@ static int vmap_try_huge_pmd(pmd_t *pmd, unsigned long addr, unsigned long end, if (!IS_ALIGNED(phys_addr, PMD_SIZE)) return 0; - if (!pmd_present(*pmd)) + if (!pmd_present(pmdp_get(pmd))) return pmd_set_huge(pmd, phys_addr, prot); /* @@ -223,7 +223,7 @@ static int vmap_try_huge_pud(pud_t *pud, unsigned long addr, unsigned long end, if (!IS_ALIGNED(phys_addr, PUD_SIZE)) return 0; - if (!pud_present(*pud)) + if (!pud_present(pudp_get(pud))) return pud_set_huge(pud, phys_addr, prot); /* See comment in vmap_try_huge_pmd(). */ @@ -280,7 +280,7 @@ static int vmap_try_huge_p4d(p4d_t *p4d, unsigned long addr, unsigned long end, if (!IS_ALIGNED(phys_addr, P4D_SIZE)) return 0; - if (!p4d_present(*p4d)) + if (!p4d_present(p4dp_get(p4d))) return p4d_set_huge(p4d, phys_addr, prot); /* See comment in vmap_try_huge_pmd(). */ @@ -402,7 +402,8 @@ static void vunmap_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end, if (size != PAGE_SIZE) { if (WARN_ON(!IS_ALIGNED(addr, size))) { addr = ALIGN_DOWN(addr, size); - pte = PTR_ALIGN_DOWN(pte, sizeof(*pte) * (size >> PAGE_SHIFT)); + pte = PTR_ALIGN_DOWN(pte, + sizeof(ptep_get(pte)) * (size >> PAGE_SHIFT)); } ptent = huge_ptep_get_and_clear(&init_mm, addr, pte, size); if (WARN_ON(end - addr < size)) @@ -429,7 +430,7 @@ static void vunmap_pmd_range(pud_t *pud, unsigned long addr, unsigned long end, next = pmd_addr_end(addr, end); cleared = pmd_clear_huge(pmd); - if (cleared || pmd_bad(*pmd)) + if (cleared || pmd_bad(pmdp_get(pmd))) *mask |= PGTBL_PMD_MODIFIED; if (cleared) { @@ -456,7 +457,7 @@ static void vunmap_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end, next = pud_addr_end(addr, end); cleared = pud_clear_huge(pud); - if (cleared || pud_bad(*pud)) + if (cleared || pud_bad(pudp_get(pud))) *mask |= PGTBL_PUD_MODIFIED; if (cleared) { @@ -480,7 +481,7 @@ static void vunmap_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end, next = p4d_addr_end(addr, end); p4d_clear_huge(p4d); - if (p4d_bad(*p4d)) + if (p4d_bad(p4dp_get(p4d))) *mask |= PGTBL_P4D_MODIFIED; if (p4d_none_or_clear_bad(p4d)) @@ -512,7 +513,7 @@ void __vunmap_range_noflush(unsigned long start, unsigned long end) pgd = pgd_offset_k(addr); do { next = pgd_addr_end(addr, end); - if (pgd_bad(*pgd)) + if (pgd_bad(pgdp_get(pgd))) mask |= PGTBL_PGD_MODIFIED; if (pgd_none_or_clear_bad(pgd)) continue; @@ -657,7 +658,7 @@ static int vmap_small_pages_range_noflush(unsigned long addr, unsigned long end, pgd = pgd_offset_k(addr); do { next = pgd_addr_end(addr, end); - if (pgd_bad(*pgd)) + if (pgd_bad(pgdp_get(pgd))) mask |= PGTBL_PGD_MODIFIED; err = vmap_pages_p4d_range(pgd, addr, next, prot, pages, &nr, &mask); if (err) @@ -835,35 +836,35 @@ struct page *vmalloc_to_page(const void *vmalloc_addr) */ VIRTUAL_BUG_ON(!is_vmalloc_or_module_addr(vmalloc_addr)); - if (pgd_none(*pgd)) + if (pgd_none(pgdp_get(pgd))) return NULL; - if (WARN_ON_ONCE(pgd_leaf(*pgd))) + if (WARN_ON_ONCE(pgd_leaf(pgdp_get(pgd)))) return NULL; /* XXX: no allowance for huge pgd */ - if (WARN_ON_ONCE(pgd_bad(*pgd))) + if (WARN_ON_ONCE(pgd_bad(pgdp_get(pgd)))) return NULL; p4d = p4d_offset(pgd, addr); - if (p4d_none(*p4d)) + if (p4d_none(p4dp_get(p4d))) return NULL; - if (p4d_leaf(*p4d)) - return p4d_page(*p4d) + ((addr & ~P4D_MASK) >> PAGE_SHIFT); - if (WARN_ON_ONCE(p4d_bad(*p4d))) + if (p4d_leaf(p4dp_get(p4d))) + return p4d_page(p4dp_get(p4d)) + ((addr & ~P4D_MASK) >> PAGE_SHIFT); + if (WARN_ON_ONCE(p4d_bad(p4dp_get(p4d)))) return NULL; pud = pud_offset(p4d, addr); - if (pud_none(*pud)) + if (pud_none(pudp_get(pud))) return NULL; - if (pud_leaf(*pud)) - return pud_page(*pud) + ((addr & ~PUD_MASK) >> PAGE_SHIFT); - if (WARN_ON_ONCE(pud_bad(*pud))) + if (pud_leaf(pudp_get(pud))) + return pud_page(pudp_get(pud)) + ((addr & ~PUD_MASK) >> PAGE_SHIFT); + if (WARN_ON_ONCE(pud_bad(pudp_get(pud)))) return NULL; pmd = pmd_offset(pud, addr); - if (pmd_none(*pmd)) + if (pmd_none(pmdp_get(pmd))) return NULL; - if (pmd_leaf(*pmd)) - return pmd_page(*pmd) + ((addr & ~PMD_MASK) >> PAGE_SHIFT); - if (WARN_ON_ONCE(pmd_bad(*pmd))) + if (pmd_leaf(pmdp_get(pmd))) + return pmd_page(pmdp_get(pmd)) + ((addr & ~PMD_MASK) >> PAGE_SHIFT); + if (WARN_ON_ONCE(pmd_bad(pmdp_get(pmd)))) return NULL; ptep = pte_offset_kernel(pmd, addr); diff --git a/mm/vmscan.c b/mm/vmscan.c index f11491ee9ed5..4a14ea4833df 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -3640,7 +3640,7 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area DEFINE_MAX_SEQ(walk->lruvec); int gen = lru_gen_from_seq(max_seq); - VM_WARN_ON_ONCE(pud_leaf(*pud)); + VM_WARN_ON_ONCE(pud_leaf(pudp_get(pud))); /* try to batch at most 1+MIN_LRU_BATCH+1 entries */ if (*first == -1) { @@ -3670,17 +3670,17 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area /* don't round down the first address */ addr = i ? (*first & PMD_MASK) + i * PMD_SIZE : *first; - if (!pmd_present(pmd[i])) + if (!pmd_present(pmdp_get(pmd + i))) goto next; - if (!pmd_trans_huge(pmd[i])) { + if (!pmd_trans_huge(pmdp_get(pmd + i))) { if (!walk->force_scan && should_clear_pmd_young() && !mm_has_notifiers(args->mm)) pmdp_test_and_clear_young(vma, addr, pmd + i); goto next; } - pfn = get_pmd_pfn(pmd[i], vma, addr, pgdat); + pfn = get_pmd_pfn(pmdp_get(pmd + i), vma, addr, pgdat); if (pfn == -1) goto next; @@ -3698,7 +3698,7 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area dirty = false; } - if (pmd_dirty(pmd[i])) + if (pmd_dirty(pmdp_get(pmd + i))) dirty = true; walk->mm_stats[MM_LEAF_YOUNG]++; @@ -3727,7 +3727,7 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end, struct lru_gen_mm_walk *walk = args->private; struct lru_gen_mm_state *mm_state = get_mm_state(walk->lruvec); - VM_WARN_ON_ONCE(pud_leaf(*pud)); + VM_WARN_ON_ONCE(pud_leaf(pudp_get(pud))); /* * Finish an entire PMD in two passes: the first only reaches to PTE @@ -3796,7 +3796,7 @@ static int walk_pud_range(p4d_t *p4d, unsigned long start, unsigned long end, unsigned long next; struct lru_gen_mm_walk *walk = args->private; - VM_WARN_ON_ONCE(p4d_leaf(*p4d)); + VM_WARN_ON_ONCE(p4d_leaf(p4dp_get(p4d))); pud = pud_offset(p4d, start & P4D_MASK); restart: diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl index 8a7787d228a6..9d1cb0a762c9 100755 --- a/scripts/checkpatch.pl +++ b/scripts/checkpatch.pl @@ -7840,6 +7840,13 @@ sub process { ERROR("UNINITIALIZED_PTR_WITH_FREE", "pointer '$1' with __free attribute should be initialized\n" . $herecurr); } + +# check for raw dereferences of hardware page table pointers + if ($realfile !~ m@^arch/@ && + $line =~ /(?))?(pte|p[mu4g]d)p?\b/) { + WARN("PAGE_TABLE_ACCESSORS", + "Use $3p_get()/set_$3() instead of dereferencing page table pointers\n" . $herecurr); + } } # If we have no input at all, then there is nothing to report on