diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 19f4cc6..6bdc70c 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -36,8 +36,8 @@ config ARM select HAVE_ARCH_SECCOMP_FILTER if (AEABI && !OABI_COMPAT) select HAVE_ARCH_TRACEHOOK select HAVE_BPF_JIT + select HAVE_CONTEXT_TRACKING if !IPIPE select HAVE_CC_STACKPROTECTOR - select HAVE_CONTEXT_TRACKING select HAVE_C_RECORDMCOUNT select HAVE_DEBUG_KMEMLEAK select HAVE_DMA_API_DEBUG @@ -71,6 +71,7 @@ config ARM select HAVE_SYSCALL_TRACEPOINTS select HAVE_UID16 select HAVE_VIRT_CPU_ACCOUNTING_GEN + select IPIPE_WANT_PTE_PINNING if IPIPE && !(CPU_V6K || CPU_V7) select IRQ_FORCED_THREADING select MODULES_USE_ELF_REL select NO_BOOTMEM @@ -496,6 +497,7 @@ config ARCH_IXP4XX select CLKSRC_MMIO select CPU_XSCALE select DMABOUNCE if PCI + select IPIPE_ARM_KUSER_TSC if IPIPE select GENERIC_CLOCKEVENTS select MIGHT_HAVE_PCI select NEED_MACH_IO_H @@ -682,6 +684,7 @@ config ARCH_SA1100 config ARCH_S3C24XX bool "Samsung S3C24XX SoCs" + select IPIPE_ARM_KUSER_TSC if IPIPE select ARCH_REQUIRE_GPIOLIB select ATAGS select CLKDEV_LOOKUP @@ -981,6 +984,14 @@ config ARM_TIMER_SP804 select CLKSRC_MMIO select CLKSRC_OF if OF +if IPIPE +config IPIPE_ARM_KUSER_TSC + bool + select GENERIC_TIME_VSYSCALL + select IPIPE_HAVE_HOSTRT if IPIPE + default y if ARCH_AT91 || ARM_TIMER_SP804 || ARCH_MXC || ARCH_OMAP || PLAT_PXA || PLAT_S3C24XX || ARCH_SA1100 +endif + source "arch/arm/firmware/Kconfig" source arch/arm/mm/Kconfig @@ -1492,6 +1503,8 @@ config ARCH_NR_GPIO If unsure, leave the default value. +source kernel/ipipe/Kconfig + source kernel/Kconfig.preempt config HZ_FIXED @@ -1718,6 +1731,7 @@ config ALIGNMENT_TRAP config UACCESS_WITH_MEMCPY bool "Use kernel mem{cpy,set}() for {copy_to,clear}_user()" depends on MMU + depends on !IPIPE default y if CPU_FEROCEON help Implement faster copy_to_user and clear_user methods for CPU diff --git a/arch/arm/boot/compressed/decompress.c b/arch/arm/boot/compressed/decompress.c index a0765e7..381ad50 100644 --- a/arch/arm/boot/compressed/decompress.c +++ b/arch/arm/boot/compressed/decompress.c @@ -1,10 +1,8 @@ -#define _LINUX_STRING_H_ - #include /* for inline */ #include /* for size_t */ #include /* for NULL */ #include -#include +#include extern unsigned long free_mem_ptr; extern unsigned long free_mem_end_ptr; diff --git a/arch/arm/boot/compressed/head.S b/arch/arm/boot/compressed/head.S index 2c45b57..c0c8f9b 100644 --- a/arch/arm/boot/compressed/head.S +++ b/arch/arm/boot/compressed/head.S @@ -1330,6 +1330,15 @@ memdump: mov r12, r0 mov pc, r10 #endif +#ifdef CONFIG_IPIPE_TRACE_MCOUNT + .text + .align 0 + .type mcount %function + .global mcount +mcount: + mov pc, lr @ just return +#endif + .ltorg #ifdef CONFIG_ARM_VIRT_EXT diff --git a/arch/arm/boot/compressed/string.c b/arch/arm/boot/compressed/string.c index 36e53ef..3651693 100644 --- a/arch/arm/boot/compressed/string.c +++ b/arch/arm/boot/compressed/string.c @@ -93,6 +93,23 @@ int strcmp(const char *cs, const char *ct) return res; } +char *strstr(const char *s1, const char *s2) +{ + size_t l1, l2; + + l2 = strlen(s2); + if (!l2) + return (char *)s1; + l1 = strlen(s1); + while (l1 >= l2) { + l1--; + if (!memcmp(s1, s2, l2)) + return (char *)s1; + s1++; + } + return NULL; +} + void *memchr(const void *s, int c, size_t count) { const unsigned char *p = s; diff --git a/arch/arm/boot/dts/Makefile b/arch/arm/boot/dts/Makefile index 992736b..1def578 100644 --- a/arch/arm/boot/dts/Makefile +++ b/arch/arm/boot/dts/Makefile @@ -1,3 +1,9 @@ +omap3-igep0020-3430.dts: omap3-igep0020.dts omap3430-igep.dtsi + sed 's/omap3-igep\.dtsi/omap3430-igep.dtsi/' $< > $@ + +omap3430-igep.dtsi: omap3-igep.dtsi Makefile + sed 's/omap36xx\.dtsi/omap34xx.dtsi/' $< > $@ + ifeq ($(CONFIG_OF),y) dtb-$(CONFIG_ARCH_ALPINE) += \ diff --git a/arch/arm/boot/dts/imx6qdl.dtsi b/arch/arm/boot/dts/imx6qdl.dtsi index 38c7860..45dbecc 100644 --- a/arch/arm/boot/dts/imx6qdl.dtsi +++ b/arch/arm/boot/dts/imx6qdl.dtsi @@ -127,6 +127,14 @@ clocks = <&clks IMX6QDL_CLK_TWD>; }; + timer@00a00200 { + compatible = "arm,cortex-a9-global-timer"; + reg = <0x00a00200 0x20>; + interrupts = <1 11 0xf01>; + interrupt-parent = <&intc>; + clocks = <&clks IMX6QDL_CLK_TWD>; + }; + L2: l2-cache@00a02000 { compatible = "arm,pl310-cache"; reg = <0x00a02000 0x1000>; diff --git a/arch/arm/boot/dts/omap3-igep0020-3430.dts b/arch/arm/boot/dts/omap3-igep0020-3430.dts new file mode 100644 index 0000000..e2cffbd --- /dev/null +++ b/arch/arm/boot/dts/omap3-igep0020-3430.dts @@ -0,0 +1,280 @@ +/* + * Device Tree Source for IGEPv2 Rev. (TI OMAP AM/DM37x) + * + * Copyright (C) 2012 Javier Martinez Canillas + * Copyright (C) 2012 Enric Balletbo i Serra + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include "omap3430-igep.dtsi" +#include "omap-gpmc-smsc9221.dtsi" + +/ { + model = "IGEPv2 (TI OMAP AM/DM35x)"; + compatible = "isee,omap3-igep0020", "ti,omap35xx", "ti,omap3"; + + leds { + pinctrl-names = "default"; + pinctrl-0 = <&leds_pins>; + compatible = "gpio-leds"; + + boot { + label = "omap3:green:boot"; + gpios = <&gpio1 26 GPIO_ACTIVE_HIGH>; + default-state = "on"; + }; + + user0 { + label = "omap3:red:user0"; + gpios = <&gpio1 27 GPIO_ACTIVE_HIGH>; + default-state = "off"; + }; + + user1 { + label = "omap3:red:user1"; + gpios = <&gpio1 28 GPIO_ACTIVE_HIGH>; + default-state = "off"; + }; + + user2 { + label = "omap3:green:user1"; + gpios = <&twl_gpio 19 GPIO_ACTIVE_LOW>; + }; + }; + + /* HS USB Port 1 Power */ + hsusb1_power: hsusb1_power_reg { + compatible = "regulator-fixed"; + regulator-name = "hsusb1_vbus"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + gpio = <&twl_gpio 18 GPIO_ACTIVE_LOW>; /* GPIO LEDA */ + startup-delay-us = <70000>; + }; + + /* HS USB Host PHY on PORT 1 */ + hsusb1_phy: hsusb1_phy { + compatible = "usb-nop-xceiv"; + reset-gpios = <&gpio1 24 GPIO_ACTIVE_LOW>; /* gpio_24 */ + vcc-supply = <&hsusb1_power>; + }; + + tfp410: encoder@0 { + compatible = "ti,tfp410"; + powerdown-gpios = <&gpio6 10 GPIO_ACTIVE_LOW>; /* gpio_170 */ + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + tfp410_in: endpoint@0 { + remote-endpoint = <&dpi_out>; + }; + }; + + port@1 { + reg = <1>; + + tfp410_out: endpoint@0 { + remote-endpoint = <&dvi_connector_in>; + }; + }; + }; + }; + + dvi0: connector@0 { + compatible = "dvi-connector"; + label = "dvi"; + + digital; + + ddc-i2c-bus = <&i2c3>; + + port { + dvi_connector_in: endpoint { + remote-endpoint = <&tfp410_out>; + }; + }; + }; +}; + +&omap3_pmx_core { + pinctrl-names = "default"; + pinctrl-0 = < + &tfp410_pins + &dss_dpi_pins + >; + + tfp410_pins: pinmux_tfp410_pins { + pinctrl-single,pins = < + 0x196 (PIN_OUTPUT | MUX_MODE4) /* hdq_sio.gpio_170 */ + >; + }; + + dss_dpi_pins: pinmux_dss_dpi_pins { + pinctrl-single,pins = < + 0x0a4 (PIN_OUTPUT | MUX_MODE0) /* dss_pclk.dss_pclk */ + 0x0a6 (PIN_OUTPUT | MUX_MODE0) /* dss_hsync.dss_hsync */ + 0x0a8 (PIN_OUTPUT | MUX_MODE0) /* dss_vsync.dss_vsync */ + 0x0aa (PIN_OUTPUT | MUX_MODE0) /* dss_acbias.dss_acbias */ + 0x0ac (PIN_OUTPUT | MUX_MODE0) /* dss_data0.dss_data0 */ + 0x0ae (PIN_OUTPUT | MUX_MODE0) /* dss_data1.dss_data1 */ + 0x0b0 (PIN_OUTPUT | MUX_MODE0) /* dss_data2.dss_data2 */ + 0x0b2 (PIN_OUTPUT | MUX_MODE0) /* dss_data3.dss_data3 */ + 0x0b4 (PIN_OUTPUT | MUX_MODE0) /* dss_data4.dss_data4 */ + 0x0b6 (PIN_OUTPUT | MUX_MODE0) /* dss_data5.dss_data5 */ + 0x0b8 (PIN_OUTPUT | MUX_MODE0) /* dss_data6.dss_data6 */ + 0x0ba (PIN_OUTPUT | MUX_MODE0) /* dss_data7.dss_data7 */ + 0x0bc (PIN_OUTPUT | MUX_MODE0) /* dss_data8.dss_data8 */ + 0x0be (PIN_OUTPUT | MUX_MODE0) /* dss_data9.dss_data9 */ + 0x0c0 (PIN_OUTPUT | MUX_MODE0) /* dss_data10.dss_data10 */ + 0x0c2 (PIN_OUTPUT | MUX_MODE0) /* dss_data11.dss_data11 */ + 0x0c4 (PIN_OUTPUT | MUX_MODE0) /* dss_data12.dss_data12 */ + 0x0c6 (PIN_OUTPUT | MUX_MODE0) /* dss_data13.dss_data13 */ + 0x0c8 (PIN_OUTPUT | MUX_MODE0) /* dss_data14.dss_data14 */ + 0x0ca (PIN_OUTPUT | MUX_MODE0) /* dss_data15.dss_data15 */ + 0x0cc (PIN_OUTPUT | MUX_MODE0) /* dss_data16.dss_data16 */ + 0x0ce (PIN_OUTPUT | MUX_MODE0) /* dss_data17.dss_data17 */ + 0x0d0 (PIN_OUTPUT | MUX_MODE0) /* dss_data18.dss_data18 */ + 0x0d2 (PIN_OUTPUT | MUX_MODE0) /* dss_data19.dss_data19 */ + 0x0d4 (PIN_OUTPUT | MUX_MODE0) /* dss_data20.dss_data20 */ + 0x0d6 (PIN_OUTPUT | MUX_MODE0) /* dss_data21.dss_data21 */ + 0x0d8 (PIN_OUTPUT | MUX_MODE0) /* dss_data22.dss_data22 */ + 0x0da (PIN_OUTPUT | MUX_MODE0) /* dss_data23.dss_data23 */ + >; + }; +}; + +&omap3_pmx_core2 { + pinctrl-names = "default"; + pinctrl-0 = < + &hsusbb1_pins + >; + + hsusbb1_pins: pinmux_hsusbb1_pins { + pinctrl-single,pins = < + OMAP3630_CORE2_IOPAD(0x25da, PIN_OUTPUT | MUX_MODE3) /* etk_ctl.hsusb1_clk */ + OMAP3630_CORE2_IOPAD(0x25d8, PIN_OUTPUT | MUX_MODE3) /* etk_clk.hsusb1_stp */ + OMAP3630_CORE2_IOPAD(0x25ec, PIN_INPUT_PULLDOWN | MUX_MODE3) /* etk_d8.hsusb1_dir */ + OMAP3630_CORE2_IOPAD(0x25ee, PIN_INPUT_PULLDOWN | MUX_MODE3) /* etk_d9.hsusb1_nxt */ + OMAP3630_CORE2_IOPAD(0x25dc, PIN_INPUT_PULLDOWN | MUX_MODE3) /* etk_d0.hsusb1_data0 */ + OMAP3630_CORE2_IOPAD(0x25de, PIN_INPUT_PULLDOWN | MUX_MODE3) /* etk_d1.hsusb1_data1 */ + OMAP3630_CORE2_IOPAD(0x25e0, PIN_INPUT_PULLDOWN | MUX_MODE3) /* etk_d2.hsusb1_data2 */ + OMAP3630_CORE2_IOPAD(0x25e2, PIN_INPUT_PULLDOWN | MUX_MODE3) /* etk_d3.hsusb1_data7 */ + OMAP3630_CORE2_IOPAD(0x25e4, PIN_INPUT_PULLDOWN | MUX_MODE3) /* etk_d4.hsusb1_data4 */ + OMAP3630_CORE2_IOPAD(0x25e6, PIN_INPUT_PULLDOWN | MUX_MODE3) /* etk_d5.hsusb1_data5 */ + OMAP3630_CORE2_IOPAD(0x25e8, PIN_INPUT_PULLDOWN | MUX_MODE3) /* etk_d6.hsusb1_data6 */ + OMAP3630_CORE2_IOPAD(0x25ea, PIN_INPUT_PULLDOWN | MUX_MODE3) /* etk_d7.hsusb1_data3 */ + >; + }; + + leds_pins: pinmux_leds_pins { + pinctrl-single,pins = < + OMAP3630_CORE2_IOPAD(0x25f4, PIN_OUTPUT | MUX_MODE4) /* etk_d12.gpio_26 */ + OMAP3630_CORE2_IOPAD(0x25f6, PIN_OUTPUT | MUX_MODE4) /* etk_d13.gpio_27 */ + OMAP3630_CORE2_IOPAD(0x25f8, PIN_OUTPUT | MUX_MODE4) /* etk_d14.gpio_28 */ + >; + }; +}; + +&i2c3 { + clock-frequency = <100000>; + + /* + * Display monitor features are burnt in the EEPROM + * as EDID data. + */ + eeprom@50 { + compatible = "ti,eeprom"; + reg = <0x50>; + }; +}; + +&gpmc { + ranges = <0 0 0x00000000 0x20000000>, + <5 0 0x2c000000 0x01000000>; + + nand@0,0 { + linux,mtd-name= "micron,mt29c4g96maz"; + reg = <0 0 0>; + nand-bus-width = <16>; + ti,nand-ecc-opt = "bch8"; + + gpmc,sync-clk-ps = <0>; + gpmc,cs-on-ns = <0>; + gpmc,cs-rd-off-ns = <44>; + gpmc,cs-wr-off-ns = <44>; + gpmc,adv-on-ns = <6>; + gpmc,adv-rd-off-ns = <34>; + gpmc,adv-wr-off-ns = <44>; + gpmc,we-off-ns = <40>; + gpmc,oe-off-ns = <54>; + gpmc,access-ns = <64>; + gpmc,rd-cycle-ns = <82>; + gpmc,wr-cycle-ns = <82>; + gpmc,wr-access-ns = <40>; + gpmc,wr-data-mux-bus-ns = <0>; + + #address-cells = <1>; + #size-cells = <1>; + + partition@0 { + label = "SPL"; + reg = <0 0x100000>; + }; + partition@80000 { + label = "U-Boot"; + reg = <0x100000 0x180000>; + }; + partition@1c0000 { + label = "Environment"; + reg = <0x280000 0x100000>; + }; + partition@280000 { + label = "Kernel"; + reg = <0x380000 0x300000>; + }; + partition@780000 { + label = "Filesystem"; + reg = <0x680000 0x1f980000>; + }; + }; + + ethernet@gpmc { + pinctrl-names = "default"; + pinctrl-0 = <&smsc9221_pins>; + reg = <5 0 0xff>; + interrupt-parent = <&gpio6>; + interrupts = <16 IRQ_TYPE_LEVEL_LOW>; + }; +}; + +&usbhshost { + port1-mode = "ehci-phy"; +}; + +&usbhsehci { + phys = <&hsusb1_phy>; +}; + +&vpll2 { + /* Needed for DSS */ + regulator-name = "vdds_dsi"; +}; + +&dss { + status = "ok"; + + port { + dpi_out: endpoint { + remote-endpoint = <&tfp410_in>; + data-lines = <24>; + }; + }; +}; diff --git a/arch/arm/boot/dts/omap3430-igep.dtsi b/arch/arm/boot/dts/omap3430-igep.dtsi new file mode 100644 index 0000000..3f47b65 --- /dev/null +++ b/arch/arm/boot/dts/omap3430-igep.dtsi @@ -0,0 +1,222 @@ +/* + * Common device tree for IGEP boards based on AM/DM37x + * + * Copyright (C) 2012 Javier Martinez Canillas + * Copyright (C) 2012 Enric Balletbo i Serra + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +/dts-v1/; + +#include "omap34xx.dtsi" + +/ { + memory { + device_type = "memory"; + reg = <0x80000000 0x20000000>; /* 512 MB */ + }; + + sound { + compatible = "ti,omap-twl4030"; + ti,model = "igep2"; + ti,mcbsp = <&mcbsp2>; + ti,codec = <&twl_audio>; + }; + + vdd33: regulator-vdd33 { + compatible = "regulator-fixed"; + regulator-name = "vdd33"; + regulator-always-on; + }; + + lbee1usjyc_vmmc: lbee1usjyc_vmmc { + pinctrl-names = "default"; + pinctrl-0 = <&lbee1usjyc_pins>; + compatible = "regulator-fixed"; + regulator-name = "regulator-lbee1usjyc"; + regulator-min-microvolt = <3300000>; + regulator-max-microvolt = <3300000>; + gpio = <&gpio5 10 GPIO_ACTIVE_HIGH>; /* gpio_138 WIFI_PDN */ + startup-delay-us = <10000>; + enable-active-high; + vin-supply = <&vdd33>; + }; +}; + +&omap3_pmx_core { + uart1_pins: pinmux_uart1_pins { + pinctrl-single,pins = < + 0x152 (PIN_INPUT | MUX_MODE0) /* uart1_rx.uart1_rx */ + 0x14c (PIN_OUTPUT |MUX_MODE0) /* uart1_tx.uart1_tx */ + >; + }; + + uart2_pins: pinmux_uart2_pins { + pinctrl-single,pins = < + 0x14a (PIN_INPUT | MUX_MODE0) /* uart2_rx.uart2_rx */ + 0x148 (PIN_OUTPUT | MUX_MODE0) /* uart2_tx.uart2_tx */ + >; + }; + + uart3_pins: pinmux_uart3_pins { + pinctrl-single,pins = < + 0x16e (PIN_INPUT | MUX_MODE0) /* uart3_rx.uart3_rx */ + 0x170 (PIN_OUTPUT | MUX_MODE0) /* uart3_tx.uart3_tx */ + >; + }; + + /* WiFi/BT combo */ + lbee1usjyc_pins: pinmux_lbee1usjyc_pins { + pinctrl-single,pins = < + 0x136 (PIN_OUTPUT | MUX_MODE4) /* sdmmc2_dat5.gpio_137 */ + 0x138 (PIN_OUTPUT | MUX_MODE4) /* sdmmc2_dat6.gpio_138 */ + 0x13a (PIN_OUTPUT | MUX_MODE4) /* sdmmc2_dat7.gpio_139 */ + >; + }; + + mcbsp2_pins: pinmux_mcbsp2_pins { + pinctrl-single,pins = < + 0x10c (PIN_INPUT | MUX_MODE0) /* mcbsp2_fsx.mcbsp2_fsx */ + 0x10e (PIN_INPUT | MUX_MODE0) /* mcbsp2_clkx.mcbsp2_clkx */ + 0x110 (PIN_INPUT | MUX_MODE0) /* mcbsp2_dr.mcbsp2.dr */ + 0x112 (PIN_OUTPUT | MUX_MODE0) /* mcbsp2_dx.mcbsp2_dx */ + >; + }; + + mmc1_pins: pinmux_mmc1_pins { + pinctrl-single,pins = < + 0x114 (PIN_INPUT_PULLUP | MUX_MODE0) /* sdmmc1_clk.sdmmc1_clk */ + 0x116 (PIN_INPUT_PULLUP | MUX_MODE0) /* sdmmc1_cmd.sdmmc1_cmd */ + 0x118 (PIN_INPUT_PULLUP | MUX_MODE0) /* sdmmc1_dat0.sdmmc1_dat0 */ + 0x11a (PIN_INPUT_PULLUP | MUX_MODE0) /* sdmmc1_dat1.sdmmc1_dat1 */ + 0x11c (PIN_INPUT_PULLUP | MUX_MODE0) /* sdmmc1_dat2.sdmmc1_dat2 */ + 0x11e (PIN_INPUT_PULLUP | MUX_MODE0) /* sdmmc1_dat3.sdmmc1_dat3 */ + >; + }; + + mmc2_pins: pinmux_mmc2_pins { + pinctrl-single,pins = < + 0x128 (PIN_INPUT_PULLUP | MUX_MODE0) /* sdmmc2_clk.sdmmc2_clk */ + 0x12a (PIN_INPUT_PULLUP | MUX_MODE0) /* sdmmc2_cmd.sdmmc2_cmd */ + 0x12c (PIN_INPUT_PULLUP | MUX_MODE0) /* sdmmc2_dat0.sdmmc2_dat0 */ + 0x12e (PIN_INPUT_PULLUP | MUX_MODE0) /* sdmmc2_dat1.sdmmc2_dat1 */ + 0x130 (PIN_INPUT_PULLUP | MUX_MODE0) /* sdmmc2_dat2.sdmmc2_dat2 */ + 0x132 (PIN_INPUT_PULLUP | MUX_MODE0) /* sdmmc2_dat3.sdmmc2_dat3 */ + >; + }; + + smsc9221_pins: pinmux_smsc9221_pins { + pinctrl-single,pins = < + 0x1a2 (PIN_INPUT | MUX_MODE4) /* mcspi1_cs2.gpio_176 */ + >; + }; + + i2c1_pins: pinmux_i2c1_pins { + pinctrl-single,pins = < + 0x18a (PIN_INPUT | MUX_MODE0) /* i2c1_scl.i2c1_scl */ + 0x18c (PIN_INPUT | MUX_MODE0) /* i2c1_sda.i2c1_sda */ + >; + }; + + i2c2_pins: pinmux_i2c2_pins { + pinctrl-single,pins = < + 0x18e (PIN_INPUT | MUX_MODE0) /* i2c2_scl.i2c2_scl */ + 0x190 (PIN_INPUT | MUX_MODE0) /* i2c2_sda.i2c2_sda */ + >; + }; + + i2c3_pins: pinmux_i2c3_pins { + pinctrl-single,pins = < + 0x192 (PIN_INPUT | MUX_MODE0) /* i2c3_scl.i2c3_scl */ + 0x194 (PIN_INPUT | MUX_MODE0) /* i2c3_sda.i2c3_sda */ + >; + }; +}; + +&i2c1 { + pinctrl-names = "default"; + pinctrl-0 = <&i2c1_pins>; + clock-frequency = <2600000>; + + twl: twl@48 { + reg = <0x48>; + interrupts = <7>; /* SYS_NIRQ cascaded to intc */ + interrupt-parent = <&intc>; + + twl_audio: audio { + compatible = "ti,twl4030-audio"; + codec { + }; + }; + }; +}; + +#include "twl4030.dtsi" +#include "twl4030_omap3.dtsi" + +&i2c2 { + pinctrl-names = "default"; + pinctrl-0 = <&i2c2_pins>; + clock-frequency = <400000>; +}; + +&i2c3 { + pinctrl-names = "default"; + pinctrl-0 = <&i2c3_pins>; +}; + +&mcbsp2 { + pinctrl-names = "default"; + pinctrl-0 = <&mcbsp2_pins>; + status = "okay"; +}; + +&mmc1 { + pinctrl-names = "default"; + pinctrl-0 = <&mmc1_pins>; + vmmc-supply = <&vmmc1>; + vmmc_aux-supply = <&vsim>; + bus-width = <4>; +}; + +&mmc2 { + pinctrl-names = "default"; + pinctrl-0 = <&mmc2_pins>; + vmmc-supply = <&lbee1usjyc_vmmc>; + bus-width = <4>; + non-removable; +}; + +&mmc3 { + status = "disabled"; +}; + +&uart1 { + pinctrl-names = "default"; + pinctrl-0 = <&uart1_pins>; +}; + +&uart2 { + pinctrl-names = "default"; + pinctrl-0 = <&uart2_pins>; +}; + +&uart3 { + pinctrl-names = "default"; + pinctrl-0 = <&uart3_pins>; +}; + +&twl_gpio { + ti,use-leds; +}; + +&usb_otg_hs { + interface-type = <0>; + usb-phy = <&usb2_phy>; + phys = <&usb2_phy>; + phy-names = "usb2-phy"; + mode = <3>; + power = <50>; +}; diff --git a/arch/arm/boot/dts/omap4.dtsi b/arch/arm/boot/dts/omap4.dtsi index 84be9da..a0cb96e 100644 --- a/arch/arm/boot/dts/omap4.dtsi +++ b/arch/arm/boot/dts/omap4.dtsi @@ -82,6 +82,14 @@ interrupt-parent = <&gic>; }; + global_timer: timer@48240200 { + compatible = "arm,cortex-a9-global-timer"; + reg = <0x48240200 0x20>; + clocks = <&mpu_periphclk>; + interrupts = ; + interrupt-parent = <&gic>; + }; + /* * The soc node represents the soc top level view. It is used for IPs * that are not memory mapped in the MPU view or for the MPU itself. diff --git a/arch/arm/common/it8152.c b/arch/arm/common/it8152.c index 5114b68..f84157d 100644 --- a/arch/arm/common/it8152.c +++ b/arch/arm/common/it8152.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include @@ -124,21 +125,21 @@ void it8152_irq_demux(unsigned int irq, struct irq_desc *desc) bits_pd &= ((1 << IT8152_PD_IRQ_COUNT) - 1); while (bits_pd) { i = __ffs(bits_pd); - generic_handle_irq(IT8152_PD_IRQ(i)); + ipipe_handle_demuxed_irq(IT8152_PD_IRQ(i)); bits_pd &= ~(1 << i); } bits_lp &= ((1 << IT8152_LP_IRQ_COUNT) - 1); while (bits_lp) { i = __ffs(bits_lp); - generic_handle_irq(IT8152_LP_IRQ(i)); + ipipe_handle_demuxed_irq(IT8152_LP_IRQ(i)); bits_lp &= ~(1 << i); } bits_ld &= ((1 << IT8152_LD_IRQ_COUNT) - 1); while (bits_ld) { i = __ffs(bits_ld); - generic_handle_irq(IT8152_LD_IRQ(i)); + ipipe_handle_demuxed_irq(IT8152_LD_IRQ(i)); bits_ld &= ~(1 << i); } } diff --git a/arch/arm/common/timer-sp.c b/arch/arm/common/timer-sp.c index 1921132..98f4283 100644 --- a/arch/arm/common/timer-sp.c +++ b/arch/arm/common/timer-sp.c @@ -29,10 +29,24 @@ #include #include #include +#include +#include +#include #include #include +#ifdef CONFIG_IPIPE +static struct __ipipe_tscinfo tsc_info = { + .type = IPIPE_TSC_TYPE_FREERUNNING_COUNTDOWN, + .u = { + { + .mask = 0xffffffff, + }, + }, +}; +#endif /* CONFIG_IPIPE */ + static long __init sp804_get_clock_rate(struct clk *clk) { long rate; @@ -72,6 +86,7 @@ static u64 notrace sp804_read(void) } void __init __sp804_clocksource_and_sched_clock_init(void __iomem *base, + unsigned long phys, const char *name, struct clk *clk, int use_sched_clock) @@ -106,12 +121,25 @@ void __init __sp804_clocksource_and_sched_clock_init(void __iomem *base, sched_clock_base = base; sched_clock_register(sp804_read, 32, rate); } + +#ifdef CONFIG_IPIPE + tsc_info.freq = rate; + tsc_info.counter_vaddr = (unsigned long)base + TIMER_VALUE; + tsc_info.u.counter_paddr = phys + TIMER_VALUE; + __ipipe_tsc_register(&tsc_info); +#endif } static void __iomem *clkevt_base; static unsigned long clkevt_reload; +static inline void sp804_timer_ack(void) +{ + /* clear the interrupt */ + writel(1, clkevt_base + TIMER_INTCLR); +} + /* * IRQ handler for the timer */ @@ -119,8 +147,8 @@ static irqreturn_t sp804_timer_interrupt(int irq, void *dev_id) { struct clock_event_device *evt = dev_id; - /* clear the interrupt */ - writel(1, clkevt_base + TIMER_INTCLR); + if (!clockevent_ipipe_stolen(evt)) + sp804_timer_ack(); evt->event_handler(evt); @@ -165,12 +193,21 @@ static int sp804_set_next_event(unsigned long next, return 0; } +#ifdef CONFIG_IPIPE +static struct ipipe_timer sp804_itimer = { + .ack = sp804_timer_ack, +}; +#endif /* CONFIG_IPIPE */ + static struct clock_event_device sp804_clockevent = { .features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT | CLOCK_EVT_FEAT_DYNIRQ, .set_mode = sp804_set_mode, .set_next_event = sp804_set_next_event, .rating = 300, +#ifdef CONFIG_IPIPE + .ipipe_timer = &sp804_itimer, +#endif /* CONFIG_IPIPE */ }; static struct irqaction sp804_timer_irq = { @@ -205,6 +242,10 @@ void __init __sp804_clockevents_init(void __iomem *base, unsigned int irq, struc writel(0, base + TIMER_CTRL); +#ifdef CONFIG_IPIPE + sp804_itimer.irq = irq; +#endif /* CONFIG_IPIPE */ + setup_irq(irq, &sp804_timer_irq); clockevents_config_and_register(evt, rate, 0xf, 0xffffffff); } @@ -212,6 +253,7 @@ void __init __sp804_clockevents_init(void __iomem *base, unsigned int irq, struc static void __init sp804_of_init(struct device_node *np) { static bool initialized = false; + struct resource res; void __iomem *base; int irq; u32 irq_num = 0; @@ -248,13 +290,18 @@ static void __init sp804_of_init(struct device_node *np) if (irq <= 0) goto err; + if (of_address_to_resource(np, 0, &res)) + res.start = 0; + of_property_read_u32(np, "arm,sp804-has-irq", &irq_num); if (irq_num == 2) { __sp804_clockevents_init(base + TIMER_2_BASE, irq, clk2, name); - __sp804_clocksource_and_sched_clock_init(base, name, clk1, 1); + __sp804_clocksource_and_sched_clock_init(base, res.start, + name, clk1, 1); } else { __sp804_clockevents_init(base, irq, clk1 , name); __sp804_clocksource_and_sched_clock_init(base + TIMER_2_BASE, + res.start + TIMER_2_BASE, name, clk2, 1); } initialized = true; @@ -268,6 +315,7 @@ CLOCKSOURCE_OF_DECLARE(sp804, "arm,sp804", sp804_of_init); static void __init integrator_cp_of_init(struct device_node *np) { static int init_count = 0; + struct resource res; void __iomem *base; int irq; const char *name = of_get_property(np, "compatible", NULL); @@ -286,8 +334,12 @@ static void __init integrator_cp_of_init(struct device_node *np) if (init_count == 2 || !of_device_is_available(np)) goto err; + if (of_address_to_resource(np, 0, &res)) + res.start = 0; + if (!init_count) - __sp804_clocksource_and_sched_clock_init(base, name, clk, 0); + __sp804_clocksource_and_sched_clock_init(base, res.start, + name, clk, 0); else { irq = irq_of_parse_and_map(np, 0); if (irq <= 0) diff --git a/arch/arm/include/asm/arch_timer.h b/arch/arm/include/asm/arch_timer.h index d4ebf56..160af48 100644 --- a/arch/arm/include/asm/arch_timer.h +++ b/arch/arm/include/asm/arch_timer.h @@ -105,6 +105,10 @@ static inline u32 arch_timer_get_cntkctl(void) static inline void arch_timer_set_cntkctl(u32 cntkctl) { +#ifdef CONFIG_IPIPE + /* Enable access to user-space (may not be needed) */ + cntkctl |= ARCH_TIMER_USR_PCT_ACCESS_EN; +#endif asm volatile("mcr p15, 0, %0, c14, c1, 0" : : "r" (cntkctl)); } diff --git a/arch/arm/include/asm/assembler.h b/arch/arm/include/asm/assembler.h index 186270b..62eb54e 100644 --- a/arch/arm/include/asm/assembler.h +++ b/arch/arm/include/asm/assembler.h @@ -98,6 +98,18 @@ .macro enable_irq_notrace cpsie i .endm + + .macro disable_irq_cond +#ifdef CONFIG_IPIPE + cpsid i +#endif /* CONFIG_IPIPE */ + .endm + + .macro enable_irq_cond +#ifdef CONFIG_IPIPE + cpsie i +#endif /* CONFIG_IPIPE */ + .endm #else .macro disable_irq_notrace msr cpsr_c, #PSR_I_BIT | SVC_MODE @@ -106,10 +118,22 @@ .macro enable_irq_notrace msr cpsr_c, #SVC_MODE .endm + + .macro disable_irq_cond +#ifdef CONFIG_IPIPE + msr cpsr_c, #PSR_I_BIT | SVC_MODE +#endif /* CONFIG_IPIPE */ + .endm + + .macro enable_irq_cond +#ifdef CONFIG_IPIPE + msr cpsr_c, #SVC_MODE +#endif /* CONFIG_IPIPE */ + .endm #endif .macro asm_trace_hardirqs_off -#if defined(CONFIG_TRACE_IRQFLAGS) +#if defined(CONFIG_TRACE_IRQFLAGS) && !defined(CONFIG_IPIPE) stmdb sp!, {r0-r3, ip, lr} bl trace_hardirqs_off ldmia sp!, {r0-r3, ip, lr} @@ -117,7 +141,7 @@ .endm .macro asm_trace_hardirqs_on_cond, cond -#if defined(CONFIG_TRACE_IRQFLAGS) +#if defined(CONFIG_TRACE_IRQFLAGS) && !defined(CONFIG_IPIPE) /* * actually the registers should be pushed and pop'd conditionally, but * after bl the flags are certainly clobbered diff --git a/arch/arm/include/asm/atomic.h b/arch/arm/include/asm/atomic.h index e22c119..c86438a 100644 --- a/arch/arm/include/asm/atomic.h +++ b/arch/arm/include/asm/atomic.h @@ -142,9 +142,9 @@ static inline void atomic_##op(int i, atomic_t *v) \ { \ unsigned long flags; \ \ - raw_local_irq_save(flags); \ + flags = hard_local_irq_save(); \ v->counter c_op i; \ - raw_local_irq_restore(flags); \ + hard_local_irq_restore(flags); \ } \ #define ATOMIC_OP_RETURN(op, c_op, asm_op) \ @@ -153,10 +153,10 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ unsigned long flags; \ int val; \ \ - raw_local_irq_save(flags); \ + flags = hard_local_irq_save(); \ v->counter c_op i; \ val = v->counter; \ - raw_local_irq_restore(flags); \ + hard_local_irq_restore(flags); \ \ return val; \ } @@ -166,11 +166,11 @@ static inline int atomic_cmpxchg(atomic_t *v, int old, int new) int ret; unsigned long flags; - raw_local_irq_save(flags); + flags = hard_local_irq_save(); ret = v->counter; if (likely(ret == old)) v->counter = new; - raw_local_irq_restore(flags); + hard_local_irq_restore(flags); return ret; } @@ -346,7 +346,6 @@ static inline long long atomic64_cmpxchg(atomic64_t *ptr, long long old, : "cc"); } while (res); - smp_mb(); return oldval; } diff --git a/arch/arm/include/asm/barrier.h b/arch/arm/include/asm/barrier.h index d2f81e6..e06c8c5 100644 --- a/arch/arm/include/asm/barrier.h +++ b/arch/arm/include/asm/barrier.h @@ -16,27 +16,35 @@ #if __LINUX_ARM_ARCH__ >= 7 #define isb(option) __asm__ __volatile__ ("isb " #option : : : "memory") #define dsb(option) __asm__ __volatile__ ("dsb " #option : : : "memory") -#define dmb(option) __asm__ __volatile__ ("dmb " #option : : : "memory") +#define DMB(option) "dmb " #option +#define DMB_IN + #elif defined(CONFIG_CPU_XSC3) || __LINUX_ARM_ARCH__ == 6 #define isb(x) __asm__ __volatile__ ("mcr p15, 0, %0, c7, c5, 4" \ : : "r" (0) : "memory") #define dsb(x) __asm__ __volatile__ ("mcr p15, 0, %0, c7, c10, 4" \ : : "r" (0) : "memory") -#define dmb(x) __asm__ __volatile__ ("mcr p15, 0, %0, c7, c10, 5" \ - : : "r" (0) : "memory") +#define DMB(option) "mcr p15, 0, %0, c7, c10, 5" +#define DMB_IN "r" (0) + #elif defined(CONFIG_CPU_FA526) #define isb(x) __asm__ __volatile__ ("mcr p15, 0, %0, c7, c5, 4" \ : : "r" (0) : "memory") #define dsb(x) __asm__ __volatile__ ("mcr p15, 0, %0, c7, c10, 4" \ : : "r" (0) : "memory") -#define dmb(x) __asm__ __volatile__ ("" : : : "memory") #else #define isb(x) __asm__ __volatile__ ("" : : : "memory") #define dsb(x) __asm__ __volatile__ ("mcr p15, 0, %0, c7, c10, 4" \ : : "r" (0) : "memory") -#define dmb(x) __asm__ __volatile__ ("" : : : "memory") #endif +#ifdef DMB +#define dmb(option) __asm__ __volatile__(DMB(option) : : DMB_IN : "memory") +#else +#define dmb(option) __asm__ __volatile__("" : : : "memory") +#endif + + #ifdef CONFIG_ARCH_HAS_BARRIERS #include #elif defined(CONFIG_ARM_DMA_MEM_BUFFERABLE) || defined(CONFIG_SMP) @@ -58,9 +66,35 @@ #define smp_rmb() barrier() #define smp_wmb() barrier() #else -#define smp_mb() dmb(ish) -#define smp_rmb() smp_mb() -#define smp_wmb() dmb(ishst) + +#ifdef CONFIG_THUMB2_KERNEL +#define SMP_ONLY(smp) \ + "9998: " smp "\n" \ + " .pushsection \".alt.smp.init\", \"a\"\n" \ + " .long 9998b\n" \ + " nop.w\n" \ + " .popsection\n" +#else +#define SMP_ONLY(smp) \ + "9998: " smp "\n" \ + " .pushsection \".alt.smp.init\", \"a\"\n" \ + " .long 9998b\n" \ + " nop\n" \ + " .popsection\n" +#endif + +#ifdef DMB +#define smp_mb() \ + __asm__ __volatile__ (SMP_ONLY(DMB(ish)) : : DMB_IN : "memory") +#define smp_rmb() \ + __asm__ __volatile__ (SMP_ONLY(DMB(ish)) : : DMB_IN : "memory") +#define smp_wmb() \ + __asm__ __volatile__ (SMP_ONLY(DMB(ishst)) : : DMB_IN : "memory") +#else +#define smp_mb() barrier() +#define smp_rmb() barrier() +#define smp_wmb() barrier() +#endif #endif #define smp_store_release(p, v) \ diff --git a/arch/arm/include/asm/bitops.h b/arch/arm/include/asm/bitops.h index 5638099..821ded0 100644 --- a/arch/arm/include/asm/bitops.h +++ b/arch/arm/include/asm/bitops.h @@ -39,9 +39,9 @@ static inline void ____atomic_set_bit(unsigned int bit, volatile unsigned long * p += bit >> 5; - raw_local_irq_save(flags); + flags = hard_local_irq_save(); *p |= mask; - raw_local_irq_restore(flags); + hard_local_irq_restore(flags); } static inline void ____atomic_clear_bit(unsigned int bit, volatile unsigned long *p) @@ -51,9 +51,9 @@ static inline void ____atomic_clear_bit(unsigned int bit, volatile unsigned long p += bit >> 5; - raw_local_irq_save(flags); + flags = hard_local_irq_save(); *p &= ~mask; - raw_local_irq_restore(flags); + hard_local_irq_restore(flags); } static inline void ____atomic_change_bit(unsigned int bit, volatile unsigned long *p) @@ -63,9 +63,9 @@ static inline void ____atomic_change_bit(unsigned int bit, volatile unsigned lon p += bit >> 5; - raw_local_irq_save(flags); + flags = hard_local_irq_save(); *p ^= mask; - raw_local_irq_restore(flags); + hard_local_irq_restore(flags); } static inline int @@ -77,10 +77,10 @@ ____atomic_test_and_set_bit(unsigned int bit, volatile unsigned long *p) p += bit >> 5; - raw_local_irq_save(flags); + flags = hard_local_irq_save(); res = *p; *p = res | mask; - raw_local_irq_restore(flags); + hard_local_irq_restore(flags); return (res & mask) != 0; } @@ -94,10 +94,10 @@ ____atomic_test_and_clear_bit(unsigned int bit, volatile unsigned long *p) p += bit >> 5; - raw_local_irq_save(flags); + flags = hard_local_irq_save(); res = *p; *p = res & ~mask; - raw_local_irq_restore(flags); + hard_local_irq_restore(flags); return (res & mask) != 0; } @@ -111,10 +111,10 @@ ____atomic_test_and_change_bit(unsigned int bit, volatile unsigned long *p) p += bit >> 5; - raw_local_irq_save(flags); + flags = hard_local_irq_save(); res = *p; *p = res ^ mask; - raw_local_irq_restore(flags); + hard_local_irq_restore(flags); return (res & mask) != 0; } diff --git a/arch/arm/include/asm/cacheflush.h b/arch/arm/include/asm/cacheflush.h index 2d46862..297ce99 100644 --- a/arch/arm/include/asm/cacheflush.h +++ b/arch/arm/include/asm/cacheflush.h @@ -11,11 +11,13 @@ #define _ASMARM_CACHEFLUSH_H #include +#include #include #include #include #include +#include #define CACHE_COLOUR(vaddr) ((vaddr & (SHMLBA - 1)) >> PAGE_SHIFT) @@ -167,6 +169,27 @@ extern void dmac_flush_range(const void *, const void *); #endif +#ifdef CONFIG_ARM_FCSE +#define FCSE_CACHE_MASK (~(L1_CACHE_BYTES - 1)) +#define FCSE_CACHE_ALIGN(addr) (((addr) + ~FCSE_CACHE_MASK) & FCSE_CACHE_MASK) + +static inline void +fcse_flush_cache_user_range(struct vm_area_struct *vma, + unsigned long start, unsigned long end) +{ + if (cache_is_vivt() + && fcse_mm_in_cache(vma->vm_mm)) { + start = fcse_va_to_mva(vma->vm_mm, start & FCSE_CACHE_MASK); + end = fcse_va_to_mva(vma->vm_mm, FCSE_CACHE_ALIGN(end)); + __cpuc_flush_user_range(start, end, vma->vm_flags); + } +} +#undef FCSE_CACHE_MASK +#undef FCSE_CACHE_ALIGN +#else /* ! CONFIG_ARM_FCSE */ +#define fcse_flush_cache_user_range(vma, start, end) do { } while (0) +#endif /* ! CONFIG_ARM_FCSE */ + /* * Copy user data from/to a page which is mapped into a different * processes address space. Really, we want to allow our "user @@ -174,9 +197,10 @@ extern void dmac_flush_range(const void *, const void *); */ extern void copy_to_user_page(struct vm_area_struct *, struct page *, unsigned long, void *, const void *, unsigned long); -#define copy_from_user_page(vma, page, vaddr, dst, src, len) \ - do { \ - memcpy(dst, src, len); \ +#define copy_from_user_page(vma, page, vaddr, dst, src, len) \ + do { \ + fcse_flush_cache_user_range(vma, vaddr, vaddr + len); \ + memcpy(dst, src, len); \ } while (0) /* @@ -224,8 +248,11 @@ static inline void __flush_icache_all(void) static inline void vivt_flush_cache_mm(struct mm_struct *mm) { - if (cpumask_test_cpu(smp_processor_id(), mm_cpumask(mm))) + if (fcse_mm_in_cache(mm)) { + unsigned seq = fcse_flush_all_start(); __cpuc_flush_user_all(); + fcse_flush_all_done(seq, 1); + } } static inline void @@ -233,9 +260,11 @@ vivt_flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned { struct mm_struct *mm = vma->vm_mm; - if (!mm || cpumask_test_cpu(smp_processor_id(), mm_cpumask(mm))) - __cpuc_flush_user_range(start & PAGE_MASK, PAGE_ALIGN(end), - vma->vm_flags); + if (!mm || fcse_mm_in_cache(mm)) { + start = fcse_va_to_mva(mm, start & PAGE_MASK); + end = fcse_va_to_mva(mm, PAGE_ALIGN(end)); + __cpuc_flush_user_range(start, end, vma->vm_flags); + } } static inline void @@ -243,8 +272,9 @@ vivt_flush_cache_page(struct vm_area_struct *vma, unsigned long user_addr, unsig { struct mm_struct *mm = vma->vm_mm; - if (!mm || cpumask_test_cpu(smp_processor_id(), mm_cpumask(mm))) { - unsigned long addr = user_addr & PAGE_MASK; + if (!mm || fcse_mm_in_cache(mm)) { + unsigned long addr; + addr = fcse_va_to_mva(mm, user_addr) & PAGE_MASK; __cpuc_flush_user_range(addr, addr + PAGE_SIZE, vma->vm_flags); } } @@ -269,13 +299,22 @@ extern void flush_cache_page(struct vm_area_struct *vma, unsigned long user_addr * Harvard caches are synchronised for the user space address range. * This is used for the ARM private sys_cacheflush system call. */ -#define flush_cache_user_range(s,e) __cpuc_coherent_user_range(s,e) +#define flush_cache_user_range(s,e) \ + ({ \ + struct mm_struct *_mm = current->mm; \ + unsigned long _s, _e; \ + _s = fcse_va_to_mva(_mm, s); \ + _e = fcse_va_to_mva(_mm, e); \ + __cpuc_coherent_user_range(_s,_e); \ + }) /* * Perform necessary cache operations to ensure that data previously * stored within this range of addresses can be executed by the CPU. */ -#define flush_icache_range(s,e) __cpuc_coherent_kern_range(s,e) +#define flush_icache_range(s,e) \ + __cpuc_coherent_kern_range(fcse_va_to_mva(current->mm, (s)), \ + fcse_va_to_mva(current->mm, (e))) /* * Perform necessary cache operations to ensure that the TLB will @@ -316,7 +355,8 @@ static inline void flush_anon_page(struct vm_area_struct *vma, extern void __flush_anon_page(struct vm_area_struct *vma, struct page *, unsigned long); if (PageAnon(page)) - __flush_anon_page(vma, page, vmaddr); + __flush_anon_page(vma, page, + fcse_va_to_mva(vma->vm_mm, vmaddr)); } #define ARCH_HAS_FLUSH_KERNEL_DCACHE_PAGE @@ -345,9 +385,11 @@ extern void flush_kernel_dcache_page(struct page *); */ static inline void flush_cache_vmap(unsigned long start, unsigned long end) { - if (!cache_is_vipt_nonaliasing()) + if (!cache_is_vipt_nonaliasing()) { + unsigned seq = fcse_flush_all_start(); flush_cache_all(); - else + fcse_flush_all_done(seq, 1); + } else /* * set_pte_at() called from vmap_pte_range() does not * have a DSB after cleaning the cache line. @@ -357,8 +399,11 @@ static inline void flush_cache_vmap(unsigned long start, unsigned long end) static inline void flush_cache_vunmap(unsigned long start, unsigned long end) { - if (!cache_is_vipt_nonaliasing()) + if (!cache_is_vipt_nonaliasing()) { + unsigned seq = fcse_flush_all_start(); flush_cache_all(); + fcse_flush_all_done(seq, 1); + } } /* diff --git a/arch/arm/include/asm/cmpxchg.h b/arch/arm/include/asm/cmpxchg.h index abb2c37..620c394 100644 --- a/arch/arm/include/asm/cmpxchg.h +++ b/arch/arm/include/asm/cmpxchg.h @@ -65,17 +65,17 @@ static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size #error SMP is not supported on this platform #endif case 1: - raw_local_irq_save(flags); + flags = hard_local_irq_save(); ret = *(volatile unsigned char *)ptr; *(volatile unsigned char *)ptr = x; - raw_local_irq_restore(flags); + hard_local_irq_restore(flags); break; case 4: - raw_local_irq_save(flags); + flags = hard_local_irq_save(); ret = *(volatile unsigned long *)ptr; *(volatile unsigned long *)ptr = x; - raw_local_irq_restore(flags); + hard_local_irq_restore(flags); break; #else case 1: diff --git a/arch/arm/include/asm/entry-macro-multi.S b/arch/arm/include/asm/entry-macro-multi.S index 469a2b3..b34f8eb 100644 --- a/arch/arm/include/asm/entry-macro-multi.S +++ b/arch/arm/include/asm/entry-macro-multi.S @@ -11,7 +11,11 @@ @ routine called with r0 = irq number, r1 = struct pt_regs * @ adrne lr, BSYM(1b) +#ifdef CONFIG_IPIPE + bne __ipipe_grab_irq +#else bne asm_do_IRQ +#endif #ifdef CONFIG_SMP /* @@ -24,8 +28,12 @@ ALT_UP_B(9997f) movne r1, sp adrne lr, BSYM(1b) +#ifdef CONFIG_IPIPE + bne __ipipe_grab_ipi +#else bne do_IPI #endif +#endif 9997: .endm diff --git a/arch/arm/include/asm/fcse.h b/arch/arm/include/asm/fcse.h new file mode 100644 index 0000000..986fed9 --- /dev/null +++ b/arch/arm/include/asm/fcse.h @@ -0,0 +1,211 @@ +/* + * arch/arm/include/asm/fcse.h + * + * Helper header for using the ARM Fast Context Switch Extension with + * processors supporting it, lifted from the Fast Address Space + * Switching (FASS) patch for ARM Linux. + * + * Copyright (C) 2001, 2002 Adam Wiggins + * Copyright (C) 2007 Sebastian Smolorz + * Copyright (C) 2008 Richard Cochran + * Copyright (C) 2009-2011 Gilles Chanteperdrix + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#ifndef __ASM_ARM_FCSE_H +#define __ASM_ARM_FCSE_H + +#ifdef CONFIG_ARM_FCSE_DEBUG +#define FCSE_BUG_ON(expr) BUG_ON(expr) +#else /* !CONFIG_ARM_FCSE_DEBUG */ +#define FCSE_BUG_ON(expr) do { } while(0) +#endif /* !CONFIG_ARM_FCSE_DEBUG */ + +#ifdef CONFIG_ARM_FCSE + +#include /* For struct mm_struct */ +#include +#include + +#include +#include + +#define FCSE_PID_SHIFT 25 + +/* Size of PID relocation area */ +#define FCSE_PID_TASK_SIZE (1UL << FCSE_PID_SHIFT) + +/* Mask to get rid of PID from relocated address */ +#define FCSE_PID_MASK (FCSE_PID_TASK_SIZE - 1) + +#define FCSE_PID_INVALID (~0 << FCSE_PID_SHIFT) + +#define FCSE_NR_PIDS (TASK_SIZE / FCSE_PID_TASK_SIZE) +#define FCSE_PID_MAX (FCSE_NR_PIDS - 1) + +struct vm_unmapped_area_info; + +extern unsigned long fcse_pids_cache_dirty[]; +#ifdef CONFIG_ARM_FCSE_BEST_EFFORT +extern struct mm_struct *fcse_large_process; +#endif + +int fcse_pid_alloc(struct mm_struct *mm); +void fcse_pid_free(struct mm_struct *mm); +unsigned fcse_flush_all_start(void); +void fcse_flush_all_done(unsigned seq, unsigned dirty); +unsigned long +fcse_check_mmap_inner(struct mm_struct *mm, + struct vm_unmapped_area_info *info, + unsigned long addr, unsigned long flags); + +/* Sets the CPU's PID Register */ +static inline void fcse_pid_set(unsigned long pid) +{ + __asm__ __volatile__ ("mcr p15, 0, %0, c13, c0, 0" + : /* */: "r" (pid) : "cc", "memory"); +} + +static inline unsigned long fcse_pid_get(void) +{ + unsigned long pid; + __asm__ __volatile__ ("mrc p15, 0, %0, c13, c0, 0" + : "=r"(pid) : /* */ : "cc", "memory"); + return pid; +} + +static inline unsigned long fcse_mva_to_va(unsigned long mva) +{ + unsigned long va; + + if (!cache_is_vivt()) + return mva; + + va = fcse_pid_get() ^ mva; + return (va & 0xfe000000) ? mva : va; +} + +static inline unsigned long +fcse_va_to_mva(struct mm_struct *mm, unsigned long va) +{ + if (cache_is_vivt() && va < FCSE_PID_TASK_SIZE) { + return mm->context.fcse.pid | va; + } + return va; +} + + +static inline unsigned long +fcse_check_mmap_addr(struct mm_struct *mm, + unsigned long addr, unsigned long len, + struct vm_unmapped_area_info *info, unsigned long flags) +{ + if ((addr & ~PAGE_MASK) == 0 && addr + len <= FCSE_TASK_SIZE) + return addr; + + return fcse_check_mmap_inner(mm, info, addr, flags); +} + +static inline void __fcse_mark_dirty(struct mm_struct *mm) +{ + __set_bit(FCSE_PID_MAX - (mm->context.fcse.pid >> FCSE_PID_SHIFT), + fcse_pids_cache_dirty); +#ifdef CONFIG_ARM_FCSE_BEST_EFFORT + if (mm->context.fcse.large) + fcse_large_process = mm; +#endif +} + +static inline void fcse_mark_dirty(struct mm_struct *mm) +{ + if (cache_is_vivt()) { + set_bit(FCSE_PID_MAX - (mm->context.fcse.pid >> FCSE_PID_SHIFT), + fcse_pids_cache_dirty); +#ifdef CONFIG_ARM_FCSE_BEST_EFFORT + if (mm->context.fcse.large) + fcse_large_process = mm; +#endif + } +} + +#ifdef CONFIG_ARM_FCSE_BEST_EFFORT +struct fcse_user { + struct mm_struct *mm; + unsigned count; +}; +extern struct fcse_user fcse_pids_user[]; +int fcse_switch_mm_start_inner(struct mm_struct *next); +void fcse_switch_mm_end_inner(struct mm_struct *next); +void fcse_pid_reference(struct mm_struct *mm); + +static inline int fcse_switch_mm_start(struct mm_struct *next) +{ + if (!cache_is_vivt()) + return 0; + + return fcse_switch_mm_start_inner(next); +} + +static inline void fcse_switch_mm_end(struct mm_struct *next) +{ + if (!cache_is_vivt()) + return; + + fcse_switch_mm_end_inner(next); +} + +static inline int fcse_mm_in_cache(struct mm_struct *mm) +{ + unsigned fcse_pid = mm->context.fcse.pid >> FCSE_PID_SHIFT; + int res; + res = test_bit(FCSE_PID_MAX - fcse_pid, fcse_pids_cache_dirty) + && fcse_pids_user[fcse_pid].mm == mm; + return res; +} +#else /* CONFIG_ARM_FCSE_GUARANTEED */ +static inline int fcse_switch_mm_start(struct mm_struct *next) +{ + return 0; +} + +static inline void fcse_switch_mm_end(struct mm_struct *next) +{ + if (!cache_is_vivt()) + return; + + fcse_mark_dirty(next); + fcse_pid_set(next->context.fcse.pid); +} + +static inline int fcse_mm_in_cache(struct mm_struct *mm) +{ + unsigned fcse_pid = mm->context.fcse.pid >> FCSE_PID_SHIFT; + return test_bit(FCSE_PID_MAX - fcse_pid, fcse_pids_cache_dirty); +} +#endif /* CONFIG_ARM_FCSE_GUARANTEED */ + +#define fcse() (cache_is_vivt()) +#else /* ! CONFIG_ARM_FCSE */ +#define fcse_switch_mm_start(next) 1 +#define fcse_switch_mm_end(next) do { (void)(next); } while(0) +#define fcse_mva_to_va(mva) (mva) +#define fcse_va_to_mva(mm, x) ({ (void)(mm); (x); }) +#define fcse_mark_dirty(mm) do { (void)(mm); } while(0) +#define fcse_flush_all_start() (0) +#define fcse_flush_all_done(seq, dirty) do { (void)(seq); } while (0) +#define fcse_mm_in_cache(mm) \ + (cpumask_test_cpu(smp_processor_id(), mm_cpumask(mm))) +#define fcse_check_mmap_addr(mm, addr, len, info, flags) (addr) +#define fcse() (0) +#endif /* ! CONFIG_ARM_FCSE */ + +#ifdef CONFIG_ARM_FCSE_MESSAGES +void fcse_notify_segv(struct mm_struct *mm, + unsigned long addr, struct pt_regs *regs); +#else /* !FCSE_MESSAGES */ +#define fcse_notify_segv(mm, addr, regs) do { } while(0) +#endif /* !FCSE_MESSAGES */ + +#endif /* __ASM_ARM_FCSE_H */ diff --git a/arch/arm/include/asm/hardware/timer-sp.h b/arch/arm/include/asm/hardware/timer-sp.h index bb28af7..780ca50 100644 --- a/arch/arm/include/asm/hardware/timer-sp.h +++ b/arch/arm/include/asm/hardware/timer-sp.h @@ -1,19 +1,22 @@ struct clk; void __sp804_clocksource_and_sched_clock_init(void __iomem *, + unsigned long, const char *, struct clk *, int); void __sp804_clockevents_init(void __iomem *, unsigned int, struct clk *, const char *); -static inline void sp804_clocksource_init(void __iomem *base, const char *name) +static inline void sp804_clocksource_init(void __iomem *base, + unsigned long phys, const char *name) { - __sp804_clocksource_and_sched_clock_init(base, name, NULL, 0); + __sp804_clocksource_and_sched_clock_init(base, phys, name, NULL, 0); } static inline void sp804_clocksource_and_sched_clock_init(void __iomem *base, + unsigned long phys, const char *name) { - __sp804_clocksource_and_sched_clock_init(base, name, NULL, 1); + __sp804_clocksource_and_sched_clock_init(base, phys, name, NULL, 1); } static inline void sp804_clockevents_init(void __iomem *base, unsigned int irq, const char *name) diff --git a/arch/arm/include/asm/ipipe.h b/arch/arm/include/asm/ipipe.h new file mode 100644 index 0000000..d7f0ec3 --- /dev/null +++ b/arch/arm/include/asm/ipipe.h @@ -0,0 +1,298 @@ +/* -*- linux-c -*- + * arch/arm/include/asm/ipipe.h + * + * Copyright (C) 2002-2005 Philippe Gerum. + * Copyright (C) 2005 Stelian Pop. + * Copyright (C) 2006-2008 Gilles Chanteperdrix. + * Copyright (C) 2010 Philippe Gerum (SMP port). + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139, + * USA; either version 2 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ + +#ifndef __ARM_IPIPE_H +#define __ARM_IPIPE_H + +#include + +#ifdef CONFIG_IPIPE + +#define BROKEN_BUILTIN_RETURN_ADDRESS +#undef __BUILTIN_RETURN_ADDRESS0 +#undef __BUILTIN_RETURN_ADDRESS1 +#ifdef CONFIG_FRAME_POINTER +#define __BUILTIN_RETURN_ADDRESS0 arm_return_addr(0) +#define __BUILTIN_RETURN_ADDRESS1 arm_return_addr(1) +extern unsigned long arm_return_addr(int level); +#else +#define __BUILTIN_RETURN_ADDRESS0 ((unsigned long)__builtin_return_address(0)) +#define __BUILTIN_RETURN_ADDRESS1 (0) +#endif + +#include +#include + +#define IPIPE_CORE_RELEASE 9 + +struct ipipe_domain; +struct timekeeper; + +#define IPIPE_TSC_TYPE_NONE 0 +#define IPIPE_TSC_TYPE_FREERUNNING 1 +#define IPIPE_TSC_TYPE_DECREMENTER 2 +#define IPIPE_TSC_TYPE_FREERUNNING_COUNTDOWN 3 +#define IPIPE_TSC_TYPE_FREERUNNING_TWICE 4 +#define IPIPE_TSC_TYPE_FREERUNNING_ARCH 5 + +/* tscinfo, exported to user-space */ +struct __ipipe_tscinfo { + unsigned type; + unsigned freq; + unsigned long counter_vaddr; + union { + struct { + unsigned long counter_paddr; + unsigned long long mask; + }; + struct { + unsigned *counter; /* Hw counter physical address */ + unsigned long long mask; /* Significant bits in the hw counter. */ + unsigned long long *tsc; /* 64 bits tsc value. */ + } fr; + struct { + unsigned *counter; /* Hw counter physical address */ + unsigned long long mask; /* Significant bits in the hw counter. */ + unsigned *last_cnt; /* Counter value when updating + tsc value. */ + unsigned long long *tsc; /* 64 bits tsc value. */ + } dec; + } u; + unsigned int (*refresh_freq)(void); +}; + +struct ipipe_arch_sysinfo { + struct __ipipe_tscinfo tsc; +}; + + +/* arch specific stuff */ +extern char __ipipe_tsc_area[]; +void __ipipe_mach_get_tscinfo(struct __ipipe_tscinfo *info); + +#ifdef CONFIG_IPIPE_ARM_KUSER_TSC +unsigned long long __ipipe_tsc_get(void) __attribute__((long_call)); +void __ipipe_tsc_register(struct __ipipe_tscinfo *info); +void __ipipe_tsc_update(void); +void __ipipe_update_vsyscall(struct timekeeper *tk); +extern unsigned long __ipipe_kuser_tsc_freq; +#define __ipipe_hrclock_freq __ipipe_kuser_tsc_freq +#else /* ! generic tsc */ +unsigned long long __ipipe_mach_get_tsc(void); +#define __ipipe_tsc_get() __ipipe_mach_get_tsc() +static inline void __ipipe_update_vsyscall(struct timekeeper *tk) {} +#ifndef __ipipe_hrclock_freq +extern unsigned long __ipipe_hrtimer_freq; +#define __ipipe_hrclock_freq __ipipe_hrtimer_freq +#endif /* !__ipipe_mach_hrclock_freq */ +#endif /* ! generic tsc */ + +#ifdef CONFIG_IPIPE_DEBUG_INTERNAL +extern void (*__ipipe_mach_hrtimer_debug)(unsigned irq); +#endif /* CONFIG_IPIPE_DEBUG_INTERNAL */ + +#ifdef CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH + +#define ipipe_mm_switch_protect(flags) \ + do { \ + (void)(flags); \ + } while(0) + +#define ipipe_mm_switch_unprotect(flags) \ + do { \ + (void)(flags); \ + } while(0) + +#else /* !CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH */ + +#define ipipe_mm_switch_protect(flags) \ + flags = hard_cond_local_irq_save() + +#define ipipe_mm_switch_unprotect(flags) \ + hard_cond_local_irq_restore(flags) + +#endif /* !CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH */ + +#define ipipe_get_active_mm() (__this_cpu_read(ipipe_percpu.active_mm)) + +#define ipipe_read_tsc(t) do { t = __ipipe_tsc_get(); } while(0) +#define __ipipe_read_timebase() __ipipe_tsc_get() + +#define ipipe_tsc2ns(t) \ +({ \ + unsigned long long delta = (t)*1000; \ + do_div(delta, __ipipe_hrclock_freq / 1000000 + 1); \ + (unsigned long)delta; \ +}) +#define ipipe_tsc2us(t) \ +({ \ + unsigned long long delta = (t); \ + do_div(delta, __ipipe_hrclock_freq / 1000000 + 1); \ + (unsigned long)delta; \ +}) + +static inline const char *ipipe_clock_name(void) +{ + return "ipipe_tsc"; +} + +/* Private interface -- Internal use only */ + +#define __ipipe_enable_irq(irq) enable_irq(irq) +#define __ipipe_disable_irq(irq) disable_irq(irq) + +/* PIC muting */ +struct ipipe_mach_pic_muter { + void (*enable_irqdesc)(struct ipipe_domain *ipd, unsigned irq); + void (*disable_irqdesc)(struct ipipe_domain *ipd, unsigned irq); + void (*mute)(void); + void (*unmute)(void); +}; + +extern struct ipipe_mach_pic_muter ipipe_pic_muter; + +void ipipe_pic_muter_register(struct ipipe_mach_pic_muter *muter); + +void __ipipe_enable_irqdesc(struct ipipe_domain *ipd, unsigned irq); + +void __ipipe_disable_irqdesc(struct ipipe_domain *ipd, unsigned irq); + +static inline void ipipe_mute_pic(void) +{ + if (ipipe_pic_muter.mute) + ipipe_pic_muter.mute(); +} + +static inline void ipipe_unmute_pic(void) +{ + if (ipipe_pic_muter.unmute) + ipipe_pic_muter.unmute(); +} + +#define ipipe_notify_root_preemption() do { } while(0) + +#ifdef CONFIG_SMP +void __ipipe_early_core_setup(void); +void __ipipe_hook_critical_ipi(struct ipipe_domain *ipd); +void __ipipe_root_localtimer(unsigned int irq, void *cookie); +void __ipipe_send_vnmi(void (*fn)(void *), cpumask_t cpumask, void *arg); +void __ipipe_do_vnmi(unsigned int irq, void *cookie); +void __ipipe_grab_ipi(unsigned svc, struct pt_regs *regs); +void __ipipe_ipis_alloc(void); +void __ipipe_ipis_request(void); + +static inline void ipipe_handle_multi_ipi(int irq, struct pt_regs *regs) +{ + __ipipe_grab_ipi(irq, regs); +} + +#ifdef CONFIG_SMP_ON_UP +extern struct static_key __ipipe_smp_key; +#define ipipe_smp_p (static_key_true(&__ipipe_smp_key)) +#endif /* SMP_ON_UP */ +#else /* !CONFIG_SMP */ +#define __ipipe_early_core_setup() do { } while(0) +#define __ipipe_hook_critical_ipi(ipd) do { } while(0) +#endif /* !CONFIG_SMP */ +#ifndef __ipipe_mach_init_platform +#define __ipipe_mach_init_platform() do { } while(0) +#endif + +void __ipipe_enable_pipeline(void); + +void __ipipe_do_critical_sync(unsigned irq, void *cookie); + +void __ipipe_grab_irq(int irq, struct pt_regs *regs); + +void __ipipe_exit_irq(struct pt_regs *regs); + +static inline void ipipe_handle_multi_irq(int irq, struct pt_regs *regs) +{ + __ipipe_grab_irq(irq, regs); +} + +static inline unsigned long __ipipe_ffnz(unsigned long ul) +{ + return ffs(ul) - 1; +} + +#define __ipipe_root_tick_p(regs) (!arch_irqs_disabled_flags(regs->ARM_cpsr)) + +#ifdef CONFIG_IRQ_DOMAIN +static inline +int ipipe_handle_domain_irq(struct irq_domain *domain, + unsigned int hwirq, struct pt_regs *regs) +{ + unsigned int irq; + irq = irq_find_mapping(domain, hwirq); + ipipe_handle_multi_irq(irq, regs); + + return 0; +} +#endif /* irq domains */ + +#else /* !CONFIG_IPIPE */ + +#include +#include + +#define __ipipe_tsc_update() do { } while(0) + +#define hard_smp_processor_id() smp_processor_id() + +#define ipipe_mm_switch_protect(flags) \ + do { \ + (void) (flags); \ + } while(0) + +#define ipipe_mm_switch_unprotect(flags) \ + do { \ + (void) (flags); \ + } while(0) + +static inline void ipipe_handle_multi_irq(int irq, struct pt_regs *regs) +{ + handle_IRQ(irq, regs); +} + +#ifdef CONFIG_SMP +static inline void ipipe_handle_multi_ipi(int irq, struct pt_regs *regs) +{ + handle_IPI(irq, regs); +} +#endif /* CONFIG_SMP */ + +static inline +int ipipe_handle_domain_irq(struct irq_domain *domain, + unsigned int hwirq, struct pt_regs *regs) +{ + return handle_domain_irq(domain, hwirq, regs); +} + +struct timekeeper; +static inline void __ipipe_update_vsyscall(struct timekeeper *tk) {} + +#endif /* !CONFIG_IPIPE */ + +#endif /* !__ARM_IPIPE_H */ diff --git a/arch/arm/include/asm/ipipe_base.h b/arch/arm/include/asm/ipipe_base.h new file mode 100644 index 0000000..e1f46e9 --- /dev/null +++ b/arch/arm/include/asm/ipipe_base.h @@ -0,0 +1,177 @@ +/* -*- linux-c -*- + * arch/arm/include/asm/ipipe_base.h + * + * Copyright (C) 2007 Gilles Chanteperdrix. + * Copyright (C) 2010 Philippe Gerum (SMP port). + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139, + * USA; either version 2 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ + +#ifndef __ASM_ARM_IPIPE_BASE_H +#define __ASM_ARM_IPIPE_BASE_H + +#include /* For NR_IRQS */ + +#ifdef CONFIG_IPIPE + +#define IPIPE_NR_ROOT_IRQS 1024 + +#define IPIPE_NR_XIRQS IPIPE_NR_ROOT_IRQS + +#ifdef CONFIG_SMP + +extern unsigned __ipipe_first_ipi; + +#define IPIPE_CRITICAL_IPI __ipipe_first_ipi +#define IPIPE_HRTIMER_IPI (IPIPE_CRITICAL_IPI + 1) +#define IPIPE_RESCHEDULE_IPI (IPIPE_CRITICAL_IPI + 2) +#define IPIPE_SERVICE_VNMI (IPIPE_CRITICAL_IPI + 3) + +#define IPIPE_LAST_IPI IPIPE_SERVICE_VNMI + +#ifdef CONFIG_IPIPE_LEGACY +#define hard_smp_processor_id() \ + ({ \ + unsigned int cpunum; \ + __asm__ __volatile__ ("\n" \ + "1: mrc p15, 0, %0, c0, c0, 5\n" \ + " .pushsection \".alt.smp.init\", \"a\"\n" \ + " .long 1b\n" \ + " mov %0, #0\n" \ + " .popsection" \ + : "=r" (cpunum)); \ + cpunum &= 0xFF; \ + }) +extern u32 __cpu_reverse_map[]; +#define ipipe_processor_id() (__cpu_reverse_map[hard_smp_processor_id()]) + +#else /* !legacy */ +#define hard_smp_processor_id() raw_smp_processor_id() + +#ifdef CONFIG_SMP_ON_UP +unsigned __ipipe_processor_id(void); + +#define ipipe_processor_id() \ + ({ \ + register unsigned int cpunum __asm__ ("r0"); \ + register unsigned int r1 __asm__ ("r1"); \ + register unsigned int r2 __asm__ ("r2"); \ + register unsigned int r3 __asm__ ("r3"); \ + register unsigned int ip __asm__ ("ip"); \ + register unsigned int lr __asm__ ("lr"); \ + __asm__ __volatile__ ("\n" \ + "1: bl __ipipe_processor_id\n" \ + " .pushsection \".alt.smp.init\", \"a\"\n" \ + " .long 1b\n" \ + " mov %0, #0\n" \ + " .popsection" \ + : "=r"(cpunum), "=r"(r1), "=r"(r2), "=r"(r3), \ + "=r"(ip), "=r"(lr) \ + : /* */ : "cc"); \ + cpunum; \ + }) +#else /* !SMP_ON_UP */ +#define ipipe_processor_id() raw_smp_processor_id() +#endif /* !SMP_ON_UP */ +#endif /* !legacy */ + +#define IPIPE_ARCH_HAVE_VIRQ_IPI + +#else /* !CONFIG_SMP */ +#define ipipe_processor_id() (0) +#endif /* !CONFIG_IPIPE */ + +/* ARM traps */ +#define IPIPE_TRAP_ACCESS 0 /* Data or instruction access exception */ +#define IPIPE_TRAP_SECTION 1 /* Section fault */ +#define IPIPE_TRAP_DABT 2 /* Generic data abort */ +#define IPIPE_TRAP_UNKNOWN 3 /* Unknown exception */ +#define IPIPE_TRAP_BREAK 4 /* Instruction breakpoint */ +#define IPIPE_TRAP_FPU 5 /* Floating point exception */ +#define IPIPE_TRAP_VFP 6 /* VFP floating point exception */ +#define IPIPE_TRAP_UNDEFINSTR 7 /* Undefined instruction */ +#define IPIPE_TRAP_ALIGNMENT 8 /* Unaligned access exception */ +#define IPIPE_TRAP_MAYDAY 9 /* Internal recovery trap */ +#define IPIPE_NR_FAULTS 10 + +#ifndef __ASSEMBLY__ + +#ifdef CONFIG_SMP + +void ipipe_stall_root(void); + +unsigned long ipipe_test_and_stall_root(void); + +unsigned long ipipe_test_root(void); + +#else /* !CONFIG_SMP */ + +#include + +#if __GNUC__ >= 4 +/* Alias to ipipe_root_cpudom_var(status) */ +extern unsigned long __ipipe_root_status; +#else +extern unsigned long *const __ipipe_root_status_addr; +#define __ipipe_root_status (*__ipipe_root_status_addr) +#endif + +static inline void ipipe_stall_root(void) +{ + unsigned long flags; + + flags = hard_local_irq_save(); + __ipipe_root_status |= 1; + hard_local_irq_restore(flags); +} + +static inline unsigned ipipe_test_root(void) +{ + return __ipipe_root_status & 1; +} + +static inline unsigned ipipe_test_and_stall_root(void) +{ + unsigned long flags, res; + + flags = hard_local_irq_save(); + res = __ipipe_root_status; + __ipipe_root_status = res | 1; + hard_local_irq_restore(flags); + + return res & 1; +} + +#endif /* !CONFIG_SMP */ + +#endif /* !__ASSEMBLY__ */ + +#ifdef CONFIG_IPIPE_LEGACY +#define __IPIPE_FEATURE_PREEMPTIBLE_SWITCH 1 +#define __IPIPE_FEATURE_SYSINFO_V2 1 + +#ifdef CONFIG_VFP +#define __IPIPE_FEATURE_VFP_SAFE 1 +#endif + +#ifdef CONFIG_IPIPE_ARM_KUSER_TSC +#define __IPIPE_FEATURE_KUSER_TSC 1 +#endif +#endif /* CONFIG_IPIPE_LEGACY */ + +#endif /* CONFIG_IPIPE */ + +#endif /* __ASM_ARM_IPIPE_BASE_H */ diff --git a/arch/arm/include/asm/ipipe_hwirq.h b/arch/arm/include/asm/ipipe_hwirq.h new file mode 100644 index 0000000..bd8cda1 --- /dev/null +++ b/arch/arm/include/asm/ipipe_hwirq.h @@ -0,0 +1,269 @@ +/* -*- linux-c -*- + * arch/arm/include/asm/ipipe_hwirq.h + * + * Copyright (C) 2002-2005 Philippe Gerum. + * Copyright (C) 2005 Stelian Pop. + * Copyright (C) 2006-2008 Gilles Chanteperdrix. + * Copyright (C) 2010 Philippe Gerum (SMP port). + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139, + * USA; either version 2 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ + +#ifndef _ASM_ARM_IPIPE_HWIRQ_H +#define _ASM_ARM_IPIPE_HWIRQ_H + +#define hard_local_irq_restore_notrace(x) \ + __asm__ __volatile__( \ + "msr cpsr_c, %0 @ hard_local_irq_restore\n" \ + : \ + : "r" (x) \ + : "memory", "cc") + +static inline void hard_local_irq_disable_notrace(void) +{ +#if __LINUX_ARM_ARCH__ >= 6 + __asm__("cpsid i @ __cli" : : : "memory", "cc"); +#else /* linux arch <= 5 */ + unsigned long temp; + __asm__ __volatile__( + "mrs %0, cpsr @ hard_local_irq_disable\n" + "orr %0, %0, #128\n" + "msr cpsr_c, %0" + : "=r" (temp) + : + : "memory", "cc"); +#endif /* linux arch <= 5 */ +} + +static inline void hard_local_irq_enable_notrace(void) +{ +#if __LINUX_ARM_ARCH__ >= 6 + __asm__("cpsie i @ __sti" : : : "memory", "cc"); +#else /* linux arch <= 5 */ + unsigned long temp; + __asm__ __volatile__( + "mrs %0, cpsr @ hard_local_irq_enable\n" + "bic %0, %0, #128\n" + "msr cpsr_c, %0" + : "=r" (temp) + : + : "memory", "cc"); +#endif /* linux arch <= 5 */ +} + +static inline void hard_local_fiq_disable_notrace(void) +{ +#if __LINUX_ARM_ARCH__ >= 6 + __asm__("cpsid f @ __clf" : : : "memory", "cc"); +#else /* linux arch <= 5 */ + unsigned long temp; + __asm__ __volatile__( + "mrs %0, cpsr @ clf\n" + "orr %0, %0, #64\n" + "msr cpsr_c, %0" + : "=r" (temp) + : + : "memory", "cc"); +#endif /* linux arch <= 5 */ +} + +static inline void hard_local_fiq_enable_notrace(void) +{ +#if __LINUX_ARM_ARCH__ >= 6 + __asm__("cpsie f @ __stf" : : : "memory", "cc"); +#else /* linux arch <= 5 */ + unsigned long temp; + __asm__ __volatile__( + "mrs %0, cpsr @ stf\n" + "bic %0, %0, #64\n" + "msr cpsr_c, %0" + : "=r" (temp) + : + : "memory", "cc"); +#endif /* linux arch <= 5 */ +} + +static inline unsigned long hard_local_irq_save_notrace(void) +{ + unsigned long res; +#if __LINUX_ARM_ARCH__ >= 6 + __asm__ __volatile__( + "mrs %0, cpsr @ hard_local_irq_save\n" + "cpsid i" + : "=r" (res) : : "memory", "cc"); +#else /* linux arch <= 5 */ + unsigned long temp; + __asm__ __volatile__( + "mrs %0, cpsr @ hard_local_irq_save\n" + "orr %1, %0, #128\n" + "msr cpsr_c, %1" + : "=r" (res), "=r" (temp) + : + : "memory", "cc"); +#endif /* linux arch <= 5 */ + return res; +} + +#ifdef CONFIG_IPIPE + +#include + +static inline int arch_irqs_disabled_flags(unsigned long flags) +{ + return (int)((flags) & PSR_I_BIT); +} + +static inline unsigned long hard_local_save_flags(void) +{ + unsigned long flags; + __asm__ __volatile__( + "mrs %0, cpsr @ hard_local_save_flags" + : "=r" (flags) : : "memory", "cc"); + return flags; +} + +#define hard_irqs_disabled_flags(flags) arch_irqs_disabled_flags(flags) + +static inline int hard_irqs_disabled(void) +{ + return hard_irqs_disabled_flags(hard_local_save_flags()); +} + +#ifdef CONFIG_IPIPE_TRACE_IRQSOFF + +static inline void hard_local_irq_disable(void) +{ + if (!hard_irqs_disabled()) { + hard_local_irq_disable_notrace(); + ipipe_trace_begin(0x80000000); + } +} + +static inline void hard_local_irq_enable(void) +{ + if (hard_irqs_disabled()) { + ipipe_trace_end(0x80000000); + hard_local_irq_enable_notrace(); + } +} + +static inline unsigned long hard_local_irq_save(void) +{ + unsigned long flags; + + flags = hard_local_irq_save_notrace(); + if (!arch_irqs_disabled_flags(flags)) + ipipe_trace_begin(0x80000001); + + return flags; +} + +static inline void hard_local_irq_restore(unsigned long x) +{ + if (!arch_irqs_disabled_flags(x)) + ipipe_trace_end(0x80000001); + + hard_local_irq_restore_notrace(x); +} + +#else /* !CONFIG_IPIPE_TRACE_IRQSOFF */ + +#define hard_local_irq_disable hard_local_irq_disable_notrace +#define hard_local_irq_enable hard_local_irq_enable_notrace +#define hard_local_irq_save hard_local_irq_save_notrace +#define hard_local_irq_restore hard_local_irq_restore_notrace + +#endif /* CONFIG_IPIPE_TRACE_IRQSOFF */ + +#define arch_local_irq_disable() \ + ({ \ + ipipe_stall_root(); \ + barrier(); \ + }) + +#define arch_local_irq_enable() \ + do { \ + barrier(); \ + ipipe_unstall_root(); \ + } while (0) + +#define local_fiq_enable() hard_local_fiq_enable_notrace() + +#define local_fiq_disable() hard_local_fiq_disable_notrace() + +#define arch_local_irq_restore(flags) \ + do { \ + if (!arch_irqs_disabled_flags(flags)) \ + arch_local_irq_enable(); \ + } while (0) + +#define arch_local_irq_save() \ + ({ \ + unsigned long _flags; \ + _flags = ipipe_test_and_stall_root() << 7; \ + barrier(); \ + _flags; \ + }) + +#define arch_local_save_flags() \ + ({ \ + unsigned long _flags; \ + _flags = ipipe_test_root() << 7; \ + barrier(); \ + _flags; \ + }) + +#define arch_irqs_disabled() ipipe_test_root() +#define hard_irq_disable() hard_local_irq_disable() + +static inline unsigned long arch_mangle_irq_bits(int virt, unsigned long real) +{ + /* Merge virtual and real interrupt mask bits into a single + 32bit word. */ + return (real & ~(1L << 8)) | ((virt != 0) << 8); +} + +static inline int arch_demangle_irq_bits(unsigned long *x) +{ + int virt = (*x & (1 << 8)) != 0; + *x &= ~(1L << 8); + return virt; +} + +#else /* !CONFIG_IPIPE */ + +#define hard_local_irq_save() arch_local_irq_save() +#define hard_local_irq_restore(x) arch_local_irq_restore(x) +#define hard_local_irq_enable() arch_local_irq_enable() +#define hard_local_irq_disable() arch_local_irq_disable() +#define hard_irqs_disabled() irqs_disabled() + +#define hard_cond_local_irq_enable() do { } while(0) +#define hard_cond_local_irq_disable() do { } while(0) +#define hard_cond_local_irq_save() 0 +#define hard_cond_local_irq_restore(flags) do { (void)(flags); } while(0) + +#endif /* !CONFIG_IPIPE */ + +#if defined(CONFIG_SMP) && defined(CONFIG_IPIPE) +#define hard_smp_local_irq_save() hard_local_irq_save() +#define hard_smp_local_irq_restore(flags) hard_local_irq_restore(flags) +#else /* !CONFIG_SMP */ +#define hard_smp_local_irq_save() 0 +#define hard_smp_local_irq_restore(flags) do { (void)(flags); } while(0) +#endif /* CONFIG_SMP */ + +#endif /* _ASM_ARM_IPIPE_HWIRQ_H */ diff --git a/arch/arm/include/asm/irq.h b/arch/arm/include/asm/irq.h index 53c15de..28fc521 100644 --- a/arch/arm/include/asm/irq.h +++ b/arch/arm/include/asm/irq.h @@ -6,9 +6,14 @@ #ifndef CONFIG_SPARSE_IRQ #include #else +#if !defined(CONFIG_IPIPE) || defined(CONFIG_IRQ_DOMAIN) #define NR_IRQS NR_IRQS_LEGACY +#else +#define NR_IRQS 512 +#endif #endif + #ifndef irq_canonicalize #define irq_canonicalize(i) (i) #endif @@ -38,4 +43,3 @@ extern void set_handle_irq(void (*handle_irq)(struct pt_regs *)); #endif #endif - diff --git a/arch/arm/include/asm/irqflags.h b/arch/arm/include/asm/irqflags.h index 3b763d6..597b133 100644 --- a/arch/arm/include/asm/irqflags.h +++ b/arch/arm/include/asm/irqflags.h @@ -5,6 +5,10 @@ #include +#include + +#ifndef CONFIG_IPIPE + /* * CPU interrupt mask handling. */ @@ -161,5 +165,6 @@ static inline int arch_irqs_disabled_flags(unsigned long flags) return flags & IRQMASK_I_BIT; } +#endif /* ifndef IPIPE */ #endif /* ifdef __KERNEL__ */ #endif /* ifndef __ASM_ARM_IRQFLAGS_H */ diff --git a/arch/arm/include/asm/memory.h b/arch/arm/include/asm/memory.h index 184def0..f53755d 100644 --- a/arch/arm/include/asm/memory.h +++ b/arch/arm/include/asm/memory.h @@ -40,7 +40,12 @@ * TASK_UNMAPPED_BASE - the lower boundary of the mmap VM area */ #define TASK_SIZE (UL(CONFIG_PAGE_OFFSET) - UL(SZ_16M)) +#ifndef CONFIG_ARM_FCSE #define TASK_UNMAPPED_BASE ALIGN(TASK_SIZE / 3, SZ_16M) +#else /* CONFIG_ARM_FCSE */ +#define TASK_UNMAPPED_BASE UL(0x00800000) +#endif /* CONFIG_ARM_FCSE */ +#define FCSE_TASK_SIZE UL(0x02000000) /* * The maximum size of a 26-bit user space task. diff --git a/arch/arm/include/asm/mmu.h b/arch/arm/include/asm/mmu.h index a5b47421..980f704 100644 --- a/arch/arm/include/asm/mmu.h +++ b/arch/arm/include/asm/mmu.h @@ -9,6 +9,17 @@ typedef struct { #else int switch_pending; #endif +#ifdef CONFIG_ARM_FCSE + struct { + unsigned long pid; +#ifdef CONFIG_ARM_FCSE_BEST_EFFORT + unsigned shared_dirty_pages; + unsigned large : 1; + unsigned high_pages; + unsigned long highest_pid; +#endif /* CONFIG_ARM_FCSE_BEST_EFFORT */ + } fcse; +#endif /* CONFIG_ARM_FCSE */ unsigned int vmalloc_seq; unsigned long sigpage; #ifdef CONFIG_VDSO diff --git a/arch/arm/include/asm/mmu_context.h b/arch/arm/include/asm/mmu_context.h index 9b32f76..697ac1e 100644 --- a/arch/arm/include/asm/mmu_context.h +++ b/arch/arm/include/asm/mmu_context.h @@ -20,12 +20,14 @@ #include #include #include +#include void __check_vmalloc_seq(struct mm_struct *mm); #ifdef CONFIG_CPU_HAS_ASID -void check_and_switch_context(struct mm_struct *mm, struct task_struct *tsk); +int check_and_switch_context(struct mm_struct *mm, + struct task_struct *tsk, bool may_defer); #define init_new_context(tsk,mm) ({ atomic64_set(&mm->context.id, 0); 0; }) #ifdef CONFIG_ARM_ERRATA_798181 @@ -42,13 +44,14 @@ static inline void a15_erratum_get_cpumask(int this_cpu, struct mm_struct *mm, #ifdef CONFIG_MMU -static inline void check_and_switch_context(struct mm_struct *mm, - struct task_struct *tsk) +static inline int +check_and_switch_context(struct mm_struct *mm, + struct task_struct *tsk, bool may_defer) { if (unlikely(mm->context.vmalloc_seq != init_mm.context.vmalloc_seq)) __check_vmalloc_seq(mm); - if (irqs_disabled()) + if (may_defer && irqs_disabled()) { /* * cpu_switch_mm() needs to flush the VIVT caches. To avoid * high interrupt latencies, defer the call and continue @@ -57,9 +60,23 @@ static inline void check_and_switch_context(struct mm_struct *mm, * finish_arch_post_lock_switch() call. */ mm->context.switch_pending = 1; - else - cpu_switch_mm(mm->pgd, mm); + return -EAGAIN; + } else { + cpu_switch_mm(mm->pgd, mm, fcse_switch_mm_start(mm)); + } + + return 0; +} + +#ifdef CONFIG_IPIPE +extern void deferred_switch_mm(struct mm_struct *mm); +#else /* !I-pipe */ +static inline void deferred_switch_mm(struct mm_struct *next) +{ + cpu_switch_mm(next->pgd, next, fcse_switch_mm_start(next)); + fcse_switch_mm_end(next); } +#endif /* !I-pipe */ #define finish_arch_post_lock_switch \ finish_arch_post_lock_switch @@ -76,21 +93,45 @@ static inline void finish_arch_post_lock_switch(void) */ preempt_disable(); if (mm->context.switch_pending) { + unsigned long flags; mm->context.switch_pending = 0; - cpu_switch_mm(mm->pgd, mm); + ipipe_mm_switch_protect(flags); + deferred_switch_mm(mm); + ipipe_mm_switch_unprotect(flags); } preempt_enable_no_resched(); } } - #endif /* CONFIG_MMU */ -#define init_new_context(tsk,mm) 0 - -#endif /* CONFIG_CPU_HAS_ASID */ +static inline int +init_new_context(struct task_struct *tsk, struct mm_struct *mm) +{ +#ifdef CONFIG_ARM_FCSE_BEST_EFFORT + if (!mm->context.fcse.large) + fcse_pid_alloc(mm); + else { + /* We are normally forking a process vith a virtual address + space larger than 32 MB, so its pid should be 0. */ + FCSE_BUG_ON(mm->context.fcse.pid); + fcse_pid_reference(mm); + } + /* If we are forking, set_pte_at will restore the correct high pages + count, and shared writable pages are write-protected again. */ + mm->context.fcse.high_pages = 0; + mm->context.fcse.highest_pid = 0; + mm->context.fcse.shared_dirty_pages = 0; +#elif defined(CONFIG_ARM_FCSE_GUARANTEED) + int err = fcse_pid_alloc(mm); + if (err < 0) + return err; +#endif /* CONFIG_ARM_FCSE_GUARANTEED */ + FCSE_BUG_ON(fcse_mm_in_cache(mm)); + + return 0; +} -#define destroy_context(mm) do { } while(0) -#define activate_mm(prev,next) switch_mm(prev, next, NULL) +#endif /* !CONFIG_CPU_HAS_ASID */ /* * This is called when "tsk" is about to enter lazy TLB mode. @@ -112,12 +153,12 @@ enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk) * calling the CPU specific function when the mm hasn't * actually changed. */ -static inline void -switch_mm(struct mm_struct *prev, struct mm_struct *next, - struct task_struct *tsk) +static inline int +__do_switch_mm(struct mm_struct *prev, struct mm_struct *next, + struct task_struct *tsk, bool may_defer) { #ifdef CONFIG_MMU - unsigned int cpu = smp_processor_id(); + const unsigned int cpu = ipipe_processor_id(); /* * __sync_icache_dcache doesn't broadcast the I-cache invalidation, @@ -130,13 +171,84 @@ switch_mm(struct mm_struct *prev, struct mm_struct *next, __flush_icache_all(); if (!cpumask_test_and_set_cpu(cpu, mm_cpumask(next)) || prev != next) { - check_and_switch_context(next, tsk); - if (cache_is_vivt()) + int rc = check_and_switch_context(next, tsk, may_defer); +#ifdef CONFIG_IPIPE + if (rc < 0) { + cpumask_clear_cpu(cpu, mm_cpumask(next)); + return rc; + } +#ifdef CONFIG_ARM_FCSE + if (tsk) + set_tsk_thread_flag(tsk, TIF_SWITCHED); +#endif /* CONFIG_ARM_FCSE */ +#else /* !CONFIG_IPIPE */ + if (rc == 0) + fcse_switch_mm_end(next); +#endif /* CONFIG_IPIPE */ + if (cache_is_vivt() && prev) cpumask_clear_cpu(cpu, mm_cpumask(prev)); - } -#endif + } else + fcse_mark_dirty(next); +#endif /* CONFIG_MMU */ + return 0; +} + +#if defined(CONFIG_IPIPE) && defined(CONFIG_MMU) +extern void __switch_mm_inner(struct mm_struct *prev, struct mm_struct *next, + struct task_struct *tsk); +#else /* !I-pipe || !MMU */ +#define __switch_mm_inner(prev, next, tsk) \ + __do_switch_mm(prev, next, tsk, true) +#endif /* !I-pipe || !MMU */ + +static inline void +ipipe_switch_mm_head(struct mm_struct *prev, struct mm_struct *next, + struct task_struct *tsk) +{ + __do_switch_mm(prev, next, tsk, false); + fcse_switch_mm_end(next); +} + +static inline void +__switch_mm(struct mm_struct *prev, struct mm_struct *next, + struct task_struct *tsk) +{ + __switch_mm_inner(prev, next, tsk); +} + +static inline void +switch_mm(struct mm_struct *prev, struct mm_struct *next, + struct task_struct *tsk) +{ +#ifdef CONFIG_MMU + unsigned long flags; + ipipe_mm_switch_protect(flags); + __switch_mm(prev, next, tsk); + ipipe_mm_switch_unprotect(flags); +#endif /* CONFIG_MMU */ } #define deactivate_mm(tsk,mm) do { } while (0) +#ifndef CONFIG_ARM_FCSE_BEST_EFFORT +#define activate_mm(prev,next) __switch_mm(prev, next, NULL) +#else /* CONFIG_ARM_FCSE_BEST_EFFORT */ +#define activate_mm(prev,next) \ + ({ \ + __switch_mm(prev, next, NULL); \ + FCSE_BUG_ON(current->mm == next && next->context.switch_pending == 0 && !fcse_mm_in_cache(next)); \ + }) +#endif /* CONFIG_ARM_FCSE_BEST_EFFORT */ +static inline void destroy_context(struct mm_struct *mm) +{ +#ifdef CONFIG_ARM_FCSE +#ifdef CONFIG_ARM_FCSE_BEST_EFFORT + FCSE_BUG_ON(mm->context.fcse.shared_dirty_pages); + FCSE_BUG_ON(mm->context.fcse.high_pages); +#endif /* CONFIG_ARM_FCSE_BEST_EFFORT */ + if (mm->context.fcse.pid != FCSE_PID_INVALID) + fcse_pid_free(mm); +#endif /* CONFIG_ARM_FCSE */ +} + #endif diff --git a/arch/arm/include/asm/percpu.h b/arch/arm/include/asm/percpu.h index a89b407..dc029a2 100644 --- a/arch/arm/include/asm/percpu.h +++ b/arch/arm/include/asm/percpu.h @@ -20,7 +20,9 @@ * Same as asm-generic/percpu.h, except that we store the per cpu offset * in the TPIDRPRW. TPIDRPRW only exists on V6K and V7 */ -#if defined(CONFIG_SMP) && !defined(CONFIG_CPU_V6) +#if defined(CONFIG_SMP) && !defined(CONFIG_CPU_V6) && \ + (!defined(CONFIG_IPIPE) || \ + (!defined(CONFIG_SMP_ON_UP) && !defined(CONFIG_IPIPE_TRACE))) static inline void set_my_cpu_offset(unsigned long off) { /* Set TPIDRPRW */ @@ -43,6 +45,10 @@ static inline unsigned long __my_cpu_offset(void) } #define __my_cpu_offset __my_cpu_offset() #else +#if defined(CONFIG_SMP) && defined(CONFIG_IPIPE) +#define __my_cpu_offset (per_cpu_offset(ipipe_processor_id())) +#endif /* SMP && IPIPE */ + #define set_my_cpu_offset(x) do {} while(0) #endif /* CONFIG_SMP */ diff --git a/arch/arm/include/asm/pgtable.h b/arch/arm/include/asm/pgtable.h index f403541..aaa68d9 100644 --- a/arch/arm/include/asm/pgtable.h +++ b/arch/arm/include/asm/pgtable.h @@ -48,6 +48,8 @@ #define LIBRARY_TEXT_START 0x0c000000 #ifndef __ASSEMBLY__ +#include + extern void __pte_error(const char *file, int line, pte_t); extern void __pmd_error(const char *file, int line, pmd_t); extern void __pgd_error(const char *file, int line, pgd_t); @@ -163,6 +165,46 @@ extern pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, #define __S111 __PAGE_SHARED_EXEC #ifndef __ASSEMBLY__ +#ifdef CONFIG_ARM_FCSE_BEST_EFFORT +#define fcse_account_page_removal(mm, addr, val) do { \ + struct mm_struct *_mm = (mm); \ + unsigned long _addr = (addr); \ + unsigned long _val = (val); \ + if (pte_present(_val) && ((_val) & L_PTE_SHARED)) \ + --_mm->context.fcse.shared_dirty_pages; \ + if (pte_present(_val) && _addr < TASK_SIZE) { \ + if (_addr >= FCSE_TASK_SIZE \ + && 0 == --_mm->context.fcse.high_pages) \ + _mm->context.fcse.highest_pid = 0; \ + } \ +} while (0) + +#define fcse_account_page_addition(mm, addr, val) ({ \ + struct mm_struct *_mm = (mm); \ + unsigned long _addr = (addr); \ + unsigned long _val = (val); \ + if (pte_present(_val) && (_val & L_PTE_SHARED)) { \ + if ((_val & (PTE_CACHEABLE | L_PTE_RDONLY | L_PTE_DIRTY)) \ + != (PTE_CACHEABLE | L_PTE_DIRTY)) \ + _val &= ~L_PTE_SHARED; \ + else \ + ++_mm->context.fcse.shared_dirty_pages; \ + } \ + if (pte_present(_val) \ + && _addr < TASK_SIZE && _addr >= FCSE_TASK_SIZE) { \ + unsigned long pid = _addr / FCSE_TASK_SIZE; \ + ++_mm->context.fcse.high_pages; \ + BUG_ON(mm->context.fcse.large == 0); \ + if (pid > mm->context.fcse.highest_pid) \ + mm->context.fcse.highest_pid = pid; \ + } \ + _val; \ +}) +#else /* CONFIG_ARM_FCSE_GUARANTEED || !CONFIG_ARM_FCSE */ +#define fcse_account_page_removal(mm, addr, val) do { } while (0) +#define fcse_account_page_addition(mm, addr, val) (val) +#endif /* CONFIG_ARM_FCSE_GUARANTEED || !CONFIG_ARM_FCSE */ + /* * ZERO_PAGE is a global shared page that is always zero: used * for zero-mapped memory areas etc.. @@ -176,10 +218,14 @@ extern pgd_t swapper_pg_dir[PTRS_PER_PGD]; /* to find an entry in a page-table-directory */ #define pgd_index(addr) ((addr) >> PGDIR_SHIFT) -#define pgd_offset(mm, addr) ((mm)->pgd + pgd_index(addr)) +#define pgd_offset(mm, addr) \ + ({ \ + struct mm_struct *_mm = (mm); \ + (_mm->pgd + pgd_index(fcse_va_to_mva(_mm, (addr)))); \ + }) /* to find an entry in a kernel page-table-directory */ -#define pgd_offset_k(addr) pgd_offset(&init_mm, addr) +#define pgd_offset_k(addr) (init_mm.pgd + pgd_index(addr)) #define pmd_none(pmd) (!pmd_val(pmd)) #define pmd_present(pmd) (pmd_val(pmd)) @@ -212,7 +258,13 @@ static inline pte_t *pmd_page_vaddr(pmd_t pmd) #define pte_page(pte) pfn_to_page(pte_pfn(pte)) #define mk_pte(page,prot) pfn_pte(page_to_pfn(page), prot) -#define pte_clear(mm,addr,ptep) set_pte_ext(ptep, __pte(0), 0) +#define pte_clear(mm,addr,ptep) \ + do { \ + struct mm_struct *_mm = (mm); \ + if (_mm) \ + fcse_account_page_removal(_mm, addr, pte_val(*ptep)); \ + set_pte_ext(ptep, __pte(0), 0); \ + } while (0) #define pte_isset(pte, val) ((u32)(val) == (val) ? pte_val(pte) & (val) \ : !!(pte_val(pte) & (val))) @@ -239,10 +291,16 @@ extern void __sync_icache_dcache(pte_t pteval); #endif static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, - pte_t *ptep, pte_t pteval) + pte_t *ptep, pte_t pteval) { unsigned long ext = 0; + if (mm) { + fcse_account_page_removal(mm, addr, pte_val(*ptep)); + pte_val(pteval) = + fcse_account_page_addition(mm, addr, pte_val(pteval)); + } + if (addr < TASK_SIZE && pte_valid_user(pteval)) { if (!pte_special(pteval)) __sync_icache_dcache(pteval); diff --git a/arch/arm/include/asm/proc-fns.h b/arch/arm/include/asm/proc-fns.h index 5324c11..8d6f390 100644 --- a/arch/arm/include/asm/proc-fns.h +++ b/arch/arm/include/asm/proc-fns.h @@ -60,7 +60,12 @@ extern struct processor { /* * Set the page table */ +#ifndef CONFIG_ARM_FCSE_BEST_EFFORT void (*switch_mm)(phys_addr_t pgd_phys, struct mm_struct *mm); +#else /* !CONFIG_ARM_FCSE_BEST_EFFORT */ + void (*switch_mm)(phys_addr_t pgd_phys, + struct mm_struct *mm, unsigned flush); +#endif /* !CONFIG_ARM_FCSE_BEST_EFFORT */ /* * Set a possibly extended PTE. Non-extended PTEs should * ignore 'ext'. @@ -82,7 +87,12 @@ extern void cpu_proc_init(void); extern void cpu_proc_fin(void); extern int cpu_do_idle(void); extern void cpu_dcache_clean_area(void *, int); +#ifndef CONFIG_ARM_FCSE_BEST_EFFORT extern void cpu_do_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm); +#else /* !CONFIG_ARM_FCSE_BEST_EFFORT */ +extern void cpu_do_switch_mm(phys_addr_t pgd_phys, + struct mm_struct *mm, unsigned flush); +#endif /* !CONFIG_ARM_FCSE_BEST_EFFORT */ #ifdef CONFIG_ARM_LPAE extern void cpu_set_pte_ext(pte_t *ptep, pte_t pte); #else @@ -113,7 +123,16 @@ extern void cpu_resume(void); #ifdef CONFIG_MMU -#define cpu_switch_mm(pgd,mm) cpu_do_switch_mm(virt_to_phys(pgd),mm) +#ifndef CONFIG_ARM_FCSE_BEST_EFFORT +#define cpu_switch_mm(pgd,mm,fcse_switch) \ + ({ \ + (void)(fcse_switch); \ + cpu_do_switch_mm(virt_to_phys(pgd), (mm)); \ + }) +#else /* CONFIG_ARM_FCSE_BEST_EFFORT */ +#define cpu_switch_mm(pgd,mm,fcse_switch) \ + cpu_do_switch_mm(virt_to_phys(pgd), (mm), (fcse_switch)) +#endif /* CONFIG_ARM_FCSE_BEST_EFFORT */ #ifdef CONFIG_ARM_LPAE @@ -142,7 +161,7 @@ extern void cpu_resume(void); #define cpu_get_pgd() \ ({ \ unsigned long pg; \ - __asm__("mrc p15, 0, %0, c2, c0, 0" \ + __asm__ __volatile__ ("mrc p15, 0, %0, c2, c0, 0" \ : "=r" (pg) : : "cc"); \ pg &= ~0x3fff; \ (pgd_t *)phys_to_virt(pg); \ diff --git a/arch/arm/include/asm/processor.h b/arch/arm/include/asm/processor.h index 8a1e8e9..a1c3f22 100644 --- a/arch/arm/include/asm/processor.h +++ b/arch/arm/include/asm/processor.h @@ -25,9 +25,14 @@ #include #ifdef __KERNEL__ +#ifndef CONFIG_ARM_FCSE #define STACK_TOP ((current->personality & ADDR_LIMIT_32BIT) ? \ TASK_SIZE : TASK_SIZE_26) #define STACK_TOP_MAX TASK_SIZE +#else /* CONFIG_ARM_FCSE */ +#define STACK_TOP FCSE_TASK_SIZE +#define STACK_TOP_MAX FCSE_TASK_SIZE +#endif /* CONFIG_ARM_FCSE */ #endif struct debug_info { diff --git a/arch/arm/include/asm/resource.h b/arch/arm/include/asm/resource.h new file mode 100644 index 0000000..6579eec --- /dev/null +++ b/arch/arm/include/asm/resource.h @@ -0,0 +1,16 @@ +#ifndef _ARM_RESOURCE_H +#define _ARM_RESOURCE_H + +/* + * When FCSE is enabled, reduce the default stack size to 1MB, and maximum + * to 16MB, the address space is only 32MB. + */ +#ifdef CONFIG_ARM_FCSE +#define _STK_LIM (1024*1024) + +#define _STK_LIM_MAX (16*1024*1024) +#endif /* CONFIG_ARM_FCSE */ + +#include + +#endif diff --git a/arch/arm/include/asm/setup.h b/arch/arm/include/asm/setup.h index e0adb9f..5876f4b 100644 --- a/arch/arm/include/asm/setup.h +++ b/arch/arm/include/asm/setup.h @@ -25,4 +25,10 @@ extern int arm_add_memory(u64 start, u64 size); extern void early_print(const char *str, ...); extern void dump_machine_table(void); +#ifdef CONFIG_IPIPE +void smp_build_cpu_revmap(void); +#else +static inline void smp_build_cpu_revmap(void) { } +#endif + #endif diff --git a/arch/arm/include/asm/switch_to.h b/arch/arm/include/asm/switch_to.h index c99e259..317b6a4 100644 --- a/arch/arm/include/asm/switch_to.h +++ b/arch/arm/include/asm/switch_to.h @@ -20,9 +20,18 @@ */ extern struct task_struct *__switch_to(struct task_struct *, struct thread_info *, struct thread_info *); +#ifdef CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH #define switch_to(prev,next,last) \ do { \ + hard_cond_local_irq_disable(); \ last = __switch_to(prev,task_thread_info(prev), task_thread_info(next)); \ + hard_cond_local_irq_enable(); \ } while (0) +#else /* !CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH */ +#define switch_to(prev,next,last) \ +do { \ + last = __switch_to(prev,task_thread_info(prev), task_thread_info(next)); \ +} while (0) +#endif /* !CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH */ #endif /* __ASM_ARM_SWITCH_TO_H */ diff --git a/arch/arm/include/asm/thread_info.h b/arch/arm/include/asm/thread_info.h index bd32ede..5523aca 100644 --- a/arch/arm/include/asm/thread_info.h +++ b/arch/arm/include/asm/thread_info.h @@ -26,6 +26,7 @@ struct task_struct; #include #include +#include typedef unsigned long mm_segment_t; @@ -66,6 +67,10 @@ struct thread_info { #ifdef CONFIG_ARM_THUMBEE unsigned long thumbee_state; /* ThumbEE Handler Base register */ #endif +#ifdef CONFIG_IPIPE + unsigned long ipipe_flags; +#endif + struct ipipe_threadinfo ipipe_data; }; #define INIT_THREAD_INFO(tsk) \ @@ -156,6 +161,10 @@ extern int vfp_restore_user_hwstate(struct user_vfp __user *, #define TIF_USING_IWMMXT 17 #define TIF_MEMDIE 18 /* is terminating due to OOM killer */ #define TIF_RESTORE_SIGMASK 20 +#define TIF_SWITCH_MM 22 /* deferred switch_mm */ + +#define TIF_SWITCHED 23 /* FCSE */ +#define TIF_MMSWITCH_INT 24 /* MMU context switch preempted */ #define _TIF_SIGPENDING (1 << TIF_SIGPENDING) #define _TIF_NEED_RESCHED (1 << TIF_NEED_RESCHED) @@ -167,6 +176,9 @@ extern int vfp_restore_user_hwstate(struct user_vfp __user *, #define _TIF_SECCOMP (1 << TIF_SECCOMP) #define _TIF_USING_IWMMXT (1 << TIF_USING_IWMMXT) +#define _TIF_SWITCHED (1 << TIF_SWITCHED) +#define _TIF_MMSWITCH_INT (1 << TIF_MMSWITCH_INT) + /* Checks for any syscall work in entry-common.S */ #define _TIF_SYSCALL_WORK (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | \ _TIF_SYSCALL_TRACEPOINT | _TIF_SECCOMP) @@ -177,5 +189,14 @@ extern int vfp_restore_user_hwstate(struct user_vfp __user *, #define _TIF_WORK_MASK (_TIF_NEED_RESCHED | _TIF_SIGPENDING | \ _TIF_NOTIFY_RESUME | _TIF_UPROBE) +/* ti->ipipe_flags */ +#define TIP_MAYDAY 0 /* MAYDAY call is pending */ +#define TIP_NOTIFY 1 /* Notify head domain about kernel events */ +#define TIP_HEAD 2 /* Runs in head domain */ + +#define _TIP_MAYDAY (1 << TIP_MAYDAY) +#define _TIP_NOTIFY (1 << TIP_NOTIFY) +#define _TIP_HEAD (1 << TIP_HEAD) + #endif /* __KERNEL__ */ #endif /* __ASM_ARM_THREAD_INFO_H */ diff --git a/arch/arm/include/asm/tlbflush.h b/arch/arm/include/asm/tlbflush.h index def9e57..90333ee 100644 --- a/arch/arm/include/asm/tlbflush.h +++ b/arch/arm/include/asm/tlbflush.h @@ -202,6 +202,7 @@ #ifndef __ASSEMBLY__ #include +#include struct cpu_tlb_fns { void (*flush_user_range)(unsigned long, unsigned long, struct vm_area_struct *); @@ -421,7 +422,8 @@ __local_flush_tlb_page(struct vm_area_struct *vma, unsigned long uaddr) const int zero = 0; const unsigned int __tlb_flag = __cpu_tlb_flags; - uaddr = (uaddr & PAGE_MASK) | ASID(vma->vm_mm); + uaddr = (fcse_va_to_mva(vma->vm_mm, uaddr) & PAGE_MASK) + | ASID(vma->vm_mm); if (possible_tlb_flags & (TLB_V4_U_PAGE|TLB_V4_D_PAGE|TLB_V4_I_PAGE|TLB_V4_I_FULL) && cpumask_test_cpu(smp_processor_id(), mm_cpumask(vma->vm_mm))) { @@ -600,7 +602,15 @@ static inline void clean_pmd_entry(void *pmd) /* * Convert calls to our calling convention. */ -#define local_flush_tlb_range(vma,start,end) __cpu_flush_user_tlb_range(start,end,vma) +#define local_flush_tlb_range(vma, start, end) \ + ({ \ + struct mm_struct *_mm = (vma)->vm_mm; \ + unsigned long _start, _end; \ + _start = fcse_va_to_mva(_mm, start); \ + _end = fcse_va_to_mva(_mm, end); \ + __cpu_flush_user_tlb_range(_start, _end, vma); \ + }) + #define local_flush_tlb_kernel_range(s,e) __cpu_flush_kern_tlb_range(s,e) #ifndef CONFIG_SMP diff --git a/arch/arm/include/asm/uaccess.h b/arch/arm/include/asm/uaccess.h index 74b17d0..6e04650 100644 --- a/arch/arm/include/asm/uaccess.h +++ b/arch/arm/include/asm/uaccess.h @@ -13,6 +13,7 @@ */ #include #include +#include #include #include #include @@ -198,7 +199,7 @@ extern int __get_user_64t_4(void *); #define get_user(x, p) \ ({ \ - might_fault(); \ + __ipipe_uaccess_might_fault(); \ __get_user_check(x, p); \ }) @@ -244,7 +245,7 @@ extern int __put_user_8(void *, unsigned long long); #define put_user(x, p) \ ({ \ - might_fault(); \ + __ipipe_uaccess_might_fault(); \ __put_user_check(x, p); \ }) @@ -301,7 +302,7 @@ do { \ unsigned long __gu_addr = (unsigned long)(ptr); \ unsigned long __gu_val; \ __chk_user_ptr(ptr); \ - might_fault(); \ + __ipipe_uaccess_might_fault(); \ switch (sizeof(*(ptr))) { \ case 1: __get_user_asm_byte(__gu_val, __gu_addr, err); break; \ case 2: __get_user_asm_half(__gu_val, __gu_addr, err); break; \ @@ -383,7 +384,7 @@ do { \ unsigned long __pu_addr = (unsigned long)(ptr); \ __typeof__(*(ptr)) __pu_val = (x); \ __chk_user_ptr(ptr); \ - might_fault(); \ + __ipipe_uaccess_might_fault(); \ switch (sizeof(*(ptr))) { \ case 1: __put_user_asm_byte(__pu_val, __pu_addr, err); break; \ case 2: __put_user_asm_half(__pu_val, __pu_addr, err); break; \ @@ -472,7 +473,6 @@ do { \ : "r" (x), "i" (-EFAULT) \ : "cc") - #ifdef CONFIG_MMU extern unsigned long __must_check __copy_from_user(void *to, const void __user *from, unsigned long n); extern unsigned long __must_check __copy_to_user(void __user *to, const void *from, unsigned long n); diff --git a/arch/arm/include/uapi/asm/mman.h b/arch/arm/include/uapi/asm/mman.h index 41f99c5..a1a116f 100644 --- a/arch/arm/include/uapi/asm/mman.h +++ b/arch/arm/include/uapi/asm/mman.h @@ -1,3 +1,5 @@ +#define MAP_BRK 0x80000 + #include #define arch_mmap_check(addr, len, flags) \ diff --git a/arch/arm/include/uapi/asm/unistd.h b/arch/arm/include/uapi/asm/unistd.h index 0c3f5a0..02e8330 100644 --- a/arch/arm/include/uapi/asm/unistd.h +++ b/arch/arm/include/uapi/asm/unistd.h @@ -426,6 +426,12 @@ #define __ARM_NR_set_tls (__ARM_NR_BASE+5) /* + * This SWI is IPIPE private, for dispatching syscalls to the head + * domain. + */ +#define __ARM_NR_ipipe (__ARM_NR_BASE+66) + +/* * The following syscalls are obsolete and no longer available for EABI. */ #if !defined(__KERNEL__) diff --git a/arch/arm/kernel/Makefile b/arch/arm/kernel/Makefile index 752725d..6a8f267 100644 --- a/arch/arm/kernel/Makefile +++ b/arch/arm/kernel/Makefile @@ -83,6 +83,9 @@ endif head-y := head$(MMUEXT).o obj-$(CONFIG_DEBUG_LL) += debug.o obj-$(CONFIG_EARLY_PRINTK) += early_printk.o +obj-$(CONFIG_RAW_PRINTK) += raw_printk.o +obj-$(CONFIG_IPIPE) += ipipe.o +obj-$(CONFIG_IPIPE_ARM_KUSER_TSC) += ipipe_tsc.o ipipe_tsc_asm.o obj-$(CONFIG_ARM_VIRT_EXT) += hyp-stub.o ifeq ($(CONFIG_ARM_PSCI),y) diff --git a/arch/arm/kernel/asm-offsets.c b/arch/arm/kernel/asm-offsets.c index 871b826..d42b2c7 100644 --- a/arch/arm/kernel/asm-offsets.c +++ b/arch/arm/kernel/asm-offsets.c @@ -64,6 +64,9 @@ int main(void) #endif BLANK(); DEFINE(TI_FLAGS, offsetof(struct thread_info, flags)); +#ifdef CONFIG_IPIPE + DEFINE(TI_IPIPE, offsetof(struct thread_info, ipipe_flags)); +#endif DEFINE(TI_PREEMPT, offsetof(struct thread_info, preempt_count)); DEFINE(TI_ADDR_LIMIT, offsetof(struct thread_info, addr_limit)); DEFINE(TI_TASK, offsetof(struct thread_info, task)); @@ -73,6 +76,9 @@ int main(void) DEFINE(TI_USED_CP, offsetof(struct thread_info, used_cp)); DEFINE(TI_TP_VALUE, offsetof(struct thread_info, tp_value)); DEFINE(TI_FPSTATE, offsetof(struct thread_info, fpstate)); +#ifdef CONFIG_IPIPE + DEFINE(TI_IPIPE, offsetof(struct thread_info, ipipe_flags)); +#endif #ifdef CONFIG_VFP DEFINE(TI_VFPSTATE, offsetof(struct thread_info, vfpstate)); #ifdef CONFIG_SMP diff --git a/arch/arm/kernel/devtree.c b/arch/arm/kernel/devtree.c index 11c54de..b2518c4 100644 --- a/arch/arm/kernel/devtree.c +++ b/arch/arm/kernel/devtree.c @@ -174,6 +174,8 @@ void __init arm_dt_init_cpu_maps(void) cpu_logical_map(i) = tmp_map[i]; pr_debug("cpu logical map 0x%x\n", cpu_logical_map(i)); } + + smp_build_cpu_revmap(); } bool arch_match_cpu_phys_id(int cpu, u64 phys_id) diff --git a/arch/arm/kernel/entry-armv.S b/arch/arm/kernel/entry-armv.S index 570306c..a8868ff 100644 --- a/arch/arm/kernel/entry-armv.S +++ b/arch/arm/kernel/entry-armv.S @@ -4,6 +4,7 @@ * Copyright (C) 1996,1997,1998 Russell King. * ARM700 fix by Matthew Godbolt (linux-user@willothewisp.demon.co.uk) * nommu support by Hyok S. Choi (hyok.choi@samsung.com) + * Copyright (C) 2005 Stelian Pop. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -46,6 +47,10 @@ arch_irq_handler_default #endif 9997: +#ifdef CONFIG_IPIPE + bl __ipipe_check_root_interruptible + cmp r0, #1 +#endif /* CONFIG_IPIPE */ .endm .macro pabt_helper @@ -187,6 +192,14 @@ ENDPROC(__und_invalid) #ifdef CONFIG_TRACE_IRQFLAGS bl trace_hardirqs_off #endif +#ifdef CONFIG_IPIPE_TRACE_IRQSOFF + mov r0, #1 /* IPIPE_TRACE_BEGIN */ + mov r3, #0x90000000 + ldr r2, [sp, #S_PC] + mov r1, pc + bl ipipe_trace_asm + ldmia r7, {r2 - r6} +#endif /* CONFIG_IPIPE_TRACE_IRQSOFF */ .endif .endm @@ -204,6 +217,9 @@ ENDPROC(__dabt_svc) __irq_svc: svc_entry irq_handler +#ifdef CONFIG_IPIPE + bne __ipipe_fast_svc_irq_exit +#endif #ifdef CONFIG_PREEMPT get_thread_info tsk @@ -215,6 +231,9 @@ __irq_svc: blne svc_preempt #endif +#ifdef CONFIG_IPIPE +__ipipe_fast_svc_irq_exit: +#endif svc_exit r5, irq = 1 @ return from exception UNWIND(.fnend ) ENDPROC(__irq_svc) @@ -224,12 +243,16 @@ ENDPROC(__irq_svc) #ifdef CONFIG_PREEMPT svc_preempt: mov r8, lr +#ifndef CONFIG_IPIPE 1: bl preempt_schedule_irq @ irq en/disable is done inside +#else /* CONFIG_IPIPE */ +1: bl __ipipe_preempt_schedule_irq @ irq en/disable is done inside +#endif /* CONFIG_IPIPE */ ldr r0, [tsk, #TI_FLAGS] @ get new tasks TI_FLAGS tst r0, #_TIF_NEED_RESCHED reteq r8 @ go again b 1b -#endif +#endif /* CONFIG_PREEMPT */ __und_fault: @ Correct the PC such that it is pointing at the instruction @@ -254,6 +277,14 @@ __und_svc: #else svc_entry #endif + +#ifdef CONFIG_IPIPE + mov r0, #7 @ r0 = IPIPE_TRAP_UNDEFINSTR + mov r1, sp @ r1 = ®s + bl __ipipe_notify_trap @ branch to trap handler + cmp r0, #0 + bne __und_svc_finish +#endif /* CONFIG_IPIPE */ @ @ call emulation code, which returns using r9 if it has emulated @ the instruction, or the more conventional lr if we are to treat @@ -372,6 +403,15 @@ ENDPROC(__fiq_abt) sub sp, sp, #S_FRAME_SIZE ARM( stmib sp, {r1 - r12} ) THUMB( stmia sp, {r0 - r12} ) +#ifdef CONFIG_IPIPE_TRACE_IRQSOFF + mov r4, r0 + mov r0, #1 /* IPIPE_TRACE_BEGIN */ + mov r3, #0x90000000 + ldr r2, [r4, #4] /* lr_ */ + mov r1, pc + bl ipipe_trace_asm + mov r0, r4 +#endif /* CONFIG_IPIPE_TRACE_IRQSOFF */ ATRAP( mrc p15, 0, r7, c1, c0, 0) ATRAP( ldr r8, .LCcralign) @@ -446,6 +486,10 @@ __irq_usr: usr_entry kuser_cmpxchg_check irq_handler +#ifdef CONFIG_IPIPE + THUMB( it ne) + bne __ipipe_ret_to_user_irqs_disabled +#endif /* CONFIG_IPIPE */ get_thread_info tsk mov why, #0 b ret_to_user_from_irq @@ -458,6 +502,14 @@ ENDPROC(__irq_usr) __und_usr: usr_entry +#ifdef CONFIG_IPIPE + mov r0, #7 @ r0 = IPIPE_TRAP_UNDEFINSTR + mov r1, sp @ r1 = ®s + bl __ipipe_notify_trap @ branch to trap handler + cmp r0, #0 + bne ret_from_exception +#endif /* CONFIG_IPIPE */ + mov r2, r4 mov r3, r5 @@ -734,7 +786,24 @@ __pabt_usr: ENTRY(ret_from_exception) UNWIND(.fnstart ) UNWIND(.cantunwind ) +#ifdef CONFIG_IPIPE + disable_irq +#ifdef CONFIG_IPIPE_LEGACY + bl __ipipe_check_root + cmp r0, #1 + THUMB( it ne) + bne __ipipe_ret_to_user_irqs_disabled @ Fast exit path over non-root domains + get_thread_info tsk +#else /* !CONFIG_IPIPE_LEGACY */ get_thread_info tsk + ldr r0, [tsk, #TI_IPIPE] + tst r0, #_TIP_HEAD + THUMB( it ne) + bne __ipipe_ret_to_user_irqs_disabled @ Fast exit path over non-root domains +#endif /* !CONFIG_IPIPE_LEGACY */ +#else /* !CONFIG_IPIPE */ + get_thread_info tsk +#endif /* !CONFIG_IPIPE */ mov why, #0 b ret_to_user UNWIND(.fnend ) @@ -783,7 +852,11 @@ ENTRY(__switch_to) add r4, r2, #TI_CPU_SAVE ldr r0, =thread_notify_head mov r1, #THREAD_NOTIFY_SWITCH +#ifndef CONFIG_IPIPE bl atomic_notifier_call_chain +#else /* CONFIG_IPIPE */ + bl __ipipe_switch_to_notifier_call_chain +#endif /* CONFIG_IPIPE */ #if defined(CONFIG_CC_STACKPROTECTOR) && !defined(CONFIG_SMP) str r7, [r8] #endif @@ -818,6 +891,50 @@ ENDPROC(__switch_to) #endif .endm +#ifdef CONFIG_IPIPE +/* + I-pipe tsc area, here we store data shared with user-space for + tsc-emulation. If CONFIG_IPIPE_ARM_KUSER_TSC is enabled + __ipipe_kuser_get_tsc will be overwritten with the real TSC + emulation code. +*/ + .globl __ipipe_tsc_area + .equ __ipipe_tsc_area, CONFIG_VECTORS_BASE + 0x1000 + __ipipe_tsc_area_start - __kuser_helper_end + +#ifdef CONFIG_IPIPE_ARM_KUSER_TSC + .globl __ipipe_tsc_addr + .equ __ipipe_tsc_addr, CONFIG_VECTORS_BASE + 0x1000 + .LCcntr_addr - __kuser_helper_end + + .globl __ipipe_tsc_get + .equ __ipipe_tsc_get, CONFIG_VECTORS_BASE + 0x1000 + __ipipe_kuser_get_tsc - __kuser_helper_end +#endif + + .align 5 + .globl __ipipe_tsc_area_start +__ipipe_tsc_area_start: + .rep 3 + .word 0 + .endr + +#ifdef CONFIG_IPIPE_ARM_KUSER_TSC + .rep 4 + .word 0 + .endr +.LCcntr_addr: + .word 0 + + .align 5 +__ipipe_kuser_get_tsc: + nop + mov r0, #0 + mov r1, #0 + usr_ret lr + .rep 20 + .word 0 + .endr +#endif +#endif + .macro kuser_pad, sym, size .if (. - \sym) & 3 .rept 4 - (. - \sym) & 3 diff --git a/arch/arm/kernel/entry-common.S b/arch/arm/kernel/entry-common.S index 4e7f40c..e8f16ad 100644 --- a/arch/arm/kernel/entry-common.S +++ b/arch/arm/kernel/entry-common.S @@ -2,6 +2,7 @@ * linux/arch/arm/kernel/entry-common.S * * Copyright (C) 2000 Russell King + * Copyright (C) 2005 Stelian Pop. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2 as @@ -45,6 +46,42 @@ ret_fast_syscall: ct_user_enter restore_user_regs fast = 1, offset = S_OFF + +#ifdef CONFIG_IPIPE + +.macro ipipe_oabi_save_sysnr, tmp +#ifdef CONFIG_OABI_COMPAT + ldr \tmp, [sp, #S_R7 + S_OFF] + stmdb sp!, {\tmp} + ldr \tmp, =sys_oabi_call_table + cmp \tmp, tbl + moveq \tmp, scno + addeq \tmp, #__NR_SYSCALL_BASE + streq \tmp, [sp, #S_R7 + S_OFF + 4] @ head domain expects sycall number in r7 +#elif !defined(CONFIG_AEABI) + ldr \tmp, [sp, #S_R7 + S_OFF] + stmdb sp!, {\tmp} + mov \tmp, scno + add \tmp, #__NR_SYSCALL_BASE + str \tmp, [sp, #S_R7 + S_OFF + 4] +#endif +.endm + +.macro ipipe_oabi_restore_sysnr, tmp +#if defined(CONFIG_OABI_COMPAT) || !defined(CONFIG_AEABI) + ldmia sp!, {\tmp} + str \tmp, [sp, #S_R7 + S_OFF] +#endif +.endm + +__ipipe_ret_to_user: + disable_irq @ disable interrupts +ENTRY(__ipipe_ret_to_user_irqs_disabled) + slow_restore_user_regs +ENDPROC(__ipipe_ret_to_user_irqs_disabled) + +#endif /* CONFIG_IPIPE */ + UNWIND(.fnend ) /* @@ -74,12 +111,7 @@ ENTRY(ret_to_user_from_irq) bne work_pending no_work_pending: asm_trace_hardirqs_on - - /* perform architecture specific actions before user return */ - arch_ret_to_user r1, lr - ct_user_enter save = 0 - - restore_user_regs fast = 0, offset = 0 + slow_restore_user_regs ENDPROC(ret_to_user_from_irq) ENDPROC(ret_to_user) @@ -87,6 +119,7 @@ ENDPROC(ret_to_user) * This is how we return from a fork. */ ENTRY(ret_from_fork) + enable_irq_cond bl schedule_tail cmp r5, #0 movne r0, r4 @@ -133,6 +166,16 @@ ENTRY(vector_swi) str r0, [sp, #S_OLD_R0] @ Save OLD_R0 #endif zero_fp +#ifdef CONFIG_IPIPE_TRACE_IRQSOFF + mov r4, lr + mov r0, #1 /* IPIPE_TRACE_BEGIN */ + mov r3, #0x90000000 + sub r2, lr, #4 /* calling PC */ + mov r1, pc + bl ipipe_trace_asm + mov lr, r4 + ldm sp, {r0 - r4} +#endif /* CONFIG_IPIPE_TRACE_IRQSOFF */ alignment_trap r10, ip, __cr_alignment enable_irq ct_user_exit @@ -191,6 +234,57 @@ ENTRY(vector_swi) #endif local_restart: +#ifdef CONFIG_IPIPE + ldr r10, [tsk, #TI_IPIPE] + ldr r0, =(__ARM_NR_ipipe - __NR_SYSCALL_BASE) +#ifndef CONFIG_IPIPE_LEGACY + cmp scno, r0 + bne slow_path + tst r10, #_TIP_HEAD + beq slow_path + mov r0, sp + ipipe_oabi_save_sysnr r10 @ caution: affects sp + bl ipipe_fastcall_hook @ __IPIPE_SYSCALL_E is assumed + ipipe_oabi_restore_sysnr r10 + cmp r0, #0 + blt no_fastcall + get_thread_info tsk + ldr r10, [tsk, #TI_IPIPE] + tst r10, #_TIP_HEAD + bne fastcall_exit_check @ check for MAYDAY + bl __ipipe_root_sync + b ret_slow_syscall +fastcall_exit_check: + tst r10, #_TIP_MAYDAY + beq __ipipe_ret_to_user + mov r0, sp + bl __ipipe_call_mayday + b __ipipe_ret_to_user +no_fastcall: + get_thread_info tsk + ldr r0, =(__ARM_NR_ipipe - __NR_SYSCALL_BASE) + ldr r10, [tsk, #TI_IPIPE] +slow_path: +#endif /* !CONFIG_IPIPE_LEGACY */ + tst r10, #_TIP_NOTIFY + bne pipeline_syscall + cmp scno, r0 + bne root_syscall +pipeline_syscall: + mov r0, sp + ipipe_oabi_save_sysnr r10 @ caution: affects sp + bl __ipipe_notify_syscall + ipipe_oabi_restore_sysnr r10 + get_thread_info tsk + ldr r10, [tsk, #TI_IPIPE] + tst r10, #_TIP_HEAD + bne __ipipe_ret_to_user + cmp r0, #0 + bgt ret_slow_syscall +root_syscall: + ldmia sp, { r0 - r3 } +#endif /* CONFIG_IPIPE */ + ldr r10, [tsk, #TI_FLAGS] @ check for syscall tracing stmdb sp!, {r4, r5} @ push fifth and sixth args @@ -386,3 +480,28 @@ ENTRY(sys_oabi_call_table) #endif + +#if defined(CONFIG_FRAME_POINTER) && (CONFIG_IPIPE_TRACE) + + .text + .align 0 + .type arm_return_addr %function + .global arm_return_addr + +arm_return_addr: + mov ip, r0 + mov r0, fp +3: + cmp r0, #0 + beq 1f @ frame list hit end, bail + cmp ip, #0 + beq 2f @ reached desired frame + ldr r0, [r0, #-12] @ else continue, get next fp + sub ip, ip, #1 + b 3b +2: + ldr r0, [r0, #-4] @ get target return address +1: + mov pc, lr + +#endif diff --git a/arch/arm/kernel/entry-header.S b/arch/arm/kernel/entry-header.S index 1a0045a..df0dc3b 100644 --- a/arch/arm/kernel/entry-header.S +++ b/arch/arm/kernel/entry-header.S @@ -23,7 +23,7 @@ @ #define S_OFF 8 -/* +/* * The SWI code relies on the fact that R0 is at the bottom of the stack * (due to slow/fast restore user regs). */ @@ -200,6 +200,9 @@ .macro svc_exit, rpsr, irq = 0 .if \irq != 0 @ IRQs already off +#ifdef CONFIG_IPIPE_DEBUG_INTERNAL + bl __ipipe_bugon_irqs_enabled +#endif #ifdef CONFIG_TRACE_IRQFLAGS @ The parent context IRQs must have been enabled to get here in @ the first place, so there's no point checking the PSR I bit. @@ -215,6 +218,14 @@ blne trace_hardirqs_off #endif .endif + +#ifdef CONFIG_IPIPE_TRACE_IRQSOFF + mov r0, #2 /* IPIPE_TRACE_END */ + mov r3, #0x90000000 + ldr r2, [sp, #S_PC] + mov r1, pc + bl ipipe_trace_asm +#endif /* CONFIG_IPIPE_TRACE_IRQSOFF */ msr spsr_cxsf, \rpsr #if defined(CONFIG_CPU_V6) || defined(CONFIG_CPU_32v6K) @ We must avoid clrex due to Cortex-A15 erratum #830321 @@ -253,6 +264,22 @@ .endm .macro restore_user_regs, fast = 0, offset = 0 +#ifdef CONFIG_IPIPE_DEBUG_INTERNAL + bl __ipipe_bugon_irqs_enabled +#endif +#ifdef CONFIG_IPIPE_TRACE_IRQSOFF + .if \fast + mov r4, r0 + .endif + mov r0, #2 /* IPIPE_TRACE_END */ + mov r3, #0x90000000 + ldr r2, [sp, #\offset + S_PC] + mov r1, pc + bl ipipe_trace_asm + .if \fast + mov r0, r4 + .endif +#endif /* CONFIG_IPIPE_TRACE_IRQSOFF */ mov r2, sp ldr r1, [r2, #\offset + S_PSR] @ get calling cpsr ldr lr, [r2, #\offset + S_PC]! @ get pc @@ -276,6 +303,9 @@ .macro svc_exit, rpsr, irq = 0 .if \irq != 0 @ IRQs already off +#ifdef CONFIG_IPIPE_DEBUG_INTERNAL + bl __ipipe_bugon_irqs_enabled +#endif #ifdef CONFIG_TRACE_IRQFLAGS @ The parent context IRQs must have been enabled to get here in @ the first place, so there's no point checking the PSR I bit. @@ -291,6 +321,14 @@ blne trace_hardirqs_off #endif .endif + +#ifdef CONFIG_IPIPE_TRACE_IRQSOFF + mov r0, #2 /* IPIPE_TRACE_END */ + mov r3, #0x90000000 + ldr r2, [sp, #S_PC] + mov r1, pc + bl ipipe_trace_asm +#endif /* CONFIG_IPIPE_TRACE_IRQSOFF */ ldr lr, [sp, #S_SP] @ top of the stack ldrd r0, r1, [sp, #S_LR] @ calling lr and pc @@ -336,6 +374,22 @@ .endm #else /* ifdef CONFIG_CPU_V7M */ .macro restore_user_regs, fast = 0, offset = 0 +#ifdef CONFIG_IPIPE_DEBUG_INTERNAL + bl __ipipe_bugon_irqs_enabled +#endif +#ifdef CONFIG_IPIPE_TRACE_IRQSOFF + .if \fast + mov r4, r0 + .endif + mov r0, #2 /* IPIPE_TRACE_END */ + mov r3, #0x90000000 + ldr r2, [sp, #\offset + S_PC] + mov r1, pc + bl ipipe_trace_asm + .if \fast + mov r0, r4 + .endif +#endif /* CONFIG_IPIPE_TRACE_IRQSOFF */ mov r2, sp load_user_sp_lr r2, r3, \offset + S_SP @ calling sp, lr ldr r1, [sp, #\offset + S_PSR] @ get calling cpsr @@ -385,6 +439,13 @@ #endif .endm + .macro slow_restore_user_regs + /* perform architecture specific actions before user return */ + arch_ret_to_user r1, lr + ct_user_enter save = 0 + restore_user_regs fast = 0, offset = 0 + .endm + /* * These are the registers used in the syscall handler, and allow us to * have in theory up to 7 arguments to a function - r0 to r6. diff --git a/arch/arm/kernel/hibernate.c b/arch/arm/kernel/hibernate.c index a71501f..2570808 100644 --- a/arch/arm/kernel/hibernate.c +++ b/arch/arm/kernel/hibernate.c @@ -83,7 +83,7 @@ static void notrace arch_restore_image(void *unused) { struct pbe *pbe; - cpu_switch_mm(idmap_pgd, &init_mm); + cpu_switch_mm(idmap_pgd, &init_mm, 1); for (pbe = restore_pblist; pbe; pbe = pbe->next) copy_page(pbe->orig_address, pbe->address); diff --git a/arch/arm/kernel/ipipe.c b/arch/arm/kernel/ipipe.c new file mode 100644 index 0000000..ba209ff --- /dev/null +++ b/arch/arm/kernel/ipipe.c @@ -0,0 +1,526 @@ +/* -*- linux-c -*- + * linux/arch/arm/kernel/ipipe.c + * + * Copyright (C) 2002-2005 Philippe Gerum. + * Copyright (C) 2004 Wolfgang Grandegger (Adeos/arm port over 2.4). + * Copyright (C) 2005 Heikki Lindholm (PowerPC 970 fixes). + * Copyright (C) 2005 Stelian Pop. + * Copyright (C) 2006-2008 Gilles Chanteperdrix. + * Copyright (C) 2010 Philippe Gerum (SMP port). + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139, + * USA; either version 2 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * + * Architecture-dependent I-PIPE support for ARM. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static void __ipipe_do_IRQ(unsigned irq, void *cookie); + +#ifdef CONFIG_IPIPE_DEBUG_INTERNAL +void (*__ipipe_mach_hrtimer_debug)(unsigned irq); +#endif + +#ifdef CONFIG_SMP + +struct __ipipe_vnmidata { + void (*fn)(void *); + void *arg; + cpumask_t cpumask; +}; + +static struct __ipipe_vnmislot { + ipipe_spinlock_t lock; + struct __ipipe_vnmidata *data; + ipipe_rwlock_t data_lock; +} __ipipe_vnmi __cacheline_aligned_in_smp = { + .lock = IPIPE_SPIN_LOCK_UNLOCKED, + .data = NULL, + .data_lock = IPIPE_RW_LOCK_UNLOCKED, +}; + +void __ipipe_early_core_setup(void) +{ + __ipipe_mach_init_platform(); +} + +void ipipe_stall_root(void) +{ + unsigned long flags; + + ipipe_root_only(); + flags = hard_smp_local_irq_save(); + __set_bit(IPIPE_STALL_FLAG, &__ipipe_root_status); + hard_smp_local_irq_restore(flags); +} +EXPORT_SYMBOL_GPL(ipipe_stall_root); + +unsigned long ipipe_test_and_stall_root(void) +{ + unsigned long flags; + int x; + + ipipe_root_only(); + flags = hard_smp_local_irq_save(); + x = __test_and_set_bit(IPIPE_STALL_FLAG, &__ipipe_root_status); + hard_smp_local_irq_restore(flags); + + return x; +} +EXPORT_SYMBOL_GPL(ipipe_test_and_stall_root); + +unsigned long ipipe_test_root(void) +{ + unsigned long flags; + int x; + + flags = hard_smp_local_irq_save(); + x = test_bit(IPIPE_STALL_FLAG, &__ipipe_root_status); + hard_smp_local_irq_restore(flags); + + return x; +} +EXPORT_SYMBOL_GPL(ipipe_test_root); + +void __ipipe_do_vnmi(unsigned int irq, void *cookie) +{ + int cpu = ipipe_processor_id(); + struct __ipipe_vnmidata *data; + + read_lock(&__ipipe_vnmi.data_lock); + + data = __ipipe_vnmi.data; + if (likely(data && cpumask_test_cpu(cpu, &data->cpumask))) { + data->fn(data->arg); + cpumask_clear_cpu(cpu, &data->cpumask); + } + + read_unlock(&__ipipe_vnmi.data_lock); +} + +static inline void +hook_internal_ipi(struct ipipe_domain *ipd, int virq, + void (*handler)(unsigned int irq, void *cookie)) +{ + ipd->irqs[virq].ackfn = NULL; + ipd->irqs[virq].handler = handler; + ipd->irqs[virq].cookie = NULL; + /* Immediately handle in the current domain but *never* pass */ + ipd->irqs[virq].control = IPIPE_HANDLE_MASK|IPIPE_STICKY_MASK; +} + +void __ipipe_hook_critical_ipi(struct ipipe_domain *ipd) +{ + __ipipe_ipis_alloc(); + hook_internal_ipi(ipd, IPIPE_CRITICAL_IPI, __ipipe_do_critical_sync); + hook_internal_ipi(ipd, IPIPE_SERVICE_VNMI, __ipipe_do_vnmi); +} + +void ipipe_set_irq_affinity(unsigned int irq, cpumask_t cpumask) +{ + if (ipipe_virtual_irq_p(irq) || + irq_get_chip(irq)->irq_set_affinity == NULL) + return; + + cpumask_and(&cpumask, &cpumask, cpu_online_mask); + if (WARN_ON_ONCE(cpumask_empty(&cpumask))) + return; + + irq_get_chip(irq)->irq_set_affinity(irq_get_irq_data(irq), &cpumask, true); +} +EXPORT_SYMBOL_GPL(ipipe_set_irq_affinity); + +void __ipipe_send_vnmi(void (*fn)(void *), cpumask_t cpumask, void *arg) +{ + struct __ipipe_vnmidata data; + unsigned long flags; + int cpu; + + data.fn = fn; + data.arg = arg; + data.cpumask = cpumask; + + while (!spin_trylock_irqsave(&__ipipe_vnmi.lock, flags)) { + if (hard_irqs_disabled()) + __ipipe_do_vnmi(IPIPE_SERVICE_VNMI, NULL); + cpu_relax(); + } + + cpu = ipipe_processor_id(); + cpumask_clear_cpu(cpu, &data.cpumask); + if (cpumask_empty(&data.cpumask)) { + spin_unlock_irqrestore(&__ipipe_vnmi.lock, flags); + return; + } + + write_lock(&__ipipe_vnmi.data_lock); + __ipipe_vnmi.data = &data; + write_unlock(&__ipipe_vnmi.data_lock); + + ipipe_send_ipi(IPIPE_SERVICE_VNMI, data.cpumask); + while (!cpumask_empty(&data.cpumask)) + cpu_relax(); + + write_lock(&__ipipe_vnmi.data_lock); + __ipipe_vnmi.data = NULL; + write_unlock(&__ipipe_vnmi.data_lock); + + spin_unlock_irqrestore(&__ipipe_vnmi.lock, flags); +} +EXPORT_SYMBOL_GPL(__ipipe_send_vnmi); +#endif /* CONFIG_SMP */ + +#ifdef CONFIG_SMP_ON_UP +struct static_key __ipipe_smp_key = STATIC_KEY_INIT_TRUE; +EXPORT_SYMBOL_GPL(__ipipe_smp_key); + +unsigned notrace __ipipe_processor_id(void) +{ + return raw_smp_processor_id(); +} +EXPORT_SYMBOL_GPL(__ipipe_processor_id); + +static int ipipe_disable_smp(void) +{ + if (num_online_cpus() == 1) { + unsigned long flags; + + printk("I-pipe: disabling SMP code\n"); + + flags = hard_local_irq_save(); + static_key_slow_dec(&__ipipe_smp_key); + hard_local_irq_restore(flags); + } + return 0; +} +arch_initcall(ipipe_disable_smp); + +extern unsigned int smp_on_up; +EXPORT_SYMBOL_GPL(smp_on_up); +#endif /* SMP_ON_UP */ + +int ipipe_get_sysinfo(struct ipipe_sysinfo *info) +{ + info->sys_nr_cpus = num_online_cpus(); + info->sys_cpu_freq = __ipipe_hrclock_freq; + info->sys_hrtimer_irq = per_cpu(ipipe_percpu.hrtimer_irq, 0); + info->sys_hrtimer_freq = __ipipe_hrtimer_freq; + info->sys_hrclock_freq = __ipipe_hrclock_freq; + __ipipe_mach_get_tscinfo(&info->arch.tsc); + + return 0; +} +EXPORT_SYMBOL_GPL(ipipe_get_sysinfo); + +struct ipipe_mach_pic_muter ipipe_pic_muter; +EXPORT_SYMBOL_GPL(ipipe_pic_muter); + +void ipipe_pic_muter_register(struct ipipe_mach_pic_muter *muter) +{ + ipipe_pic_muter = *muter; +} + +void __ipipe_enable_irqdesc(struct ipipe_domain *ipd, unsigned irq) +{ + /* With sparse IRQs, some irqs may not have a descriptor */ + if (irq_to_desc(irq) == NULL) + return; + + if (ipipe_pic_muter.enable_irqdesc) + ipipe_pic_muter.enable_irqdesc(ipd, irq); +} +EXPORT_SYMBOL_GPL(__ipipe_enable_irqdesc); + +void __ipipe_disable_irqdesc(struct ipipe_domain *ipd, unsigned irq) +{ + if (ipipe_pic_muter.disable_irqdesc) + ipipe_pic_muter.disable_irqdesc(ipd, irq); +} +EXPORT_SYMBOL_GPL(__ipipe_disable_irqdesc); + +/* + * __ipipe_enable_pipeline() -- We are running on the boot CPU, hw + * interrupts are off, and secondary CPUs are still lost in space. + */ +void __ipipe_enable_pipeline(void) +{ + unsigned long flags; + unsigned int irq; + +#ifdef CONFIG_CPU_ARM926T + /* + * We do not want "wfi" to be called in arm926ejs based + * processor, as this causes Linux to disable the I-cache + * when idle. + */ + extern void cpu_arm926_proc_init(void); + if (likely(cpu_proc_init == &cpu_arm926_proc_init)) { + printk("I-pipe: ARM926EJ-S detected, disabling wfi instruction" + " in idle loop\n"); + cpu_idle_poll_ctrl(true); + } +#endif + flags = ipipe_critical_enter(NULL); + + /* virtualize all interrupts from the root domain. */ + for (irq = 0; irq < IPIPE_NR_ROOT_IRQS; irq++) + ipipe_request_irq(ipipe_root_domain, + irq, + (ipipe_irq_handler_t)__ipipe_do_IRQ, + NULL, NULL); + +#ifdef CONFIG_SMP + __ipipe_ipis_request(); +#endif /* CONFIG_SMP */ + + ipipe_critical_exit(flags); +} + +#ifdef CONFIG_IPIPE_DEBUG_INTERNAL +unsigned asmlinkage __ipipe_bugon_irqs_enabled(unsigned x) +{ + BUG_ON(!hard_irqs_disabled()); + return x; /* Preserve r0 */ +} +#endif + +asmlinkage int __ipipe_check_root_interruptible(void) +{ + return __ipipe_root_p && !irqs_disabled(); +} + +__kprobes int +__ipipe_switch_to_notifier_call_chain(struct atomic_notifier_head *nh, + unsigned long val, void *v) +{ + unsigned long flags; + int ret; + + local_irq_save(flags); + ret = atomic_notifier_call_chain(nh, val, v); + __ipipe_restore_root_nosync(flags); + + return ret; +} + +void __ipipe_exit_irq(struct pt_regs *regs) +{ + /* + * Testing for user_regs() eliminates foreign stack contexts, + * including from legacy domains which did not set the foreign + * stack bit (foreign stacks are always kernel-based). + */ + if (user_mode(regs) && + ipipe_test_thread_flag(TIP_MAYDAY)) { + /* + * MAYDAY is never raised under normal circumstances, + * so prefer test then maybe clear over + * test_and_clear. + */ + ipipe_clear_thread_flag(TIP_MAYDAY); + __ipipe_notify_trap(IPIPE_TRAP_MAYDAY, regs); + } +} + +/* hw irqs off */ +asmlinkage void __exception __ipipe_grab_irq(int irq, struct pt_regs *regs) +{ + struct ipipe_percpu_data *p = __ipipe_raw_cpu_ptr(&ipipe_percpu); + + ipipe_trace_irq_entry(irq); + + if (p->hrtimer_irq == -1) + goto copy_regs; + + if (irq == p->hrtimer_irq) { + /* + * Given our deferred dispatching model for regular IRQs, we + * only record CPU regs for the last timer interrupt, so that + * the timer handler charges CPU times properly. It is assumed + * that other interrupt handlers don't actually care for such + * information. + */ +#ifdef CONFIG_IPIPE_DEBUG_INTERNAL + if (__ipipe_mach_hrtimer_debug) + __ipipe_mach_hrtimer_debug(irq); +#endif /* CONFIG_IPIPE_DEBUG_INTERNAL */ + copy_regs: + p->tick_regs.ARM_cpsr = + (p->curr == &p->root + ? regs->ARM_cpsr + : regs->ARM_cpsr | PSR_I_BIT); + p->tick_regs.ARM_pc = regs->ARM_pc; + } + + __ipipe_dispatch_irq(irq, 0); + + ipipe_trace_irq_exit(irq); + + __ipipe_exit_irq(regs); +} + +static void __ipipe_do_IRQ(unsigned irq, void *cookie) +{ + handle_IRQ(irq, raw_cpu_ptr(&ipipe_percpu.tick_regs)); +} + +#ifdef CONFIG_MMU +void __switch_mm_inner(struct mm_struct *prev, struct mm_struct *next, + struct task_struct *tsk) +{ + struct mm_struct ** const active_mm = + raw_cpu_ptr(&ipipe_percpu.active_mm); +#ifdef CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH + struct thread_info *const tip = current_thread_info(); + prev = *active_mm; + clear_bit(TIF_MMSWITCH_INT, &tip->flags); + barrier(); + *active_mm = NULL; + barrier(); + for (;;) { + unsigned long flags; +#endif /* CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH */ + + int rc __maybe_unused = __do_switch_mm(prev, next, tsk, true); + +#ifdef CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH + /* + * Reading thread_info flags and setting active_mm + * must be done atomically. + */ + flags = hard_local_irq_save(); + if (__test_and_clear_bit(TIF_MMSWITCH_INT, &tip->flags) == 0) { + if (rc < 0) + *active_mm = prev; + else { + *active_mm = next; + fcse_switch_mm_end(next); + } + hard_local_irq_restore(flags); + return; + } + hard_local_irq_restore(flags); + + if (rc < 0) + /* + * We were interrupted by head domain, which + * may have changed the mm context, mm context + * is now unknown, but will be switched in + * deferred_switch_mm + */ + return; + + prev = NULL; + } +#else + if (rc < 0) + *active_mm = prev; + else { + *active_mm = next; + fcse_switch_mm_end(next); + } +#endif /* !IPIPE_WANT_PREEMPTIBLE_SWITCH */ +} + +#ifdef finish_arch_post_lock_switch +void deferred_switch_mm(struct mm_struct *next) +{ + struct mm_struct ** const active_mm = + raw_cpu_ptr(&ipipe_percpu.active_mm); + struct mm_struct *prev = *active_mm; +#ifdef CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH + struct thread_info *const tip = current_thread_info(); + clear_bit(TIF_MMSWITCH_INT, &tip->flags); + barrier(); + *active_mm = NULL; + barrier(); + for (;;) { + unsigned long flags; +#endif /* CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH */ + + __do_switch_mm(prev, next, NULL, false); + +#ifdef CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH + /* + * Reading thread_info flags and setting active_mm + * must be done atomically. + */ + flags = hard_local_irq_save(); + if (__test_and_clear_bit(TIF_MMSWITCH_INT, &tip->flags) == 0) { + *active_mm = next; + fcse_switch_mm_end(next); + hard_local_irq_restore(flags); + return; + } + hard_local_irq_restore(flags); + prev = NULL; + } +#else + *active_mm = next; + fcse_switch_mm_end(next); +#endif /* CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH */ +} +#endif +#endif /* CONFIG_MMU */ + +EXPORT_SYMBOL_GPL(do_munmap); +EXPORT_SYMBOL_GPL(show_stack); +EXPORT_SYMBOL_GPL(init_mm); +#ifndef MULTI_CPU +EXPORT_SYMBOL_GPL(cpu_do_switch_mm); +#endif +EXPORT_SYMBOL_GPL(__check_vmalloc_seq); +#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK) +EXPORT_SYMBOL_GPL(tasklist_lock); +#endif /* CONFIG_SMP || CONFIG_DEBUG_SPINLOCK */ + +#ifndef CONFIG_SPARSE_IRQ +EXPORT_SYMBOL_GPL(irq_desc); +#endif + +#ifdef CONFIG_CPU_HAS_ASID +EXPORT_SYMBOL_GPL(check_and_switch_context); +#endif /* CONFIG_CPU_HAS_ASID */ + +#if defined(CONFIG_SMP) && defined(CONFIG_IPIPE_LEGACY) +EXPORT_SYMBOL_GPL(__cpu_logical_map); +#endif /* CONFIG_IPIPE */ + +EXPORT_SYMBOL_GPL(cpu_architecture); diff --git a/arch/arm/kernel/ipipe_tsc.c b/arch/arm/kernel/ipipe_tsc.c new file mode 100644 index 0000000..4ac6fe8 --- /dev/null +++ b/arch/arm/kernel/ipipe_tsc.c @@ -0,0 +1,270 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +typedef unsigned long long __ipipe_tsc_t(void); + +extern __ipipe_tsc_t __ipipe_freerunning_64, + __ipipe_freerunning_32, + __ipipe_freerunning_countdown_32, + __ipipe_freerunning_16, + __ipipe_freerunning_countdown_16, + __ipipe_decrementer_16, + __ipipe_freerunning_twice_16, + __ipipe_freerunning_arch; +extern unsigned long __ipipe_tsc_addr; + +static struct __ipipe_tscinfo tsc_info; + +static struct clocksource clksrc = { + .name = "ipipe_tsc", + .rating = 0x7fffffff, + .read = (typeof(clksrc.read))__ipipe_tsc_get, + .mask = CLOCKSOURCE_MASK(64), + .flags = CLOCK_SOURCE_IS_CONTINUOUS, +}; + +struct ipipe_tsc_value_t { + unsigned long long last_tsc; + unsigned last_cnt; +}; + +unsigned long __ipipe_kuser_tsc_freq; + +struct ipipe_tsc_value_t *ipipe_tsc_value; +static struct timer_list ipipe_tsc_update_timer; + +static void __ipipe_tsc_update_fn(unsigned long cookie) +{ + __ipipe_tsc_update(); + ipipe_tsc_update_timer.expires += cookie; + add_timer(&ipipe_tsc_update_timer); +} + +void __init __ipipe_tsc_register(struct __ipipe_tscinfo *info) +{ + struct ipipe_tsc_value_t *vector_tsc_value; + unsigned long long wrap_ms; + unsigned long *tsc_addr; + __ipipe_tsc_t *implem; + unsigned long flags; + int registered; + char *tsc_area; + +#if !defined(CONFIG_CPU_USE_DOMAINS) + extern char __ipipe_tsc_area_start[], __kuser_helper_end[]; + + tsc_area = (char *)vectors_page + 0x1000 + + (__ipipe_tsc_area_start - __kuser_helper_end); + tsc_addr = (unsigned long *) + (tsc_area + ((char *)&__ipipe_tsc_addr - __ipipe_tsc_area)); +#else + tsc_area = __ipipe_tsc_area; + tsc_addr = &__ipipe_tsc_addr; +#endif + registered = ipipe_tsc_value != NULL; + + if (registered && info->freq < tsc_info.freq) + return; + + ipipe_tsc_value = (struct ipipe_tsc_value_t *)tsc_area; + vector_tsc_value = (struct ipipe_tsc_value_t *)__ipipe_tsc_area; + + switch(info->type) { + case IPIPE_TSC_TYPE_FREERUNNING: + switch(info->u.mask) { + case 0xffff: + implem = &__ipipe_freerunning_16; + break; + case 0xffffffff: + implem = &__ipipe_freerunning_32; + break; + case 0xffffffffffffffffULL: + implem = &__ipipe_freerunning_64; + break; + default: + goto unimplemented; + } + break; + + case IPIPE_TSC_TYPE_DECREMENTER: + if (info->u.mask != 0xffff) + goto unimplemented; + implem = &__ipipe_decrementer_16; + break; + + case IPIPE_TSC_TYPE_FREERUNNING_COUNTDOWN: + switch(info->u.mask) { + case 0xffff: + implem = &__ipipe_freerunning_countdown_16; + break; + case 0xffffffff: + implem = &__ipipe_freerunning_countdown_32; + break; + default: + goto unimplemented; + } + break; + + case IPIPE_TSC_TYPE_FREERUNNING_TWICE: + if (info->u.mask != 0xffff) + goto unimplemented; + implem = &__ipipe_freerunning_twice_16; + break; + + case IPIPE_TSC_TYPE_FREERUNNING_ARCH: + implem = &__ipipe_freerunning_arch; + break; + + default: + unimplemented: + printk("I-pipe: Unimplemented tsc configuration, " + "type: %d, mask: 0x%08Lx\n", info->type, info->u.mask); + BUG(); + } + + tsc_info = *info; + *tsc_addr = tsc_info.counter_vaddr; + if (tsc_info.type == IPIPE_TSC_TYPE_DECREMENTER) { + tsc_info.u.dec.last_cnt = &vector_tsc_value->last_cnt; + tsc_info.u.dec.tsc = &vector_tsc_value->last_tsc; + } else + tsc_info.u.fr.tsc = &vector_tsc_value->last_tsc; + + flags = hard_local_irq_save(); + ipipe_tsc_value->last_tsc = 0; + memcpy(tsc_area + 0x20, implem, 0x60); + flush_icache_range((unsigned long)(tsc_area), + (unsigned long)(tsc_area + 0x80)); + hard_local_irq_restore(flags); + + __ipipe_kuser_tsc_freq = tsc_info.freq; + + wrap_ms = info->u.mask; + do_div(wrap_ms, tsc_info.freq / 1000); + + printk(KERN_INFO "I-pipe, %u.%03u MHz clocksource, wrap in %Lu ms\n", + tsc_info.freq / 1000000, (tsc_info.freq % 1000000) / 1000, + wrap_ms); + + if (!registered) { + init_timer(&ipipe_tsc_update_timer); + clocksource_register_hz(&clksrc, tsc_info.freq); + } else + __clocksource_update_freq_hz(&clksrc, tsc_info.freq); + + wrap_ms *= HZ / 2; + do_div(wrap_ms, 1000); + if (wrap_ms > 0x7fffffff) + wrap_ms = 0x7fffffff; + ipipe_tsc_update_timer.data = wrap_ms; + ipipe_tsc_update_timer.function = __ipipe_tsc_update_fn; + mod_timer(&ipipe_tsc_update_timer, + jiffies + ipipe_tsc_update_timer.data); + + __ipipe_tracer_hrclock_initialized(); +} + +void __ipipe_mach_get_tscinfo(struct __ipipe_tscinfo *info) +{ + *info = tsc_info; +} + +void __ipipe_tsc_update(void) +{ + if (tsc_info.type == IPIPE_TSC_TYPE_DECREMENTER) { + unsigned cnt = *(unsigned *)tsc_info.counter_vaddr; + int offset = ipipe_tsc_value->last_cnt - cnt; + if (offset < 0) + offset += tsc_info.u.dec.mask + 1; + ipipe_tsc_value->last_tsc += offset; + ipipe_tsc_value->last_cnt = cnt; + return; + } + + /* Update last_tsc, in order to remain compatible with legacy + user-space 32 bits free-running counter implementation */ + ipipe_tsc_value->last_tsc = __ipipe_tsc_get() - 1; +} +EXPORT_SYMBOL(__ipipe_tsc_get); + +void __ipipe_update_vsyscall(struct timekeeper *tk) +{ + if (tk->tkr_mono.clock == &clksrc) + ipipe_update_hostrt(tk); +} + +#if !IS_ENABLED(CONFIG_VDSO) +void update_vsyscall(struct timekeeper *tk) +{ + __ipipe_update_vsyscall(tk); +} + +void update_vsyscall_tz(void) +{ +} +#endif + +#ifdef CONFIG_CPU_FREQ + +static __init void update_timer_freq(void *data) +{ + unsigned int hrclock_freq = *(unsigned int *)data; + + __ipipe_timer_refresh_freq(hrclock_freq); +} + +static __init int cpufreq_transition_handler(struct notifier_block *nb, + unsigned long state, void *data) +{ + struct cpufreq_freqs *freqs = data; + unsigned int freq; + + if (state == CPUFREQ_POSTCHANGE && tsc_info.refresh_freq) { + freq = tsc_info.refresh_freq(); + if (freqs->cpu == 0) { + int oldrate; + tsc_info.freq = freq; + __ipipe_tsc_register(&tsc_info); + __ipipe_report_clockfreq_update(freq); + /* force timekeeper to recalculate the clocksource */ + oldrate = clksrc.rating; + clocksource_change_rating(&clksrc, 0); + clocksource_change_rating(&clksrc, oldrate); + } + smp_call_function_single(freqs->cpu, update_timer_freq, + &freq, 1); + } + + return NOTIFY_OK; +} + +static struct notifier_block __initdata cpufreq_nb = { + .notifier_call = cpufreq_transition_handler, +}; + +static __init int register_cpufreq_notifier(void) +{ + cpufreq_register_notifier(&cpufreq_nb, + CPUFREQ_TRANSITION_NOTIFIER); + return 0; +} +core_initcall(register_cpufreq_notifier); + +static __init int unregister_cpufreq_notifier(void) +{ + cpufreq_unregister_notifier(&cpufreq_nb, + CPUFREQ_TRANSITION_NOTIFIER); + return 0; +} +late_initcall(unregister_cpufreq_notifier); + +#endif /* CONFIG_CPUFREQ */ diff --git a/arch/arm/kernel/ipipe_tsc_asm.S b/arch/arm/kernel/ipipe_tsc_asm.S new file mode 100644 index 0000000..3f0999d --- /dev/null +++ b/arch/arm/kernel/ipipe_tsc_asm.S @@ -0,0 +1,298 @@ +#include +#include +#include + + .macro usr_ret, reg +#ifdef CONFIG_ARM_THUMB + bx \reg +#else + mov pc, \reg +#endif + .endm + + .macro usr_reteq, reg +#ifdef CONFIG_ARM_THUMB + bxeq \reg +#else + moveq pc, \reg +#endif + .endm + + .macro myldrd, rd1, rd2, rtmp, label +#if __LINUX_ARM_ARCH__ < 5 + adr \rtmp, \label + ldm \rtmp, { \rd1, \rd2 } +#else + ldrd \rd1, \label +#endif + .endm + +/* + We use the same mechanism as Linux user helpers to store + variables and functions related to TSC emulation, so that they + can also be used in user-space. + + The function ipipe_tsc_register will copy the proper + implemntation to the vectors page. We repeat the data area so + that the PC relative operations are computed correctly. +*/ + + .section .init.text, "ax", %progbits + THUMB( .arm ) + + .align 5 + .rep 7 + .word 0 + .endr +.LCfr64_cntr_addr: + .word 0 + + .align 5 + .globl __ipipe_freerunning_64 +__ipipe_freerunning_64: + ldr r0, .LCfr64_cntr_addr +/* User-space entry-point: r0 is the hardware counter virtual address */ + mov r2, r0 +#ifndef CONFIG_CPU_BIG_ENDIAN +/* Little endian */ + ldr r1, [r2, #4] +1: ldr r0, [r2] + ldr r3, [r2, #4] + cmp r3, r1 + usr_reteq lr + mov r1, r3 + b 1b +#else /* Big endian */ + ldr r0, [r2] +1: ldr r1, [r2, #4] + ldr r3, [r2] + cmp r3, r0 + usr_reteq lr + mov r0, r3 + b 1b +#endif /* Big endian */ + + .align 5 +.LCfr32_last_tsc: + .rep 7 + .word 0 + .endr +.LCfr32_cntr_addr: + .word 0 + + .align 5 + .globl __ipipe_freerunning_32 +__ipipe_freerunning_32: + ldr r0, .LCfr32_cntr_addr +/* User-space entry-point: r0 is the hardware counter virtual address */ + myldrd r2, r3, r1, .LCfr32_last_tsc +#ifndef CONFIG_CPU_BIG_ENDIAN +/* Little endian */ + ldr r0, [r0] + cmp r2, r0 + adc r1, r3, #0 +#else /* Big endian */ + ldr r1, [r0] + cmp r3, r1 + adc r0, r2, #0 +#endif /* Big endian */ + usr_ret lr + + .align 5 +.LCfrcd32_last_tsc: + .rep 7 + .word 0 + .endr +.LCfrcd32_cntr_addr: + .word 0 + + .align 5 + .globl __ipipe_freerunning_countdown_32 +__ipipe_freerunning_countdown_32: + ldr r0, .LCfrcd32_cntr_addr +/* User-space entry-point: r0 is the hardware counter virtual address */ + myldrd r2, r3, r1, .LCfrcd32_last_tsc +#ifndef CONFIG_CPU_BIG_ENDIAN +/* Little endian */ + ldr r0, [r0] + mvn r0, r0 + cmp r2, r0 + adc r1, r3, #0 +#else /* Big endian */ + ldr r1, [r0] + mvn r1, r1 + cmp r3, r1 + adc r0, r2, #0 +#endif /* Big endian */ + usr_ret lr + + .align 5 +.LCfr16_last_tsc: + .rep 7 + .word 0 + .endr +.LCfr16_cntr_addr: + .word 0 + + .align 5 + .globl __ipipe_freerunning_16 +__ipipe_freerunning_16: + ldr r0, .LCfr16_cntr_addr +/* User-space entry-point: r0 is the hardware counter virtual address */ +1: myldrd r2, r3, r1, .LCfr16_last_tsc + ldrh ip, [r0] +#ifndef CONFIG_CPU_BIG_ENDIAN +/* Little endian */ + ldr r1, .LCfr16_last_tsc + cmp r1, r2 + mov r1, r2, lsr #16 + bne 1b + orr r0, ip, r1, lsl #16 + cmp r2, r0 + addhis r0, r0, #0x10000 + adc r1, r3, #0 +#else /* Big endian */ + ldr r1, .LCfr16_last_tsc + 4 + cmp r1, r3 + mov r1, r3, lsr #16 + bne 1b + orr r1, ip, r1, lsl #16 + cmp r3, r1 + addhis r1, r1, #0x10000 + adc r0, r2, #0 +#endif /* Big endian */ + usr_ret lr + + .align 5 +.LCfrcd16_last_tsc: + .rep 7 + .word 0 + .endr +.LCfrcd16_cntr_addr: + .word 0 + + .align 5 + .globl __ipipe_freerunning_countdown_16 +__ipipe_freerunning_countdown_16: + ldr r0, .LCfrcd16_cntr_addr +/* User-space entry-point: r0 is the hardware counter virtual address */ +1: myldrd r2, r3, r1, .LCfrcd16_last_tsc + ldrh ip, [r0] +#ifndef CONFIG_CPU_BIG_ENDIAN +/* Little endian */ + ldr r1, .LCfrcd16_last_tsc + rsb ip, ip, #0x10000 + cmp r1, r2 + mov r1, r2, lsr #16 + bne 1b + orr r0, ip, r1, lsl #16 + cmp r2, r0 + addhis r0, r0, #0x10000 + adc r1, r3, #0 +#else /* Big endian */ + ldr r1, .LCfrcd16_last_tsc + 4 + rsb ip, ip, #0x10000 + cmp r1, r3 + mov r1, r3, lsr #16 + bne 1b + orr r1, ip, r1, lsl #16 + cmp r3, r1 + addhis r1, r1, #0x10000 + adc r0, r2, #0 +#endif /* Big endian */ + usr_ret lr + + .align 5 +.LCfrt16_last_tsc: + .rep 7 + .word 0 + .endr +.LCfrt16_cntr_addr: + .word 0 + + .align 5 + .globl __ipipe_freerunning_twice_16 +__ipipe_freerunning_twice_16: + ldr r0, .LCfrt16_cntr_addr +/* User-space entry-point: r0 is the hardware counter virtual address */ +1: myldrd r2, r3, r1, .LCfrt16_last_tsc +2: ldrh ip, [r0] + ldrh r1, [r0] + cmp r1, ip + bne 2b +#ifndef CONFIG_CPU_BIG_ENDIAN +/* Little endian */ + ldr r1, .LCfrt16_last_tsc + cmp r1, r2 + mov r1, r2, lsr #16 + bne 1b + orr r0, ip, r1, lsl #16 + cmp r2, r0 + addhis r0, r0, #0x10000 + adc r1, r3, #0 +#else /* Big endian */ + ldr r1, .LCfrt16_last_tsc + 4 + cmp r1, r3 + mov r1, r3, lsr #16 + bne 1b + orr r1, ip, r1, lsl #16 + cmp r3, r1 + addhis r1, r1, #0x10000 + adc r0, r2, #0 +#endif /* Big endian */ + usr_ret lr + + .align 5 +.LCdec16_last_tsc: + .rep 2 + .word 0 + .endr +.LCdec16_last_cnt: + .rep 5 + .word 0 + .endr +.LCdec16_cntr_addr: + .word 0 + + .align 5 + .globl __ipipe_decrementer_16 +__ipipe_decrementer_16: + ldr r0, .LCdec16_cntr_addr +/* User-space entry-point: r0 is the hardware counter virtual address */ +#ifndef CONFIG_CPU_BIG_ENDIAN +/* Little endian */ +1: ldr r1, .LCdec16_last_tsc + ldrh ip, [r0] + ldr r2, .LCdec16_last_cnt + subs ip, r2, ip + addcc ip, ip, #0x10000 + myldrd r2, r3, r3, .LCdec16_last_tsc + cmp r1, r2 + bne 1b + adds r0, ip, r2 + adc r1, r3, #0 +#else /* Big endian */ +1: ldr r1, .LCdec16_last_tsc + 4 + ldrh ip, [r0] + ldr r2, .LCdec16_last_cnt + subs ip, r2, ip + addcc ip, ip, #0x10000 + myldrd r2, r3, r3, .LCdec16_last_tsc + cmp r1, r3 + bne 1b + adds r1, ip, r3 + adc r0, r2, #0 +#endif /* Big endian */ + usr_ret lr + + .align 5 + .globl __ipipe_freerunning_arch +__ipipe_freerunning_arch: + nop +#ifdef CONFIG_ARM_ARCH_TIMER + mrrc p15, 0, r0, r1, c14 +#else + mov r0, #0 + mov r1, #0 +#endif + usr_ret lr diff --git a/arch/arm/kernel/perf_callchain.c b/arch/arm/kernel/perf_callchain.c index 4e02ae5..c4c88ce 100644 --- a/arch/arm/kernel/perf_callchain.c +++ b/arch/arm/kernel/perf_callchain.c @@ -104,6 +104,9 @@ perf_callchain_kernel(struct perf_callchain_entry *entry, struct pt_regs *regs) return; } + if (IS_ENABLED(CONFIG_IPIPE)) + return; + arm_get_current_stackframe(regs, &fr); walk_stackframe(&fr, callchain_trace, entry); } diff --git a/arch/arm/kernel/process.c b/arch/arm/kernel/process.c index f192a2a..ed87693 100644 --- a/arch/arm/kernel/process.c +++ b/arch/arm/kernel/process.c @@ -54,22 +54,51 @@ static const char *isa_modes[] __maybe_unused = { "ARM" , "Thumb" , "Jazelle", "ThumbEE" }; -/* - * This is our default idle handler. - */ - void (*arm_pm_idle)(void); -/* - * Called from the core idle loop. - */ +#ifdef CONFIG_IPIPE +static void __ipipe_halt_root(void) +{ + struct ipipe_percpu_domain_data *p; -void arch_cpu_idle(void) + /* + * Emulate idle entry sequence over the root domain, which is + * stalled on entry. + */ + hard_local_irq_disable(); + + p = ipipe_this_cpu_root_context(); + __clear_bit(IPIPE_STALL_FLAG, &p->status); + + if (unlikely(__ipipe_ipending_p(p))) + __ipipe_sync_stage(); + else { + if (arm_pm_idle) + arm_pm_idle(); + else + cpu_do_idle(); + } +} +#else /* !CONFIG_IPIPE */ +static void __ipipe_halt_root(void) { if (arm_pm_idle) arm_pm_idle(); else cpu_do_idle(); +} +#endif /* !CONFIG_IPIPE */ + +/* + * Called from the core idle loop. + */ + +void arch_cpu_idle(void) +{ + if (!need_resched()) + __ipipe_halt_root(); + + /* This will re-enable hard_irqs also with IPIPE */ local_irq_enable(); } diff --git a/arch/arm/kernel/ptrace.c b/arch/arm/kernel/ptrace.c index ef9119f..91f5d3b 100644 --- a/arch/arm/kernel/ptrace.c +++ b/arch/arm/kernel/ptrace.c @@ -214,6 +214,10 @@ void ptrace_break(struct task_struct *tsk, struct pt_regs *regs) static int break_trap(struct pt_regs *regs, unsigned int instr) { + + if (__ipipe_report_trap(IPIPE_TRAP_BREAK,regs)) + return 0; + ptrace_break(current, regs); return 0; } diff --git a/arch/arm/kernel/raw_printk.c b/arch/arm/kernel/raw_printk.c new file mode 100644 index 0000000..bc97571 --- /dev/null +++ b/arch/arm/kernel/raw_printk.c @@ -0,0 +1,32 @@ +#include +#include +#include + +void printch(int); + +static void raw_console_write(struct console *co, + const char *s, unsigned count) +{ + while (count-- > 0) { + if (*s == '\n') + printch('\r'); + printch(*s++); + } +} + +static struct console raw_console = { + .name = "rawcon", + .write = raw_console_write, + .write_raw = raw_console_write, + .flags = CON_PRINTBUFFER | CON_RAW, + .index = -1, +}; + +static int __init raw_console_init(void) +{ + register_console(&raw_console); + + return 0; +} + +console_initcall(raw_console_init); diff --git a/arch/arm/kernel/setup.c b/arch/arm/kernel/setup.c index 6c777e9..cfbd4e9 100644 --- a/arch/arm/kernel/setup.c +++ b/arch/arm/kernel/setup.c @@ -31,6 +31,7 @@ #include #include #include +#include #include #include @@ -506,7 +507,39 @@ void notrace cpu_init(void) #endif } -u32 __cpu_logical_map[NR_CPUS] = { [0 ... NR_CPUS-1] = MPIDR_INVALID }; +u32 __cpu_logical_map[16] = { [0 ... 15] = MPIDR_INVALID }; +#ifdef CONFIG_IPIPE_LEGACY +#if NR_CPUS > 16 +u32 __cpu_reverse_map[NR_CPUS]; +#else /* NR_CPUS < 16 */ +u32 __cpu_reverse_map[16]; +#endif /* NR_CPUS < 16 */ +EXPORT_SYMBOL(__cpu_reverse_map); +#endif /* CONFIG_IPIPE_LEGACY */ + +#ifdef CONFIG_IPIPE + +void __init smp_build_cpu_revmap(void) +{ +#ifdef CONFIG_IPIPE_LEGACY + int i; + u32 mpidr = is_smp() ? read_cpuid_mpidr() & MPIDR_HWID_BITMASK : 0; + u32 cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0); + u32 max = cpu + 1 > nr_cpu_ids ? cpu + 1 : nr_cpu_ids; + + BUG_ON(max > ARRAY_SIZE(__cpu_reverse_map)); +#endif /* CONFIG_IPIPE_LEGACY */ + + /* printk on I-pipe needs per cpu data */ + set_my_cpu_offset(per_cpu_offset(0)); + +#ifdef CONFIG_IPIPE_LEGACY + for (i = 0; i < nr_cpu_ids; ++i) + __cpu_reverse_map[cpu_logical_map(i) & 0xff] = i; +#endif /* CONFIG_IPIPE_LEGACY */ +} + +#endif void __init smp_setup_processor_id(void) { diff --git a/arch/arm/kernel/signal.c b/arch/arm/kernel/signal.c index 586eef2..a2d9bfb 100644 --- a/arch/arm/kernel/signal.c +++ b/arch/arm/kernel/signal.c @@ -569,11 +569,13 @@ do_work_pending(struct pt_regs *regs, unsigned int thread_flags, int syscall) { do { if (likely(thread_flags & _TIF_NEED_RESCHED)) { + local_irq_disable(); + hard_cond_local_irq_enable(); schedule(); } else { if (unlikely(!user_mode(regs))) return 0; - local_irq_enable(); + hard_local_irq_enable(); if (thread_flags & _TIF_SIGPENDING) { int restart = do_signal(regs, syscall); if (unlikely(restart)) { @@ -592,7 +594,7 @@ do_work_pending(struct pt_regs *regs, unsigned int thread_flags, int syscall) tracehook_notify_resume(regs); } } - local_irq_disable(); + hard_local_irq_disable(); thread_flags = current_thread_info()->flags; } while (thread_flags & _TIF_WORK_MASK); return 0; diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index f11d825..68981b8 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c @@ -72,8 +72,24 @@ enum ipi_msg_type { IPI_CPU_STOP, IPI_IRQ_WORK, IPI_COMPLETION, + IPI_CPU_DUMP, +#ifdef CONFIG_IPIPE + IPI_IPIPE_FIRST, +#endif /* CONFIG_IPIPE */ }; +#ifdef CONFIG_IPIPE +#define noipipe_irq_enter() \ + do { \ + } while(0) +#define noipipe_irq_exit() \ + do { \ + } while(0) +#else /* !CONFIG_IPIPE */ +#define noipipe_irq_enter() irq_enter() +#define noipipe_irq_exit() irq_exit() +#endif /* !CONFIG_IPIPE */ + static DECLARE_COMPLETION(cpu_running); static struct smp_operations smp_ops; @@ -340,10 +356,17 @@ asmlinkage void secondary_start_kernel(void) * The identity mapping is uncached (strongly ordered), so * switch away from it before attempting any exclusive accesses. */ - cpu_switch_mm(mm->pgd, mm); + cpu_switch_mm(mm->pgd, mm, 1); local_flush_bp_all(); enter_lazy_tlb(mm, current); local_flush_tlb_all(); +#ifdef CONFIG_IPIPE + /* + * With CONFIG_IPIPE debug_smp_processor_id requires access + * to percpu data. + */ + set_my_cpu_offset(per_cpu_offset(ipipe_processor_id())); +#endif /* * All kernel threads share the same mm context; grab a @@ -519,6 +542,91 @@ void arch_irq_work_raise(void) #endif #ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST + +static inline void ipi_timer(void) +{ +#if defined(CONFIG_IPIPE_LEGACY) && !defined(CONFIG_IPIPE_ARM_KUSER_TSC) + __ipipe_mach_update_tsc(); +#endif + + tick_receive_broadcast(); +} + +#endif + +#ifdef CONFIG_IPIPE +#define IPIPE_IPI_BASE IPIPE_VIRQ_BASE + +unsigned __ipipe_first_ipi; +EXPORT_SYMBOL_GPL(__ipipe_first_ipi); + +static void __ipipe_do_IPI(unsigned virq, void *cookie) +{ + enum ipi_msg_type msg = virq - IPIPE_IPI_BASE; + handle_IPI(msg, raw_cpu_ptr(&ipipe_percpu.tick_regs)); +} + +void __ipipe_ipis_alloc(void) +{ + unsigned virq, _virq; + unsigned ipi_nr; + + if (__ipipe_first_ipi) + return; + + /* __ipipe_first_ipi is 0 here */ + ipi_nr = IPI_IPIPE_FIRST + IPIPE_LAST_IPI + 1; + + for (virq = IPIPE_IPI_BASE; virq < IPIPE_IPI_BASE + ipi_nr; virq++) { + _virq = ipipe_alloc_virq(); + if (virq != _virq) + panic("I-pipe: cannot reserve virq #%d (got #%d)\n", + virq, _virq); + + if (virq - IPIPE_IPI_BASE == IPI_IPIPE_FIRST) + __ipipe_first_ipi = virq; + } +} + +void __ipipe_ipis_request(void) +{ + unsigned virq; + + for (virq = IPIPE_IPI_BASE; virq < __ipipe_first_ipi; virq++) + ipipe_request_irq(ipipe_root_domain, + virq, + (ipipe_irq_handler_t)__ipipe_do_IPI, + NULL, NULL); +} +void ipipe_send_ipi(unsigned ipi, cpumask_t cpumask) +{ + enum ipi_msg_type msg = ipi - IPIPE_IPI_BASE; + smp_cross_call(&cpumask, msg); +} +EXPORT_SYMBOL_GPL(ipipe_send_ipi); + + /* hw IRQs off */ +asmlinkage void __exception __ipipe_grab_ipi(unsigned svc, struct pt_regs *regs) +{ + int virq = IPIPE_IPI_BASE + svc; + + /* + * Virtual NMIs ignore the root domain's stall + * bit. When caught over high priority + * domains, virtual VMIs are pipelined the + * usual way as normal interrupts. + */ + if (virq == IPIPE_SERVICE_VNMI && __ipipe_root_p) + __ipipe_do_vnmi(IPIPE_SERVICE_VNMI, NULL); + else + __ipipe_dispatch_irq(virq, IPIPE_IRQF_NOACK); + + __ipipe_exit_irq(regs); +} + +#endif /* CONFIG_IPIPE */ + +#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST void tick_broadcast(const struct cpumask *mask) { smp_cross_call(mask, IPI_TIMER); @@ -586,9 +694,9 @@ void handle_IPI(int ipinr, struct pt_regs *regs) #ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST case IPI_TIMER: - irq_enter(); - tick_receive_broadcast(); - irq_exit(); + noipipe_irq_enter(); + ipi_timer(); + noipipe_irq_exit(); break; #endif @@ -597,35 +705,35 @@ void handle_IPI(int ipinr, struct pt_regs *regs) break; case IPI_CALL_FUNC: - irq_enter(); + noipipe_irq_enter(); generic_smp_call_function_interrupt(); - irq_exit(); + noipipe_irq_exit(); break; case IPI_CALL_FUNC_SINGLE: - irq_enter(); + noipipe_irq_enter(); generic_smp_call_function_single_interrupt(); - irq_exit(); + noipipe_irq_exit(); break; case IPI_CPU_STOP: - irq_enter(); + noipipe_irq_enter(); ipi_cpu_stop(cpu); - irq_exit(); + noipipe_irq_exit(); break; #ifdef CONFIG_IRQ_WORK case IPI_IRQ_WORK: - irq_enter(); + noipipe_irq_enter(); irq_work_run(); - irq_exit(); + noipipe_irq_exit(); break; #endif case IPI_COMPLETION: - irq_enter(); + noipipe_irq_enter(); ipi_complete(cpu); - irq_exit(); + noipipe_irq_exit(); break; default: diff --git a/arch/arm/kernel/smp_twd.c b/arch/arm/kernel/smp_twd.c index 172c6a05..64c3c5c 100644 --- a/arch/arm/kernel/smp_twd.c +++ b/arch/arm/kernel/smp_twd.c @@ -20,14 +20,19 @@ #include #include #include +#include #include #include +#include +#include #include #include +#include /* set up by the platform code */ static void __iomem *twd_base; +static struct clk *twd_clk; static struct clk *twd_clk; static unsigned long twd_timer_rate; @@ -36,6 +41,38 @@ static DEFINE_PER_CPU(bool, percpu_setup_called); static struct clock_event_device __percpu *twd_evt; static int twd_ppi; +#if defined(CONFIG_IPIPE) && defined(CONFIG_SMP) +static DEFINE_PER_CPU(struct ipipe_timer, twd_itimer); + +void __iomem *gt_base; + +static void twd_ack(void) +{ + writel_relaxed(1, twd_base + TWD_TIMER_INTSTAT); +} + +static void twd_get_clock(struct device_node *np); +static void __cpuinit twd_calibrate_rate(void); + +#ifdef CONFIG_IPIPE_DEBUG_INTERNAL + +static DEFINE_PER_CPU(int, irqs); + +void twd_hrtimer_debug(unsigned int irq) /* hw interrupt off */ +{ + int cpu = ipipe_processor_id(); + + if ((++per_cpu(irqs, cpu) % HZ) == 0) { +#if 0 + raw_printk("%c", 'A' + cpu); +#else + do { } while (0); +#endif + } +} +#endif /* CONFIG_IPIPE_DEBUG_INTERNAL */ +#endif /* CONFIG_IPIPE && CONFIG_SMP */ + static void twd_set_mode(enum clock_event_mode mode, struct clock_event_device *clk) { @@ -189,6 +226,13 @@ core_initcall(twd_cpufreq_init); #endif +#ifdef CONFIG_IPIPE +static unsigned int twd_refresh_freq(void) +{ + return clk_get_rate(twd_clk); +} +#endif + static void twd_calibrate_rate(void) { unsigned long count; @@ -232,7 +276,11 @@ static irqreturn_t twd_handler(int irq, void *dev_id) { struct clock_event_device *evt = dev_id; + if (clockevent_ipipe_stolen(evt)) + goto handle; + if (twd_timer_ack()) { + handle: evt->event_handler(evt); return IRQ_HANDLED; } @@ -299,6 +347,18 @@ static void twd_timer_setup(void) clk->set_mode = twd_set_mode; clk->set_next_event = twd_set_next_event; clk->irq = twd_ppi; + +#if defined(CONFIG_IPIPE) && defined(CONFIG_SMP) + printk(KERN_INFO "I-pipe, %lu.%03lu MHz timer\n", + twd_timer_rate / 1000000, + (twd_timer_rate % 1000000) / 1000); + clk->ipipe_timer = raw_cpu_ptr(&twd_itimer); + clk->ipipe_timer->irq = clk->irq; + clk->ipipe_timer->ack = twd_ack; + clk->ipipe_timer->min_delay_ticks = 0xf; + clk->ipipe_timer->refresh_freq = twd_refresh_freq; +#endif + clk->cpumask = cpumask_of(cpu); clockevents_config_and_register(clk, twd_timer_rate, @@ -357,6 +417,10 @@ static int __init twd_local_timer_common_register(struct device_node *np) else late_time_init = twd_timer_setup; +#ifdef CONFIG_IPIPE_DEBUG_INTERNAL + __ipipe_mach_hrtimer_debug = &twd_hrtimer_debug; +#endif /* CONFIG_IPIPE_DEBUG_INTERNAL */ + return 0; out_irq: @@ -380,6 +444,7 @@ int __init twd_local_timer_register(struct twd_local_timer *tlt) if (!twd_base) return -ENOMEM; + return twd_local_timer_common_register(NULL); } @@ -403,6 +468,7 @@ static void __init twd_local_timer_of_register(struct device_node *np) goto out; } + err = twd_local_timer_common_register(np); out: diff --git a/arch/arm/kernel/suspend.c b/arch/arm/kernel/suspend.c index 9a2f882..363cbf7 100644 --- a/arch/arm/kernel/suspend.c +++ b/arch/arm/kernel/suspend.c @@ -31,7 +31,7 @@ int cpu_suspend(unsigned long arg, int (*fn)(unsigned long)) */ ret = __cpu_suspend(arg, fn, __mpidr); if (ret == 0) { - cpu_switch_mm(mm->pgd, mm); + cpu_switch_mm(mm->pgd, mm, 1); local_flush_bp_all(); local_flush_tlb_all(); } diff --git a/arch/arm/kernel/traps.c b/arch/arm/kernel/traps.c index 3dce1a3..c46c87e 100644 --- a/arch/arm/kernel/traps.c +++ b/arch/arm/kernel/traps.c @@ -25,6 +25,7 @@ #include #include #include +#include #include #include @@ -494,6 +495,14 @@ asmlinkage void __exception_irq_entry handle_fiq_as_nmi(struct pt_regs *regs) */ asmlinkage void bad_mode(struct pt_regs *regs, int reason) { + if (__ipipe_report_trap(IPIPE_TRAP_UNKNOWN,regs)) + return; + +#ifdef CONFIG_IPIPE + ipipe_stall_root(); + hard_local_irq_enable(); +#endif + console_verbose(); pr_crit("Bad mode in %s handler detected\n", handler[reason]); @@ -830,10 +839,21 @@ void __init trap_init(void) #ifdef CONFIG_KUSER_HELPERS static void __init kuser_init(void *vectors) { +#ifndef CONFIG_IPIPE extern char __kuser_helper_start[], __kuser_helper_end[]; int kuser_sz = __kuser_helper_end - __kuser_helper_start; +#else /* !CONFIG_IPIPE */ + extern char __ipipe_tsc_area_start[], __kuser_helper_end[]; + int kuser_sz = __kuser_helper_end - __ipipe_tsc_area_start; + extern char __vectors_start[], __vectors_end[]; +#endif /* !CONFIG_IPIPE */ +#ifndef CONFIG_IPIPE memcpy(vectors + 0x1000 - kuser_sz, __kuser_helper_start, kuser_sz); +#else /* !CONFIG_IPIPE */ + BUG_ON(0x1000 - kuser_sz < __vectors_end - __vectors_start); + memcpy(vectors + 0x1000 - kuser_sz, __ipipe_tsc_area_start, kuser_sz); +#endif /* !CONFIG_IPIPE */ /* * vectors + 0xfe0 = __kuser_get_tls diff --git a/arch/arm/kernel/vdso.c b/arch/arm/kernel/vdso.c index efe17dd..42a59bc 100644 --- a/arch/arm/kernel/vdso.c +++ b/arch/arm/kernel/vdso.c @@ -299,6 +299,8 @@ void update_vsyscall(struct timekeeper *tk) struct timespec xtime_coarse; struct timespec64 *wtm = &tk->wall_to_monotonic; + __ipipe_update_vsyscall(tk); + if (!cntvct_ok) { /* The entry points have been zeroed, so there is no * point in updating the data page. diff --git a/arch/arm/mach-at91/Kconfig b/arch/arm/mach-at91/Kconfig index fd95f34..fae5b2c 100644 --- a/arch/arm/mach-at91/Kconfig +++ b/arch/arm/mach-at91/Kconfig @@ -74,6 +74,17 @@ config SOC_AT91SAM9 AT91SAM9X35 AT91SAM9XE +config IPIPE_AT91_TC + depends on IPIPE + int "use AT91 TC as I-pipe hrclock" + default 0 + range 0 5 + help + When the interrupt pipeline is enabled, TC0 is used by default + as time base, but you can use TC1 or TC2 by setting this + variable to 1 or 2. This should only be needed to avoid + conflicts with other drivers. + config HAVE_AT91_UTMI bool diff --git a/arch/arm/mach-at91/Makefile b/arch/arm/mach-at91/Makefile index 4fa8b45..e2e5c85 100644 --- a/arch/arm/mach-at91/Makefile +++ b/arch/arm/mach-at91/Makefile @@ -23,3 +23,5 @@ endif ifeq ($(CONFIG_PM_DEBUG),y) CFLAGS_pm.o += -DDEBUG endif + +obj-$(CONFIG_IPIPE) += at91_ipipe.o diff --git a/arch/arm/mach-at91/at91_ipipe.c b/arch/arm/mach-at91/at91_ipipe.c new file mode 100644 index 0000000..2e87329 --- /dev/null +++ b/arch/arm/mach-at91/at91_ipipe.c @@ -0,0 +1,239 @@ +/* + * linux/arch/arm/mach-at91/at91_ipipe.c + * + * Copyright (C) 2007,2014 Gilles Chanteperdrix + * + * Adaptation to AT91SAM926x: + * Copyright (C) 2007 Gregory CLEMENT, Adeneo + * + * Adaptation to AT91SAM9G45: + * Copyright (C) 2011 Gregory CLEMENT, Free Electrons + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define AT91_SLOW_CLOCK 32768 +#define TCNXCNS(timer,v) ((v) << ((timer)<<1)) +#define AT91_TC_REG_MASK (0xffff) + +#define at91_tc_read(reg) \ + __raw_readl(at91_tc_base + ATMEL_TC_REG(CONFIG_IPIPE_AT91_TC % 3, reg)) + +#define at91_tc_write(reg, value) \ + __raw_writel(value, at91_tc_base + ATMEL_TC_REG(CONFIG_IPIPE_AT91_TC % 3, reg)) + +#define read_CV() at91_tc_read(CV) +#define read_RC() at91_tc_read(RC) +#define write_RC(value) at91_tc_write(RC, value) + +static void __iomem *at91_tc_base; +static unsigned max_delta_ticks; + +static int at91_tc_set_16(unsigned long evt, void *timer); + +/* + * IRQ handler for the timer. + */ +static void at91_tc_ack(void) +{ + at91_tc_read(SR); +} + +static void at91_tc_request(struct ipipe_timer *timer, int steal) +{ + /* Enable CPCS interrupt. */ + at91_tc_write(IER, ATMEL_TC_CPCS); +} + +static void at91_tc_release(struct ipipe_timer *timer) +{ + /* Disable all interrupts. */ + at91_tc_write(IDR, ~0ul); +} + +static struct ipipe_timer at91_itimer = { + .request = at91_tc_request, + .set = at91_tc_set_16, + .ack = at91_tc_ack, + .release = at91_tc_release, + + .name = "at91_tc" __stringify(CONFIG_IPIPE_AT91_TC), + .rating = 250, +}; + +/* + * Reprogram the timer + */ +static int at91_tc_set_16(unsigned long evt, void *timer) +{ + unsigned short next_tick; + + if (evt > max_delta_ticks) + evt = max_delta_ticks; + + __ipipe_tsc_update(); + + next_tick = read_CV() + evt; + write_RC(next_tick); + if (evt >= AT91_TC_REG_MASK / 2 + || (short)(next_tick - read_CV()) > 0) + return 0; + + at91_itimer.min_delay_ticks = evt; + return -ETIME; +} + +static int at91_tc_set_32(unsigned long evt, void *timer) +{ + unsigned long next_tick; + + next_tick = read_CV() + evt; + write_RC(next_tick); + if (evt >= 0x7fffffff + || (long)(next_tick - read_CV()) > 0) + return 0; + + at91_itimer.min_delay_ticks = evt; + return -ETIME; +} + +static struct __ipipe_tscinfo tsc_info = { + .type = IPIPE_TSC_TYPE_FREERUNNING, + .u = { + { + .mask = AT91_TC_REG_MASK, + }, + }, +}; + +static int __init at91_ipipe_init(void) +{ + unsigned master_freq, divided_freq = 0; + unsigned divisor, width32, target; + unsigned long at91_tc_pbase = 0; + unsigned long long wrap_ns; + unsigned index, block; + struct atmel_tc *tc; + struct resource *r; + int tc_timer_clock; + unsigned short v; + int ret; + + index = CONFIG_IPIPE_AT91_TC % 3; + block = CONFIG_IPIPE_AT91_TC / 3; + + tc = atmel_tc_alloc(block); + if (tc == NULL) { + printk(KERN_ERR "I-pipe: could not reserve TC block %d\n", + block); + return -ENODEV; + } + at91_tc_base = tc->regs; + r = platform_get_resource(tc->pdev, IORESOURCE_MEM, 0); + at91_tc_pbase = r->start; + at91_itimer.irq = tc->irq[index]; + + ret = clk_prepare_enable(tc->clk[index]); + if (ret < 0) + goto err_free_tc; + + master_freq = clk_get_rate(tc->clk[index]); + + width32 = tc->tcb_config && tc->tcb_config->counter_width == 32; + target = width32 ? 5000000 : 1000000; + + /* Find the first frequency above 1 or 5 MHz */ + for (tc_timer_clock = ARRAY_SIZE(atmel_tc_divisors) - 1; + tc_timer_clock >= 0; tc_timer_clock--) { + divisor = atmel_tc_divisors[tc_timer_clock]; + divided_freq = + (divisor ? master_freq / divisor : AT91_SLOW_CLOCK); + if (divided_freq > target) + break; + } + + if (divided_freq < target) + printk(KERN_INFO "AT91 I-pipe warning: could not find a" + " frequency greater than %dMHz\n", target / 1000000); + + if (width32) { + at91_itimer.set = at91_tc_set_32; + tsc_info.u.mask = 0xffffffffU; + wrap_ns = 0; + } else { + wrap_ns = (unsigned long long)(AT91_TC_REG_MASK + 1); + wrap_ns *= NSEC_PER_SEC; + do_div(wrap_ns, divided_freq); + + /* + * Add a 1ms margin. It means that when an interrupt + * occurs, update_tsc must be called within + * 1ms. update_tsc is called through set_dec. + */ + wrap_ns -= 1000000; + } + + printk(KERN_INFO "AT91 I-pipe timer: using TC%d, div: %u, " + "freq: %u.%06u MHz\n", + CONFIG_IPIPE_AT91_TC, divisor, + divided_freq / 1000000, divided_freq % 1000000); + + /* Disable the channel */ + at91_tc_write(CCR, ATMEL_TC_CLKDIS); + + /* Disable all interrupts. */ + at91_tc_write(IDR, ~0ul); + + /* No Sync. */ + at91_tc_write(BCR, 0); + + /* program NO signal on XCN */ + v = __raw_readl(at91_tc_base + ATMEL_TC_BMR); + v &= ~TCNXCNS(index, 3); + v |= TCNXCNS(index, 1); /* AT91_TC_TCNXCNS_NONE */ + __raw_writel(v, at91_tc_base + ATMEL_TC_BMR); + + /* Use the clock selected as input clock. */ + at91_tc_write(CMR, tc_timer_clock); + + /* Load the TC register C. */ + write_RC(0xffff); + + /* Enable the channel. */ + at91_tc_write(CCR, ATMEL_TC_CLKEN | ATMEL_TC_SWTRG); + + at91_itimer.freq = divided_freq; + at91_itimer.min_delay_ticks = ipipe_timer_ns2ticks(&at91_itimer, 2000); + if (wrap_ns) + max_delta_ticks = ipipe_timer_ns2ticks(&at91_itimer, wrap_ns); + ipipe_timer_register(&at91_itimer); + + tsc_info.counter_vaddr = + (unsigned long)(at91_tc_base + ATMEL_TC_REG(index, CV)); + tsc_info.u.counter_paddr = (at91_tc_pbase + ATMEL_TC_REG(index, CV)); + tsc_info.freq = divided_freq; + __ipipe_tsc_register(&tsc_info); + + return 0; + +err_free_tc: + atmel_tc_free(tc); + return ret; +} +subsys_initcall(at91_ipipe_init); diff --git a/arch/arm/mach-at91/at91_ipipe.h b/arch/arm/mach-at91/at91_ipipe.h new file mode 100644 index 0000000..057e493 --- /dev/null +++ b/arch/arm/mach-at91/at91_ipipe.h @@ -0,0 +1,12 @@ +#ifndef AT91_IPIPE_H +#define AT91_IPIPE_H + +#include + +#ifdef CONFIG_IPIPE +void at91_pic_muter_register(void); +#else /* !CONFIG_IPIPE */ +#define at91_pic_muter_register() do { } while (0) +#endif /* CONFIG_IPIPE */ + +#endif /* AT91_IPIPE_TIME_H */ diff --git a/arch/arm/mach-at91/at91sam926x_time.c b/arch/arm/mach-at91/at91sam926x_time.c new file mode 100644 index 0000000..da10238 --- /dev/null +++ b/arch/arm/mach-at91/at91sam926x_time.c @@ -0,0 +1,297 @@ +/* + * at91sam926x_time.c - Periodic Interval Timer (PIT) for at91sam926x + * + * Copyright (C) 2005-2006 M. Amine SAYA, ATMEL Rousset, France + * Revision 2005 M. Nicolas Diremdjian, ATMEL Rousset, France + * Converted to ClockSource/ClockEvents by David Brownell. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include "at91_ipipe.h" + +#define AT91_PIT_MR 0x00 /* Mode Register */ +#define AT91_PIT_PITIEN (1 << 25) /* Timer Interrupt Enable */ +#define AT91_PIT_PITEN (1 << 24) /* Timer Enabled */ +#define AT91_PIT_PIV (0xfffff) /* Periodic Interval Value */ + +#define AT91_PIT_SR 0x04 /* Status Register */ +#define AT91_PIT_PITS (1 << 0) /* Timer Status */ + +#define AT91_PIT_PIVR 0x08 /* Periodic Interval Value Register */ +#define AT91_PIT_PIIR 0x0c /* Periodic Interval Image Register */ +#define AT91_PIT_PICNT (0xfff << 20) /* Interval Counter */ +#define AT91_PIT_CPIV (0xfffff) /* Inverval Value */ + +#define PIT_CPIV(x) ((x) & AT91_PIT_CPIV) +#define PIT_PICNT(x) (((x) & AT91_PIT_PICNT) >> 20) + +static u32 pit_cycle; /* write-once */ +static u32 pit_cnt; /* access only w/system irq blocked */ +static void __iomem *pit_base_addr __read_mostly; +static struct clk *mck; + +static inline unsigned int pit_read(unsigned int reg_offset) +{ + return __raw_readl(pit_base_addr + reg_offset); +} + +static inline void pit_write(unsigned int reg_offset, unsigned long value) +{ + __raw_writel(value, pit_base_addr + reg_offset); +} + +/* + * Clocksource: just a monotonic counter of MCK/16 cycles. + * We don't care whether or not PIT irqs are enabled. + */ +static cycle_t read_pit_clk(struct clocksource *cs) +{ + unsigned long flags; + u32 elapsed; + u32 t; + + raw_local_irq_save(flags); + elapsed = pit_cnt; + t = pit_read(AT91_PIT_PIIR); + raw_local_irq_restore(flags); + + elapsed += PIT_PICNT(t) * pit_cycle; + elapsed += PIT_CPIV(t); + return elapsed; +} + +static struct clocksource pit_clk = { + .name = "pit", + .rating = 175, + .read = read_pit_clk, + .flags = CLOCK_SOURCE_IS_CONTINUOUS, +}; + + +/* + * Clockevent device: interrupts every 1/HZ (== pit_cycles * MCK/16) + */ +static void +pit_clkevt_mode(enum clock_event_mode mode, struct clock_event_device *dev) +{ + switch (mode) { + case CLOCK_EVT_MODE_PERIODIC: + /* update clocksource counter */ + pit_cnt += pit_cycle * PIT_PICNT(pit_read(AT91_PIT_PIVR)); + pit_write(AT91_PIT_MR, (pit_cycle - 1) | AT91_PIT_PITEN + | AT91_PIT_PITIEN); + break; + case CLOCK_EVT_MODE_ONESHOT: + BUG(); + /* FALLTHROUGH */ + case CLOCK_EVT_MODE_SHUTDOWN: + case CLOCK_EVT_MODE_UNUSED: + /* disable irq, leaving the clocksource active */ + pit_write(AT91_PIT_MR, (pit_cycle - 1) | AT91_PIT_PITEN); + break; + case CLOCK_EVT_MODE_RESUME: + break; + } +} + +static void at91sam926x_pit_suspend(struct clock_event_device *cedev) +{ + /* Disable timer */ + pit_write(AT91_PIT_MR, 0); +} + +static void at91sam926x_pit_reset(void) +{ + /* Disable timer and irqs */ + pit_write(AT91_PIT_MR, 0); + + /* Clear any pending interrupts, wait for PIT to stop counting */ + while (PIT_CPIV(pit_read(AT91_PIT_PIVR)) != 0) + cpu_relax(); + + /* Start PIT but don't enable IRQ */ + pit_write(AT91_PIT_MR, (pit_cycle - 1) | AT91_PIT_PITEN); +} + +static void at91sam926x_pit_resume(struct clock_event_device *cedev) +{ + at91sam926x_pit_reset(); +} + +static struct clock_event_device pit_clkevt = { + .name = "pit", + .features = CLOCK_EVT_FEAT_PERIODIC, + .shift = 32, + .rating = 100, + .set_mode = pit_clkevt_mode, + .suspend = at91sam926x_pit_suspend, + .resume = at91sam926x_pit_resume, +}; + + +/* + * IRQ handler for the timer. + */ +static irqreturn_t at91sam926x_pit_interrupt(int irq, void *dev_id) +{ + /* + * irqs should be disabled here, but as the irq is shared they are only + * guaranteed to be off if the timer irq is registered first. + */ + WARN_ON_ONCE(!irqs_disabled()); + + /* The PIT interrupt may be disabled, and is shared */ + if ((pit_clkevt.mode == CLOCK_EVT_MODE_PERIODIC) + && (pit_read(AT91_PIT_SR) & AT91_PIT_PITS)) { + unsigned nr_ticks; + + /* Get number of ticks performed before irq, and ack it */ + nr_ticks = PIT_PICNT(pit_read(AT91_PIT_PIVR)); + do { + pit_cnt += pit_cycle; + pit_clkevt.event_handler(&pit_clkevt); + nr_ticks--; + } while (nr_ticks); + + return IRQ_HANDLED; + } + + return IRQ_NONE; +} + +static struct irqaction at91sam926x_pit_irq = { + .name = "at91_tick", + .flags = IRQF_SHARED | IRQF_TIMER | IRQF_IRQPOLL, + .handler = at91sam926x_pit_interrupt, + .irq = NR_IRQS_LEGACY + AT91_ID_SYS, +}; + +#ifdef CONFIG_OF +static struct of_device_id pit_timer_ids[] = { + { .compatible = "atmel,at91sam9260-pit" }, + { /* sentinel */ } +}; + +static int __init of_at91sam926x_pit_init(void) +{ + struct device_node *np; + int ret; + + np = of_find_matching_node(NULL, pit_timer_ids); + if (!np) + goto err; + + pit_base_addr = of_iomap(np, 0); + if (!pit_base_addr) + goto node_err; + + mck = of_clk_get(np, 0); + + /* Get the interrupts property */ + ret = irq_of_parse_and_map(np, 0); + if (!ret) { + pr_crit("AT91: PIT: Unable to get IRQ from DT\n"); + if (!IS_ERR(mck)) + clk_put(mck); + goto ioremap_err; + } + at91sam926x_pit_irq.irq = ret; + + of_node_put(np); + + return 0; + +ioremap_err: + iounmap(pit_base_addr); +node_err: + of_node_put(np); +err: + return -EINVAL; +} +#else +static int __init of_at91sam926x_pit_init(void) +{ + return -EINVAL; +} +#endif + +/* + * Set up both clocksource and clockevent support. + */ +void __init at91sam926x_pit_init(void) +{ + unsigned long pit_rate; + unsigned bits; + int ret; + + mck = ERR_PTR(-ENOENT); + + /* For device tree enabled device: initialize here */ + of_at91sam926x_pit_init(); + + /* + * Use our actual MCK to figure out how many MCK/16 ticks per + * 1/HZ period (instead of a compile-time constant LATCH). + */ + if (IS_ERR(mck)) + mck = clk_get(NULL, "mck"); + + if (IS_ERR(mck)) + panic("AT91: PIT: Unable to get mck clk\n"); + pit_rate = clk_get_rate(mck) / 16; + pit_cycle = (pit_rate + HZ/2) / HZ; + WARN_ON(((pit_cycle - 1) & ~AT91_PIT_PIV) != 0); + + /* Initialize and enable the timer */ + at91sam926x_pit_reset(); + + /* + * Register clocksource. The high order bits of PIV are unused, + * so this isn't a 32-bit counter unless we get clockevent irqs. + */ + bits = 12 /* PICNT */ + ilog2(pit_cycle) /* PIV */; + pit_clk.mask = CLOCKSOURCE_MASK(bits); + clocksource_register_hz(&pit_clk, pit_rate); + + /* Set up irq handler */ + ret = setup_irq(at91sam926x_pit_irq.irq, &at91sam926x_pit_irq); + if (ret) + pr_crit("AT91: PIT: Unable to setup IRQ\n"); + + /* Set up and register clockevents */ + pit_clkevt.mult = div_sc(pit_rate, NSEC_PER_SEC, pit_clkevt.shift); + pit_clkevt.cpumask = cpumask_of(0); + clockevents_register_device(&pit_clkevt); + + at91_pic_muter_register(); +} + +void __init at91sam926x_ioremap_pit(u32 addr) +{ +#if defined(CONFIG_OF) + struct device_node *np = + of_find_matching_node(NULL, pit_timer_ids); + + if (np) { + of_node_put(np); + return; + } +#endif + pit_base_addr = ioremap(addr, 16); + + if (!pit_base_addr) + panic("Impossible to ioremap PIT\n"); +} diff --git a/arch/arm/mach-at91/include/mach/at91_ipipe.h b/arch/arm/mach-at91/include/mach/at91_ipipe.h new file mode 100644 index 0000000..057e493 --- /dev/null +++ b/arch/arm/mach-at91/include/mach/at91_ipipe.h @@ -0,0 +1,12 @@ +#ifndef AT91_IPIPE_H +#define AT91_IPIPE_H + +#include + +#ifdef CONFIG_IPIPE +void at91_pic_muter_register(void); +#else /* !CONFIG_IPIPE */ +#define at91_pic_muter_register() do { } while (0) +#endif /* CONFIG_IPIPE */ + +#endif /* AT91_IPIPE_TIME_H */ diff --git a/arch/arm/mach-davinci/Kconfig b/arch/arm/mach-davinci/Kconfig index dd8f531..3f2eac9 100644 --- a/arch/arm/mach-davinci/Kconfig +++ b/arch/arm/mach-davinci/Kconfig @@ -42,6 +42,7 @@ config ARCH_DAVINCI_DA850 depends on !ARCH_DAVINCI_DMx || AUTO_ZRELADDR select ARCH_DAVINCI_DA8XX select CP_INTC + select IPIPE_ARM_KUSER_TSC if IPIPE config ARCH_DAVINCI_DA8XX bool diff --git a/arch/arm/mach-davinci/cp_intc.c b/arch/arm/mach-davinci/cp_intc.c index 006dae8..816affe 100644 --- a/arch/arm/mach-davinci/cp_intc.c +++ b/arch/arm/mach-davinci/cp_intc.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include diff --git a/arch/arm/mach-davinci/time.c b/arch/arm/mach-davinci/time.c index 160c960..e89f114 100644 --- a/arch/arm/mach-davinci/time.c +++ b/arch/arm/mach-davinci/time.c @@ -19,6 +19,10 @@ #include #include #include +#ifdef CONFIG_IPIPE +#include +#include +#endif /* CONFIG_IPIPE */ #include #include @@ -94,9 +98,15 @@ struct timer_s { unsigned long opts; unsigned long flags; void __iomem *base; +#ifdef CONFIG_IPIPE + void *pbase; +#endif /*CONFIG_IPIPE */ unsigned long tim_off; unsigned long prd_off; unsigned long enamode_shift; +#ifdef CONFIG_IPIPE + int irq; +#endif /* CONFIG_IPIPE */ struct irqaction irqaction; }; static struct timer_s timers[]; @@ -241,6 +251,9 @@ static void __init timer_init(void) t->base = base[timer]; if (!t->base) continue; +#ifdef CONFIG_IPIPE + t->pbase = (void *)dtip[timer].base; +#endif /* CONFIG_IPIPE */ if (IS_TIMER_BOT(t->id)) { t->enamode_shift = 6; @@ -262,6 +275,9 @@ static void __init timer_init(void) irq = USING_COMPARE(t) ? dtip[i].cmp_irq : irq; setup_irq(irq, &t->irqaction); } +#ifdef CONFIG_IPIPE + t->irq = irq; +#endif /* CONFIG_IPIPE */ } } @@ -329,10 +345,26 @@ static void davinci_set_mode(enum clock_event_mode mode, } } +#ifdef CONFIG_IPIPE +static struct ipipe_timer davinci_itimer; + +static struct __ipipe_tscinfo tsc_info = { + .type = IPIPE_TSC_TYPE_FREERUNNING, + .u = { + { + .mask = 0xffffffff, + }, + }, +}; +#endif /* CONFIG_IPIPE */ + static struct clock_event_device clockevent_davinci = { .features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT, .set_next_event = davinci_set_next_event, .set_mode = davinci_set_mode, +#ifdef CONFIG_IPIPE + .ipipe_timer = &davinci_itimer, +#endif /* CONFIG_IPIPE */ }; @@ -397,6 +429,17 @@ void __init davinci_timer_init(void) clockevent_davinci.name = id_to_name[timers[TID_CLOCKEVENT].id]; clockevent_davinci.cpumask = cpumask_of(0); +#ifdef CONFIG_IPIPE + tsc_info.freq = davinci_clock_tick_rate; + tsc_info.counter_vaddr = (void *)(timers[TID_CLOCKSOURCE].base + + timers[TID_CLOCKSOURCE].tim_off); + tsc_info.u.counter_paddr = timers[TID_CLOCKSOURCE].pbase + + timers[TID_CLOCKSOURCE].tim_off; + __ipipe_tsc_register(&tsc_info); + + davinci_itimer.irq = timers[TID_CLOCKEVENT].irq; + davinci_itimer.min_delay_ticks = 3; +#endif /* CONFIG_IPIPE */ clockevents_config_and_register(&clockevent_davinci, davinci_clock_tick_rate, 1, 0xfffffffe); diff --git a/arch/arm/mach-imx/3ds_debugboard.c b/arch/arm/mach-imx/3ds_debugboard.c index 1343773..0a478c8 100644 --- a/arch/arm/mach-imx/3ds_debugboard.c +++ b/arch/arm/mach-imx/3ds_debugboard.c @@ -20,6 +20,7 @@ #include #include #include +#include #include "hardware.h" @@ -101,7 +102,7 @@ static void mxc_expio_irq_handler(u32 irq, struct irq_desc *desc) for (; int_valid != 0; int_valid >>= 1, expio_irq++) { if ((int_valid & 1) == 0) continue; - generic_handle_irq(irq_find_mapping(domain, expio_irq)); + ipipe_handle_demuxed_irq(irq_find_mapping(domain, expio_irq)); } desc->irq_data.chip->irq_ack(&desc->irq_data); diff --git a/arch/arm/mach-imx/Kconfig b/arch/arm/mach-imx/Kconfig index 3a3d3e9..97f9c63 100644 --- a/arch/arm/mach-imx/Kconfig +++ b/arch/arm/mach-imx/Kconfig @@ -557,6 +557,7 @@ config SOC_IMX6 config SOC_IMX6Q bool "i.MX6 Quad/DualLite support" select ARM_ERRATA_764369 if SMP + select ARM_GLOBAL_TIMER if SMP && IPIPE select HAVE_ARM_SCU if SMP select HAVE_ARM_TWD if SMP select PCI_DOMAINS if PCI diff --git a/arch/arm/mach-imx/avic.c b/arch/arm/mach-imx/avic.c index 1a89323..c41f13c8 100644 --- a/arch/arm/mach-imx/avic.c +++ b/arch/arm/mach-imx/avic.c @@ -126,6 +126,9 @@ static __init void avic_init_gc(int idx, unsigned int irq_start) ct->chip.irq_mask = irq_gc_mask_clr_bit; ct->chip.irq_unmask = irq_gc_mask_set_bit; ct->chip.irq_ack = irq_gc_mask_clr_bit; +#ifdef CONFIG_IPIPE + ct->chip.irq_mask_ack = irq_gc_mask_clr_bit; +#endif /* CONFIG_IPIPE */ ct->chip.irq_set_wake = irq_gc_set_wake; ct->chip.irq_suspend = avic_irq_suspend; ct->chip.irq_resume = avic_irq_resume; @@ -144,7 +147,7 @@ static void __exception_irq_entry avic_handle_irq(struct pt_regs *regs) if (nivector == 0xffff) break; - handle_domain_irq(domain, nivector, regs); + ipipe_handle_domain_irq(domain, nivector, regs); } while (1); } diff --git a/arch/arm/mach-imx/clk-imx1.c b/arch/arm/mach-imx/clk-imx1.c index 37c307a..16bd3b8 100644 --- a/arch/arm/mach-imx/clk-imx1.c +++ b/arch/arm/mach-imx/clk-imx1.c @@ -98,7 +98,8 @@ int __init mx1_clocks_init(unsigned long fref) clk_register_clkdev(clk[IMX1_CLK_DUMMY], "ipg", "imx1-fb.0"); clk_register_clkdev(clk[IMX1_CLK_DUMMY], "ahb", "imx1-fb.0"); - mxc_timer_init(MX1_IO_ADDRESS(MX1_TIM1_BASE_ADDR), MX1_TIM1_INT); + mxc_timer_init(MX1_IO_ADDRESS(MX1_TIM1_BASE_ADDR), + MX1_TIM1_BASE_ADDR, MX1_TIM1_INT); return 0; } diff --git a/arch/arm/mach-imx/clk-imx21.c b/arch/arm/mach-imx/clk-imx21.c index 4b4c753..7433bf2 100644 --- a/arch/arm/mach-imx/clk-imx21.c +++ b/arch/arm/mach-imx/clk-imx21.c @@ -153,7 +153,8 @@ int __init mx21_clocks_init(unsigned long lref, unsigned long href) clk_register_clkdev(clk[IMX21_CLK_I2C_GATE], NULL, "imx21-i2c.0"); clk_register_clkdev(clk[IMX21_CLK_OWIRE_GATE], NULL, "mxc_w1.0"); - mxc_timer_init(MX21_IO_ADDRESS(MX21_GPT1_BASE_ADDR), MX21_INT_GPT1); + mxc_timer_init(MX21_IO_ADDRESS(MX21_GPT1_BASE_ADDR), + MX21_GPT1_BASE_ADDR, MX21_INT_GPT1); return 0; } diff --git a/arch/arm/mach-imx/clk-imx27.c b/arch/arm/mach-imx/clk-imx27.c index ab6349e..8e50aa90 100644 --- a/arch/arm/mach-imx/clk-imx27.c +++ b/arch/arm/mach-imx/clk-imx27.c @@ -229,7 +229,8 @@ int __init mx27_clocks_init(unsigned long fref) clk_register_clkdev(clk[IMX27_CLK_EMMA_AHB_GATE], "ahb", "m2m-emmaprp.0"); clk_register_clkdev(clk[IMX27_CLK_EMMA_IPG_GATE], "ipg", "m2m-emmaprp.0"); - mxc_timer_init(MX27_IO_ADDRESS(MX27_GPT1_BASE_ADDR), MX27_INT_GPT1); + mxc_timer_init(MX27_IO_ADDRESS(MX27_GPT1_BASE_ADDR), + MX27_GPT1_BASE_ADDR, MX27_INT_GPT1); return 0; } diff --git a/arch/arm/mach-imx/clk-imx31.c b/arch/arm/mach-imx/clk-imx31.c index 286ef422..4d5927c 100644 --- a/arch/arm/mach-imx/clk-imx31.c +++ b/arch/arm/mach-imx/clk-imx31.c @@ -182,7 +182,8 @@ int __init mx31_clocks_init(unsigned long fref) mx31_revision(); clk_disable_unprepare(clk[iim_gate]); - mxc_timer_init(MX31_IO_ADDRESS(MX31_GPT1_BASE_ADDR), MX31_INT_GPT); + mxc_timer_init(MX31_IO_ADDRESS(MX31_GPT1_BASE_ADDR), + MX31_GPT1_BASE_ADDR, MX31_INT_GPT); return 0; } diff --git a/arch/arm/mach-imx/clk-imx35.c b/arch/arm/mach-imx/clk-imx35.c index a0d2b57..aefef36 100644 --- a/arch/arm/mach-imx/clk-imx35.c +++ b/arch/arm/mach-imx/clk-imx35.c @@ -277,9 +277,11 @@ int __init mx35_clocks_init(void) imx_print_silicon_rev("i.MX35", mx35_revision()); #ifdef CONFIG_MXC_USE_EPIT - epit_timer_init(MX35_IO_ADDRESS(MX35_EPIT1_BASE_ADDR), MX35_INT_EPIT1); + epit_timer_init(MX35_IO_ADDRESS(MX35_EPIT1_BASE_ADDR), + MX35_EPIT1_BASE_ADDR, MX35_INT_EPIT1); #else - mxc_timer_init(MX35_IO_ADDRESS(MX35_GPT1_BASE_ADDR), MX35_INT_GPT); + mxc_timer_init(MX35_IO_ADDRESS(MX35_GPT1_BASE_ADDR), + MX35_GPT1_BASE_ADDR, MX35_INT_GPT); #endif return 0; diff --git a/arch/arm/mach-imx/clk-imx51-imx53.c b/arch/arm/mach-imx/clk-imx51-imx53.c index 0f7e536..f7e741f 100644 --- a/arch/arm/mach-imx/clk-imx51-imx53.c +++ b/arch/arm/mach-imx/clk-imx51-imx53.c @@ -313,6 +313,10 @@ static void __init mx5_clocks_common_init(void __iomem *ccm_base) clk_prepare_enable(clk[IMX5_CLK_TMAX1]); clk_prepare_enable(clk[IMX5_CLK_TMAX2]); /* esdhc2, fec */ clk_prepare_enable(clk[IMX5_CLK_TMAX3]); /* esdhc1, esdhc4 */ + +#ifdef CONFIG_IPIPE + mxc_pic_muter_register(); +#endif } static void __init mx50_clocks_init(struct device_node *np) diff --git a/arch/arm/mach-imx/common.h b/arch/arm/mach-imx/common.h index 0f04e30..ab12460 100644 --- a/arch/arm/mach-imx/common.h +++ b/arch/arm/mach-imx/common.h @@ -44,7 +44,8 @@ void imx27_soc_init(void); void imx31_soc_init(void); void imx35_soc_init(void); void epit_timer_init(void __iomem *base, int irq); -void mxc_timer_init(void __iomem *, int); +void mxc_pic_muter_register(void); +void mxc_timer_init(void __iomem *, unsigned long, int); int mx1_clocks_init(unsigned long fref); int mx21_clocks_init(unsigned long lref, unsigned long fref); int mx27_clocks_init(unsigned long fref); diff --git a/arch/arm/mach-imx/gpc.c b/arch/arm/mach-imx/gpc.c index 78b6fd0..e8dc2f5 100644 --- a/arch/arm/mach-imx/gpc.c +++ b/arch/arm/mach-imx/gpc.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -53,6 +54,7 @@ struct pu_domain { static void __iomem *gpc_base; static u32 gpc_wake_irqs[IMR_NUM]; static u32 gpc_saved_imrs[IMR_NUM]; +static IPIPE_DEFINE_RAW_SPINLOCK(gpc_lock); void imx_gpc_set_arm_power_up_timing(u32 sw2iso, u32 sw) { @@ -74,28 +76,38 @@ void imx_gpc_set_arm_power_in_lpm(bool power_off) void imx_gpc_pre_suspend(bool arm_power_off) { void __iomem *reg_imr1 = gpc_base + GPC_IMR1; + unsigned long flags; int i; /* Tell GPC to power off ARM core when suspend */ if (arm_power_off) imx_gpc_set_arm_power_in_lpm(arm_power_off); + flags = hard_cond_local_irq_save(); + for (i = 0; i < IMR_NUM; i++) { gpc_saved_imrs[i] = readl_relaxed(reg_imr1 + i * 4); writel_relaxed(~gpc_wake_irqs[i], reg_imr1 + i * 4); } + + hard_cond_local_irq_restore(flags); } void imx_gpc_post_resume(void) { void __iomem *reg_imr1 = gpc_base + GPC_IMR1; + unsigned long flags; int i; /* Keep ARM core powered on for other low-power modes */ imx_gpc_set_arm_power_in_lpm(false); + flags = hard_cond_local_irq_save(); + for (i = 0; i < IMR_NUM; i++) writel_relaxed(gpc_saved_imrs[i], reg_imr1 + i * 4); + + hard_cond_local_irq_restore(flags); } static int imx_gpc_irq_set_wake(struct irq_data *d, unsigned int on) @@ -117,22 +129,31 @@ static int imx_gpc_irq_set_wake(struct irq_data *d, unsigned int on) void imx_gpc_mask_all(void) { void __iomem *reg_imr1 = gpc_base + GPC_IMR1; + unsigned long flags; int i; + flags = hard_cond_local_irq_save(); + for (i = 0; i < IMR_NUM; i++) { gpc_saved_imrs[i] = readl_relaxed(reg_imr1 + i * 4); writel_relaxed(~0, reg_imr1 + i * 4); } + hard_cond_local_irq_restore(flags); } void imx_gpc_restore_all(void) { void __iomem *reg_imr1 = gpc_base + GPC_IMR1; + unsigned long flags; int i; + flags = hard_cond_local_irq_save(); + for (i = 0; i < IMR_NUM; i++) writel_relaxed(gpc_saved_imrs[i], reg_imr1 + i * 4); + + hard_cond_local_irq_restore(flags); } void imx_gpc_hwirq_unmask(unsigned int hwirq) @@ -159,16 +180,49 @@ void imx_gpc_hwirq_mask(unsigned int hwirq) static void imx_gpc_irq_unmask(struct irq_data *d) { + unsigned long flags; + + raw_spin_lock_irqsave(&gpc_lock, flags); imx_gpc_hwirq_unmask(d->hwirq); + __ipipe_spin_unlock_irqbegin(&gpc_lock); irq_chip_unmask_parent(d); + __ipipe_spin_unlock_irqcomplete(flags); } static void imx_gpc_irq_mask(struct irq_data *d) { + unsigned long flags; + + raw_spin_lock_irqsave(&gpc_lock, flags); + /* Parent IC will handle virtual locking */ imx_gpc_hwirq_mask(d->hwirq); + __ipipe_spin_unlock_irqbegin(&gpc_lock); irq_chip_mask_parent(d); + __ipipe_spin_unlock_irqcomplete(flags); } +#ifdef CONFIG_IPIPE + +static void imx_gpc_hold_irq(struct irq_data *d) +{ + raw_spin_lock(&gpc_lock); + imx_gpc_hwirq_mask(d->hwirq); + raw_spin_unlock(&gpc_lock); + irq_chip_hold_parent(d); +} + +static void imx_gpc_release_irq(struct irq_data *d) +{ + unsigned long flags; + + raw_spin_lock_irqsave(&gpc_lock, flags); + imx_gpc_hwirq_unmask(d->hwirq); + raw_spin_unlock_irqrestore(&gpc_lock, flags); + irq_chip_release_parent(d); +} + +#endif /* CONFIG_IPIPE */ + static struct irq_chip imx_gpc_chip = { .name = "GPC", .irq_eoi = irq_chip_eoi_parent, @@ -179,6 +233,10 @@ static struct irq_chip imx_gpc_chip = { #ifdef CONFIG_SMP .irq_set_affinity = irq_chip_set_affinity_parent, #endif +#ifdef CONFIG_IPIPE + .irq_hold = imx_gpc_hold_irq, + .irq_release = imx_gpc_release_irq, +#endif }; static int imx_gpc_domain_xlate(struct irq_domain *domain, diff --git a/arch/arm/mach-imx/mach-imx25.c b/arch/arm/mach-imx/mach-imx25.c index 9379fd0..3ae7256 100644 --- a/arch/arm/mach-imx/mach-imx25.c +++ b/arch/arm/mach-imx/mach-imx25.c @@ -34,6 +34,11 @@ static void __init mx25_init_irq(void) mxc_init_irq(avic_base); } +static void __init mx25_dt_init(void) +{ + imx_aips_allow_unprivileged_access("fsl,aips-bus"); +} + static const char * const imx25_dt_board_compat[] __initconst = { "fsl,imx25", NULL @@ -42,5 +47,6 @@ static const char * const imx25_dt_board_compat[] __initconst = { DT_MACHINE_START(IMX25_DT, "Freescale i.MX25 (Device Tree Support)") .init_early = imx25_init_early, .init_irq = mx25_init_irq, + .init_machine = mx25_dt_init, .dt_compat = imx25_dt_board_compat, MACHINE_END diff --git a/arch/arm/mach-imx/mach-imx51.c b/arch/arm/mach-imx/mach-imx51.c index b015129..b42c66b 100644 --- a/arch/arm/mach-imx/mach-imx51.c +++ b/arch/arm/mach-imx/mach-imx51.c @@ -63,6 +63,11 @@ static void __init imx51_dt_init(void) static void __init imx51_init_late(void) { mx51_neon_fixup(); +#ifdef CONFIG_IPIPE + /* Allow user-space access to emulated tsc */ + imx_set_aips(IMX_IO_ADDRESS(0x73f00000)); + imx_set_aips(IMX_IO_ADDRESS(0x83f00000)); +#endif imx51_pm_init(); } diff --git a/arch/arm/mach-imx/mach-imx53.c b/arch/arm/mach-imx/mach-imx53.c index 86316a9..e5ed9fd 100644 --- a/arch/arm/mach-imx/mach-imx53.c +++ b/arch/arm/mach-imx/mach-imx53.c @@ -39,6 +39,11 @@ static void __init imx53_dt_init(void) static void __init imx53_init_late(void) { +#ifdef CONFIG_IPIPE + /* Allow user-space access to emulated tsc */ + imx_set_aips(IMX_IO_ADDRESS(0x53f00000)); + imx_set_aips(IMX_IO_ADDRESS(0x63f00000)); +#endif imx53_pm_init(); platform_device_register_simple("cpufreq-dt", -1, NULL, 0); diff --git a/arch/arm/mach-imx/mach-imx6q.c b/arch/arm/mach-imx/mach-imx6q.c index 3ab6154..04d4e75 100644 --- a/arch/arm/mach-imx/mach-imx6q.c +++ b/arch/arm/mach-imx/mach-imx6q.c @@ -388,6 +388,11 @@ static void __init imx6q_map_io(void) static void __init imx6q_init_irq(void) { +#ifdef CONFIG_IPIPE + extern void __init mx6_pic_muter_register(void); + + mx6_pic_muter_register(); +#endif /* CONFIG_IPIPE */ imx_gpc_check_dt(); imx_init_revision_from_anatop(); imx_init_l2cache(); diff --git a/arch/arm/mach-imx/mach-mx31_3ds.c b/arch/arm/mach-imx/mach-mx31_3ds.c index 65a0dc0..8d72498 100644 --- a/arch/arm/mach-imx/mach-mx31_3ds.c +++ b/arch/arm/mach-imx/mach-mx31_3ds.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include diff --git a/arch/arm/mach-imx/mach-mx31ads.c b/arch/arm/mach-imx/mach-mx31ads.c index d08c37c..b652f9d 100644 --- a/arch/arm/mach-imx/mach-mx31ads.c +++ b/arch/arm/mach-imx/mach-mx31ads.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -168,7 +169,7 @@ static void mx31ads_expio_irq_handler(u32 irq, struct irq_desc *desc) if ((int_valid & 1) == 0) continue; - generic_handle_irq(irq_find_mapping(domain, expio_irq)); + ipipe_handle_demuxed_irq(irq_find_mapping(domain, expio_irq)); } } diff --git a/arch/arm/mach-imx/mm-imx1.c b/arch/arm/mach-imx/mm-imx1.c index e065fed..64c026b 100644 --- a/arch/arm/mach-imx/mm-imx1.c +++ b/arch/arm/mach-imx/mm-imx1.c @@ -19,6 +19,7 @@ #include #include #include +#include #include diff --git a/arch/arm/mach-imx/mm-imx27.c b/arch/arm/mach-imx/mm-imx27.c index 7d82a5a5..9bbd0f4 100644 --- a/arch/arm/mach-imx/mm-imx27.c +++ b/arch/arm/mach-imx/mm-imx27.c @@ -81,6 +81,10 @@ static const struct resource imx27_audmux_res[] __initconst = { void __init imx27_soc_init(void) { +#ifdef CONFIG_IPIPE + volatile unsigned long aips_reg; + void __iomem *aips_virt; +#endif mxc_arch_reset_init(MX27_IO_ADDRESS(MX27_WDOG_BASE_ADDR)); mxc_device_init(); @@ -95,6 +99,15 @@ void __init imx27_soc_init(void) pinctrl_provide_dummies(); imx_add_imx_dma("imx27-dma", MX27_DMA_BASE_ADDR, MX27_INT_DMACH0, 0); /* No ERR irq */ + + /* Setup AIPS register */ +#ifdef CONFIG_IPIPE + aips_virt = (void __iomem *)MX27_IO_P2V(MX27_AIPI_BASE_ADDR); + aips_reg = __raw_readl(aips_virt + 8); + aips_reg &= ~(1 << 3); + __raw_writel(aips_reg, aips_virt); +#endif + /* imx27 has the imx21 type audmux */ platform_device_register_simple("imx21-audmux", 0, imx27_audmux_res, ARRAY_SIZE(imx27_audmux_res)); diff --git a/arch/arm/mach-imx/mm-imx3.c b/arch/arm/mach-imx/mm-imx3.c index 0884ca9..9553126 100644 --- a/arch/arm/mach-imx/mm-imx3.c +++ b/arch/arm/mach-imx/mm-imx3.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -139,7 +140,11 @@ void __init imx31_init_early(void) { mxc_set_cpu_type(MXC_CPU_MX31); arch_ioremap_caller = imx3_ioremap_caller; +#ifdef CONFIG_IPIPE + cpu_idle_poll_ctrl(true); +#else /* !CONFIG_IPIPE */ arm_pm_idle = imx3_idle; +#endif /* !CONFIG_IPIPE */ mx3_ccm_base = MX31_IO_ADDRESS(MX31_CCM_BASE_ADDR); } @@ -216,7 +221,11 @@ void __init imx35_init_early(void) { mxc_set_cpu_type(MXC_CPU_MX35); mxc_iomux_v3_init(MX35_IO_ADDRESS(MX35_IOMUXC_BASE_ADDR)); +#ifdef CONFIG_IPIPE + cpu_idle_poll_ctrl(true); +#else /* !CONFIG_IPIPE */ arm_pm_idle = imx3_idle; +#endif /* !CONFIG_IPIPE */ arch_ioremap_caller = imx3_ioremap_caller; mx3_ccm_base = MX35_IO_ADDRESS(MX35_CCM_BASE_ADDR); } diff --git a/arch/arm/mach-imx/mm-imx5.c b/arch/arm/mach-imx/mm-imx5.c new file mode 100644 index 0000000..2172f98 --- /dev/null +++ b/arch/arm/mach-imx/mm-imx5.c @@ -0,0 +1,165 @@ +/* + * Copyright 2008-2010 Freescale Semiconductor, Inc. All Rights Reserved. + * + * The code contained herein is licensed under the GNU General Public + * License. You may obtain a copy of the GNU General Public License + * Version 2 or later at the following locations: + * + * http://www.opensource.org/licenses/gpl-license.html + * http://www.gnu.org/copyleft/gpl.html + * + * Create static mapping between physical to virtual memory. + */ + +#include +#include +#include +#include +#include + +#include + +#include "common.h" +#include "devices/devices-common.h" +#include "hardware.h" +#include "iomux-v3.h" + +/* + * Define the MX51 memory map. + */ +static struct map_desc mx51_io_desc[] __initdata = { + imx_map_entry(MX51, TZIC, MT_DEVICE), + imx_map_entry(MX51, IRAM, MT_DEVICE), + imx_map_entry(MX51, AIPS1, MT_DEVICE), + imx_map_entry(MX51, SPBA0, MT_DEVICE), + imx_map_entry(MX51, AIPS2, MT_DEVICE), +}; + +/* + * Define the MX53 memory map. + */ +static struct map_desc mx53_io_desc[] __initdata = { + imx_map_entry(MX53, TZIC, MT_DEVICE), + imx_map_entry(MX53, AIPS1, MT_DEVICE), + imx_map_entry(MX53, SPBA0, MT_DEVICE), + imx_map_entry(MX53, AIPS2, MT_DEVICE), +}; + +/* + * This function initializes the memory map. It is called during the + * system startup to create static physical to virtual memory mappings + * for the IO modules. + */ +void __init mx51_map_io(void) +{ + iotable_init(mx51_io_desc, ARRAY_SIZE(mx51_io_desc)); +} + +void __init mx53_map_io(void) +{ + iotable_init(mx53_io_desc, ARRAY_SIZE(mx53_io_desc)); +} + +/* + * The MIPI HSC unit has been removed from the i.MX51 Reference Manual by + * the Freescale marketing division. However this did not remove the + * hardware from the chip which still needs to be configured for proper + * IPU support. + */ +static void __init imx51_ipu_mipi_setup(void) +{ + void __iomem *hsc_addr; + hsc_addr = MX51_IO_ADDRESS(MX51_MIPI_HSC_BASE_ADDR); + + /* setup MIPI module to legacy mode */ + __raw_writel(0xf00, hsc_addr); + + /* CSI mode: reserved; DI control mode: legacy (from Freescale BSP) */ + __raw_writel(__raw_readl(hsc_addr + 0x800) | 0x30ff, + hsc_addr + 0x800); +} + +void __init imx51_init_early(void) +{ + imx51_ipu_mipi_setup(); + mxc_set_cpu_type(MXC_CPU_MX51); + mxc_iomux_v3_init(MX51_IO_ADDRESS(MX51_IOMUXC_BASE_ADDR)); + imx_src_init(); +} + +void __init imx53_init_early(void) +{ + mxc_set_cpu_type(MXC_CPU_MX53); + imx_src_init(); +} + +void __init mx51_init_irq(void) +{ + tzic_init_irq(MX51_IO_ADDRESS(MX51_TZIC_BASE_ADDR)); +} + +void __init mx53_init_irq(void) +{ + struct device_node *np; + void __iomem *base; + + np = of_find_compatible_node(NULL, NULL, "fsl,imx53-tzic"); + base = of_iomap(np, 0); + WARN_ON(!base); + + tzic_init_irq(base); +} + +static struct sdma_platform_data imx51_sdma_pdata __initdata = { + .fw_name = "sdma-imx51.bin", +}; + +static const struct resource imx51_audmux_res[] __initconst = { + DEFINE_RES_MEM(MX51_AUDMUX_BASE_ADDR, SZ_16K), +}; + +void __init imx51_soc_init(void) +{ + mxc_arch_reset_init(MX51_IO_ADDRESS(MX51_WDOG1_BASE_ADDR)); + mxc_device_init(); + + /* i.mx51 has the i.mx35 type gpio */ + mxc_register_gpio("imx35-gpio", 0, MX51_GPIO1_BASE_ADDR, SZ_16K, MX51_INT_GPIO1_LOW, MX51_INT_GPIO1_HIGH); + mxc_register_gpio("imx35-gpio", 1, MX51_GPIO2_BASE_ADDR, SZ_16K, MX51_INT_GPIO2_LOW, MX51_INT_GPIO2_HIGH); + mxc_register_gpio("imx35-gpio", 2, MX51_GPIO3_BASE_ADDR, SZ_16K, MX51_INT_GPIO3_LOW, MX51_INT_GPIO3_HIGH); + mxc_register_gpio("imx35-gpio", 3, MX51_GPIO4_BASE_ADDR, SZ_16K, MX51_INT_GPIO4_LOW, MX51_INT_GPIO4_HIGH); + + pinctrl_provide_dummies(); + + /* i.mx51 has the i.mx35 type sdma */ + imx_add_imx_sdma("imx35-sdma", MX51_SDMA_BASE_ADDR, MX51_INT_SDMA, &imx51_sdma_pdata); + + /* Setup AIPS registers */ + imx_set_aips(MX51_IO_ADDRESS(MX51_AIPS1_BASE_ADDR)); + imx_set_aips(MX51_IO_ADDRESS(MX51_AIPS2_BASE_ADDR)); + + /* i.mx51 has the i.mx31 type audmux */ + platform_device_register_simple("imx31-audmux", 0, imx51_audmux_res, + ARRAY_SIZE(imx51_audmux_res)); +} + +void __init imx51_init_late(void) +{ + mx51_neon_fixup(); +#ifdef CONFIG_IPIPE + /* Allow user-space access to emulated tsc */ + imx_set_aips(MX51_IO_ADDRESS(MX51_AIPS1_BASE_ADDR)); + imx_set_aips(MX51_IO_ADDRESS(MX51_AIPS2_BASE_ADDR)); +#endif + imx5_pm_init(); +} + +void __init imx53_init_late(void) +{ +#ifdef CONFIG_IPIPE + /* Allow user-space access to emulated tsc */ + imx_set_aips(MX51_IO_ADDRESS(MX53_AIPS1_BASE_ADDR)); + imx_set_aips(MX51_IO_ADDRESS(MX53_AIPS2_BASE_ADDR)); +#endif + imx5_pm_init(); +} diff --git a/arch/arm/mach-imx/time.c b/arch/arm/mach-imx/time.c index 15d18e1..0d12e51 100644 --- a/arch/arm/mach-imx/time.c +++ b/arch/arm/mach-imx/time.c @@ -31,6 +31,8 @@ #include #include #include +#include +#include #include @@ -83,6 +85,7 @@ static struct clock_event_device clockevent_mxc; static enum clock_event_mode clockevent_mode = CLOCK_EVT_MODE_UNUSED; static void __iomem *timer_base; +static unsigned long timer_base_phys; static inline void gpt_irq_disable(void) { @@ -248,12 +251,8 @@ static void mxc_set_mode(enum clock_event_mode mode, } } -/* - * IRQ handler for the timer - */ -static irqreturn_t mxc_timer_interrupt(int irq, void *dev_id) +static inline void mxc_timer_ack(void) { - struct clock_event_device *evt = &clockevent_mxc; uint32_t tstat; if (timer_is_v2()) @@ -262,12 +261,38 @@ static irqreturn_t mxc_timer_interrupt(int irq, void *dev_id) tstat = __raw_readl(timer_base + MX1_2_TSTAT); gpt_irq_acknowledge(); +} + +/* + * IRQ handler for the timer + */ +static irqreturn_t mxc_timer_interrupt(int irq, void *dev_id) +{ + struct clock_event_device *evt = &clockevent_mxc; + + if (!clockevent_ipipe_stolen(evt)) + mxc_timer_ack(); evt->event_handler(evt); return IRQ_HANDLED; } +#ifdef CONFIG_IPIPE +static struct __ipipe_tscinfo tsc_info = { + .type = IPIPE_TSC_TYPE_FREERUNNING, + .u = { + { + .mask = 0xffffffff, + }, + }, +}; + +static struct ipipe_timer mxc_itimer = { + .ack = mxc_timer_ack, +}; +#endif + static struct irqaction mxc_timer_irq = { .name = "i.MX Timer Tick", .flags = IRQF_TIMER | IRQF_IRQPOLL, @@ -280,6 +305,9 @@ static struct clock_event_device clockevent_mxc = { .set_mode = mxc_set_mode, .set_next_event = mx1_2_set_next_event, .rating = 200, +#ifdef CONFIG_IPIPE + .ipipe_timer = &mxc_itimer, +#endif }; static int __init mxc_clockevent_init(struct clk *timer_clk) @@ -338,18 +366,40 @@ static void __init _mxc_timer_init(int irq, /* init and register the timer to the framework */ mxc_clocksource_init(clk_per); +#ifdef CONFIG_IPIPE + { + unsigned long phys = timer_base_phys; + unsigned long virt = (unsigned long)timer_base; + + if (timer_is_v1()) { + tsc_info.u.counter_paddr = phys + MX1_2_TCN; + tsc_info.counter_vaddr = virt + MX1_2_TCN; + } else { + tsc_info.u.counter_paddr = phys + V2_TCN; + tsc_info.counter_vaddr = virt + V2_TCN; + } + tsc_info.freq = clk_get_rate(clk_per); + __ipipe_tsc_register(&tsc_info); + } + + mxc_itimer.irq = irq; + mxc_itimer.freq = clk_get_rate(clk_per); + mxc_itimer.min_delay_ticks = ipipe_timer_ns2ticks(&mxc_itimer, 2000); +#endif /* CONFIG_IPIPE */ mxc_clockevent_init(clk_per); /* Make irqs happen */ setup_irq(irq, &mxc_timer_irq); } -void __init mxc_timer_init(void __iomem *base, int irq) +void __init +mxc_timer_init(void __iomem *base, unsigned long phys, int irq) { struct clk *clk_per = clk_get_sys("imx-gpt.0", "per"); struct clk *clk_ipg = clk_get_sys("imx-gpt.0", "ipg"); timer_base = base; + timer_base_phys = phys; _mxc_timer_init(irq, clk_per, clk_ipg); } @@ -357,6 +407,7 @@ void __init mxc_timer_init(void __iomem *base, int irq) static void __init mxc_timer_init_dt(struct device_node *np) { struct clk *clk_per, *clk_ipg; + struct resource res; int irq; if (timer_base) @@ -366,6 +417,11 @@ static void __init mxc_timer_init_dt(struct device_node *np) WARN_ON(!timer_base); irq = irq_of_parse_and_map(np, 0); + if (of_address_to_resource(np, 0, &res)) + res.start = 0; + + timer_base_phys = res.start; + clk_ipg = of_clk_get_by_name(np, "ipg"); /* Try osc_per first, and fall back to per otherwise */ diff --git a/arch/arm/mach-imx/tzic.c b/arch/arm/mach-imx/tzic.c index 4de65ee..8828773 100644 --- a/arch/arm/mach-imx/tzic.c +++ b/arch/arm/mach-imx/tzic.c @@ -117,6 +117,9 @@ static __init void tzic_init_gc(int idx, unsigned int irq_start) ct = gc->chip_types; ct->chip.irq_mask = irq_gc_mask_disable_reg; ct->chip.irq_unmask = irq_gc_unmask_enable_reg; +#ifdef CONFIG_IPIPE + ct->chip.irq_mask_ack = irq_gc_mask_disable_reg; +#endif /* CONFIG_IPIPE */ ct->chip.irq_set_wake = irq_gc_set_wake; ct->chip.irq_suspend = tzic_irq_suspend; ct->chip.irq_resume = tzic_irq_resume; @@ -141,13 +144,34 @@ static void __exception_irq_entry tzic_handle_irq(struct pt_regs *regs) while (stat) { handled = 1; irqofs = fls(stat) - 1; - handle_domain_irq(domain, irqofs + i * 32, regs); + ipipe_handle_domain_irq(domain, irqofs + i * 32, regs); stat &= ~(1 << irqofs); } } } while (handled); } + +#if defined(CONFIG_IPIPE) +void tzic_set_irq_prio(unsigned irq, unsigned hi) +{ + if (irq >= TZIC_NUM_IRQS) + return; + + __raw_writeb(hi ? 0 : 0x80, tzic_base + TZIC_PRIORITY0 + irq); +} + +void tzic_mute_pic(void) +{ + __raw_writel(0x10, tzic_base + TZIC_PRIOMASK); +} + +void tzic_unmute_pic(void) +{ + __raw_writel(0xf0, tzic_base + TZIC_PRIOMASK); +} +#endif /* CONFIG_IPIPE */ + /* * This function initializes the TZIC hardware and disables all the * interrupts. It registers the interrupt enable and disable functions @@ -169,8 +193,13 @@ void __init tzic_init_irq(void) i = __raw_readl(tzic_base + TZIC_INTCNTL); __raw_writel(0x80010001, tzic_base + TZIC_INTCNTL); +#ifndef CONFIG_IPIPE __raw_writel(0x1f, tzic_base + TZIC_PRIOMASK); __raw_writel(0x02, tzic_base + TZIC_SYNCCTRL); +#else + __raw_writel(0xf0, tzic_base + TZIC_PRIOMASK); + __raw_writel(0, tzic_base + TZIC_SYNCCTRL); +#endif for (i = 0; i < 4; i++) __raw_writel(0xFFFFFFFF, tzic_base + TZIC_INTSEC0(i)); diff --git a/arch/arm/mach-integrator/core.c b/arch/arm/mach-integrator/core.c index 948872a..ab58ce9 100644 --- a/arch/arm/mach-integrator/core.c +++ b/arch/arm/mach-integrator/core.c @@ -2,6 +2,7 @@ * linux/arch/arm/mach-integrator/core.c * * Copyright (C) 2000-2003 Deep Blue Solutions Ltd + * Copyright (C) 2005 Stelian Pop. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 2, as diff --git a/arch/arm/mach-integrator/integrator_cp.c b/arch/arm/mach-integrator/integrator_cp.c index b5fb71a..40edcef 100644 --- a/arch/arm/mach-integrator/integrator_cp.c +++ b/arch/arm/mach-integrator/integrator_cp.c @@ -2,6 +2,7 @@ * linux/arch/arm/mach-integrator/integrator_cp.c * * Copyright (C) 2003 Deep Blue Solutions Ltd + * Copyright (C) 2005 Stelian Pop. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/arch/arm/mach-ixp4xx/common.c b/arch/arm/mach-ixp4xx/common.c index 8537d4c..a6c44fc 100644 --- a/arch/arm/mach-ixp4xx/common.c +++ b/arch/arm/mach-ixp4xx/common.c @@ -6,10 +6,10 @@ * Maintainer: Deepak Saxena * * Copyright 2002 (c) Intel Corporation - * Copyright 2003-2004 (c) MontaVista, Software, Inc. - * - * This file is licensed under the terms of the GNU General Public - * License version 2. This program is licensed "as is" without any + * Copyright 2003-2004 (c) MontaVista, Software, Inc. + * + * This file is licensed under the terms of the GNU General Public + * License version 2. This program is licensed "as is" without any * warranty of any kind, whether express or implied. */ @@ -31,6 +31,8 @@ #include #include #include +#include +#include #include #include #include @@ -98,7 +100,7 @@ void __init ixp4xx_map_io(void) */ /* GPIO pin types */ #define IXP4XX_GPIO_OUT 0x1 -#define IXP4XX_GPIO_IN 0x2 +#define IXP4XX_GPIO_IN 0x2 /* GPIO signal types */ #define IXP4XX_GPIO_LOW 0 @@ -110,10 +112,14 @@ void __init ixp4xx_map_io(void) static void gpio_line_config(u8 line, u32 direction) { + unsigned long flags; + + flags = hard_local_irq_save(); if (direction == IXP4XX_GPIO_IN) *IXP4XX_GPIO_GPOER |= (1 << line); else *IXP4XX_GPIO_GPOER &= ~(1 << line); + hard_local_irq_restore(flags); } static void gpio_line_get(u8 line, int *value) @@ -123,10 +129,14 @@ static void gpio_line_get(u8 line, int *value) static void gpio_line_set(u8 line, int value) { + unsigned long flags; + + flags = hard_local_irq_save(); if (value == IXP4XX_GPIO_HIGH) *IXP4XX_GPIO_GPOUTR |= (1 << line); else if (value == IXP4XX_GPIO_LOW) *IXP4XX_GPIO_GPOUTR &= ~(1 << line); + hard_local_irq_restore(flags); } /************************************************************************* @@ -282,7 +292,7 @@ void __init ixp4xx_init_irq(void) *IXP4XX_ICLR = 0x0; /* Disable all interrupt */ - *IXP4XX_ICMR = 0x0; + *IXP4XX_ICMR = 0x0; if (cpu_is_ixp46x() || cpu_is_ixp43x()) { /* Route upper 32 sources to IRQ instead of FIQ */ @@ -292,7 +302,7 @@ void __init ixp4xx_init_irq(void) *IXP4XX_ICMR2 = 0x00; } - /* Default to all level triggered */ + /* Default to all level triggered */ for(i = 0; i < NR_IRQS; i++) { irq_set_chip_and_handler(i, &ixp4xx_irq_chip, handle_level_irq); @@ -300,10 +310,15 @@ void __init ixp4xx_init_irq(void) } } +static inline void ixp4xx_timer_ack(void) +{ + /* Clear Pending Interrupt by writing '1' to it */ + *IXP4XX_OSST = IXP4XX_OSST_TIMER_1_PEND; +} /************************************************************************* * IXP4xx timer tick - * We use OS timer1 on the CPU for the timer tick and the timestamp + * We use OS timer1 on the CPU for the timer tick and the timestamp * counter as a source of real clock ticks to account for missed jiffies. *************************************************************************/ @@ -311,8 +326,8 @@ static irqreturn_t ixp4xx_timer_interrupt(int irq, void *dev_id) { struct clock_event_device *evt = dev_id; - /* Clear Pending Interrupt by writing '1' to it */ - *IXP4XX_OSST = IXP4XX_OSST_TIMER_1_PEND; + if (!clockevent_ipipe_stolen(evt)) + ixp4xx_timer_ack(); evt->event_handler(evt); @@ -500,12 +515,31 @@ static cycle_t ixp4xx_clocksource_read(struct clocksource *c) unsigned long ixp4xx_timer_freq = IXP4XX_TIMER_FREQ; EXPORT_SYMBOL(ixp4xx_timer_freq); + +#ifdef CONFIG_IPIPE +static struct __ipipe_tscinfo tsc_info = { + .type = IPIPE_TSC_TYPE_FREERUNNING, + .freq = IXP4XX_TIMER_FREQ, + .counter_vaddr = (unsigned long)IXP4XX_OSTS, + .u = { + { + .mask = 0xffffffff, + .counter_paddr = IXP4XX_TIMER_BASE_PHYS, + }, + }, +}; +#endif /* CONFIG_IPIPE */ + static void __init ixp4xx_clocksource_init(void) { sched_clock_register(ixp4xx_read_sched_clock, 32, ixp4xx_timer_freq); clocksource_mmio_init(NULL, "OSTS", ixp4xx_timer_freq, 200, 32, ixp4xx_clocksource_read); + +#ifdef CONFIG_IPIPE + __ipipe_tsc_register(&tsc_info); +#endif } /* @@ -552,12 +586,23 @@ static void ixp4xx_set_mode(enum clock_event_mode mode, *IXP4XX_OSRT1 = osrt | opts; } +#ifdef CONFIG_IPIPE +static struct ipipe_timer ixp4xx_itimer = { + .irq = IRQ_IXP4XX_TIMER1, + .min_delay_ticks = 333, /* 5 usec with the 66.66 MHz system clock */ + .ack = ixp4xx_timer_ack, +}; +#endif /* CONFIG_IPIPE */ + static struct clock_event_device clockevent_ixp4xx = { .name = "ixp4xx timer1", .features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT, .rating = 200, .set_mode = ixp4xx_set_mode, .set_next_event = ixp4xx_set_next_event, +#ifdef CONFIG_IPIPE + .ipipe_timer = &ixp4xx_itimer, +#endif /* CONFIG_IPIPE */ }; static void __init ixp4xx_clockevent_init(void) diff --git a/arch/arm/mach-ixp4xx/include/mach/platform.h b/arch/arm/mach-ixp4xx/include/mach/platform.h index 75c4c65..a02a406 100644 --- a/arch/arm/mach-ixp4xx/include/mach/platform.h +++ b/arch/arm/mach-ixp4xx/include/mach/platform.h @@ -74,8 +74,8 @@ extern unsigned long ixp4xx_exp_bus_size; /* * Clock Speed Definitions. */ -#define IXP4XX_PERIPHERAL_BUS_CLOCK (66) /* 66Mhzi APB BUS */ -#define IXP4XX_UART_XTAL 14745600 +#define IXP4XX_PERIPHERAL_BUS_CLOCK (66) /* 66Mhzi APB BUS */ +#define IXP4XX_UART_XTAL 14745600 /* * This structure provide a means for the board setup code @@ -132,4 +132,3 @@ extern int ixp4xx_setup(int nr, struct pci_sys_data *sys); extern struct pci_ops ixp4xx_ops; #endif // __ASSEMBLY__ - diff --git a/arch/arm/mach-mxs/Kconfig b/arch/arm/mach-mxs/Kconfig index 8479413..01c3790 100644 --- a/arch/arm/mach-mxs/Kconfig +++ b/arch/arm/mach-mxs/Kconfig @@ -11,6 +11,7 @@ config SOC_IMX28 select ARM_CPU_SUSPEND if PM select CPU_ARM926T select PINCTRL_IMX28 + select IPIPE_ARM_KUSER_TSC if IPIPE config ARCH_MXS bool "Freescale MXS (i.MX23, i.MX28) support" diff --git a/arch/arm/mach-omap2/Kconfig b/arch/arm/mach-omap2/Kconfig index 6468f15..8133f8d 100644 --- a/arch/arm/mach-omap2/Kconfig +++ b/arch/arm/mach-omap2/Kconfig @@ -35,6 +35,7 @@ config ARCH_OMAP4 select PM if CPU_IDLE select ARM_ERRATA_754322 select ARM_ERRATA_775420 + select ARM_GLOBAL_TIMER if IPIPE && SMP config SOC_OMAP5 bool "TI OMAP5" diff --git a/arch/arm/mach-omap2/io.c b/arch/arm/mach-omap2/io.c index 820dde8..c5c4bef 100644 --- a/arch/arm/mach-omap2/io.c +++ b/arch/arm/mach-omap2/io.c @@ -21,9 +21,11 @@ #include #include #include +#include #include #include +#include #include @@ -460,6 +462,9 @@ void __init omap3_init_early(void) omap3xxx_clockdomains_init(); omap3xxx_hwmod_init(); omap_hwmod_init_postsetup(); +#ifdef CONFIG_IPIPE + cpu_idle_poll_ctrl(true); +#endif if (!of_have_populated_dt()) { omap3_control_legacy_iomap_init(); if (soc_is_am35xx()) @@ -557,6 +562,9 @@ void __init ti814x_init_early(void) ti81xx_clockdomains_init(); ti81xx_hwmod_init(); omap_hwmod_init_postsetup(); +#ifdef CONFIG_IPIPE + cpu_idle_poll_ctrl(true); +#endif if (of_have_populated_dt()) omap_clk_soc_init = ti81xx_dt_clk_init; } diff --git a/arch/arm/mach-omap2/irq.c b/arch/arm/mach-omap2/irq.c new file mode 100644 index 0000000..1b2441b --- /dev/null +++ b/arch/arm/mach-omap2/irq.c @@ -0,0 +1,427 @@ +/* + * linux/arch/arm/mach-omap2/irq.c + * + * Interrupt handler for OMAP2 boards. + * + * Copyright (C) 2005 Nokia Corporation + * Author: Paul Mundt + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + */ +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "soc.h" +#include "iomap.h" +#include "common.h" + +/* selected INTC register offsets */ + +#define INTC_REVISION 0x0000 +#define INTC_SYSCONFIG 0x0010 +#define INTC_SYSSTATUS 0x0014 +#define INTC_SIR 0x0040 +#define INTC_CONTROL 0x0048 +#define INTC_PROTECTION 0x004C +#define INTC_IDLE 0x0050 +#define INTC_THRESHOLD 0x0068 +#define INTC_MIR0 0x0084 +#define INTC_MIR_CLEAR0 0x0088 +#define INTC_MIR_SET0 0x008c +#define INTC_PENDING_IRQ0 0x0098 +#define INTC_PRIO 0x0100 +/* Number of IRQ state bits in each MIR register */ +#define IRQ_BITS_PER_REG 32 + +#define OMAP2_IRQ_BASE OMAP2_L4_IO_ADDRESS(OMAP24XX_IC_BASE) +#define OMAP3_IRQ_BASE OMAP2_L4_IO_ADDRESS(OMAP34XX_IC_BASE) +#define INTCPS_SIR_IRQ_OFFSET 0x0040 /* omap2/3 active interrupt offset */ +#define ACTIVEIRQ_MASK 0x7f /* omap2/3 active interrupt bits */ +#define INTCPS_NR_MIR_REGS 3 +#define INTCPS_NR_IRQS 96 + +#if !defined(MULTI_OMAP1) && !defined(MULTI_OMAP2) +#define inline_single inline +#else +#define inline_single +#endif + +/* + * OMAP2 has a number of different interrupt controllers, each interrupt + * controller is identified as its own "bank". Register definitions are + * fairly consistent for each bank, but not all registers are implemented + * for each bank.. when in doubt, consult the TRM. + */ +static struct omap_irq_bank { + void __iomem *base_reg; + unsigned int nr_irqs; +} __attribute__ ((aligned(4))) irq_banks[] = { + { + /* MPU INTC */ + .nr_irqs = 96, + }, +}; + +static struct irq_domain *domain; + +/* Structure to save interrupt controller context */ +struct omap3_intc_regs { + u32 sysconfig; + u32 protection; + u32 idle; + u32 threshold; + u32 ilr[INTCPS_NR_IRQS]; + u32 mir[INTCPS_NR_MIR_REGS]; +}; + +/* INTC bank register get/set */ + +static inline_single void intc_bank_write_reg(u32 val, struct omap_irq_bank *bank, u16 reg) +{ + writel_relaxed(val, bank->base_reg + reg); +} + +static inline_single u32 intc_bank_read_reg(struct omap_irq_bank *bank, u16 reg) +{ + return readl_relaxed(bank->base_reg + reg); +} + +/* XXX: FIQ and additional INTC support (only MPU at the moment) */ +static inline_single void omap_ack_irq(struct irq_data *d) +{ + intc_bank_write_reg(0x1, &irq_banks[0], INTC_CONTROL); + dsb(); +} + +static void omap_mask_ack_irq(struct irq_data *d) +{ + irq_gc_mask_disable_reg(d); + omap_ack_irq(d); +} + +static void __init omap_irq_bank_init_one(struct omap_irq_bank *bank) +{ + unsigned long tmp; + + tmp = intc_bank_read_reg(bank, INTC_REVISION) & 0xff; + pr_info("IRQ: Found an INTC at 0x%p (revision %ld.%ld) with %d interrupts\n", + bank->base_reg, tmp >> 4, tmp & 0xf, bank->nr_irqs); + + tmp = intc_bank_read_reg(bank, INTC_SYSCONFIG); + tmp |= 1 << 1; /* soft reset */ + intc_bank_write_reg(tmp, bank, INTC_SYSCONFIG); + + while (!(intc_bank_read_reg(bank, INTC_SYSSTATUS) & 0x1)) + /* Wait for reset to complete */; + +#ifndef CONFIG_IPIPE + /* Enable autoidle */ + intc_bank_write_reg(1 << 0, bank, INTC_SYSCONFIG); + intc_bank_write_reg(0x2, bank, INTC_IDLE); +#else /* CONFIG_IPIPE */ + /* Disable autoidle */ + intc_bank_write_reg(0, bank, INTC_SYSCONFIG); + intc_bank_write_reg(0x1, bank, INTC_IDLE); +#endif /* CONFIG_IPIPE */ +} + +int omap_irq_pending(void) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(irq_banks); i++) { + struct omap_irq_bank *bank = irq_banks + i; + int irq; + + for (irq = 0; irq < bank->nr_irqs; irq += 32) + if (intc_bank_read_reg(bank, INTC_PENDING_IRQ0 + + ((irq >> 5) << 5))) + return 1; + } + return 0; +} + +static __init void +omap_alloc_gc(void __iomem *base, unsigned int irq_start, unsigned int num) +{ + struct irq_chip_generic *gc; + struct irq_chip_type *ct; + + gc = irq_alloc_generic_chip("INTC", 1, irq_start, base, + handle_level_irq); + ct = gc->chip_types; + ct->chip.irq_ack = omap_mask_ack_irq; + ct->chip.irq_mask = irq_gc_mask_disable_reg; +#ifdef CONFIG_IPIPE + ct->chip.irq_mask_ack = omap_mask_ack_irq; +#endif + ct->chip.irq_unmask = irq_gc_unmask_enable_reg; + ct->chip.flags |= IRQCHIP_SKIP_SET_WAKE; + + ct->regs.enable = INTC_MIR_CLEAR0; + ct->regs.disable = INTC_MIR_SET0; + irq_setup_generic_chip(gc, IRQ_MSK(num), IRQ_GC_INIT_MASK_CACHE, + IRQ_NOREQUEST | IRQ_NOPROBE, 0); +} + +static void __init omap_init_irq(u32 base, int nr_irqs, + struct device_node *node) +{ + void __iomem *omap_irq_base; + unsigned long nr_of_irqs = 0; + unsigned int nr_banks = 0; + int i, j, irq_base; + + omap_irq_base = ioremap(base, SZ_4K); + if (WARN_ON(!omap_irq_base)) + return; + + irq_base = irq_alloc_descs(-1, 0, nr_irqs, 0); + if (irq_base < 0) { + pr_warn("Couldn't allocate IRQ numbers\n"); + irq_base = 0; + } + + domain = irq_domain_add_legacy(node, nr_irqs, irq_base, 0, + &irq_domain_simple_ops, NULL); + + for (i = 0; i < ARRAY_SIZE(irq_banks); i++) { + struct omap_irq_bank *bank = irq_banks + i; + + bank->nr_irqs = nr_irqs; + + /* Static mapping, never released */ + bank->base_reg = ioremap(base, SZ_4K); + if (!bank->base_reg) { + pr_err("Could not ioremap irq bank%i\n", i); + continue; + } + + omap_irq_bank_init_one(bank); + + for (j = 0; j < bank->nr_irqs; j += 32) + omap_alloc_gc(bank->base_reg + j, j + irq_base, 32); + + nr_of_irqs += bank->nr_irqs; + nr_banks++; + } + + pr_info("Total of %ld interrupts on %d active controller%s\n", + nr_of_irqs, nr_banks, nr_banks > 1 ? "s" : ""); +} + +void __init omap2_init_irq(void) +{ + omap_init_irq(OMAP24XX_IC_BASE, 96, NULL); +} + +void __init omap3_init_irq(void) +{ + omap_init_irq(OMAP34XX_IC_BASE, 96, NULL); +} + +void __init ti81xx_init_irq(void) +{ + omap_init_irq(OMAP34XX_IC_BASE, 128, NULL); +} + +static inline void omap_intc_handle_irq(void __iomem *base_addr, struct pt_regs *regs) +{ + u32 irqnr; + int handled_irq = 0; + + do { + irqnr = readl_relaxed(base_addr + 0x98); + if (irqnr) + goto out; + + irqnr = readl_relaxed(base_addr + 0xb8); + if (irqnr) + goto out; + + irqnr = readl_relaxed(base_addr + 0xd8); +#if IS_ENABLED(CONFIG_SOC_TI81XX) || IS_ENABLED(CONFIG_SOC_AM33XX) + if (irqnr) + goto out; + irqnr = readl_relaxed(base_addr + 0xf8); +#endif + +out: + if (!irqnr) + break; + + irqnr = readl_relaxed(base_addr + INTCPS_SIR_IRQ_OFFSET); + irqnr &= ACTIVEIRQ_MASK; + + if (irqnr) { + ipipe_handle_domain_irq(domain, irqnr, regs); + handled_irq = 1; + } + } while (irqnr); + + /* If an irq is masked or deasserted while active, we will + * keep ending up here with no irq handled. So remove it from + * the INTC with an ack.*/ + if (!handled_irq) + omap_ack_irq(NULL); +} + +asmlinkage void __exception_irq_entry omap2_intc_handle_irq(struct pt_regs *regs) +{ + void __iomem *base_addr = OMAP2_IRQ_BASE; + omap_intc_handle_irq(base_addr, regs); +} + +int __init intc_of_init(struct device_node *node, + struct device_node *parent) +{ + struct resource res; + u32 nr_irq = 96; + + if (WARN_ON(!node)) + return -ENODEV; + + if (of_address_to_resource(node, 0, &res)) { + WARN(1, "unable to get intc registers\n"); + return -EINVAL; + } + + if (of_property_read_u32(node, "ti,intc-size", &nr_irq)) + pr_warn("unable to get intc-size, default to %d\n", nr_irq); + + omap_init_irq(res.start, nr_irq, of_node_get(node)); + + return 0; +} + +static struct of_device_id irq_match[] __initdata = { + { .compatible = "ti,omap2-intc", .data = intc_of_init, }, + { } +}; + +void __init omap_intc_of_init(void) +{ + of_irq_init(irq_match); +} + + +#if defined(CONFIG_IPIPE) && defined(CONFIG_ARCH_OMAP2PLUS) +#if defined(CONFIG_ARCH_OMAP3) || defined(CONFIG_SOC_AM33XX) +void omap3_intc_mute(void) +{ + struct omap_irq_bank *bank = &irq_banks[0]; + + intc_bank_write_reg(0x1, bank, INTC_THRESHOLD); + intc_bank_write_reg(0x1, bank, INTC_CONTROL); +} + +void omap3_intc_unmute(void) +{ + struct omap_irq_bank *bank = &irq_banks[0]; + + intc_bank_write_reg(0xff, bank, INTC_THRESHOLD); +} + +void omap3_intc_set_irq_prio(int irq, int hi) +{ + struct omap_irq_bank *bank = &irq_banks[0]; + + if (irq >= INTCPS_NR_MIR_REGS * 32) + return; + intc_bank_write_reg(hi ? 0 : 0xfc, bank, INTC_PRIO + 4 * irq); +} +#endif /* CONFIG_ARCH_OMAP3 */ +#endif /* CONFIG_IPIPE && ARCH_OMAP2PLUS */ + +#if defined(CONFIG_ARCH_OMAP3) || defined(CONFIG_SOC_AM33XX) +static struct omap3_intc_regs intc_context[ARRAY_SIZE(irq_banks)]; + +void omap_intc_save_context(void) +{ + int ind = 0, i = 0; + for (ind = 0; ind < ARRAY_SIZE(irq_banks); ind++) { + struct omap_irq_bank *bank = irq_banks + ind; + intc_context[ind].sysconfig = + intc_bank_read_reg(bank, INTC_SYSCONFIG); + intc_context[ind].protection = + intc_bank_read_reg(bank, INTC_PROTECTION); + intc_context[ind].idle = + intc_bank_read_reg(bank, INTC_IDLE); + intc_context[ind].threshold = + intc_bank_read_reg(bank, INTC_THRESHOLD); + for (i = 0; i < INTCPS_NR_IRQS; i++) + intc_context[ind].ilr[i] = + intc_bank_read_reg(bank, (0x100 + 0x4*i)); + for (i = 0; i < INTCPS_NR_MIR_REGS; i++) + intc_context[ind].mir[i] = + intc_bank_read_reg(&irq_banks[0], INTC_MIR0 + + (0x20 * i)); + } +} + +void omap_intc_restore_context(void) +{ + int ind = 0, i = 0; + + for (ind = 0; ind < ARRAY_SIZE(irq_banks); ind++) { + struct omap_irq_bank *bank = irq_banks + ind; + intc_bank_write_reg(intc_context[ind].sysconfig, + bank, INTC_SYSCONFIG); + intc_bank_write_reg(intc_context[ind].sysconfig, + bank, INTC_SYSCONFIG); + intc_bank_write_reg(intc_context[ind].protection, + bank, INTC_PROTECTION); + intc_bank_write_reg(intc_context[ind].idle, + bank, INTC_IDLE); + intc_bank_write_reg(intc_context[ind].threshold, + bank, INTC_THRESHOLD); + for (i = 0; i < INTCPS_NR_IRQS; i++) + intc_bank_write_reg(intc_context[ind].ilr[i], + bank, (0x100 + 0x4*i)); + for (i = 0; i < INTCPS_NR_MIR_REGS; i++) + intc_bank_write_reg(intc_context[ind].mir[i], + &irq_banks[0], INTC_MIR0 + (0x20 * i)); + } + /* MIRs are saved and restore with other PRCM registers */ +} + +void omap3_intc_suspend(void) +{ + /* A pending interrupt would prevent OMAP from entering suspend */ + omap_ack_irq(NULL); +} + +void omap3_intc_prepare_idle(void) +{ + /* + * Disable autoidle as it can stall interrupt controller, + * cf. errata ID i540 for 3430 (all revisions up to 3.1.x) + */ + intc_bank_write_reg(0, &irq_banks[0], INTC_SYSCONFIG); +} + +void omap3_intc_resume_idle(void) +{ + /* Re-enable autoidle */ + intc_bank_write_reg(1, &irq_banks[0], INTC_SYSCONFIG); +} + +asmlinkage void __exception_irq_entry omap3_intc_handle_irq(struct pt_regs *regs) +{ + void __iomem *base_addr = OMAP3_IRQ_BASE; + omap_intc_handle_irq(base_addr, regs); +} +#endif /* CONFIG_ARCH_OMAP3 */ diff --git a/arch/arm/mach-omap2/mux.c b/arch/arm/mach-omap2/mux.c index 176eef6..8d05dc3 100644 --- a/arch/arm/mach-omap2/mux.c +++ b/arch/arm/mach-omap2/mux.c @@ -34,6 +34,7 @@ #include #include #include +#include #include "omap_hwmod.h" @@ -394,7 +395,7 @@ static bool omap_hwmod_mux_scan_wakeups(struct omap_hwmod_mux_info *hmux, handled_irqs |= 1 << irq; - generic_handle_irq(mpu_irqs[irq].irq); + ipipe_handle_demuxed_irq(mpu_irqs[irq].irq); } return false; @@ -411,7 +412,7 @@ static int _omap_hwmod_mux_handle_irq(struct omap_hwmod *oh, void *data) if (!oh->mux || !oh->mux->enabled) return 0; if (omap_hwmod_mux_scan_wakeups(oh->mux, oh->mpu_irqs)) - generic_handle_irq(oh->mpu_irqs[0].irq); + ipipe_handle_demuxed_irq(oh->mpu_irqs[0].irq); return 0; } diff --git a/arch/arm/mach-omap2/omap-wakeupgen.c b/arch/arm/mach-omap2/omap-wakeupgen.c index 6833df4..76c334b 100644 --- a/arch/arm/mach-omap2/omap-wakeupgen.c +++ b/arch/arm/mach-omap2/omap-wakeupgen.c @@ -51,7 +51,7 @@ static void __iomem *wakeupgen_base; static void __iomem *sar_base; -static DEFINE_RAW_SPINLOCK(wakeupgen_lock); +static IPIPE_DEFINE_RAW_SPINLOCK(wakeupgen_lock); static unsigned int irq_target_cpu[MAX_IRQS]; static unsigned int irq_banks = DEFAULT_NR_REG_BANKS; static unsigned int max_irqs = DEFAULT_IRQS; @@ -141,6 +141,26 @@ static void wakeupgen_unmask(struct irq_data *d) irq_chip_unmask_parent(d); } +#ifdef CONFIG_IPIPE +static void wakeupgen_hold(struct irq_data *d) +{ + raw_spin_lock(&wakeupgen_lock); + _wakeupgen_clear(d->hwirq, irq_target_cpu[d->hwirq]); + raw_spin_unlock(&wakeupgen_lock); + irq_chip_hold_parent(d); +} + +static void wakeupgen_release(struct irq_data *d) +{ + unsigned long flags; + + raw_spin_lock_irqsave(&wakeupgen_lock, flags); + _wakeupgen_set(d->hwirq, irq_target_cpu[d->hwirq]); + raw_spin_unlock_irqrestore(&wakeupgen_lock, flags); + irq_chip_release_parent(d); +} +#endif + #ifdef CONFIG_HOTPLUG_CPU static DEFINE_PER_CPU(u32 [MAX_NR_REG_BANKS], irqmasks); @@ -391,6 +411,10 @@ static struct irq_chip wakeupgen_chip = { .irq_eoi = irq_chip_eoi_parent, .irq_mask = wakeupgen_mask, .irq_unmask = wakeupgen_unmask, +#ifdef CONFIG_IPIPE + .irq_hold = wakeupgen_hold, + .irq_release = wakeupgen_release, +#endif .irq_retrigger = irq_chip_retrigger_hierarchy, .irq_set_type = irq_chip_set_type_parent, .flags = IRQCHIP_SKIP_SET_WAKE | IRQCHIP_MASK_ON_SUSPEND, diff --git a/arch/arm/mach-omap2/pm34xx.c b/arch/arm/mach-omap2/pm34xx.c index 87b98bf9..ae5a08d 100644 --- a/arch/arm/mach-omap2/pm34xx.c +++ b/arch/arm/mach-omap2/pm34xx.c @@ -298,6 +298,10 @@ void omap_sram_idle(void) static void omap3_pm_idle(void) { +#ifdef CONFIG_IPIPE + BUG(); +#endif /* CONFIG_IPIPE */ + if (omap_irq_pending()) return; diff --git a/arch/arm/mach-omap2/pm44xx.c b/arch/arm/mach-omap2/pm44xx.c index d697cec..09345c0 100644 --- a/arch/arm/mach-omap2/pm44xx.c +++ b/arch/arm/mach-omap2/pm44xx.c @@ -155,7 +155,13 @@ static int __init pwrdms_setup(struct powerdomain *pwrdm, void *unused) */ static void omap_default_idle(void) { + hard_local_irq_disable(); + hard_local_fiq_disable_notrace(); + omap_do_wfi(); + + hard_local_fiq_enable_notrace(); + hard_local_irq_enable(); } /* diff --git a/arch/arm/mach-omap2/prm_common.c b/arch/arm/mach-omap2/prm_common.c index 7add799..71d842f 100644 --- a/arch/arm/mach-omap2/prm_common.c +++ b/arch/arm/mach-omap2/prm_common.c @@ -145,11 +145,11 @@ static void omap_prcm_irq_handler(unsigned int irq, struct irq_desc *desc) /* Serve priority events first */ for_each_set_bit(virtirq, priority_pending, nr_irq) - generic_handle_irq(prcm_irq_setup->base_irq + virtirq); + ipipe_handle_demuxed_irq(prcm_irq_setup->base_irq + virtirq); /* Serve normal events next */ for_each_set_bit(virtirq, pending, nr_irq) - generic_handle_irq(prcm_irq_setup->base_irq + virtirq); + ipipe_handle_demuxed_irq(prcm_irq_setup->base_irq + virtirq); } if (chip->irq_ack) chip->irq_ack(&desc->irq_data); diff --git a/arch/arm/mach-omap2/timer.c b/arch/arm/mach-omap2/timer.c index cac46d8..090584c 100644 --- a/arch/arm/mach-omap2/timer.c +++ b/arch/arm/mach-omap2/timer.c @@ -42,6 +42,9 @@ #include #include #include +#include +#include +#include #include #include @@ -63,11 +66,23 @@ #define INCREMENTER_DENUMERATOR_RELOAD_OFFSET 0x14 #define NUMERATOR_DENUMERATOR_MASK 0xfffff000 +#ifdef CONFIG_IPIPE +void __init omap3_pic_muter_register(void); +void __init omap4_pic_muter_register(void); +#endif /* CONFIG_IPIPE */ + /* Clockevent code */ static struct omap_dm_timer clkev; static struct clock_event_device clockevent_gpt; + +static void omap2_gp_timer_ack(void) +{ + __omap_dm_timer_write_status(&clkev, OMAP_TIMER_INT_OVERFLOW); + __omap_dm_timer_read_status(&clkev); +} + #ifdef CONFIG_SOC_HAS_REALTIME_COUNTER static unsigned long arch_timer_freq; @@ -81,7 +96,8 @@ static irqreturn_t omap2_gp_timer_interrupt(int irq, void *dev_id) { struct clock_event_device *evt = &clockevent_gpt; - __omap_dm_timer_write_status(&clkev, OMAP_TIMER_INT_OVERFLOW); + if (!clockevent_ipipe_stolen(evt)) + omap2_gp_timer_ack(); evt->event_handler(evt); return IRQ_HANDLED; @@ -97,7 +113,7 @@ static int omap2_gp_timer_set_next_event(unsigned long cycles, struct clock_event_device *evt) { __omap_dm_timer_load_start(&clkev, OMAP_TIMER_CTRL_ST, - 0xffffffff - cycles, OMAP_TIMER_POSTED); + 0xffffffff - cycles, OMAP_TIMER_NONPOSTED); return 0; } @@ -107,7 +123,7 @@ static void omap2_gp_timer_set_mode(enum clock_event_mode mode, { u32 period; - __omap_dm_timer_stop(&clkev, OMAP_TIMER_POSTED, clkev.rate); + __omap_dm_timer_stop(&clkev, OMAP_TIMER_NONPOSTED, clkev.rate); switch (mode) { case CLOCK_EVT_MODE_PERIODIC: @@ -115,10 +131,10 @@ static void omap2_gp_timer_set_mode(enum clock_event_mode mode, period -= 1; /* Looks like we need to first set the load value separately */ __omap_dm_timer_write(&clkev, OMAP_TIMER_LOAD_REG, - 0xffffffff - period, OMAP_TIMER_POSTED); + 0xffffffff - period, OMAP_TIMER_NONPOSTED); __omap_dm_timer_load_start(&clkev, OMAP_TIMER_CTRL_AR | OMAP_TIMER_CTRL_ST, - 0xffffffff - period, OMAP_TIMER_POSTED); + 0xffffffff - period, OMAP_TIMER_NONPOSTED); break; case CLOCK_EVT_MODE_ONESHOT: break; @@ -230,7 +246,7 @@ static int __init omap_dm_timer_init_one(struct omap_dm_timer *timer, const char *fck_source, const char *property, const char **timer_name, - int posted) + int posted, int ipipe) { char name[10]; /* 10 = sizeof("gptXX_Xck0") */ const char *oh_name = NULL; @@ -254,6 +270,9 @@ static int __init omap_dm_timer_init_one(struct omap_dm_timer *timer, return -ENXIO; timer->io_base = of_iomap(np, 0); + if (of_address_to_resource(np, 0, &mem)) + mem.start = 0; + timer->phys_base = mem.start; of_node_put(np); } else { @@ -283,6 +302,7 @@ static int __init omap_dm_timer_init_one(struct omap_dm_timer *timer, return -ENXIO; /* Static mapping, never released */ + timer->phys_base = mem.start; timer->io_base = ioremap(mem.start, mem.end - mem.start); } @@ -310,6 +330,15 @@ static int __init omap_dm_timer_init_one(struct omap_dm_timer *timer, omap_hwmod_setup_one(oh_name); omap_hwmod_enable(oh); __omap_dm_timer_init_regs(timer); +#ifdef CONFIG_IPIPE + if (ipipe) { + u32 l; + + l = __raw_readl(timer->io_base + OMAP_TIMER_OCP_CFG_OFFSET); + l = (0x3 << 8) | (l & (1 << 5)) | (0x1 << 3) | (1 << 2); + __raw_writel(l, timer->io_base + OMAP_TIMER_OCP_CFG_OFFSET); + } +#endif if (posted) __omap_dm_timer_enable_posted(timer); @@ -324,11 +353,63 @@ static int __init omap_dm_timer_init_one(struct omap_dm_timer *timer, return r; } +#ifdef CONFIG_IPIPE +static struct ipipe_timer omap_shared_itimer = { + .ack = omap2_gp_timer_ack, + .min_delay_ticks = 3, +}; + +#define IPIPE_GPTIMER 3 + +static struct omap_dm_timer itimer; +static void omap3_itimer_request(struct ipipe_timer *timer, int steal) +{ + __omap_dm_timer_stop(&itimer, 0, itimer.rate); +} + +static int omap3_itimer_set(unsigned long cycles, void *timer) +{ + __omap_dm_timer_load_start(&itimer, OMAP_TIMER_CTRL_ST, + 0xffffffff - cycles, OMAP_TIMER_NONPOSTED); + return 0; +} + +static void omap3_itimer_ack(void) +{ + __omap_dm_timer_write_status(&itimer, OMAP_TIMER_INT_OVERFLOW); + __omap_dm_timer_read_status(&itimer); +} + +static void omap3_itimer_release(struct ipipe_timer *timer) +{ + __omap_dm_timer_stop(&itimer, 0, itimer.rate); +} + +static struct ipipe_timer omap3_itimer = { + .request = omap3_itimer_request, + .set = omap3_itimer_set, + .ack = omap3_itimer_ack, + .release = omap3_itimer_release, + + .rating = 100, + .min_delay_ticks = 3, +}; + +static struct __ipipe_tscinfo __maybe_unused tsc_info = { + .type = IPIPE_TSC_TYPE_FREERUNNING, + .u = { + { + .mask = 0xffffffff, + }, + }, +}; +#endif /* CONFIG_IPIPE */ + static void __init omap2_gp_clockevent_init(int gptimer_id, const char *fck_source, const char *property) { - int res; + int res, ipipe = 0; clkev.id = gptimer_id; clkev.errata = omap_dm_timer_get_errata(); @@ -340,8 +421,33 @@ static void __init omap2_gp_clockevent_init(int gptimer_id, */ __omap_dm_timer_override_errata(&clkev, OMAP_TIMER_ERRATA_I103_I767); +#ifdef CONFIG_IPIPE + if (cpu_is_omap34xx()) { + itimer.id = IPIPE_GPTIMER; + itimer.errata = omap_dm_timer_get_errata(); + __omap_dm_timer_override_errata(&itimer, + OMAP_TIMER_ERRATA_I103_I767); + res = omap_dm_timer_init_one(&itimer, + "timer_sys_ck", + NULL, + &omap3_itimer.name, + OMAP_TIMER_POSTED, 1); + BUG_ON(res); + + __omap_dm_timer_int_enable(&itimer, OMAP_TIMER_INT_OVERFLOW); + omap3_itimer.irq = itimer.irq; + omap3_itimer.freq = itimer.rate; + omap3_itimer.cpumask = cpumask_of(0); + + ipipe_timer_register(&omap3_itimer); + } + if ((cpu_is_omap44xx() && num_possible_cpus() == 1) || soc_is_am33xx()) + ipipe = 1; +#endif /* CONFIG_IPIPE */ + res = omap_dm_timer_init_one(&clkev, fck_source, property, - &clockevent_gpt.name, OMAP_TIMER_POSTED); + &clockevent_gpt.name, + OMAP_TIMER_POSTED, ipipe); BUG_ON(res); omap2_gp_timer_irq.dev_id = &clkev; @@ -351,6 +457,16 @@ static void __init omap2_gp_clockevent_init(int gptimer_id, clockevent_gpt.cpumask = cpu_possible_mask; clockevent_gpt.irq = omap_dm_timer_get_irq(&clkev); + +#ifdef CONFIG_IPIPE + if (ipipe) { + omap_shared_itimer.irq = clkev.irq; + omap_shared_itimer.min_delay_ticks = 3; + + clockevent_gpt.ipipe_timer = &omap_shared_itimer; + } +#endif + clockevents_config_and_register(&clockevent_gpt, clkev.rate, 3, /* Timer internal resynch latency */ 0xffffffff); @@ -454,18 +570,21 @@ static int __init __maybe_unused omap2_sync32k_clocksource_init(void) return ret; } +/* Setup free-running counter for clocksource */ static void __init omap2_gptimer_clocksource_init(int gptimer_id, const char *fck_source, const char *property) { - int res; + int res, ipipe = IS_ENABLED(CONFIG_IPIPE); clksrc.id = gptimer_id; clksrc.errata = omap_dm_timer_get_errata(); + __omap_dm_timer_override_errata(&clksrc, OMAP_TIMER_ERRATA_I103_I767); res = omap_dm_timer_init_one(&clksrc, fck_source, property, &clocksource_gpt.name, - OMAP_TIMER_NONPOSTED); + (ipipe ? OMAP_TIMER_POSTED + : OMAP_TIMER_NONPOSTED), ipipe); BUG_ON(res); __omap_dm_timer_load_start(&clksrc, @@ -473,6 +592,21 @@ static void __init omap2_gptimer_clocksource_init(int gptimer_id, OMAP_TIMER_NONPOSTED); sched_clock_register(dmtimer_read_sched_clock, 32, clksrc.rate); +#ifdef CONFIG_IPIPE + { + unsigned long off; + + off = OMAP_TIMER_COUNTER_REG & 0xff; + if (clksrc.revision == 2) + off += OMAP_TIMER_V2_FUNC_OFFSET; + + tsc_info.freq = clksrc.rate; + tsc_info.counter_vaddr = (unsigned long)clksrc.io_base + off; + tsc_info.u.counter_paddr = clksrc.phys_base + off; + __ipipe_tsc_register(&tsc_info); + } +#endif + if (clocksource_register_hz(&clocksource_gpt, clksrc.rate)) pr_err("Could not register clocksource %s\n", clocksource_gpt.name); @@ -608,6 +742,7 @@ void __init omap##name##_gptimer_timer_init(void) \ clksrc_prop); \ } +#ifndef CONFIG_IPIPE #define OMAP_SYS_32K_TIMER_INIT(name, clkev_nr, clkev_src, clkev_prop, \ clksrc_nr, clksrc_src, clksrc_prop) \ void __init omap##name##_sync32k_timer_init(void) \ @@ -622,6 +757,34 @@ void __init omap##name##_sync32k_timer_init(void) \ else \ omap2_sync32k_clocksource_init(); \ } +#else +#define OMAP_SYS_32K_TIMER_INIT(name, clkev_nr, clkev_src, clkev_prop, \ + clksrc_nr, clksrc_src, clksrc_prop) \ +void __init omap##name##_sync32k_timer_init(void) \ +{ \ + const char *clk = clkev_src; \ + \ + if (num_possible_cpus() == 1 && !soc_is_omap54xx()) { \ + use_gptimer_clksrc = 1; \ + if (cpu_is_omap44xx()) \ + clk = "timer_sys_ck"; \ + } \ + \ + omap_clk_init(); \ + omap_dmtimer_init(); \ + omap2_gp_clockevent_init((clkev_nr), clk, clkev_prop); \ + /* Enable the use of clocksource="gp_timer" kernel parameter */ \ + if (use_gptimer_clksrc) \ + omap2_gptimer_clocksource_init((clksrc_nr), clksrc_src, \ + clksrc_prop); \ + else \ + omap2_sync32k_clocksource_init(); \ + if (cpu_is_omap34xx()) \ + omap3_pic_muter_register(); \ + else if (cpu_is_omap44xx() || soc_is_omap54xx()) \ + omap4_pic_muter_register(); \ +} +#endif #ifdef CONFIG_ARCH_OMAP2 OMAP_SYS_32K_TIMER_INIT(2, 1, "timer_32k_ck", "ti,timer-alwon", diff --git a/arch/arm/mach-pxa/irq.c b/arch/arm/mach-pxa/irq.c index 89a7c06..03cf9b0 100644 --- a/arch/arm/mach-pxa/irq.c +++ b/arch/arm/mach-pxa/irq.c @@ -88,6 +88,9 @@ static struct irq_chip pxa_internal_irq_chip = { .name = "SC", .irq_ack = pxa_mask_irq, .irq_mask = pxa_mask_irq, +#ifdef CONFIG_IPIPE + .irq_mask_ack = pxa_mask_irq, +#endif /* CONFIG_IPIPE */ .irq_unmask = pxa_unmask_irq, }; @@ -103,7 +106,7 @@ asmlinkage void __exception_irq_entry icip_handle_irq(struct pt_regs *regs) if (mask == 0) break; - handle_IRQ(PXA_IRQ(fls(mask) - 1), regs); + ipipe_handle_multi_irq(PXA_IRQ(fls(mask) - 1), regs); } while (1); } @@ -117,7 +120,7 @@ asmlinkage void __exception_irq_entry ichp_handle_irq(struct pt_regs *regs) if ((ichp & ICHP_VAL_IRQ) == 0) break; - handle_IRQ(PXA_IRQ(ICHP_IRQ(ichp)), regs); + ipipe_handle_multi_irq(PXA_IRQ(ICHP_IRQ(ichp)), regs); } while (1); } diff --git a/arch/arm/mach-pxa/lpd270.c b/arch/arm/mach-pxa/lpd270.c index eaee2c2..433fafb 100644 --- a/arch/arm/mach-pxa/lpd270.c +++ b/arch/arm/mach-pxa/lpd270.c @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -130,7 +131,7 @@ static void lpd270_irq_handler(unsigned int irq, struct irq_desc *desc) desc->irq_data.chip->irq_ack(&desc->irq_data); if (likely(pending)) { irq = LPD270_IRQ(0) + __ffs(pending); - generic_handle_irq(irq); + ipipe_handle_demuxed_irq(irq); pending = __raw_readw(LPD270_INT_STATUS) & lpd270_irq_enabled; diff --git a/arch/arm/mach-pxa/pcm990-baseboard.c b/arch/arm/mach-pxa/pcm990-baseboard.c index 2897da2..d1961dc 100644 --- a/arch/arm/mach-pxa/pcm990-baseboard.c +++ b/arch/arm/mach-pxa/pcm990-baseboard.c @@ -25,6 +25,7 @@ #include #include #include +#include #include #include @@ -296,7 +297,7 @@ static void pcm990_irq_handler(unsigned int irq, struct irq_desc *desc) desc->irq_data.chip->irq_ack(&desc->irq_data); if (likely(pending)) { irq = PCM027_IRQ(0) + __ffs(pending); - generic_handle_irq(irq); + ipipe_handle_demuxed_irq(irq); } pending = ~pcm990_cpld_readb(PCM990_CTRL_INTSETCLR); pending &= pcm990_irq_enabled; diff --git a/arch/arm/mach-pxa/time.c b/arch/arm/mach-pxa/time.c new file mode 100644 index 0000000..1a0b158 --- /dev/null +++ b/arch/arm/mach-pxa/time.c @@ -0,0 +1,201 @@ +/* + * arch/arm/mach-pxa/time.c + * + * PXA clocksource, clockevents, and OST interrupt handlers. + * Copyright (c) 2007 by Bill Gatliff . + * + * Derived from Nicolas Pitre's PXA timer handler Copyright (c) 2001 + * by MontaVista Software, Inc. (Nico, your code rocks!) + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +/* + * This is PXA's sched_clock implementation. This has a resolution + * of at least 308 ns and a maximum value of 208 days. + * + * The return value is guaranteed to be monotonic in that range as + * long as there is always less than 582 seconds between successive + * calls to sched_clock() which should always be the case in practice. + */ + +static u64 notrace pxa_read_sched_clock(void) +{ + return readl_relaxed(OSCR); +} + + +#define MIN_OSCR_DELTA 16 + +static inline void pxa_ost0_ack(void) +{ + /* Disarm the compare/match, signal the event. */ + writel_relaxed(readl_relaxed(OIER) & ~OIER_E0, OIER); + writel_relaxed(OSSR_M0, OSSR); +} + +static irqreturn_t +pxa_ost0_interrupt(int irq, void *dev_id) +{ + struct clock_event_device *c = dev_id; + + if (clockevent_ipipe_stolen(c) == 0) + pxa_ost0_ack(); + + c->event_handler(c); + + return IRQ_HANDLED; +} + +static int +pxa_osmr0_set_next_event(unsigned long delta, struct clock_event_device *dev) +{ + unsigned long next, oscr; + + writel_relaxed(readl_relaxed(OIER) | OIER_E0, OIER); + next = readl_relaxed(OSCR) + delta; + writel_relaxed(next, OSMR0); + oscr = readl_relaxed(OSCR); + + return (signed)(next - oscr) <= MIN_OSCR_DELTA ? -ETIME : 0; +} + +static void +pxa_osmr0_set_mode(enum clock_event_mode mode, struct clock_event_device *dev) +{ + switch (mode) { + case CLOCK_EVT_MODE_ONESHOT: + writel_relaxed(readl_relaxed(OIER) & ~OIER_E0, OIER); + writel_relaxed(OSSR_M0, OSSR); + break; + + case CLOCK_EVT_MODE_UNUSED: + case CLOCK_EVT_MODE_SHUTDOWN: + /* initializing, released, or preparing for suspend */ + writel_relaxed(readl_relaxed(OIER) & ~OIER_E0, OIER); + writel_relaxed(OSSR_M0, OSSR); + break; + + case CLOCK_EVT_MODE_RESUME: + case CLOCK_EVT_MODE_PERIODIC: + break; + } +} + +#ifdef CONFIG_PM +static unsigned long osmr[4], oier, oscr; + +static void pxa_timer_suspend(struct clock_event_device *cedev) +{ + osmr[0] = readl_relaxed(OSMR0); + osmr[1] = readl_relaxed(OSMR1); + osmr[2] = readl_relaxed(OSMR2); + osmr[3] = readl_relaxed(OSMR3); + oier = readl_relaxed(OIER); + oscr = readl_relaxed(OSCR); +} + +static void pxa_timer_resume(struct clock_event_device *cedev) +{ + /* + * Ensure that we have at least MIN_OSCR_DELTA between match + * register 0 and the OSCR, to guarantee that we will receive + * the one-shot timer interrupt. We adjust OSMR0 in preference + * to OSCR to guarantee that OSCR is monotonically incrementing. + */ + if (osmr[0] - oscr < MIN_OSCR_DELTA) + osmr[0] += MIN_OSCR_DELTA; + + writel_relaxed(osmr[0], OSMR0); + writel_relaxed(osmr[1], OSMR1); + writel_relaxed(osmr[2], OSMR2); + writel_relaxed(osmr[3], OSMR3); + writel_relaxed(oier, OIER); + writel_relaxed(oscr, OSCR); +} +#else +#define pxa_timer_suspend NULL +#define pxa_timer_resume NULL +#endif + +#ifdef CONFIG_IPIPE +static struct ipipe_timer pxa_osmr0_itimer = { + .irq = IRQ_OST0, + .ack = pxa_ost0_ack, + .min_delay_ticks = MIN_OSCR_DELTA, +}; +#endif /* CONFIG_IPIPE */ + +static struct clock_event_device ckevt_pxa_osmr0 = { + .name = "osmr0", + .features = CLOCK_EVT_FEAT_ONESHOT, + .rating = 200, + .set_next_event = pxa_osmr0_set_next_event, + .set_mode = pxa_osmr0_set_mode, + .suspend = pxa_timer_suspend, + .resume = pxa_timer_resume, +#ifdef CONFIG_IPIPE + .ipipe_timer = &pxa_osmr0_itimer, +#endif /* CONFIG_IPIPE */ +}; + +static struct irqaction pxa_ost0_irq = { + .name = "ost0", + .flags = IRQF_TIMER | IRQF_IRQPOLL, + .handler = pxa_ost0_interrupt, + .dev_id = &ckevt_pxa_osmr0, +}; + +#ifdef CONFIG_IPIPE +static struct __ipipe_tscinfo tsc_info = { + .type = IPIPE_TSC_TYPE_FREERUNNING, + .counter_vaddr = (unsigned long)io_p2v(0x40A00010UL), + .u = { + { + .counter_paddr = 0x40A00010UL, + .mask = 0xffffffff, + }, + }, +}; +#endif /* CONFIG_IPIPE */ + +void __init pxa_timer_init(void) +{ + unsigned long clock_tick_rate = get_clock_tick_rate(); + + writel_relaxed(0, OIER); + writel_relaxed(OSSR_M0 | OSSR_M1 | OSSR_M2 | OSSR_M3, OSSR); + + sched_clock_register(pxa_read_sched_clock, 32, clock_tick_rate); + + ckevt_pxa_osmr0.cpumask = cpumask_of(0); + + setup_irq(IRQ_OST0, &pxa_ost0_irq); + + clocksource_mmio_init(OSCR, "oscr0", clock_tick_rate, 200, 32, + clocksource_mmio_readl_up); + +#ifdef CONFIG_IPIPE + tsc_info.freq = clock_tick_rate; + __ipipe_tsc_register(&tsc_info); +#endif /* CONFIG_IPIPE */ + + clockevents_config_and_register(&ckevt_pxa_osmr0, clock_tick_rate, + MIN_OSCR_DELTA * 2, 0x7fffffff); +} diff --git a/arch/arm/mach-pxa/viper.c b/arch/arm/mach-pxa/viper.c index de3b080..bed076c 100644 --- a/arch/arm/mach-pxa/viper.c +++ b/arch/arm/mach-pxa/viper.c @@ -45,6 +45,7 @@ #include #include #include +#include #include #include @@ -288,7 +289,7 @@ static void viper_irq_handler(unsigned int irq, struct irq_desc *desc) if (likely(pending)) { irq = viper_bit_to_irq(__ffs(pending)); - generic_handle_irq(irq); + ipipe_handle_demuxed_irq(irq); } pending = viper_irq_pending(); } while (pending); @@ -625,8 +626,8 @@ static struct isp116x_platform_data isp116x_platform_data = { static struct platform_device isp116x_device = { .name = "isp116x-hcd", .id = -1, - .num_resources = ARRAY_SIZE(isp116x_resources), - .resource = isp116x_resources, + .num_resources = ARRAY_SIZE(isp116x_resources), + .resource = isp116x_resources, .dev = { .platform_data = &isp116x_platform_data, }, diff --git a/arch/arm/mach-realview/core.c b/arch/arm/mach-realview/core.c index c309593..f8a1f2d 100644 --- a/arch/arm/mach-realview/core.c +++ b/arch/arm/mach-realview/core.c @@ -358,6 +358,7 @@ void __iomem *timer0_va_base; void __iomem *timer1_va_base; void __iomem *timer2_va_base; void __iomem *timer3_va_base; +void __iomem *timer3_pa_base; /* * Set up the clock source and clock events devices @@ -366,14 +367,14 @@ void __init realview_timer_init(unsigned int timer_irq) { u32 val; - /* - * set clock frequency: + /* + * set clock frequency: * REALVIEW_REFCLK is 32KHz * REALVIEW_TIMCLK is 1MHz */ val = readl(__io_address(REALVIEW_SCTL_BASE)); writel((REALVIEW_TIMCLK << REALVIEW_TIMER1_EnSel) | - (REALVIEW_TIMCLK << REALVIEW_TIMER2_EnSel) | + (REALVIEW_TIMCLK << REALVIEW_TIMER2_EnSel) | (REALVIEW_TIMCLK << REALVIEW_TIMER3_EnSel) | (REALVIEW_TIMCLK << REALVIEW_TIMER4_EnSel) | val, __io_address(REALVIEW_SCTL_BASE)); @@ -386,7 +387,7 @@ void __init realview_timer_init(unsigned int timer_irq) writel(0, timer2_va_base + TIMER_CTRL); writel(0, timer3_va_base + TIMER_CTRL); - sp804_clocksource_init(timer3_va_base, "timer3"); + sp804_clocksource_init(timer3_va_base, timer3_pa_base, "timer3"); sp804_clockevents_init(timer0_va_base, timer_irq, "timer0"); } diff --git a/arch/arm/mach-realview/core.h b/arch/arm/mach-realview/core.h index 868ece2..9afeb81 100644 --- a/arch/arm/mach-realview/core.h +++ b/arch/arm/mach-realview/core.h @@ -46,6 +46,7 @@ extern void __iomem *timer0_va_base; extern void __iomem *timer1_va_base; extern void __iomem *timer2_va_base; extern void __iomem *timer3_va_base; +extern void __iomem *timer3_pa_base; extern void realview_timer_init(unsigned int timer_irq); extern int realview_flash_register(struct resource *res, u32 num); diff --git a/arch/arm/mach-realview/realview_eb.c b/arch/arm/mach-realview/realview_eb.c index b3869cb..45b3cc5 100644 --- a/arch/arm/mach-realview/realview_eb.c +++ b/arch/arm/mach-realview/realview_eb.c @@ -408,6 +408,7 @@ static void __init realview_eb_timer_init(void) timer1_va_base = __io_address(REALVIEW_EB_TIMER0_1_BASE) + 0x20; timer2_va_base = __io_address(REALVIEW_EB_TIMER2_3_BASE); timer3_va_base = __io_address(REALVIEW_EB_TIMER2_3_BASE) + 0x20; + timer3_pa_base = REALVIEW_EB_TIMER2_3_BASE + 0x20; if (core_tile_eb11mp() || core_tile_a9mp()) timer_irq = IRQ_EB11MP_TIMER0_1; diff --git a/arch/arm/mach-realview/realview_pb1176.c b/arch/arm/mach-realview/realview_pb1176.c index ce92c18..080ff0f 100644 --- a/arch/arm/mach-realview/realview_pb1176.c +++ b/arch/arm/mach-realview/realview_pb1176.c @@ -326,6 +326,7 @@ static void __init realview_pb1176_timer_init(void) timer1_va_base = __io_address(REALVIEW_PB1176_TIMER0_1_BASE) + 0x20; timer2_va_base = __io_address(REALVIEW_PB1176_TIMER2_3_BASE); timer3_va_base = __io_address(REALVIEW_PB1176_TIMER2_3_BASE) + 0x20; + timer3_pa_base = REALVIEW_PB1176_TIMER2_3_BASE + 0x20; realview_clk_init(__io_address(REALVIEW_SYS_BASE), true); realview_timer_init(IRQ_DC1176_TIMER0); diff --git a/arch/arm/mach-realview/realview_pb11mp.c b/arch/arm/mach-realview/realview_pb11mp.c index 15c45e2..3b8f1b8 100644 --- a/arch/arm/mach-realview/realview_pb11mp.c +++ b/arch/arm/mach-realview/realview_pb11mp.c @@ -311,6 +311,7 @@ static void __init realview_pb11mp_timer_init(void) timer1_va_base = __io_address(REALVIEW_PB11MP_TIMER0_1_BASE) + 0x20; timer2_va_base = __io_address(REALVIEW_PB11MP_TIMER2_3_BASE); timer3_va_base = __io_address(REALVIEW_PB11MP_TIMER2_3_BASE) + 0x20; + timer3_pa_base = REALVIEW_PB11MP_TIMER2_3_BASE + 0x20; realview_clk_init(__io_address(REALVIEW_SYS_BASE), false); realview_timer_init(IRQ_TC11MP_TIMER0_1); diff --git a/arch/arm/mach-realview/realview_pba8.c b/arch/arm/mach-realview/realview_pba8.c index 4c64662..d963e10 100644 --- a/arch/arm/mach-realview/realview_pba8.c +++ b/arch/arm/mach-realview/realview_pba8.c @@ -260,6 +260,7 @@ static void __init realview_pba8_timer_init(void) timer1_va_base = __io_address(REALVIEW_PBA8_TIMER0_1_BASE) + 0x20; timer2_va_base = __io_address(REALVIEW_PBA8_TIMER2_3_BASE); timer3_va_base = __io_address(REALVIEW_PBA8_TIMER2_3_BASE) + 0x20; + timer3_pa_base = REALVIEW_PBA8_TIMER2_3_BASE + 0x20; realview_clk_init(__io_address(REALVIEW_SYS_BASE), false); realview_timer_init(IRQ_PBA8_TIMER0_1); diff --git a/arch/arm/mach-realview/realview_pbx.c b/arch/arm/mach-realview/realview_pbx.c index 9a22b86..29dcb65 100644 --- a/arch/arm/mach-realview/realview_pbx.c +++ b/arch/arm/mach-realview/realview_pbx.c @@ -320,6 +320,7 @@ static void __init realview_pbx_timer_init(void) timer1_va_base = __io_address(REALVIEW_PBX_TIMER0_1_BASE) + 0x20; timer2_va_base = __io_address(REALVIEW_PBX_TIMER2_3_BASE); timer3_va_base = __io_address(REALVIEW_PBX_TIMER2_3_BASE) + 0x20; + timer3_pa_base = REALVIEW_PBX_TIMER2_3_BASE + 0x20; realview_clk_init(__io_address(REALVIEW_SYS_BASE), false); realview_timer_init(IRQ_PBX_TIMER0_1); diff --git a/arch/arm/mach-s3c24xx/bast-irq.c b/arch/arm/mach-s3c24xx/bast-irq.c index cb1b791..9b72630 100644 --- a/arch/arm/mach-s3c24xx/bast-irq.c +++ b/arch/arm/mach-s3c24xx/bast-irq.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include @@ -121,7 +122,7 @@ bast_irq_pc104_demux(unsigned int irq, for (i = 0; stat != 0; i++, stat >>= 1) { if (stat & 1) { irqno = bast_pc104_irqs[i]; - generic_handle_irq(irqno); + ipipe_handle_demuxed_irq(irqno); } } } diff --git a/arch/arm/mach-sa1100/irq.c b/arch/arm/mach-sa1100/irq.c index 65aebfa..254b3a6 100644 --- a/arch/arm/mach-sa1100/irq.c +++ b/arch/arm/mach-sa1100/irq.c @@ -147,8 +147,8 @@ sa1100_handle_irq(struct pt_regs *regs) if (mask == 0) break; - handle_domain_irq(sa1100_normal_irqdomain, - ffs(mask) - 1, regs); + ipipe_handle_domain_irq(sa1100_normal_irqdomain, + ffs(mask) - 1, regs); } while (1); } diff --git a/arch/arm/mach-spear/Kconfig b/arch/arm/mach-spear/Kconfig index b6f4bda2..80e77b2 100644 --- a/arch/arm/mach-spear/Kconfig +++ b/arch/arm/mach-spear/Kconfig @@ -7,6 +7,7 @@ menuconfig PLAT_SPEAR select ARCH_REQUIRE_GPIOLIB select ARM_AMBA select CLKSRC_MMIO + select IPIPE_ARM_KUSER_TSC if IPIPE if PLAT_SPEAR diff --git a/arch/arm/mach-spear/time.c b/arch/arm/mach-spear/time.c index 26fda4e..0885f16 100644 --- a/arch/arm/mach-spear/time.c +++ b/arch/arm/mach-spear/time.c @@ -22,6 +22,8 @@ #include #include #include +#include +#include #include #include "generic.h" @@ -65,23 +67,43 @@ static __iomem void *gpt_base; static struct clk *gpt_clk; +static unsigned long gpt_phys_base; +static void spear_timer_ack(void); static void clockevent_set_mode(enum clock_event_mode mode, struct clock_event_device *clk_event_dev); static int clockevent_next_event(unsigned long evt, struct clock_event_device *clk_event_dev); +#ifdef CONFIG_IPIPE +static unsigned prescale, max_delta_ticks; + +static struct __ipipe_tscinfo __maybe_unused tsc_info = { + .type = IPIPE_TSC_TYPE_FREERUNNING_TWICE, + .u = { + { + .mask = 0x0000ffff, + }, + }, +}; +#endif /* CONFIG_IPIPE */ + static void __init spear_clocksource_init(void) { u32 tick_rate; u16 val; + tick_rate = clk_get_rate(gpt_clk); +#ifndef CONFIG_IPIPE /* program the prescaler (/256)*/ writew(CTRL_PRESCALER256, gpt_base + CR(CLKSRC)); /* find out actual clock driving Timer */ - tick_rate = clk_get_rate(gpt_clk); tick_rate >>= CTRL_PRESCALER256; +#else /* CONFIG_IPIPE */ + writew(prescale, gpt_base + CR(CLKSRC)); + tick_rate >>= prescale; +#endif /* CONFIG_IPIPE */ writew(0xFFFF, gpt_base + LOAD(CLKSRC)); @@ -93,14 +115,30 @@ static void __init spear_clocksource_init(void) /* register the clocksource */ clocksource_mmio_init(gpt_base + COUNT(CLKSRC), "tmr1", tick_rate, 200, 16, clocksource_mmio_readw_up); + +#ifdef CONFIG_IPIPE + tsc_info.u.counter_paddr = gpt_phys_base + COUNT(CLKSRC), + tsc_info.counter_vaddr = (unsigned long)(gpt_base + COUNT(CLKSRC)); + tsc_info.freq = tick_rate; + __ipipe_tsc_register(&tsc_info); +#endif /* CONFIG_IPIPE */ } +#ifdef CONFIG_IPIPE +static struct ipipe_timer spear_itimer = { + .ack = spear_timer_ack, +}; +#endif /* CONFIG_IPIPE */ + static struct clock_event_device clkevt = { .name = "tmr0", .features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT, .set_mode = clockevent_set_mode, .set_next_event = clockevent_next_event, .shift = 0, /* to be computed */ +#ifdef CONFIG_IPIPE + .ipipe_timer = &spear_itimer, +#endif /* CONFIG_IPIPE */ }; static void clockevent_set_mode(enum clock_event_mode mode, @@ -117,7 +155,11 @@ static void clockevent_set_mode(enum clock_event_mode mode, switch (mode) { case CLOCK_EVT_MODE_PERIODIC: period = clk_get_rate(gpt_clk) / HZ; +#ifndef CONFIG_IPIPE period >>= CTRL_PRESCALER16; +#else /* !CONFIG_IPIPE */ + period >>= prescale; +#endif /* !CONFIG_IPIPE */ writew(period, gpt_base + LOAD(CLKEVT)); val = readw(gpt_base + CR(CLKEVT)); @@ -148,6 +190,11 @@ static int clockevent_next_event(unsigned long cycles, { u16 val = readw(gpt_base + CR(CLKEVT)); +#ifdef CONFIG_IPIPE + if (cycles > max_delta_ticks) + cycles = max_delta_ticks; +#endif + if (val & CTRL_ENABLE) writew(val & ~CTRL_ENABLE, gpt_base + CR(CLKEVT)); @@ -159,11 +206,17 @@ static int clockevent_next_event(unsigned long cycles, return 0; } +static void spear_timer_ack(void) +{ + writew(INT_STATUS, gpt_base + IR(CLKEVT)); +} + static irqreturn_t spear_timer_interrupt(int irq, void *dev_id) { struct clock_event_device *evt = &clkevt; - writew(INT_STATUS, gpt_base + IR(CLKEVT)); + if (!clockevent_ipipe_stolen(evt)) + spear_timer_ack(); evt->event_handler(evt); @@ -180,11 +233,26 @@ static void __init spear_clockevent_init(int irq) { u32 tick_rate; - /* program the prescaler */ + tick_rate = clk_get_rate(gpt_clk); +#ifndef CONFIG_IPIPE + /* program the prescaler (/16)*/ writew(CTRL_PRESCALER16, gpt_base + CR(CLKEVT)); - tick_rate = clk_get_rate(gpt_clk); + /* find out actual clock driving Timer */ tick_rate >>= CTRL_PRESCALER16; +#else /* CONFIG_IPIPE */ + /* Find the prescaler giving a precision under 1us */ + for (prescale = CTRL_PRESCALER256; prescale != 0xffff; prescale--) + if ((tick_rate >> prescale) >= 1000000) + break; + + spear_itimer.irq = irq; + + writew(prescale, gpt_base + CR(CLKEVT)); + tick_rate >>= prescale; + + max_delta_ticks = 0xffff - tick_rate / 1000; +#endif /* CONFIG_IPIPE */ clkevt.cpumask = cpumask_of(0); @@ -201,6 +269,7 @@ const static struct of_device_id timer_of_match[] __initconst = { void __init spear_setup_of_timer(void) { struct device_node *np; + struct resource res; int irq, ret; np = of_find_matching_node(NULL, timer_of_match); @@ -221,6 +290,10 @@ void __init spear_setup_of_timer(void) return; } + if (of_address_to_resource(np, 0, &res)) + res.start = 0; + gpt_phys_base = res.start; + gpt_clk = clk_get_sys("gpt0", NULL); if (!gpt_clk) { pr_err("%s:couldn't get clk for gpt\n", __func__); diff --git a/arch/arm/mach-versatile/core.c b/arch/arm/mach-versatile/core.c index 6ea09fe..b78b79b 100644 --- a/arch/arm/mach-versatile/core.c +++ b/arch/arm/mach-versatile/core.c @@ -803,6 +803,7 @@ void __init versatile_timer_init(void) writel(0, TIMER2_VA_BASE + TIMER_CTRL); writel(0, TIMER3_VA_BASE + TIMER_CTRL); - sp804_clocksource_init(TIMER3_VA_BASE, "timer3"); + sp804_clocksource_init(TIMER3_VA_BASE, + VERSATILE_TIMER2_3_BASE + 0x20, "timer3"); sp804_clockevents_init(TIMER0_VA_BASE, IRQ_TIMERINT0_1, "timer0"); } diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig index b4f92b9..9304e7e 100644 --- a/arch/arm/mm/Kconfig +++ b/arch/arm/mm/Kconfig @@ -795,6 +795,7 @@ config NEEDS_SYSCALL_FOR_CMPXCHG config NEED_KUSER_HELPERS bool + default y if IPIPE config KUSER_HELPERS bool "Enable kuser helpers in vector page" if !NEED_KUSER_HELPERS @@ -1011,6 +1012,79 @@ config ARCH_HAS_BARRIERS This option allows the use of custom mandatory barriers included via the mach/barriers.h file. +config ARM_FCSE + bool "Fast Context Switch Extension (EXPERIMENTAL)" + depends on !SMP && (CPU_32v4 || CPU_32v4T || CPU_32v5) + help + The Fast Context Switch Extension (FCSE for short) is an extension of + some ARM processors which allows to switch contexts between processes + without flushing cache and saves a few tens of microseconds in the + worst case. + + Enabling this option makes linux use the FCSE. + + We propose two modes: + - the guaranteed mode: we guarantee that there will never be any cache + flush when switching context, but this means that there can not be + more than 95 running processes in the system, each with a virtual + memory space smaller than 32MB, and that the shared memory + mappings do not use cache; + - the best effort mode: we allow some cache flushes to happen from + time to time, but do not limit the number of processes or the + virtual memory space available for each process, and the shared + memory mappings use cache. + +if ARM_FCSE + +choice + prompt "FCSE mode" + default ARM_FCSE_BEST_EFFORT + help + This option allow setting which FCSE mode will be used. + +config ARM_FCSE_GUARANTEED + bool "guaranteed" + help + Select guaranteed mode. + +config ARM_FCSE_BEST_EFFORT + bool "best effort" + help + Select best-effort mode. + +endchoice + +config ARM_FCSE_PREEMPT_FLUSH + bool "Preemptible cache flushes" + default ARM_FCSE_GUARANTEED + help + When FCSE is enabled, some cache flushes happen with preemption + disabled by default, this allows avoiding more cache + flushes, but increases the latency. This option allows making + them preemptible. It probably only make sense in guaranteed mode. + +config ARM_FCSE_MESSAGES + bool "help messages" + default ARM_FCSE_BEST_EFFORT + help + When FCSE is enabled in best-effort mode, due to the VM space + reduction, a too large stack size limit may result in processes + exceeding the 32MB limit too easily. A too small stack size may result + in stack overflows. Enabling this option will print messages in these + situations to assist you in tuning the stack size limit. + + In guaranteed mode, this option will cause message to be printed if + one of the hard limits (95 proceses, 32 MB VM space) is exceeded. + +config ARM_FCSE_DEBUG + bool "FCSE debug" + select ARM_FCSE_MESSAGES + help + This option enables some internal debug checks. It has a high + overhead, and is only useful for debugging the FCSE code. + +endif + config ARCH_SUPPORTS_BIG_ENDIAN bool help diff --git a/arch/arm/mm/Makefile b/arch/arm/mm/Makefile index d3afdf9..683f5e1 100644 --- a/arch/arm/mm/Makefile +++ b/arch/arm/mm/Makefile @@ -100,3 +100,4 @@ obj-$(CONFIG_CACHE_FEROCEON_L2) += cache-feroceon-l2.o obj-$(CONFIG_CACHE_L2X0) += cache-l2x0.o l2c-l2x0-resume.o obj-$(CONFIG_CACHE_XSC3L2) += cache-xsc3l2.o obj-$(CONFIG_CACHE_TAUROS2) += cache-tauros2.o +obj-$(CONFIG_ARM_FCSE) += fcse.o diff --git a/arch/arm/mm/alignment.c b/arch/arm/mm/alignment.c index 9769f1e..c3a4a6f 100644 --- a/arch/arm/mm/alignment.c +++ b/arch/arm/mm/alignment.c @@ -475,7 +475,7 @@ do_alignment_ldrstr(unsigned long addr, unsigned long instr, struct pt_regs *reg * * B = rn pointer before instruction, A = rn pointer after instruction * ------ increasing address -----> - * | | r0 | r1 | ... | rx | | + * | | r0 | r1 | ... | rx | | * PU = 01 B A * PU = 11 B A * PU = 00 A B @@ -760,7 +760,10 @@ do_alignment(unsigned long addr, unsigned int fsr, struct pt_regs *regs) int thumb2_32b = 0; if (interrupts_enabled(regs)) - local_irq_enable(); + hard_local_irq_enable(); + + if (__ipipe_report_trap(IPIPE_TRAP_ALIGNMENT,regs)) + return 0; instrptr = instruction_pointer(regs); @@ -921,7 +924,7 @@ do_alignment(unsigned long addr, unsigned int fsr, struct pt_regs *regs) task_pid_nr(current), instrptr, isize << 1, isize == 2 ? tinstr : instr, - addr, fsr); + addr, fsr); if (ai_usermode & UM_FIXUP) goto fixup; @@ -948,7 +951,7 @@ do_alignment(unsigned long addr, unsigned int fsr, struct pt_regs *regs) * entry-common.S) and disable the alignment trap only if * there is no work pending for this thread. */ - raw_local_irq_disable(); + hard_local_irq_disable(); if (!(current_thread_info()->flags & _TIF_WORK_MASK)) set_cr(cr_no_alignment); } diff --git a/arch/arm/mm/cache-l2x0.c b/arch/arm/mm/cache-l2x0.c index e309c8f..48b9211 100644 --- a/arch/arm/mm/cache-l2x0.c +++ b/arch/arm/mm/cache-l2x0.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include @@ -47,9 +48,23 @@ struct l2c_init_data { #define CACHE_LINE_SIZE 32 +#ifdef CONFIG_IPIPE +#define CACHE_RANGE_ATOMIC_MAX 512UL +static int l2x0_wa = -1; +static int __init l2x0_setup_wa(char *str) +{ + l2x0_wa = !!simple_strtol(str, NULL, 0); + return 0; +} +early_param("l2x0_write_allocate", l2x0_setup_wa); +#else +#define CACHE_RANGE_ATOMIC_MAX 4096UL +static int l2x0_wa = 1; +#endif + static void __iomem *l2x0_base; static const struct l2c_init_data *l2x0_data; -static DEFINE_RAW_SPINLOCK(l2x0_lock); +static IPIPE_DEFINE_RAW_SPINLOCK(l2x0_lock); static u32 l2x0_way_mask; /* Bitmask of active ways */ static u32 l2x0_size; static unsigned long sync_reg_offset = L2X0_CACHE_SYNC; @@ -292,10 +307,10 @@ static void l2c220_op_way(void __iomem *base, unsigned reg) static unsigned long l2c220_op_pa_range(void __iomem *reg, unsigned long start, unsigned long end, unsigned long flags) { - raw_spinlock_t *lock = &l2x0_lock; + typeof(l2x0_lock) *lock = &l2x0_lock; while (start < end) { - unsigned long blk_end = start + min(end - start, 4096UL); + unsigned long blk_end = start + min(end - start, CACHE_RANGE_ATOMIC_MAX); while (start < blk_end) { l2c_wait_mask(reg, 1); @@ -498,13 +513,13 @@ static void l2c310_inv_range_erratum(unsigned long start, unsigned long end) static void l2c310_flush_range_erratum(unsigned long start, unsigned long end) { - raw_spinlock_t *lock = &l2x0_lock; + typeof(l2x0_lock) *lock = &l2x0_lock; unsigned long flags; void __iomem *base = l2x0_base; raw_spin_lock_irqsave(lock, flags); while (start < end) { - unsigned long blk_end = start + min(end - start, 4096UL); + unsigned long blk_end = start + min(end - start, CACHE_RANGE_ATOMIC_MAX); l2c_set_debug(base, 0x03); while (start < blk_end) { @@ -797,6 +812,28 @@ static int __init __l2c_init(const struct l2c_init_data *data, if (aux_val & aux_mask) pr_alert("L2C: platform provided aux values permit register corruption.\n"); + if (IS_ENABLED(CONFIG_IPIPE)) { + switch (cache_id & L2X0_CACHE_ID_PART_MASK) { + case L2X0_CACHE_ID_PART_L310: + if ((cache_id & L2X0_CACHE_ID_RTL_MASK) + >= L310_CACHE_ID_RTL_R3P2) { + l2x0_wa = 1; + pr_alert("L2C: I-pipe: revision >= L310-r3p2 detected, forcing WA.\n"); + } + case L2X0_CACHE_ID_PART_L220: + if (l2x0_wa < 0) { + l2x0_wa = 0; + pr_alert("L2C: I-pipe: l2x0_write_allocate= not specified, defaults to 0 (disabled).\n"); + } + if (!l2x0_wa) { + aux_mask &= ~L220_AUX_CTRL_FWA_MASK; + aux_val &= ~L220_AUX_CTRL_FWA_MASK; + aux_val |= 1 << L220_AUX_CTRL_FWA_SHIFT; + } else + pr_alert("L2C: I-pipe: write-allocate enabled, induces high latencies.\n"); + } + } + old_aux = aux = readl_relaxed(l2x0_base + L2X0_AUX_CTRL); aux &= aux_mask; aux |= aux_val; diff --git a/arch/arm/mm/context.c b/arch/arm/mm/context.c index 845769e..e740196 100644 --- a/arch/arm/mm/context.c +++ b/arch/arm/mm/context.c @@ -42,7 +42,7 @@ #define ASID_FIRST_VERSION (1ULL << ASID_BITS) #define NUM_USER_ASIDS ASID_FIRST_VERSION -static DEFINE_RAW_SPINLOCK(cpu_asid_lock); +static IPIPE_DEFINE_RAW_SPINLOCK(cpu_asid_lock); static atomic64_t asid_generation = ATOMIC64_INIT(ASID_FIRST_VERSION); static DECLARE_BITMAP(asid_map, NUM_USER_ASIDS); @@ -223,15 +223,18 @@ bump_gen: return asid; } -void check_and_switch_context(struct mm_struct *mm, struct task_struct *tsk) +int check_and_switch_context(struct mm_struct *mm, struct task_struct *tsk, bool root_p) { unsigned long flags; - unsigned int cpu = smp_processor_id(); + unsigned int cpu = ipipe_processor_id(); u64 asid; if (unlikely(mm->context.vmalloc_seq != init_mm.context.vmalloc_seq)) __check_vmalloc_seq(mm); +#ifdef CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH + flags = hard_local_irq_save(); +#endif /* CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH */ /* * We cannot update the pgd and the ASID atomicly with classic * MMU, so switch exclusively to global mappings to avoid @@ -244,7 +247,11 @@ void check_and_switch_context(struct mm_struct *mm, struct task_struct *tsk) && atomic64_xchg(&per_cpu(active_asids, cpu), asid)) goto switch_mm_fastpath; +#ifdef CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH + raw_spin_lock(&cpu_asid_lock); +#else /* !CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH */ raw_spin_lock_irqsave(&cpu_asid_lock, flags); +#endif /* !CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH */ /* Check that our ASID belongs to the current generation. */ asid = atomic64_read(&mm->context.id); if ((asid ^ atomic64_read(&asid_generation)) >> ASID_BITS) { @@ -259,8 +266,17 @@ void check_and_switch_context(struct mm_struct *mm, struct task_struct *tsk) atomic64_set(&per_cpu(active_asids, cpu), asid); cpumask_set_cpu(cpu, mm_cpumask(mm)); +#ifdef CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH + raw_spin_unlock(&cpu_asid_lock); +#else /* !CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH */ raw_spin_unlock_irqrestore(&cpu_asid_lock, flags); +#endif /* CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH */ switch_mm_fastpath: - cpu_switch_mm(mm->pgd, mm); + cpu_switch_mm(mm->pgd, mm, 1); +#ifdef CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH + hard_local_irq_restore(flags); +#endif /* CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH */ + + return 0; } diff --git a/arch/arm/mm/copypage-v4mc.c b/arch/arm/mm/copypage-v4mc.c index 1267e64..4ba1bc2 100644 --- a/arch/arm/mm/copypage-v4mc.c +++ b/arch/arm/mm/copypage-v4mc.c @@ -40,7 +40,7 @@ static DEFINE_RAW_SPINLOCK(minicache_lock); * instruction. If your processor does not supply this, you have to write your * own copy_user_highpage that does the right thing. */ -static void __naked +static void notrace __naked mc_copy_user_page(void *from, void *to) { asm volatile( diff --git a/arch/arm/mm/copypage-xscale.c b/arch/arm/mm/copypage-xscale.c index 0fb8502..afd7970 100644 --- a/arch/arm/mm/copypage-xscale.c +++ b/arch/arm/mm/copypage-xscale.c @@ -36,7 +36,7 @@ static DEFINE_RAW_SPINLOCK(minicache_lock); * Dcache aliasing issue. The writes will be forwarded to the write buffer, * and merged as appropriate. */ -static void __naked +static void notrace __naked mc_copy_user_page(void *from, void *to) { /* diff --git a/arch/arm/mm/fault-armv.c b/arch/arm/mm/fault-armv.c index d9e0d00..41bd962 100644 --- a/arch/arm/mm/fault-armv.c +++ b/arch/arm/mm/fault-armv.c @@ -28,6 +28,30 @@ static pteval_t shared_pte_mask = L_PTE_MT_BUFFERABLE; #if __LINUX_ARM_ARCH__ < 6 +#ifdef CONFIG_ARM_FCSE_BEST_EFFORT +static void fcse_set_pte_shared(struct vm_area_struct *vma, + unsigned long address, pte_t *ptep) +{ + pte_t entry; + + if (!(vma->vm_flags & VM_MAYSHARE) || address >= TASK_SIZE) + return; + + entry = *ptep; + if ((pte_val(entry) + & (L_PTE_PRESENT | PTE_CACHEABLE | L_PTE_RDONLY | L_PTE_DIRTY | L_PTE_SHARED)) + == (L_PTE_PRESENT | PTE_CACHEABLE | L_PTE_DIRTY)) { + pte_val(entry) |= L_PTE_SHARED; + /* Bypass set_pte_at here, we are not changing + hardware bits, flush is not needed */ + ++vma->vm_mm->context.fcse.shared_dirty_pages; + *ptep = entry; + } +} +#else /* !CONFIG_ARM_FCSE_BEST_EFFORT */ +#define fcse_set_pte_shared(vma, addr, ptep) do { } while (0) +#endif /* !CONFIG_ARM_FCSE_BEST_EFFORT */ + /* * We take the easy way out of this problem - we make the * PTE uncacheable. However, we leave the write buffer on. @@ -65,6 +89,7 @@ static int do_adjust_pte(struct vm_area_struct *vma, unsigned long address, return ret; } +#ifndef CONFIG_ARM_FCSE_GUARANTEED #if USE_SPLIT_PTE_PTLOCKS /* * If we are using split PTE locks, then we need to take the page @@ -127,11 +152,13 @@ static int adjust_pte(struct vm_area_struct *vma, unsigned long address, return ret; } +#endif /* CONFIG_ARM_FCSE_GUARANTEED */ static void make_coherent(struct address_space *mapping, struct vm_area_struct *vma, unsigned long addr, pte_t *ptep, unsigned long pfn) { +#ifndef CONFIG_ARM_FCSE_GUARANTEED struct mm_struct *mm = vma->vm_mm; struct vm_area_struct *mpnt; unsigned long offset; @@ -162,6 +189,12 @@ make_coherent(struct address_space *mapping, struct vm_area_struct *vma, flush_dcache_mmap_unlock(mapping); if (aliases) do_adjust_pte(vma, addr, pfn, ptep); + else + fcse_set_pte_shared(vma, addr, ptep); +#else /* CONFIG_ARM_FCSE_GUARANTEED */ + if (vma->vm_flags & VM_MAYSHARE) + do_adjust_pte(vma, addr, pfn, ptep); +#endif /* CONFIG_ARM_FCSE_GUARANTEED */ } /* diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c index 6333d9c..b90be1d 100644 --- a/arch/arm/mm/fault.c +++ b/arch/arm/mm/fault.c @@ -25,11 +25,45 @@ #include #include #include +#include #include "fault.h" #ifdef CONFIG_MMU +#ifdef CONFIG_IPIPE + +static inline unsigned long ipipe_fault_entry(void) +{ + unsigned long flags; + int s; + + flags = hard_local_irq_save(); + s = __test_and_set_bit(IPIPE_STALL_FLAG, &__ipipe_root_status); + hard_local_irq_enable(); + + return arch_mangle_irq_bits(s, flags); +} + +static inline void ipipe_fault_exit(unsigned long x) +{ + if (!arch_demangle_irq_bits(&x)) + local_irq_enable(); + else + hard_local_irq_restore(x); +} + +#else + +static inline unsigned long ipipe_fault_entry(void) +{ + return 0; +} + +static inline void ipipe_fault_exit(unsigned long x) { } + +#endif + #ifdef CONFIG_KPROBES static inline int notify_page_fault(struct pt_regs *regs, unsigned int fsr) { @@ -64,6 +98,11 @@ void show_pte(struct mm_struct *mm, unsigned long addr) mm = &init_mm; pr_alert("pgd = %p\n", mm->pgd); +#ifdef CONFIG_ARM_FCSE + pr_alert(KERN_ALERT "fcse pid: %ld, 0x%08lx, hw pid: 0x%08lx\n", + mm->context.fcse.pid >> FCSE_PID_SHIFT, + mm->context.fcse.pid, fcse_pid_get()); +#endif /* CONFIG_ARM_FCSE */ pgd = pgd_offset(mm, addr); pr_alert("[%08lx] *pgd=%08llx", addr, (long long)pgd_val(*pgd)); @@ -166,13 +205,15 @@ __do_user_fault(struct task_struct *tsk, unsigned long addr, #ifdef CONFIG_DEBUG_USER if (((user_debug & UDBG_SEGV) && (sig == SIGSEGV)) || ((user_debug & UDBG_BUS) && (sig == SIGBUS))) { - printk(KERN_DEBUG "%s: unhandled page fault (%d) at 0x%08lx, code 0x%03x\n", + printk("%s: unhandled page fault (%d) at 0x%08lx, code 0x%03x\n", tsk->comm, sig, addr, fsr); show_pte(tsk->mm, addr); show_regs(regs); } #endif + fcse_notify_segv(tsk->mm, addr, regs); + tsk->thread.address = addr; tsk->thread.error_code = fsr; tsk->thread.trap_no = 14; @@ -261,10 +302,16 @@ do_page_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs) struct mm_struct *mm; int fault, sig, code; unsigned int flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; + unsigned long irqflags; - if (notify_page_fault(regs, fsr)) + if (__ipipe_report_trap(IPIPE_TRAP_ACCESS, regs)) return 0; + irqflags = ipipe_fault_entry(); + + if (notify_page_fault(regs, fsr)) + goto out; + tsk = current; mm = tsk->mm; @@ -315,7 +362,7 @@ retry: * it would already be released in __lock_page_or_retry in * mm/filemap.c. */ if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) - return 0; + goto out; /* * Major/minor page fault accounting is only done on the @@ -349,7 +396,7 @@ retry: * Handle the "normal" case first - VM_FAULT_MAJOR / VM_FAULT_MINOR */ if (likely(!(fault & (VM_FAULT_ERROR | VM_FAULT_BADMAP | VM_FAULT_BADACCESS)))) - return 0; + goto out; /* * If we are in kernel mode at this point, we @@ -365,7 +412,7 @@ retry: * got oom-killed) */ pagefault_out_of_memory(); - return 0; + goto out; } if (fault & VM_FAULT_SIGBUS) { @@ -386,10 +433,13 @@ retry: } __do_user_fault(tsk, addr, fsr, sig, code, regs); - return 0; + goto out; no_context: __do_kernel_fault(mm, addr, fsr, regs); +out: + ipipe_fault_exit(irqflags); + return 0; } #else /* CONFIG_MMU */ @@ -422,11 +472,14 @@ static int __kprobes do_translation_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs) { + unsigned long irqflags; unsigned int index; pgd_t *pgd, *pgd_k; pud_t *pud, *pud_k; pmd_t *pmd, *pmd_k; + IPIPE_BUG_ON(!hard_irqs_disabled()); + if (addr < TASK_SIZE) return do_page_fault(addr, fsr, regs); @@ -474,10 +527,19 @@ do_translation_fault(unsigned long addr, unsigned int fsr, goto bad_area; copy_pmd(pmd, pmd_k); + return 0; bad_area: + if (__ipipe_report_trap(IPIPE_TRAP_ACCESS, regs)) + return 0; + + irqflags = ipipe_fault_entry(); + do_bad_area(addr, fsr, regs); + + ipipe_fault_exit(irqflags); + return 0; } #else /* CONFIG_MMU */ @@ -497,7 +559,17 @@ do_translation_fault(unsigned long addr, unsigned int fsr, static int do_sect_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs) { + unsigned long irqflags; + + if (__ipipe_report_trap(IPIPE_TRAP_SECTION, regs)) + return 0; + + irqflags = ipipe_fault_entry(); + do_bad_area(addr, fsr, regs); + + ipipe_fault_exit(irqflags); + return 0; } #endif /* CONFIG_ARM_LPAE */ @@ -508,6 +580,9 @@ do_sect_fault(unsigned long addr, unsigned int fsr, struct pt_regs *regs) static int do_bad(unsigned long addr, unsigned int fsr, struct pt_regs *regs) { + if (__ipipe_report_trap(IPIPE_TRAP_DABT,regs)) + return 0; + return 1; } @@ -545,11 +620,19 @@ asmlinkage void __exception do_DataAbort(unsigned long addr, unsigned int fsr, struct pt_regs *regs) { const struct fsr_info *inf = fsr_info + fsr_fs(fsr); + unsigned long irqflags; struct siginfo info; + addr = fcse_mva_to_va(addr); + if (!inf->fn(addr, fsr & ~FSR_LNX_PF, regs)) return; + if (__ipipe_report_trap(IPIPE_TRAP_UNKNOWN, regs)) + return; + + irqflags = ipipe_fault_entry(); + pr_alert("Unhandled fault: %s (0x%03x) at 0x%08lx\n", inf->name, fsr, addr); show_pte(current->mm, addr); @@ -559,6 +642,8 @@ do_DataAbort(unsigned long addr, unsigned int fsr, struct pt_regs *regs) info.si_code = inf->code; info.si_addr = (void __user *)addr; arm_notify_die("", regs, &info, fsr, 0); + + ipipe_fault_exit(irqflags); } void __init @@ -578,11 +663,17 @@ asmlinkage void __exception do_PrefetchAbort(unsigned long addr, unsigned int ifsr, struct pt_regs *regs) { const struct fsr_info *inf = ifsr_info + fsr_fs(ifsr); + unsigned long irqflags; struct siginfo info; if (!inf->fn(addr, ifsr | FSR_LNX_PF, regs)) return; + if (__ipipe_report_trap(IPIPE_TRAP_UNKNOWN, regs)) + return; + + irqflags = ipipe_fault_entry(); + pr_alert("Unhandled prefetch abort: %s (0x%03x) at 0x%08lx\n", inf->name, ifsr, addr); @@ -591,6 +682,8 @@ do_PrefetchAbort(unsigned long addr, unsigned int ifsr, struct pt_regs *regs) info.si_code = inf->code; info.si_addr = (void __user *)addr; arm_notify_die("", regs, &info, ifsr, 0); + + ipipe_fault_exit(irqflags); } #ifndef CONFIG_ARM_LPAE diff --git a/arch/arm/mm/fcse.c b/arch/arm/mm/fcse.c new file mode 100644 index 0000000..f6ece33 --- /dev/null +++ b/arch/arm/mm/fcse.c @@ -0,0 +1,463 @@ +/* + * arch/arm/kernel/fcse.c + * + * Helper functions for using the ARM Fast Context Switch Extension with + * processors supporting it. + * + * Copyright (C) 2008 Richard Cochran + * Copyright (C) 2009-2011 Gilles Chanteperdrix + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include /* For user_debug, UDBG_SEGV */ + +#define PIDS_LONGS ((FCSE_NR_PIDS + BITS_PER_LONG - 1) / BITS_PER_LONG) + +static IPIPE_DEFINE_RAW_SPINLOCK(fcse_lock); +static unsigned long fcse_pids_bits[PIDS_LONGS]; +unsigned long fcse_pids_cache_dirty[PIDS_LONGS]; +EXPORT_SYMBOL(fcse_pids_cache_dirty); + +#ifdef CONFIG_ARM_FCSE_BEST_EFFORT +static unsigned random_pid; +struct mm_struct *fcse_large_process; +struct fcse_user fcse_pids_user[FCSE_NR_PIDS]; +static struct mm_struct *fcse_cur_mm = &init_mm; +#endif /* CONFIG_ARM_FCSE_BEST_EFFORT */ + +static inline void fcse_pid_reference_inner(struct mm_struct *mm) +{ + unsigned fcse_pid = mm->context.fcse.pid >> FCSE_PID_SHIFT; + +#ifdef CONFIG_ARM_FCSE_BEST_EFFORT + if (++fcse_pids_user[fcse_pid].count == 1) +#endif /* CONFIG_ARM_FCSE_BEST_EFFORT */ + __set_bit(FCSE_PID_MAX - fcse_pid, fcse_pids_bits); +} + +static inline void fcse_pid_dereference(struct mm_struct *mm) +{ + unsigned fcse_pid = mm->context.fcse.pid >> FCSE_PID_SHIFT; + +#ifdef CONFIG_ARM_FCSE_BEST_EFFORT + if (--fcse_pids_user[fcse_pid].count == 0) + __clear_bit(FCSE_PID_MAX - fcse_pid, fcse_pids_bits); + + /* + * The following means we suppose that by the time this + * function is called, this mm is out of cache: + * - when the caller is destroy_context, exit_mmap is called + * by mmput before, which flushes the cache; + * - when the caller is fcse_relocate_mm_to_pid from + * fcse_switch_mm_inner, we only relocate when the mm is out + * of cache; + * - when the caller is fcse_relocate_mm_to_pid from + * fcse_relocate_mm_to_null_pid, we flush the cache in this + * function. + */ + if (fcse_pids_user[fcse_pid].mm == mm) { + fcse_pids_user[fcse_pid].mm = NULL; + __clear_bit(FCSE_PID_MAX - fcse_pid, fcse_pids_cache_dirty); + } + if (fcse_large_process == mm) + fcse_large_process = NULL; +#else /* CONFIG_ARM_FCSE_BEST_EFFORT */ + __clear_bit(FCSE_PID_MAX - fcse_pid, fcse_pids_bits); + __clear_bit(FCSE_PID_MAX - fcse_pid, fcse_pids_cache_dirty); +#endif /* CONFIG_ARM_FCSE_BEST_EFFORT */ +} + +static inline long find_free_pid(unsigned long bits[]) +{ + return FCSE_PID_MAX - find_first_zero_bit(bits, FCSE_NR_PIDS); +} + +void fcse_pid_free(struct mm_struct *mm) +{ + unsigned long flags; + + raw_spin_lock_irqsave(&fcse_lock, flags); + fcse_pid_dereference(mm); + raw_spin_unlock_irqrestore(&fcse_lock, flags); +} + +int fcse_pid_alloc(struct mm_struct *mm) +{ + unsigned long flags; + unsigned fcse_pid; + + raw_spin_lock_irqsave(&fcse_lock, flags); + fcse_pid = find_free_pid(fcse_pids_bits); + if (fcse_pid == -1) { + /* Allocate zero pid last, since zero pid is also used by + processes with address space larger than 32MB in + best-effort mode. */ +#ifdef CONFIG_ARM_FCSE_BEST_EFFORT + if(++random_pid == FCSE_NR_PIDS) { + if (fcse_large_process) { + random_pid = + fcse_large_process->context.fcse.highest_pid + 1; + if (random_pid == FCSE_NR_PIDS) + random_pid = 0; + } else + random_pid = 0; + } + fcse_pid = random_pid; +#else /* CONFIG_ARM_FCSE_GUARANTEED */ + raw_spin_unlock_irqrestore(&fcse_lock, flags); +#ifdef CONFIG_ARM_FCSE_MESSAGES + printk(KERN_WARNING "FCSE: %s[%d] would exceed the %lu processes limit.\n", + current->comm, current->pid, FCSE_NR_PIDS); +#endif /* CONFIG_ARM_FCSE_MESSAGES */ + /* + * Set mm pid to FCSE_PID_INVALID, as even when + * init_new_context fails, destroy_context is called. + */ + mm->context.fcse.pid = FCSE_PID_INVALID; + return -EAGAIN; +#endif /* CONFIG_ARM_FCSE_GUARANTEED */ + } + mm->context.fcse.pid = fcse_pid << FCSE_PID_SHIFT; + fcse_pid_reference_inner(mm); + raw_spin_unlock_irqrestore(&fcse_lock, flags); + + return 0; +} + +static inline void fcse_clear_dirty_all(void) +{ + switch(ARRAY_SIZE(fcse_pids_cache_dirty)) { + case 3: + fcse_pids_cache_dirty[2] = 0UL; + case 2: + fcse_pids_cache_dirty[1] = 0UL; + case 1: + fcse_pids_cache_dirty[0] = 0UL; + } +#ifdef CONFIG_ARM_FCSE_BEST_EFFORT + fcse_large_process = NULL; +#endif +} + +unsigned fcse_flush_all_start(void) +{ + if (!cache_is_vivt()) + return 0; + +#ifndef CONFIG_ARM_FCSE_PREEMPT_FLUSH + preempt_disable(); +#endif /* CONFIG_ARM_FCSE_PREEMPT_FLUSH */ + +#if defined(CONFIG_IPIPE) + clear_ti_thread_flag(current_thread_info(), TIF_SWITCHED); +#elif defined(CONFIG_ARM_FCSE_PREEMPT_FLUSH) + return nr_context_switches(); +#endif /* CONFIG_ARM_FCSE_PREEMPT_FLUSH */ + + return 0; +} + +noinline void +fcse_flush_all_done(unsigned seq, unsigned dirty) +{ + struct mm_struct *mm; + unsigned long flags; + + if (!cache_is_vivt()) + return; + + mm = current->mm; + + raw_spin_lock_irqsave(&fcse_lock, flags); +#if defined(CONFIG_IPIPE) + if (!test_ti_thread_flag(current_thread_info(), TIF_SWITCHED)) +#elif defined(CONFIG_ARM_FCSE_PREEMPT_FLUSH) + if (seq == nr_context_switches()) +#endif /* CONFIG_ARM_FCSE_PREEMPT_FLUSH */ + fcse_clear_dirty_all(); + + if (dirty && mm != &init_mm && mm) + __fcse_mark_dirty(mm); + raw_spin_unlock_irqrestore(&fcse_lock, flags); +#ifndef CONFIG_ARM_FCSE_PREEMPT_FLUSH + preempt_enable(); +#endif /* CONFIG_ARM_FCSE_PREEMPT_FLUSH */ +} + +#ifdef CONFIG_ARM_FCSE_BEST_EFFORT +/* Called with preemption disabled, mm->mmap_sem being held for writing. */ +static noinline int fcse_relocate_mm_to_pid(struct mm_struct *mm, int fcse_pid) +{ + const unsigned len = pgd_index(FCSE_TASK_SIZE) * sizeof(pgd_t); + unsigned long flags; + pgd_t *from, *to; + + raw_spin_lock_irqsave(&fcse_lock, flags); + fcse_pid_dereference(mm); + from = pgd_offset(mm, 0); + mm->context.fcse.pid = fcse_pid << FCSE_PID_SHIFT; + to = pgd_offset(mm, 0); + fcse_pid_reference_inner(mm); + fcse_pids_user[fcse_pid].mm = mm; + __fcse_mark_dirty(mm); + raw_spin_unlock_irqrestore(&fcse_lock, flags); + + memcpy(to, from, len); + memset(from, '\0', len); + barrier(); + clean_dcache_area(from, len); + clean_dcache_area(to, len); + + return fcse_pid; +} + +static int fcse_flush_needed_p(struct mm_struct *next) +{ + unsigned fcse_pid = next->context.fcse.pid >> FCSE_PID_SHIFT; + unsigned flush_needed = 0; + + if (next == &init_mm) + goto check_cur; + + if (fcse_pids_user[fcse_pid].mm != next) + if (fcse_pids_user[fcse_pid].mm) + flush_needed = test_bit(FCSE_PID_MAX - fcse_pid, + fcse_pids_cache_dirty); + + if (flush_needed == 0 + && fcse_large_process + && fcse_large_process != next + && fcse_pid <= fcse_large_process->context.fcse.highest_pid) + flush_needed = 1; + + check_cur: + if (flush_needed == 0 && fcse_cur_mm->context.fcse.shared_dirty_pages) + flush_needed = 1; + + return flush_needed; +} + +int fcse_switch_mm_start_inner(struct mm_struct *next) +{ + unsigned flush_needed; + unsigned long flags; + + raw_spin_lock_irqsave(&fcse_lock, flags); + flush_needed = fcse_flush_needed_p(next); + raw_spin_unlock_irqrestore(&fcse_lock, flags); + + return flush_needed; +} +EXPORT_SYMBOL_GPL(fcse_switch_mm_start_inner); + +void fcse_switch_mm_end_inner(struct mm_struct *next) +{ + unsigned fcse_pid = next->context.fcse.pid >> FCSE_PID_SHIFT; + unsigned long flags; + + raw_spin_lock_irqsave(&fcse_lock, flags); + if (fcse_flush_needed_p(next)) + fcse_clear_dirty_all(); + + fcse_pid_set(fcse_pid << FCSE_PID_SHIFT); + if (next != &init_mm) { + __fcse_mark_dirty(next); + if (fcse_pids_user[fcse_pid].mm != next) + fcse_pids_user[fcse_pid].mm = next; + } + fcse_cur_mm = next; + raw_spin_unlock_irqrestore(&fcse_lock, flags); +} +EXPORT_SYMBOL_GPL(fcse_switch_mm_end_inner); + +void fcse_pid_reference(struct mm_struct *mm) +{ + unsigned long flags; + + raw_spin_lock_irqsave(&fcse_lock, flags); + fcse_pid_reference_inner(mm); + raw_spin_unlock_irqrestore(&fcse_lock, flags); +} + +/* Called with mm->mmap_sem write-locked. */ +static noinline void fcse_relocate_mm_to_null_pid(struct mm_struct *mm) +{ + if (!cache_is_vivt()) + return; + + preempt_disable(); + while (fcse_mm_in_cache(mm)) { + unsigned seq; + + preempt_enable(); + + seq = fcse_flush_all_start(); + flush_cache_all(); + + preempt_disable(); + fcse_flush_all_done(seq, 0); + } + + fcse_relocate_mm_to_pid(mm, 0); + barrier(); + flush_tlb_mm(mm); + fcse_pid_set(0); + + preempt_enable(); +} +#endif /* CONFIG_ARM_FCSE_BEST_EFFORT */ + +unsigned long +fcse_check_mmap_inner(struct mm_struct *mm, + struct vm_unmapped_area_info *info, + unsigned long addr, unsigned long flags) +{ + if (flags & MAP_FIXED) + goto skip_retry; + + /* Try again the mmap, allowing addresses above 32 MB */ + info->flags = 0; + info->low_limit = PAGE_ALIGN(mm->start_stack); +#ifdef CONFIG_ARM_FCSE_BEST_EFFORT + info->high_limit = TASK_SIZE; +#else /* CONFIG_ARM_FCSE_GUARANTEED */ + info->high_limit = FCSE_TASK_SIZE; +#endif /* CONFIG_ARM_FCSE_GUARANTEED */ + addr = vm_unmapped_area(info); + + if ((addr & ~PAGE_MASK) == 0 && addr + info->length <= FCSE_TASK_SIZE) + return addr; + + /* Could not find an address */ + skip_retry: +#ifdef CONFIG_ARM_FCSE_BEST_EFFORT + /* Could not find an address */ + if (addr & ~PAGE_MASK) + return addr; + + /* It is not the first time this process gets addresses above 32MB */ + if (mm->context.fcse.large) + return addr; + +#ifdef CONFIG_ARM_FCSE_MESSAGES + printk(KERN_INFO "FCSE: process %u(%s) VM exceeds 32MB.\n", + current->pid, current->comm); +#endif /* CONFIG_ARM_FCSE_MESSAGES */ + mm->context.fcse.large = 1; + if (mm->context.fcse.pid) + fcse_relocate_mm_to_null_pid(mm); + + return addr; + +#else /* CONFIG_ARM_FCSE_GUARANTEED */ + /* Address above 32MB, no 32MB processes in guaranteed mode. */ +#ifdef CONFIG_ARM_FCSE_MESSAGES + if ((flags & MAP_BRK) == 0) + printk(KERN_WARNING + "FCSE: process %u(%s) VM would exceed the 32MB limit.\n", + current->pid, current->comm); +#endif /* CONFIG_ARM_FCSE_MESSAGES */ + return -ENOMEM; +#endif /* CONFIG_ARM_FCSE_GUARANTEED */ +} + +#ifdef CONFIG_ARM_FCSE_MESSAGES +#define addr_in_vma(vma, addr) \ + ({ \ + struct vm_area_struct *_vma = (vma); \ + ((unsigned long)((addr) - _vma->vm_start) \ + < (unsigned long)((_vma->vm_end - _vma->vm_start))); \ + }) + +#ifdef CONFIG_DEBUG_USER +static noinline void +dump_vmas(struct mm_struct *mm, unsigned long addr, struct pt_regs *regs) +{ + struct vm_area_struct *vma; + char path[128]; + int locked = 0; + + printk("mappings:\n"); + if (!in_atomic()) + locked = down_read_trylock(&mm->mmap_sem); + for(vma = mm->mmap; vma; vma = vma->vm_next) { + struct file *file = vma->vm_file; + int flags = vma->vm_flags; + const char *name; + + printk("0x%08lx-0x%08lx %c%c%c%c 0x%08llx ", + vma->vm_start, + vma->vm_end, + flags & VM_READ ? 'r' : '-', + flags & VM_WRITE ? 'w' : '-', + flags & VM_EXEC ? 'x' : '-', + flags & VM_MAYSHARE ? 's' : 'p', + ((loff_t)vma->vm_pgoff) << PAGE_SHIFT); + + if (file) + name = d_path(&file->f_path, path, sizeof(path)); + else if ((name = arch_vma_name(vma))) + ; + else if (!vma->vm_mm) + name = "[vdso]"; + else if (vma->vm_start <= mm->start_brk + && vma->vm_end >= mm->brk) + name = "[heap]"; + else if (vma->vm_start <= mm->start_stack && + vma->vm_end >= mm->start_stack) + name = "[stack]"; + else + name = ""; + printk("%s", name); + if (addr_in_vma(vma, regs->ARM_pc)) + printk(" <- PC"); + if (addr_in_vma(vma, regs->ARM_sp)) + printk(" <- SP"); + if (addr_in_vma(vma, addr)) + printk("%s fault", + (addr_in_vma(vma, regs->ARM_pc) + || addr_in_vma(vma, regs->ARM_sp) + ? "," : " <-")); + printk("\n"); + } + if (locked) + up_read(&mm->mmap_sem); +} +#endif /* CONFIG_DEBUG_USER */ + +void fcse_notify_segv(struct mm_struct *mm, + unsigned long addr, struct pt_regs *regs) +{ + int locked = 0; + +#if defined(CONFIG_DEBUG_USER) + if (user_debug & UDBG_SEGV) + dump_vmas(mm, addr, regs); +#endif /* CONFIG_DEBUG_USER */ + + if (!in_atomic()) + locked = down_read_trylock(&mm->mmap_sem); + if (find_vma(mm, addr) == find_vma(mm, regs->ARM_sp)) + printk(KERN_INFO "FCSE: process %u(%s) probably overflowed stack at 0x%08lx.\n", + current->pid, current->comm, regs->ARM_pc); + if (locked) + up_read(&mm->mmap_sem); +} +#endif /* CONFIG_ARM_FCSE_MESSAGES */ diff --git a/arch/arm/mm/flush.c b/arch/arm/mm/flush.c index 34b66af..23e4c7e 100644 --- a/arch/arm/mm/flush.c +++ b/arch/arm/mm/flush.c @@ -148,7 +148,7 @@ void flush_ptrace_access(struct vm_area_struct *vma, struct page *page, unsigned long uaddr, void *kaddr, unsigned long len) { unsigned int flags = 0; - if (cpumask_test_cpu(smp_processor_id(), mm_cpumask(vma->vm_mm))) + if (fcse_mm_in_cache(vma->vm_mm)) flags |= FLAG_PA_CORE_IN_MM; if (vma->vm_flags & VM_EXEC) flags |= FLAG_PA_IS_EXEC; @@ -177,6 +177,7 @@ void copy_to_user_page(struct vm_area_struct *vma, struct page *page, #ifdef CONFIG_SMP preempt_disable(); #endif + fcse_flush_cache_user_range(vma, uaddr, uaddr + len); memcpy(dst, src, len); flush_ptrace_access(vma, page, uaddr, dst, len); #ifdef CONFIG_SMP diff --git a/arch/arm/mm/idmap.c b/arch/arm/mm/idmap.c index e7a81ceb..c6ca62c 100644 --- a/arch/arm/mm/idmap.c +++ b/arch/arm/mm/idmap.c @@ -122,7 +122,7 @@ early_initcall(init_static_idmap); void setup_mm_for_reboot(void) { /* Switch to the identity mapping. */ - cpu_switch_mm(idmap_pgd, &init_mm); + cpu_switch_mm(idmap_pgd, &init_mm, 1); local_flush_bp_all(); #ifdef CONFIG_CPU_HAS_ASID diff --git a/arch/arm/mm/ioremap.c b/arch/arm/mm/ioremap.c index d1e5ad7..a1aa3a5 100644 --- a/arch/arm/mm/ioremap.c +++ b/arch/arm/mm/ioremap.c @@ -328,6 +328,7 @@ void __iomem * __arm_ioremap_pfn_caller(unsigned long pfn, } flush_cache_vmap(addr, addr + size); + __ipipe_pin_mapping_globally(addr, addr + size); return (void __iomem *) (offset + addr); } diff --git a/arch/arm/mm/mmap.c b/arch/arm/mm/mmap.c index 407dc78..01b1ecc 100644 --- a/arch/arm/mm/mmap.c +++ b/arch/arm/mm/mmap.c @@ -32,6 +32,7 @@ static int mmap_is_legacy(void) static unsigned long mmap_base(unsigned long rnd) { +#ifndef CONFIG_ARM_FCSE unsigned long gap = rlimit(RLIMIT_STACK); if (gap < MIN_GAP) @@ -40,6 +41,9 @@ static unsigned long mmap_base(unsigned long rnd) gap = MAX_GAP; return PAGE_ALIGN(TASK_SIZE - gap - rnd); +#else /* CONFIG_ARM_FCSE */ + return PAGE_ALIGN(FCSE_TASK_SIZE - rlimit(RLIMIT_STACK) - rnd); +#endif /* CONFIG_ARM_FCSE */ } /* @@ -75,12 +79,16 @@ arch_get_unmapped_area(struct file *filp, unsigned long addr, if (aliasing && flags & MAP_SHARED && (addr - (pgoff << PAGE_SHIFT)) & (SHMLBA - 1)) return -EINVAL; - return addr; + goto found_addr; } if (len > TASK_SIZE) return -ENOMEM; + info.length = len; + info.align_mask = do_align ? (PAGE_MASK & (SHMLBA - 1)) : 0; + info.align_offset = pgoff << PAGE_SHIFT; + if (addr) { if (do_align) addr = COLOUR_ALIGN(addr, pgoff); @@ -90,16 +98,18 @@ arch_get_unmapped_area(struct file *filp, unsigned long addr, vma = find_vma(mm, addr); if (TASK_SIZE - len >= addr && (!vma || addr + len <= vma->vm_start)) - return addr; + goto found_addr; } info.flags = 0; - info.length = len; info.low_limit = mm->mmap_base; - info.high_limit = TASK_SIZE; - info.align_mask = do_align ? (PAGE_MASK & (SHMLBA - 1)) : 0; - info.align_offset = pgoff << PAGE_SHIFT; - return vm_unmapped_area(&info); + info.high_limit = fcse() == 0 ? TASK_SIZE : + (PAGE_ALIGN(mm->start_stack) + - rlimit(RLIMIT_STACK) - PAGE_SIZE); + addr = vm_unmapped_area(&info); + + found_addr: + return fcse_check_mmap_addr(mm, addr, len, &info, flags); } unsigned long @@ -129,9 +139,13 @@ arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0, if (aliasing && flags & MAP_SHARED && (addr - (pgoff << PAGE_SHIFT)) & (SHMLBA - 1)) return -EINVAL; - return addr; + goto found_addr; } + info.length = len; + info.align_mask = do_align ? (PAGE_MASK & (SHMLBA - 1)) : 0; + info.align_offset = pgoff << PAGE_SHIFT; + /* requesting a specific address */ if (addr) { if (do_align) @@ -139,18 +153,31 @@ arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0, else addr = PAGE_ALIGN(addr); vma = find_vma(mm, addr); - if (TASK_SIZE - len >= addr && + if (TASK_SIZE - len > addr && (!vma || addr + len <= vma->vm_start)) - return addr; + goto found_addr; } info.flags = VM_UNMAPPED_AREA_TOPDOWN; info.length = len; info.low_limit = FIRST_USER_ADDRESS; + + if (fcse()) { + unsigned long top, bottom, shift; + + BUG_ON(mm->start_stack == 0); + top = PAGE_ALIGN(mm->start_stack); + bottom = top - rlimit(RLIMIT_STACK); + shift = FCSE_TASK_SIZE - (top - PAGE_SIZE); + if (mm->mmap_base > bottom) + mm->mmap_base -= shift; + } + info.high_limit = mm->mmap_base; - info.align_mask = do_align ? (PAGE_MASK & (SHMLBA - 1)) : 0; - info.align_offset = pgoff << PAGE_SHIFT; + addr = vm_unmapped_area(&info); + found_addr: + addr = fcse_check_mmap_addr(mm, addr, len, &info, flags); /* * A failed mmap() very likely causes application failure, @@ -158,7 +185,7 @@ arch_get_unmapped_area_topdown(struct file *filp, const unsigned long addr0, * can happen with large stack limits and large mmap() * allocations. */ - if (addr & ~PAGE_MASK) { + if (!fcse() && addr & ~PAGE_MASK) { VM_BUG_ON(addr != -ENOMEM); info.flags = 0; info.low_limit = mm->mmap_base; diff --git a/arch/arm/mm/pgd.c b/arch/arm/mm/pgd.c index a3681f1..dd0dadf 100644 --- a/arch/arm/mm/pgd.c +++ b/arch/arm/mm/pgd.c @@ -27,6 +27,173 @@ #define __pgd_free(pgd) free_pages((unsigned long)pgd, 2) #endif +#ifdef CONFIG_IPIPE_WANT_PTE_PINNING + +/* + * Provide support for pinning the PTEs referencing kernel mappings in + * the current memory context, so that we don't get minor faults when + * treading over kernel memory. For this we need to maintain a map of + * active PGDs. + */ + +#include +#include + +static DEFINE_SPINLOCK(pgd_index_lock); +static struct rb_root pgd_table = RB_ROOT; +static LIST_HEAD(pgd_list); + +struct pgd_holder { + pgd_t *pgd; + struct rb_node rb; + struct list_head next; +}; + +#define pgd_table_lock(__flags) \ + spin_lock_irqsave(&pgd_index_lock, __flags) + +#define pgd_table_unlock(__flags) \ + spin_unlock_irqrestore(&pgd_index_lock, __flags) + +static inline struct pgd_holder *pgd_holder_alloc(pgd_t *pgd) +{ + struct pgd_holder *h; + + h = kmalloc(sizeof(*h), GFP_KERNEL); + if (h == NULL) + return NULL; + + h->pgd = pgd; + + return h; +} + +static inline void pgd_holder_drop(struct pgd_holder *h) +{ + kfree(h); +} + +static inline int pgd_holder_insert(struct pgd_holder *h) +{ + struct rb_node **rbp = &pgd_table.rb_node, *parent = NULL; + struct pgd_holder *p; + + while (*rbp) { + p = rb_entry(*rbp, struct pgd_holder, rb); + parent = *rbp; + if ((unsigned long)h->pgd < (unsigned long)p->pgd) + rbp = &(*rbp)->rb_left; + else { + if (h->pgd == p->pgd) + return 0; + rbp = &(*rbp)->rb_right; + } + } + + rb_link_node(&h->rb, parent, rbp); + rb_insert_color(&h->rb, &pgd_table); + list_add(&h->next, &pgd_list); + + return 1; +} + +static inline void pgd_holder_free(pgd_t *pgd) +{ + struct rb_node *node = pgd_table.rb_node; + struct pgd_holder *p; + unsigned long flags; + + pgd_table_lock(flags); + + while (node) { + p = rb_entry(node, struct pgd_holder, rb); + if ((unsigned long)pgd < (unsigned long)p->pgd) + node = node->rb_left; + else if ((unsigned long)pgd > (unsigned long)p->pgd) + node = node->rb_right; + else { + rb_erase(node, &pgd_table); + list_del(&p->next); + pgd_table_unlock(flags); + pgd_holder_drop(p); + return; + } + } + + pgd_table_unlock(flags); +} + +static void pin_k_mapping(pgd_t *pgd, unsigned long addr) +{ + unsigned int index; + pud_t *pud, *pud_k; + pmd_t *pmd, *pmd_k; + pgd_t *pgd_k; + + index = pgd_index(addr); + pgd += index; + pgd_k = init_mm.pgd + index; + + if (pgd_none(*pgd_k)) + return; + + if (!pgd_present(*pgd)) + set_pgd(pgd, *pgd_k); + + pud = pud_offset(pgd, addr); + pud_k = pud_offset(pgd_k, addr); + + if (pud_none(*pud_k)) + return; + + if (!pud_present(*pud)) + set_pud(pud, *pud_k); + + pmd = pmd_offset(pud, addr); + pmd_k = pmd_offset(pud_k, addr); + + index = IS_ENABLED(ARM_LPAE) ? 0 : (addr >> SECTION_SHIFT) & 1; + if (!pmd_none(pmd_k[index])) + copy_pmd(pmd, pmd_k); +} + +void __ipipe_pin_mapping_globally(unsigned long start, unsigned long end) +{ + unsigned long next, addr; + struct pgd_holder *h; + unsigned long flags; + + pgd_table_lock(flags); + + list_for_each_entry(h, &pgd_list, next) { + addr = start; + do { + next = pgd_addr_end(addr, end); + pin_k_mapping(h->pgd, addr); + } while (addr = next, addr != end); + } + + pgd_table_unlock(flags); +} + +#else /* !CONFIG_IPIPE_WANT_PTE_PINNING */ + +#define pgd_table_lock(__flags) do { (void)(__flags); } while (0) +#define pgd_table_unlock(__flags) do { (void)(__flags); } while (0) + +static inline struct pgd_holder *pgd_holder_alloc(pgd_t *pgd) +{ + return NULL; +} + +static inline void pgd_holder_drop(struct pgd_holder *h) { } + +static inline int pgd_holder_insert(struct pgd_holder *h) { return 1; } + +static inline void pgd_holder_free(pgd_t *pgd) { } + +#endif /* !CONFIG_IPIPE_WANT_PTE_PINNING */ + /* * need to get a 16k page for level 1 */ @@ -36,6 +203,9 @@ pgd_t *pgd_alloc(struct mm_struct *mm) pud_t *new_pud, *init_pud; pmd_t *new_pmd, *init_pmd; pte_t *new_pte, *init_pte; + struct pgd_holder *h; + unsigned long flags; + int ret; new_pgd = __pgd_alloc(); if (!new_pgd) @@ -47,9 +217,15 @@ pgd_t *pgd_alloc(struct mm_struct *mm) * Copy over the kernel and IO PGD entries */ init_pgd = pgd_offset_k(0); + h = pgd_holder_alloc(new_pgd); + pgd_table_lock(flags); memcpy(new_pgd + USER_PTRS_PER_PGD, init_pgd + USER_PTRS_PER_PGD, (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t)); + ret = pgd_holder_insert(h); + pgd_table_unlock(flags); + if (!ret) + pgd_holder_drop(h); clean_dcache_area(new_pgd, PTRS_PER_PGD * sizeof(pgd_t)); #ifdef CONFIG_ARM_LPAE @@ -67,6 +243,10 @@ pgd_t *pgd_alloc(struct mm_struct *mm) #endif if (!vectors_high()) { +#ifdef CONFIG_ARM_FCSE + /* FCSE does not work without high vectors. */ + BUG(); +#endif /* CONFIG_ARM_FCSE */ /* * On ARM, first page must always be allocated since it * contains the machine vectors. The vectors are always high @@ -120,7 +300,7 @@ void pgd_free(struct mm_struct *mm, pgd_t *pgd_base) if (pgd_none_or_clear_bad(pgd)) goto no_pgd; - pud = pud_offset(pgd, 0); + pud = pud_offset(pgd + pgd_index(fcse_va_to_mva(mm, 0)), 0); if (pud_none_or_clear_bad(pud)) goto no_pud; @@ -140,6 +320,8 @@ no_pud: pgd_clear(pgd); pud_free(mm, pud); no_pgd: + pgd_holder_free(pgd); + #ifdef CONFIG_ARM_LPAE /* * Free modules/pkmap or identity pmd tables. diff --git a/arch/arm/mm/proc-arm920.S b/arch/arm/mm/proc-arm920.S index 7a14bd4..14a73b0 100644 --- a/arch/arm/mm/proc-arm920.S +++ b/arch/arm/mm/proc-arm920.S @@ -347,6 +347,11 @@ ENTRY(cpu_arm920_dcache_clean_area) ENTRY(cpu_arm920_switch_mm) #ifdef CONFIG_MMU mov ip, #0 +#ifdef CONFIG_ARM_FCSE_BEST_EFFORT + cmp r2, #0 + beq 3f +#endif /* CONFIG_ARM_FCSE_BEST_EFFORT */ +#ifndef CONFIG_ARM_FCSE_GUARANTEED #ifdef CONFIG_CPU_DCACHE_WRITETHROUGH mcr p15, 0, ip, c7, c6, 0 @ invalidate D cache #else @@ -364,6 +369,10 @@ ENTRY(cpu_arm920_switch_mm) #endif mcr p15, 0, ip, c7, c5, 0 @ invalidate I cache mcr p15, 0, ip, c7, c10, 4 @ drain WB +#endif /* !CONFIG_ARM_FCSE_GUARANTEED */ +#ifdef CONFIG_ARM_FCSE_BEST_EFFORT +3: +#endif /* CONFIG_ARM_FCSE_BEST_EFFORT */ mcr p15, 0, r0, c2, c0, 0 @ load page table pointer mcr p15, 0, ip, c8, c7, 0 @ invalidate I & D TLBs #endif diff --git a/arch/arm/mm/proc-arm926.S b/arch/arm/mm/proc-arm926.S index fb827c6..a00a020 100644 --- a/arch/arm/mm/proc-arm926.S +++ b/arch/arm/mm/proc-arm926.S @@ -368,6 +368,11 @@ ENTRY(cpu_arm926_dcache_clean_area) ENTRY(cpu_arm926_switch_mm) #ifdef CONFIG_MMU mov ip, #0 +#ifdef CONFIG_ARM_FCSE_BEST_EFFORT + cmp r2, #0 + beq 2f +#endif /* CONFIG_ARM_FCSE_BEST_EFFORT */ +#ifndef CONFIG_ARM_FCSE_GUARANTEED #ifdef CONFIG_CPU_DCACHE_WRITETHROUGH mcr p15, 0, ip, c7, c6, 0 @ invalidate D cache #else @@ -377,6 +382,10 @@ ENTRY(cpu_arm926_switch_mm) #endif mcr p15, 0, ip, c7, c5, 0 @ invalidate I cache mcr p15, 0, ip, c7, c10, 4 @ drain WB +#endif /* !CONFIG_ARM_FCSE_GUARANTEED */ +#ifdef CONFIG_ARM_FCSE_BEST_EFFORT +2: +#endif /* CONFIG_ARM_FCSE_BEST_EFFORT */ mcr p15, 0, r0, c2, c0, 0 @ load page table pointer mcr p15, 0, ip, c8, c7, 0 @ invalidate I & D TLBs #endif diff --git a/arch/arm/mm/proc-feroceon.S b/arch/arm/mm/proc-feroceon.S index 92e08bf..c4a6dbc 100644 --- a/arch/arm/mm/proc-feroceon.S +++ b/arch/arm/mm/proc-feroceon.S @@ -475,6 +475,12 @@ ENTRY(cpu_feroceon_dcache_clean_area) .align 5 ENTRY(cpu_feroceon_switch_mm) #ifdef CONFIG_MMU +#ifndef CONFIG_ARM_FCSE_GUARANTEED +#ifdef CONFIG_ARM_FCSE_BEST_EFFORT + cmp r2, #0 + mov r2, lr + beq 2f +#else /* !CONFIG_ARM_FCSE */ /* * Note: we wish to call __flush_whole_cache but we need to preserve * lr to do so. The only way without touching main memory is to @@ -482,12 +488,19 @@ ENTRY(cpu_feroceon_switch_mm) * compensate locally for the skipped ops if it is not set. */ mov r2, lr @ abuse r2 to preserve lr +#endif /* !CONFIG_ARM_FCSE */ bl __flush_whole_cache @ if r2 contains the VM_EXEC bit then the next 2 ops are done already tst r2, #VM_EXEC mcreq p15, 0, ip, c7, c5, 0 @ invalidate I cache mcreq p15, 0, ip, c7, c10, 4 @ drain WB +#ifdef CONFIG_ARM_FCSE +2: +#endif +#else /* CONFIG_ARM_FCSE_GUARANTEED */ + mov r2, lr +#endif /* CONFIG_ARM_FCSE_GUARANTEED */ mcr p15, 0, r0, c2, c0, 0 @ load page table pointer mcr p15, 0, ip, c8, c7, 0 @ invalidate I & D TLBs ret r2 diff --git a/arch/arm/mm/proc-xscale.S b/arch/arm/mm/proc-xscale.S index b6bbfdb..f363be2 100644 --- a/arch/arm/mm/proc-xscale.S +++ b/arch/arm/mm/proc-xscale.S @@ -471,9 +471,18 @@ ENTRY(cpu_xscale_dcache_clean_area) */ .align 5 ENTRY(cpu_xscale_switch_mm) +#ifdef CONFIG_ARM_FCSE_BEST_EFFORT + cmp r2, #0 + beq 2f +#endif /* CONFIG_ARM_FCSE_BEST_EFFORT */ +#ifndef CONFIG_ARM_FCSE_GUARANTEED clean_d_cache r1, r2 mcr p15, 0, ip, c7, c5, 0 @ Invalidate I cache & BTB mcr p15, 0, ip, c7, c10, 4 @ Drain Write (& Fill) Buffer +#endif /* CONFIG_ARM_FCSE_GUARANTEED */ +#ifdef CONFIG_ARM_FCSE_BEST_EFFORT +2: +#endif /* !CONFIG_ARM_FCSE_GUARANTEED */ mcr p15, 0, r0, c2, c0, 0 @ load page table pointer mcr p15, 0, ip, c8, c7, 0 @ invalidate I & D TLBs cpwait_ret lr, ip diff --git a/arch/arm/plat-omap/dmtimer.c b/arch/arm/plat-omap/dmtimer.c index 8ca94d3..ceb4c04 100644 --- a/arch/arm/plat-omap/dmtimer.c +++ b/arch/arm/plat-omap/dmtimer.c @@ -285,7 +285,6 @@ struct omap_dm_timer *omap_dm_timer_request_specific(int id) return _omap_dm_timer_request(REQUEST_BY_ID, &id); } -EXPORT_SYMBOL_GPL(omap_dm_timer_request_specific); /** * omap_dm_timer_request_by_cap - Request a timer by capability @@ -365,6 +364,18 @@ int omap_dm_timer_get_irq(struct omap_dm_timer *timer) } EXPORT_SYMBOL_GPL(omap_dm_timer_get_irq); +#ifdef CONFIG_IPIPE +unsigned long omap_dm_timer_get_phys_counter_addr(struct omap_dm_timer *timer) +{ + return timer->phys_base + (OMAP_TIMER_COUNTER_REG & 0xff); +} + +unsigned long omap_dm_timer_get_virt_counter_addr(struct omap_dm_timer *timer) +{ + return (unsigned long)timer->io_base + (OMAP_TIMER_COUNTER_REG & 0xff); +} +#endif /* CONFIG_IPIPE */ + #if defined(CONFIG_ARCH_OMAP1) #include /** @@ -563,7 +574,7 @@ EXPORT_SYMBOL_GPL(omap_dm_timer_set_load); /* Optimized set_load which removes costly spin wait in timer_start */ int omap_dm_timer_set_load_start(struct omap_dm_timer *timer, int autoreload, - unsigned int load) + unsigned int load) { u32 l; @@ -828,6 +839,7 @@ static int omap_dm_timer_probe(struct platform_device *pdev) } timer->fclk = ERR_PTR(-ENODEV); + timer->phys_base = mem->start; timer->io_base = devm_ioremap_resource(dev, mem); if (IS_ERR(timer->io_base)) return PTR_ERR(timer->io_base); diff --git a/arch/arm/plat-omap/include/plat/dmtimer.h b/arch/arm/plat-omap/include/plat/dmtimer.h index dd79f30..ee368c4 100644 --- a/arch/arm/plat-omap/include/plat/dmtimer.h +++ b/arch/arm/plat-omap/include/plat/dmtimer.h @@ -104,6 +104,7 @@ struct omap_dm_timer { int irq; struct clk *fclk; + unsigned long phys_base; void __iomem *io_base; void __iomem *irq_stat; /* TISR/IRQSTATUS interrupt status */ void __iomem *irq_ena; /* irq enable */ @@ -415,4 +416,9 @@ static inline void __omap_dm_timer_write_status(struct omap_dm_timer *timer, writel_relaxed(value, timer->irq_stat); } +static inline unsigned long __omap_dm_timer_read_status(struct omap_dm_timer *timer) +{ + return __raw_readl(timer->irq_stat); +} + #endif /* __ASM_ARCH_DMTIMER_H */ diff --git a/arch/arm/plat-samsung/include/plat/gpio-core.h b/arch/arm/plat-samsung/include/plat/gpio-core.h index 6ce11bf..5740526 100644 --- a/arch/arm/plat-samsung/include/plat/gpio-core.h +++ b/arch/arm/plat-samsung/include/plat/gpio-core.h @@ -74,7 +74,7 @@ struct samsung_gpio_chip { void __iomem *base; int irq_base; int group; - spinlock_t lock; + ipipe_spinlock_t lock; #ifdef CONFIG_PM u32 pm_save[4]; #endif diff --git a/arch/arm/vfp/entry.S b/arch/arm/vfp/entry.S index 2e78760..3888fbd 100644 --- a/arch/arm/vfp/entry.S +++ b/arch/arm/vfp/entry.S @@ -26,6 +26,7 @@ @ ENTRY(do_vfp) inc_preempt_count r10, r4 + disable_irq_cond ldr r4, .LCvfp ldr r11, [r10, #TI_CPU] @ CPU number add r10, r10, #TI_VFPSTATE @ r10 = workspace @@ -33,6 +34,7 @@ ENTRY(do_vfp) ENDPROC(do_vfp) ENTRY(vfp_null_entry) + enable_irq dec_preempt_count_ti r10, r4 ret lr ENDPROC(vfp_null_entry) @@ -46,6 +48,7 @@ ENDPROC(vfp_null_entry) __INIT ENTRY(vfp_testing_entry) + enable_irq dec_preempt_count_ti r10, r4 ldr r0, VFP_arch_address str r0, [r0] @ set to non-zero value diff --git a/arch/arm/vfp/vfphw.S b/arch/arm/vfp/vfphw.S index f74a8f7..5cf55d7 100644 --- a/arch/arm/vfp/vfphw.S +++ b/arch/arm/vfp/vfphw.S @@ -177,6 +177,7 @@ vfp_hw_state_valid: @ out before setting an FPEXC that @ stops us reading stuff VFPFMXR FPEXC, r1 @ Restore FPEXC last + enable_irq_cond sub r2, r2, #4 @ Retry current instruction - if Thumb str r2, [sp, #S_PC] @ mode it's two 16-bit instructions, @ else it's one 32-bit instruction, so @@ -206,6 +207,7 @@ skip: @ Fall into hand on to next handler - appropriate coproc instr @ not recognised by VFP + enable_irq_cond DBGSTR "not VFP" dec_preempt_count_ti r10, r4 ret lr diff --git a/arch/arm/vfp/vfpmodule.c b/arch/arm/vfp/vfpmodule.c index f6e4d56..e2904aa 100644 --- a/arch/arm/vfp/vfpmodule.c +++ b/arch/arm/vfp/vfpmodule.c @@ -93,6 +93,7 @@ static void vfp_force_reload(unsigned int cpu, struct thread_info *thread) static void vfp_thread_flush(struct thread_info *thread) { union vfp_state *vfp = &thread->vfpstate; + unsigned long flags; unsigned int cpu; /* @@ -103,11 +104,11 @@ static void vfp_thread_flush(struct thread_info *thread) * Do this first to ensure that preemption won't overwrite our * state saving should access to the VFP be enabled at this point. */ - cpu = get_cpu(); + cpu = __ipipe_get_cpu(flags); if (vfp_current_hw_state[cpu] == vfp) vfp_current_hw_state[cpu] = NULL; fmxr(FPEXC, fmrx(FPEXC) & ~FPEXC_EN); - put_cpu(); + __ipipe_put_cpu(flags); memset(vfp, 0, sizeof(union vfp_state)); @@ -122,11 +123,12 @@ static void vfp_thread_exit(struct thread_info *thread) { /* release case: Per-thread VFP cleanup. */ union vfp_state *vfp = &thread->vfpstate; - unsigned int cpu = get_cpu(); + unsigned long flags; + unsigned int cpu = __ipipe_get_cpu(flags); if (vfp_current_hw_state[cpu] == vfp) vfp_current_hw_state[cpu] = NULL; - put_cpu(); + __ipipe_put_cpu(flags); } static void vfp_thread_copy(struct thread_info *thread) @@ -166,6 +168,7 @@ static void vfp_thread_copy(struct thread_info *thread) static int vfp_notifier(struct notifier_block *self, unsigned long cmd, void *v) { struct thread_info *thread = v; + unsigned long flags; u32 fpexc; #ifdef CONFIG_SMP unsigned int cpu; @@ -173,8 +176,9 @@ static int vfp_notifier(struct notifier_block *self, unsigned long cmd, void *v) switch (cmd) { case THREAD_NOTIFY_SWITCH: - fpexc = fmrx(FPEXC); + flags = hard_cond_local_irq_save(); + fpexc = fmrx(FPEXC); #ifdef CONFIG_SMP cpu = thread->cpu; @@ -192,6 +196,7 @@ static int vfp_notifier(struct notifier_block *self, unsigned long cmd, void *v) * old state. */ fmxr(FPEXC, fpexc & ~FPEXC_EN); + hard_cond_local_irq_restore(flags); break; case THREAD_NOTIFY_FLUSH: @@ -335,7 +340,7 @@ static u32 vfp_emulate_instruction(u32 inst, u32 fpscr, struct pt_regs *regs) */ void VFP_bounce(u32 trigger, u32 fpexc, struct pt_regs *regs) { - u32 fpscr, orig_fpscr, fpsid, exceptions; + u32 fpscr, orig_fpscr, fpsid, exceptions, next_trigger = 0; pr_debug("VFP: bounce: trigger %08x fpexc %08x\n", trigger, fpexc); @@ -365,6 +370,7 @@ void VFP_bounce(u32 trigger, u32 fpexc, struct pt_regs *regs) /* * Synchronous exception, emulate the trigger instruction */ + hard_cond_local_irq_enable(); goto emulate; } @@ -377,7 +383,18 @@ void VFP_bounce(u32 trigger, u32 fpexc, struct pt_regs *regs) trigger = fmrx(FPINST); regs->ARM_pc -= 4; #endif - } else if (!(fpexc & FPEXC_DEX)) { + if (fpexc & FPEXC_FP2V) { + /* + * The barrier() here prevents fpinst2 being read + * before the condition above. + */ + barrier(); + next_trigger = fmrx(FPINST2); + } + } + hard_cond_local_irq_enable(); + + if (!(fpexc & (FPEXC_EX | FPEXC_DEX))) { /* * Illegal combination of bits. It can be caused by an * unallocated VFP instruction but with FPSCR.IXE set and not @@ -417,18 +434,14 @@ void VFP_bounce(u32 trigger, u32 fpexc, struct pt_regs *regs) if ((fpexc & (FPEXC_EX | FPEXC_FP2V)) != (FPEXC_EX | FPEXC_FP2V)) goto exit; - /* - * The barrier() here prevents fpinst2 being read - * before the condition above. - */ - barrier(); - trigger = fmrx(FPINST2); + trigger = next_trigger; emulate: exceptions = vfp_emulate_instruction(trigger, orig_fpscr, regs); if (exceptions) vfp_raise_exceptions(exceptions, trigger, orig_fpscr, regs); exit: + hard_cond_local_irq_enable(); preempt_enable(); } @@ -515,7 +528,8 @@ static inline void vfp_pm_init(void) { } */ void vfp_sync_hwstate(struct thread_info *thread) { - unsigned int cpu = get_cpu(); + unsigned long flags; + unsigned int cpu = __ipipe_get_cpu(flags); if (vfp_state_in_hw(cpu, thread)) { u32 fpexc = fmrx(FPEXC); @@ -528,17 +542,18 @@ void vfp_sync_hwstate(struct thread_info *thread) fmxr(FPEXC, fpexc); } - put_cpu(); + __ipipe_put_cpu(flags); } /* Ensure that the thread reloads the hardware VFP state on the next use. */ void vfp_flush_hwstate(struct thread_info *thread) { - unsigned int cpu = get_cpu(); + unsigned long flags; + unsigned int cpu = __ipipe_get_cpu(flags); vfp_force_reload(cpu, thread); - put_cpu(); + __ipipe_put_cpu(flags); } /* diff --git a/drivers/clocksource/arm_arch_timer.c b/drivers/clocksource/arm_arch_timer.c index 0aa135d..c48e8f7 100644 --- a/drivers/clocksource/arm_arch_timer.c +++ b/drivers/clocksource/arm_arch_timer.c @@ -17,6 +17,8 @@ #include #include #include +#include +#include #include #include #include @@ -137,8 +139,7 @@ u32 arch_timer_reg_read(int access, enum arch_timer_reg reg, return val; } -static __always_inline irqreturn_t timer_handler(const int access, - struct clock_event_device *evt) +static int arch_timer_ack(const int access, struct clock_event_device *evt) { unsigned long ctrl; @@ -146,6 +147,52 @@ static __always_inline irqreturn_t timer_handler(const int access, if (ctrl & ARCH_TIMER_CTRL_IT_STAT) { ctrl |= ARCH_TIMER_CTRL_IT_MASK; arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, evt); + return 1; + } + return 0; +} + +#ifdef CONFIG_IPIPE +static DEFINE_PER_CPU(struct ipipe_timer, arch_itimer); +static struct __ipipe_tscinfo tsc_info = { + .type = IPIPE_TSC_TYPE_FREERUNNING_ARCH, + .u = { + { + .mask = 0xffffffffffffffff, + }, + }, +}; + +static void arch_itimer_ack_phys(void) +{ + struct clock_event_device *evt = this_cpu_ptr(arch_timer_evt); + arch_timer_ack(ARCH_TIMER_PHYS_ACCESS, evt); +} + +static void arch_itimer_ack_virt(void) +{ + struct clock_event_device *evt = this_cpu_ptr(arch_timer_evt); + arch_timer_ack(ARCH_TIMER_VIRT_ACCESS, evt); +} +#endif /* CONFIG_IPIPE */ + +static inline irqreturn_t timer_handler(int irq, const int access, + struct clock_event_device *evt) +{ + if (clockevent_ipipe_stolen(evt)) + goto stolen; + + if (arch_timer_ack(access, evt)) { +#ifdef CONFIG_IPIPE + struct ipipe_timer *itimer = raw_cpu_ptr(&arch_itimer); + if (itimer->irq != irq) + itimer->irq = irq; +#endif /* CONFIG_IPIPE */ + stolen: + /* + * This is a 64bit clock source, no need for TSC + * update. + */ evt->event_handler(evt); return IRQ_HANDLED; } @@ -157,28 +204,28 @@ static irqreturn_t arch_timer_handler_virt(int irq, void *dev_id) { struct clock_event_device *evt = dev_id; - return timer_handler(ARCH_TIMER_VIRT_ACCESS, evt); + return timer_handler(irq, ARCH_TIMER_VIRT_ACCESS, evt); } static irqreturn_t arch_timer_handler_phys(int irq, void *dev_id) { struct clock_event_device *evt = dev_id; - return timer_handler(ARCH_TIMER_PHYS_ACCESS, evt); + return timer_handler(irq, ARCH_TIMER_PHYS_ACCESS, evt); } static irqreturn_t arch_timer_handler_phys_mem(int irq, void *dev_id) { struct clock_event_device *evt = dev_id; - return timer_handler(ARCH_TIMER_MEM_PHYS_ACCESS, evt); + return timer_handler(irq, ARCH_TIMER_MEM_PHYS_ACCESS, evt); } static irqreturn_t arch_timer_handler_virt_mem(int irq, void *dev_id) { struct clock_event_device *evt = dev_id; - return timer_handler(ARCH_TIMER_MEM_VIRT_ACCESS, evt); + return timer_handler(irq, ARCH_TIMER_MEM_VIRT_ACCESS, evt); } static __always_inline void timer_set_mode(const int access, int mode, @@ -280,6 +327,18 @@ static void __arch_timer_setup(unsigned type, clk->set_mode = arch_timer_set_mode_phys; clk->set_next_event = arch_timer_set_next_event_phys; } + +#ifdef CONFIG_IPIPE + clk->ipipe_timer = raw_cpu_ptr(&arch_itimer); + if (arch_timer_use_virtual) { + clk->ipipe_timer->irq = arch_timer_ppi[VIRT_PPI]; + clk->ipipe_timer->ack = arch_itimer_ack_virt; + } else { + clk->ipipe_timer->irq = arch_timer_ppi[PHYS_SECURE_PPI]; + clk->ipipe_timer->ack = arch_itimer_ack_phys; + } + clk->ipipe_timer->freq = arch_timer_rate; +#endif } else { clk->features |= CLOCK_EVT_FEAT_DYNIRQ; clk->name = "arch_mem_timer"; @@ -342,6 +401,9 @@ static void arch_counter_set_user_access(void) /* Enable user access to the virtual counter */ cntkctl |= ARCH_TIMER_USR_VCT_ACCESS_EN; +#ifdef CONFIG_IPIPE + cntkctl |= ARCH_TIMER_USR_PCT_ACCESS_EN; +#endif arch_timer_set_cntkctl(cntkctl); } @@ -483,6 +545,11 @@ static void __init arch_counter_register(unsigned type) clocksource_counter.name = "arch_mem_counter"; } +#ifdef CONFIG_IPIPE + tsc_info.freq = arch_timer_rate; + __ipipe_tsc_register(&tsc_info); +#endif /* CONFIG_IPIPE */ + start_count = arch_timer_read_counter(); clocksource_register_hz(&clocksource_counter, arch_timer_rate); cyclecounter.mult = clocksource_counter.mult; diff --git a/drivers/clocksource/arm_global_timer.c b/drivers/clocksource/arm_global_timer.c index e683377..37acb3b 100644 --- a/drivers/clocksource/arm_global_timer.c +++ b/drivers/clocksource/arm_global_timer.c @@ -24,6 +24,7 @@ #include #include +#include #define GT_COUNTER0 0x00 #define GT_COUNTER1 0x04 @@ -48,6 +49,8 @@ * the units for all operations. */ static void __iomem *gt_base; +static unsigned long gt_pbase; +static struct clk *gt_clk; static unsigned long gt_clk_rate; static int gt_ppi; static struct clock_event_device __percpu *gt_evt; @@ -208,8 +211,36 @@ static u64 notrace gt_sched_clock_read(void) } #endif +#ifdef CONFIG_IPIPE + +static unsigned int refresh_gt_freq(void) +{ + gt_clk_rate = clk_get_rate(gt_clk); + + __clocksource_update_freq_hz(>_clocksource, gt_clk_rate); + + return gt_clk_rate; +} + +#endif + static void __init gt_clocksource_init(void) { +#ifdef CONFIG_IPIPE + struct __ipipe_tscinfo tsc_info = { + .type = IPIPE_TSC_TYPE_FREERUNNING, + .freq = gt_clk_rate, + .counter_vaddr = (unsigned long)gt_base, + .u = { + { + .counter_paddr = gt_pbase, + .mask = 0xffffffff, + } + }, + .refresh_freq = refresh_gt_freq, + }; +#endif + writel(0, gt_base + GT_CONTROL); writel(0, gt_base + GT_COUNTER0); writel(0, gt_base + GT_COUNTER1); @@ -219,6 +250,9 @@ static void __init gt_clocksource_init(void) #ifdef CONFIG_CLKSRC_ARM_GLOBAL_TIMER_SCHED_CLOCK sched_clock_register(gt_sched_clock_read, 64, gt_clk_rate); #endif +#ifdef CONFIG_IPIPE + __ipipe_tsc_register(&tsc_info); +#endif clocksource_register_hz(>_clocksource, gt_clk_rate); } @@ -242,8 +276,8 @@ static struct notifier_block gt_cpu_nb = { static void __init global_timer_of_register(struct device_node *np) { - struct clk *gt_clk; - int err = 0; + int err = 0, usable_timer = 1; + struct resource res; /* * In A9 r2p0 the comparators for each processor with the global timer @@ -253,13 +287,15 @@ static void __init global_timer_of_register(struct device_node *np) if (read_cpuid_part() == ARM_CPU_PART_CORTEX_A9 && (read_cpuid_id() & 0xf0000f) < 0x200000) { pr_warn("global-timer: non support for this cpu version.\n"); - return; + usable_timer = 0; } - gt_ppi = irq_of_parse_and_map(np, 0); - if (!gt_ppi) { - pr_warn("global-timer: unable to parse irq\n"); - return; + if (usable_timer) { + gt_ppi = irq_of_parse_and_map(np, 0); + if (!gt_ppi) { + pr_warn("global-timer: unable to parse irq\n"); + return; + } } gt_base = of_iomap(np, 0); @@ -268,6 +304,11 @@ static void __init global_timer_of_register(struct device_node *np) return; } + if (of_address_to_resource(np, 0, &res)) + res.start = 0; + + gt_pbase = res.start; + gt_clk = of_clk_get(np, 0); if (!IS_ERR(gt_clk)) { err = clk_prepare_enable(gt_clk); @@ -280,30 +321,33 @@ static void __init global_timer_of_register(struct device_node *np) } gt_clk_rate = clk_get_rate(gt_clk); - gt_evt = alloc_percpu(struct clock_event_device); - if (!gt_evt) { - pr_warn("global-timer: can't allocate memory\n"); - err = -ENOMEM; - goto out_clk; - } - - err = request_percpu_irq(gt_ppi, gt_clockevent_interrupt, + if (usable_timer) { + gt_evt = alloc_percpu(struct clock_event_device); + if (!gt_evt) { + pr_warn("global-timer: can't allocate memory\n"); + err = -ENOMEM; + goto out_clk; + } + + err = request_percpu_irq(gt_ppi, gt_clockevent_interrupt, "gt", gt_evt); - if (err) { - pr_warn("global-timer: can't register interrupt %d (%d)\n", - gt_ppi, err); - goto out_free; - } - - err = register_cpu_notifier(>_cpu_nb); - if (err) { - pr_warn("global-timer: unable to register cpu notifier.\n"); - goto out_irq; + if (err) { + pr_warn("global-timer: can't register interrupt %d (%d)\n", + gt_ppi, err); + goto out_free; + } + + err = register_cpu_notifier(>_cpu_nb); + if (err) { + pr_warn("global-timer: unable to register cpu notifier.\n"); + goto out_irq; + } } /* Immediately configure the timer on the boot CPU */ gt_clocksource_init(); - gt_clockevents_init(this_cpu_ptr(gt_evt)); + if (usable_timer) + gt_clockevents_init(this_cpu_ptr(gt_evt)); return; diff --git a/drivers/clocksource/mxs_timer.c b/drivers/clocksource/mxs_timer.c index 445b68a..6e6ded2 100644 --- a/drivers/clocksource/mxs_timer.c +++ b/drivers/clocksource/mxs_timer.c @@ -22,6 +22,8 @@ #include #include +#include +#include #include #include #include @@ -76,10 +78,17 @@ #define BV_TIMROTv2_TIMCTRLn_SELECT__32KHZ_XTAL 0xb #define BV_TIMROTv2_TIMCTRLn_SELECT__TICK_ALWAYS 0xf +#define IPIPE_DIV_ORDER 0 /* APBX clock prescaler order */ +#define IPIPE_DIV (1 << IPIPE_DIV_ORDER) +#define BV_TIMROTv2_TIMCTRLn_PRESCALE (1 << 4) + static struct clock_event_device mxs_clockevent_device; static enum clock_event_mode mxs_clockevent_mode = CLOCK_EVT_MODE_UNUSED; static void __iomem *mxs_timrot_base; +#ifdef CONFIG_IPIPE +static unsigned long mxs_timrot_paddr; +#endif /* CONFIG_IPIPE */ static u32 timrot_major_version; static inline void timrot_irq_disable(void) @@ -128,7 +137,11 @@ static irqreturn_t mxs_timer_interrupt(int irq, void *dev_id) { struct clock_event_device *evt = dev_id; - timrot_irq_acknowledge(); + if (!clockevent_ipipe_stolen(evt)) + timrot_irq_acknowledge(); + + __ipipe_tsc_update(); + evt->event_handler(evt); return IRQ_HANDLED; @@ -193,21 +206,45 @@ static void mxs_set_mode(enum clock_event_mode mode, } } +#ifdef CONFIG_IPIPE +static struct ipipe_timer mxs_itimer = { + .ack = timrot_irq_acknowledge, +}; + +static struct __ipipe_tscinfo __maybe_unused tsc_info = { + .type = IPIPE_TSC_TYPE_FREERUNNING_COUNTDOWN, + .u = { + { + .mask = 0xffffffff, + }, + }, +}; +#endif /* CONFIG_IPIPE */ + static struct clock_event_device mxs_clockevent_device = { .name = "mxs_timrot", .features = CLOCK_EVT_FEAT_ONESHOT, .set_mode = mxs_set_mode, .set_next_event = timrotv2_set_next_event, .rating = 200, +#ifdef CONFIG_IPIPE + .ipipe_timer = &mxs_itimer, +#endif /* CONFIG_IPIPE */ }; static int __init mxs_clockevent_init(struct clk *timer_clk) { + unsigned int c = clk_get_rate(timer_clk); + +#ifdef CONFIG_IPIPE + c /= IPIPE_DIV; +#endif /* CONFIG_IPIPE */ + if (timrot_is_v1()) mxs_clockevent_device.set_next_event = timrotv1_set_next_event; mxs_clockevent_device.cpumask = cpumask_of(0); clockevents_config_and_register(&mxs_clockevent_device, - clk_get_rate(timer_clk), + c, timrot_is_v1() ? 0xf : 0x2, timrot_is_v1() ? 0xfffe : 0xfffffffe); @@ -231,11 +268,19 @@ static int __init mxs_clocksource_init(struct clk *timer_clk) { unsigned int c = clk_get_rate(timer_clk); - if (timrot_is_v1()) + if (timrot_is_v1()) { clocksource_register_hz(&clocksource_mxs, c); - else { + } else { +#ifndef CONFIG_IPIPE clocksource_mmio_init(mxs_timrot_base + HW_TIMROT_RUNNING_COUNTn(1), "mxs_timer", c, 200, 32, clocksource_mmio_readl_down); +#else /* CONFIG_IPIPE */ + c /= IPIPE_DIV; + tsc_info.freq = c; + tsc_info.counter_vaddr = (unsigned long)mxs_timrot_base + HW_TIMROT_RUNNING_COUNTn(1); + tsc_info.u.counter_paddr = mxs_timrot_paddr + HW_TIMROT_RUNNING_COUNTn(1); + __ipipe_tsc_register(&tsc_info); +#endif /* CONFIG_IPIPE */ sched_clock_register(mxs_read_sched_clock_v2, 32, c); } @@ -245,11 +290,23 @@ static int __init mxs_clocksource_init(struct clk *timer_clk) static void __init mxs_timer_init(struct device_node *np) { struct clk *timer_clk; + unsigned long xtal; int irq; mxs_timrot_base = of_iomap(np, 0); WARN_ON(!mxs_timrot_base); +#ifdef CONFIG_IPIPE + if (mxs_timrot_base){ + struct resource res; + + if (of_address_to_resource(np, 0, &res)) + res.start = 0; + + mxs_timrot_paddr = res.start; + } +#endif /* CONFIG_IPIPE */ + timer_clk = of_clk_get(np, 0); if (IS_ERR(timer_clk)) { pr_err("%s: failed to get clk\n", __func__); @@ -270,20 +327,26 @@ static void __init mxs_timer_init(struct device_node *np) MX28_TIMROT_VERSION_OFFSET)); timrot_major_version >>= BP_TIMROT_MAJOR_VERSION; + if (timrot_is_v1()) + xtal = BV_TIMROTv1_TIMCTRLn_SELECT__32KHZ_XTAL; + else { +#ifndef CONFIG_IPIPE + xtal = BV_TIMROTv2_TIMCTRLn_SELECT__TICK_ALWAYS; +#else + xtal = BV_TIMROTv2_TIMCTRLn_SELECT__TICK_ALWAYS | + (IPIPE_DIV_ORDER * BV_TIMROTv2_TIMCTRLn_PRESCALE); +#endif + } /* one for clock_event */ - __raw_writel((timrot_is_v1() ? - BV_TIMROTv1_TIMCTRLn_SELECT__32KHZ_XTAL : - BV_TIMROTv2_TIMCTRLn_SELECT__TICK_ALWAYS) | - BM_TIMROT_TIMCTRLn_UPDATE | - BM_TIMROT_TIMCTRLn_IRQ_EN, - mxs_timrot_base + HW_TIMROT_TIMCTRLn(0)); + __raw_writel(xtal | + BM_TIMROT_TIMCTRLn_UPDATE | + BM_TIMROT_TIMCTRLn_IRQ_EN, + mxs_timrot_base + HW_TIMROT_TIMCTRLn(0)); /* another for clocksource */ - __raw_writel((timrot_is_v1() ? - BV_TIMROTv1_TIMCTRLn_SELECT__32KHZ_XTAL : - BV_TIMROTv2_TIMCTRLn_SELECT__TICK_ALWAYS) | - BM_TIMROT_TIMCTRLn_RELOAD, - mxs_timrot_base + HW_TIMROT_TIMCTRLn(1)); + __raw_writel(xtal | + BM_TIMROT_TIMCTRLn_RELOAD, + mxs_timrot_base + HW_TIMROT_TIMCTRLn(1)); /* set clocksource timer fixed count to the maximum */ if (timrot_is_v1()) @@ -300,5 +363,9 @@ static void __init mxs_timer_init(struct device_node *np) /* Make irqs happen */ irq = irq_of_parse_and_map(np, 0); setup_irq(irq, &mxs_timer_irq); + +#ifdef CONFIG_IPIPE + mxs_itimer.irq = irq; +#endif } CLOCKSOURCE_OF_DECLARE(mxs, "fsl,timrot", mxs_timer_init); diff --git a/drivers/clocksource/pxa_timer.c b/drivers/clocksource/pxa_timer.c index d9438af..702dcc4 100644 --- a/drivers/clocksource/pxa_timer.c +++ b/drivers/clocksource/pxa_timer.c @@ -20,6 +20,8 @@ #include #include #include +#include +#include #include @@ -62,14 +64,21 @@ static u64 notrace pxa_read_sched_clock(void) #define MIN_OSCR_DELTA 16 +static inline void pxa_ost0_ack(void) +{ + /* Disarm the compare/match, signal the event. */ + timer_writel(timer_readl(OIER) & ~OIER_E0, OIER); + timer_writel(OSSR_M0, OSSR); +} + static irqreturn_t pxa_ost0_interrupt(int irq, void *dev_id) { struct clock_event_device *c = dev_id; - /* Disarm the compare/match, signal the event. */ - timer_writel(timer_readl(OIER) & ~OIER_E0, OIER); - timer_writel(OSSR_M0, OSSR); + if (clockevent_ipipe_stolen(c) == 0) + pxa_ost0_ack(); + c->event_handler(c); return IRQ_HANDLED; @@ -146,6 +155,13 @@ static void pxa_timer_resume(struct clock_event_device *cedev) #define pxa_timer_resume NULL #endif +#ifdef CONFIG_IPIPE +static struct ipipe_timer pxa_osmr0_itimer = { + .ack = pxa_ost0_ack, + .min_delay_ticks = MIN_OSCR_DELTA, +}; +#endif /* CONFIG_IPIPE */ + static struct clock_event_device ckevt_pxa_osmr0 = { .name = "osmr0", .features = CLOCK_EVT_FEAT_ONESHOT, @@ -154,6 +170,9 @@ static struct clock_event_device ckevt_pxa_osmr0 = { .set_mode = pxa_osmr0_set_mode, .suspend = pxa_timer_suspend, .resume = pxa_timer_resume, +#ifdef CONFIG_IPIPE + .ipipe_timer = &pxa_osmr0_itimer, +#endif /* CONFIG_IPIPE */ }; static struct irqaction pxa_ost0_irq = { @@ -163,6 +182,18 @@ static struct irqaction pxa_ost0_irq = { .dev_id = &ckevt_pxa_osmr0, }; +#ifdef CONFIG_IPIPE +static struct __ipipe_tscinfo tsc_info = { + .type = IPIPE_TSC_TYPE_FREERUNNING, + .u = { + { + .counter_paddr = 0x40A00010UL, + .mask = 0xffffffff, + }, + }, +}; +#endif /* CONFIG_IPIPE */ + static void __init pxa_timer_common_init(int irq, unsigned long clock_tick_rate) { timer_writel(0, OIER); @@ -175,7 +206,17 @@ static void __init pxa_timer_common_init(int irq, unsigned long clock_tick_rate) setup_irq(irq, &pxa_ost0_irq); clocksource_mmio_init(timer_base + OSCR, "oscr0", clock_tick_rate, 200, - 32, clocksource_mmio_readl_up); + 32, clocksource_mmio_readl_up); + +#ifdef CONFIG_IPIPE + tsc_info.freq = clock_tick_rate; + tsc_info.counter_vaddr = (unsigned long)timer_base + OSCR; + + __ipipe_tsc_register(&tsc_info); + + pxa_osmr0_itimer.irq = irq; +#endif /* CONFIG_IPIPE */ + clockevents_config_and_register(&ckevt_pxa_osmr0, clock_tick_rate, MIN_OSCR_DELTA * 2, 0x7fffffff); } @@ -212,7 +253,7 @@ CLOCKSOURCE_OF_DECLARE(pxa_timer, "marvell,pxa-timer", pxa_timer_dt_init); * Legacy timer init for non device-tree boards. */ void __init pxa_timer_nodt_init(int irq, void __iomem *base, - unsigned long clock_tick_rate) + unsigned long clock_tick_rate) { struct clk *clk; diff --git a/drivers/gpio/gpio-davinci.c b/drivers/gpio/gpio-davinci.c index c5e05c8..5f36a06 100644 --- a/drivers/gpio/gpio-davinci.c +++ b/drivers/gpio/gpio-davinci.c @@ -23,6 +23,7 @@ #include #include #include +#include struct davinci_gpio_regs { u32 dir; @@ -357,7 +358,7 @@ gpio_irq_handler(unsigned irq, struct irq_desc *desc) while (status) { bit = __ffs(status); status &= ~BIT(bit); - generic_handle_irq( + ipipe_handle_demuxed_irq( irq_find_mapping(d->irq_domain, d->chip.base + bit)); } diff --git a/drivers/gpio/gpio-mvebu.c b/drivers/gpio/gpio-mvebu.c index 1a54205..b4f7125 100644 --- a/drivers/gpio/gpio-mvebu.c +++ b/drivers/gpio/gpio-mvebu.c @@ -310,10 +310,11 @@ static void mvebu_gpio_irq_ack(struct irq_data *d) struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); struct mvebu_gpio_chip *mvchip = gc->private; u32 mask = ~(1 << (d->irq - gc->irq_base)); + unsigned long flags; - irq_gc_lock(gc); + flags = irq_gc_lock(gc); writel_relaxed(mask, mvebu_gpioreg_edge_cause(mvchip)); - irq_gc_unlock(gc); + irq_gc_unlock(gc, flags); } static void mvebu_gpio_edge_irq_mask(struct irq_data *d) @@ -322,12 +323,13 @@ static void mvebu_gpio_edge_irq_mask(struct irq_data *d) struct mvebu_gpio_chip *mvchip = gc->private; struct irq_chip_type *ct = irq_data_get_chip_type(d); u32 mask = 1 << (d->irq - gc->irq_base); + unsigned long flags; - irq_gc_lock(gc); + flags = irq_gc_lock(gc); ct->mask_cache_priv &= ~mask; writel_relaxed(ct->mask_cache_priv, mvebu_gpioreg_edge_mask(mvchip)); - irq_gc_unlock(gc); + irq_gc_unlock(gc, flags); } static void mvebu_gpio_edge_irq_unmask(struct irq_data *d) @@ -335,13 +337,14 @@ static void mvebu_gpio_edge_irq_unmask(struct irq_data *d) struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); struct mvebu_gpio_chip *mvchip = gc->private; struct irq_chip_type *ct = irq_data_get_chip_type(d); + unsigned long flags; u32 mask = 1 << (d->irq - gc->irq_base); - irq_gc_lock(gc); + flags = irq_gc_lock(gc); ct->mask_cache_priv |= mask; writel_relaxed(ct->mask_cache_priv, mvebu_gpioreg_edge_mask(mvchip)); - irq_gc_unlock(gc); + irq_gc_unlock(gc, flags); } static void mvebu_gpio_level_irq_mask(struct irq_data *d) @@ -349,13 +352,14 @@ static void mvebu_gpio_level_irq_mask(struct irq_data *d) struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); struct mvebu_gpio_chip *mvchip = gc->private; struct irq_chip_type *ct = irq_data_get_chip_type(d); + unsigned long flags; u32 mask = 1 << (d->irq - gc->irq_base); - irq_gc_lock(gc); + flags = irq_gc_lock(gc); ct->mask_cache_priv &= ~mask; writel_relaxed(ct->mask_cache_priv, mvebu_gpioreg_level_mask(mvchip)); - irq_gc_unlock(gc); + irq_gc_unlock(gc, flags); } static void mvebu_gpio_level_irq_unmask(struct irq_data *d) @@ -363,13 +367,14 @@ static void mvebu_gpio_level_irq_unmask(struct irq_data *d) struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); struct mvebu_gpio_chip *mvchip = gc->private; struct irq_chip_type *ct = irq_data_get_chip_type(d); + unsigned long flags; u32 mask = 1 << (d->irq - gc->irq_base); - irq_gc_lock(gc); + flags = irq_gc_lock(gc); ct->mask_cache_priv |= mask; writel_relaxed(ct->mask_cache_priv, mvebu_gpioreg_level_mask(mvchip)); - irq_gc_unlock(gc); + irq_gc_unlock(gc, flags); } /***************************************************************************** diff --git a/drivers/gpio/gpio-mxc.c b/drivers/gpio/gpio-mxc.c index 9f7446a..505fb0c 100644 --- a/drivers/gpio/gpio-mxc.c +++ b/drivers/gpio/gpio-mxc.c @@ -33,6 +33,7 @@ #include #include #include +#include #include enum mxc_gpio_hwtype { @@ -66,6 +67,9 @@ struct mxc_gpio_port { struct irq_domain *domain; struct bgpio_chip bgc; u32 both_edges; +#ifdef CONFIG_IPIPE + unsigned nonroot; +#endif /* CONFIG_IPIPE */ }; static struct mxc_gpio_hwdata imx1_imx21_gpio_hwdata = { @@ -265,7 +269,7 @@ static void mxc_gpio_irq_handler(struct mxc_gpio_port *port, u32 irq_stat) if (port->both_edges & (1 << irqoffset)) mxc_flip_edge(port, irqoffset); - generic_handle_irq(irq_find_mapping(port->domain, irqoffset)); + ipipe_handle_demuxed_irq(irq_find_mapping(port->domain, irqoffset)); irq_stat &= ~(1 << irqoffset); } @@ -502,6 +506,122 @@ static struct platform_driver mxc_gpio_driver = { .id_table = mxc_gpio_devtype, }; +#if defined(CONFIG_IPIPE) && \ + (defined(CONFIG_MXC_TZIC) || defined(CONFIG_SOC_IMX6Q)) +extern void tzic_set_irq_prio(int irq, int hi); +extern void tzic_mute_pic(void); +extern void tzic_unmute_pic(void); +extern void gic_mute(void); +extern void gic_unmute(void); +extern void gic_set_irq_prio(int irq, int hi); + +#ifdef CONFIG_MXC_TZIC +static unsigned is_mx5; +#endif /* CONFIG_MXC_TZIC */ +#ifdef CONFIG_SOC_IMX6Q +static unsigned is_mx6; +#endif /* CONFIG_SOC_IMX6Q */ + +static void mxc_set_irq_prio(int irq, int hi) +{ + struct irq_desc *desc = irq_to_desc(irq); + struct irq_data *idata = irq_desc_get_irq_data(desc); + +#ifdef CONFIG_SOC_IMX6Q + if (is_mx6) + gic_set_irq_prio(idata->hwirq, hi); +#endif /* CONFIG_SOC_IMX6Q */ + +#ifdef CONFIG_MXC_TZIC + if (is_mx5) + tzic_set_irq_prio(idata->hwirq, hi); +#endif /* CONFIG_MXC_TZIC */ +} + +static void mxc_enable_irqdesc(struct ipipe_domain *ipd, unsigned irq) +{ + struct irq_desc *desc = irq_to_desc(irq); + struct irq_data *idata = irq_desc_get_irq_data(desc); + struct irq_chip *chip = irq_data_get_irq_chip(idata); + + if (chip->irq_set_type == gpio_set_irq_type) { + /* It is a gpio. */ + struct irq_chip_generic *gc = irq_data_get_irq_chip_data(idata); + struct mxc_gpio_port *port = gc->private; + + if (ipd == &ipipe_root) { + port->nonroot &= ~(1 << idata->hwirq); + if (port->nonroot == 0) { + mxc_set_irq_prio(port->irq, 0); + if (port->irq_high > 0) + mxc_set_irq_prio(port->irq_high, 0); + } + } else { + port->nonroot |= (1 << idata->hwirq); + if (port->nonroot == (1 << idata->hwirq)) { + mxc_set_irq_prio(port->irq, 1); + if (port->irq_high > 0) + mxc_set_irq_prio(port->irq_high, 1); + } + } + } else + mxc_set_irq_prio(irq, ipd != &ipipe_root); +} + +static void mxc_disable_irqdesc(struct ipipe_domain *ipd, unsigned irq) +{ + struct irq_desc *desc = irq_to_desc(irq); + struct irq_data *idata = irq_desc_get_irq_data(desc); + struct irq_chip *chip = irq_data_get_irq_chip(idata); + + if (chip->irq_set_type == gpio_set_irq_type) { + /* It is a gpio. */ + struct irq_chip_generic *gc = irq_data_get_irq_chip_data(idata); + struct mxc_gpio_port *port = gc->private; + + if (ipd != &ipipe_root) { + port->nonroot &= ~(1 << idata->hwirq); + if (port->nonroot == 0) { + mxc_set_irq_prio(port->irq, 0); + if (port->irq_high > 0) + mxc_set_irq_prio(port->irq_high, 0); + } + } + } else if (ipd != &ipipe_root) + mxc_set_irq_prio(irq, 0); +} + +#ifdef CONFIG_MXC_TZIC +void __init mxc_pic_muter_register(void) +{ + struct ipipe_mach_pic_muter pic_muter = { + .enable_irqdesc = mxc_enable_irqdesc, + .disable_irqdesc = mxc_disable_irqdesc, + .mute = tzic_mute_pic, + .unmute = tzic_unmute_pic, + }; + + is_mx5 = 1; + ipipe_pic_muter_register(&pic_muter); +} +#endif + +#ifdef CONFIG_SOC_IMX6Q +void __init mx6_pic_muter_register(void) +{ + struct ipipe_mach_pic_muter pic_muter = { + .enable_irqdesc = mxc_enable_irqdesc, + .disable_irqdesc = mxc_disable_irqdesc, + .mute = gic_mute, + .unmute = gic_unmute, + }; + + is_mx6 = 1; + ipipe_pic_muter_register(&pic_muter); +} +#endif /* CONFIG_SOC_IMX6Q */ +#endif /* CONFIG_IPIPE */ + static int __init gpio_mxc_init(void) { return platform_driver_register(&mxc_gpio_driver); diff --git a/drivers/gpio/gpio-mxs.c b/drivers/gpio/gpio-mxs.c index 84cbda6..8a6e44e 100644 --- a/drivers/gpio/gpio-mxs.c +++ b/drivers/gpio/gpio-mxs.c @@ -34,6 +34,7 @@ #include #include #include +#include #define MXS_SET 0x4 #define MXS_CLR 0x8 diff --git a/drivers/gpio/gpio-omap.c b/drivers/gpio/gpio-omap.c index b232397..a77d0ce 100644 --- a/drivers/gpio/gpio-omap.c +++ b/drivers/gpio/gpio-omap.c @@ -27,6 +27,7 @@ #include #include #include +#include #define OFF_MODE 1 @@ -57,7 +58,7 @@ struct gpio_bank { u32 saved_datain; u32 level_mask; u32 toggle_mask; - spinlock_t lock; + ipipe_spinlock_t lock; struct gpio_chip chip; struct clk *dbck; u32 mod_usage; @@ -79,6 +80,10 @@ struct gpio_bank { int (*get_context_loss_count)(struct device *dev); struct omap_gpio_reg_offs *regs; +#ifdef CONFIG_IPIPE + unsigned nonroot; + unsigned muted; +#endif }; #define GPIO_MOD_CTRL_BIT BIT(0) @@ -373,8 +378,8 @@ static void omap_toggle_gpio_edge_triggering(struct gpio_bank *bank, int gpio) static void omap_toggle_gpio_edge_triggering(struct gpio_bank *bank, int gpio) {} #endif -static int omap_set_gpio_triggering(struct gpio_bank *bank, int gpio, - unsigned trigger) +static inline int omap_set_gpio_triggering(struct gpio_bank *bank, int gpio, + unsigned trigger) { void __iomem *reg = bank->base; void __iomem *base = bank->base; @@ -515,7 +520,7 @@ static int omap_gpio_irq_type(struct irq_data *d, unsigned type) return retval; } -static void omap_clear_gpio_irqbank(struct gpio_bank *bank, int gpio_mask) +static inline void omap_clear_gpio_irqbank(struct gpio_bank *bank, int gpio_mask) { void __iomem *reg = bank->base; @@ -552,7 +557,7 @@ static u32 omap_get_gpio_irqbank_mask(struct gpio_bank *bank) return l; } -static void omap_enable_gpio_irqbank(struct gpio_bank *bank, int gpio_mask) +static inline void omap_enable_gpio_irqbank(struct gpio_bank *bank, int gpio_mask) { void __iomem *reg = bank->base; u32 l; @@ -574,7 +579,7 @@ static void omap_enable_gpio_irqbank(struct gpio_bank *bank, int gpio_mask) writel_relaxed(l, reg); } -static void omap_disable_gpio_irqbank(struct gpio_bank *bank, int gpio_mask) +static inline void omap_disable_gpio_irqbank(struct gpio_bank *bank, int gpio_mask) { void __iomem *reg = bank->base; u32 l; @@ -638,7 +643,7 @@ static int omap_set_gpio_wakeup(struct gpio_bank *bank, unsigned offset, return 0; } -static void omap_reset_gpio(struct gpio_bank *bank, unsigned offset) +static inline void omap_reset_gpio(struct gpio_bank *bank, unsigned offset) { omap_set_gpio_direction(bank, offset, 1); omap_set_gpio_irqenable(bank, offset, 0); @@ -725,7 +730,10 @@ static void omap_gpio_irq_handler(unsigned int irq, struct irq_desc *desc) bank = container_of(chip, struct gpio_bank, chip); isr_reg = bank->base + bank->regs->irqstatus; + +#ifndef CONFIG_IPIPE pm_runtime_get_sync(bank->dev); +#endif if (WARN_ON(!isr_reg)) goto exit; @@ -771,7 +779,7 @@ static void omap_gpio_irq_handler(unsigned int irq, struct irq_desc *desc) if (bank->toggle_mask & (BIT(bit))) omap_toggle_gpio_edge_triggering(bank, bit); - generic_handle_irq(irq_find_mapping(bank->chip.irqdomain, + ipipe_handle_demuxed_irq(irq_find_mapping(bank->chip.irqdomain, bit)); } } @@ -782,7 +790,9 @@ static void omap_gpio_irq_handler(unsigned int irq, struct irq_desc *desc) exit: if (!unmasked) chained_irq_exit(irqchip, desc); +#ifndef CONFIG_IPIPE pm_runtime_put(bank->dev); +#endif } static unsigned int omap_gpio_irq_startup(struct irq_data *d) @@ -842,6 +852,19 @@ static void omap_gpio_mask_irq(struct irq_data *d) spin_unlock_irqrestore(&bank->lock, flags); } +static void omap_gpio_mask_ack_irq(struct irq_data *d) +{ + struct gpio_bank *bank = omap_irq_data_get_bank(d); + unsigned offset = d->hwirq; + unsigned long flags; + + spin_lock_irqsave(&bank->lock, flags); + omap_set_gpio_irqenable(bank, offset, 0); + omap_set_gpio_triggering(bank, offset, IRQ_TYPE_NONE); + spin_unlock_irqrestore(&bank->lock, flags); + omap_clear_gpio_irqstatus(bank, offset); +} + static void omap_gpio_unmask_irq(struct irq_data *d) { struct gpio_bank *bank = omap_irq_data_get_bank(d); @@ -1160,6 +1183,7 @@ static int omap_gpio_probe(struct platform_device *pdev) irqc->irq_shutdown = omap_gpio_irq_shutdown, irqc->irq_ack = omap_gpio_ack_irq, irqc->irq_mask = omap_gpio_mask_irq, + irqc->irq_mask_ack = omap_gpio_mask_ack_irq, irqc->irq_unmask = omap_gpio_unmask_irq, irqc->irq_set_type = omap_gpio_irq_type, irqc->irq_set_wake = omap_gpio_wake_enable, @@ -1235,6 +1259,164 @@ static int omap_gpio_probe(struct platform_device *pdev) #ifdef CONFIG_ARCH_OMAP2PLUS +#if defined(CONFIG_IPIPE) +extern void omap3_intc_mute(void); +extern void omap3_intc_unmute(void); +extern void omap3_intc_set_irq_prio(int irq, int hi); +extern void gic_mute(void); +extern void gic_unmute(void); +extern void gic_set_irq_prio(int irq, int hi); +static unsigned ipipe_mach_omap; + +static inline void omap2plus_pic_set_irq_prio(int irq, int hi) +{ + struct irq_desc *desc = irq_to_desc(irq); + struct irq_data *idata = irq_desc_get_irq_data(desc); + +#if defined(CONFIG_ARCH_OMAP3) || defined(CONFIG_SOC_AM33XX) + if (ipipe_mach_omap ==3) + omap3_intc_set_irq_prio(idata->hwirq, hi); +#endif /* omap3 */ +#ifdef CONFIG_ARM_GIC + if (ipipe_mach_omap == 4) + gic_set_irq_prio(idata->hwirq, hi); +#endif /* gic */ +} + +static void omap2plus_enable_irqdesc(struct ipipe_domain *ipd, unsigned irq) +{ + struct irq_desc *desc = irq_to_desc(irq); + struct irq_data *idata = irq_desc_get_irq_data(desc); + struct irq_chip *chip = irq_data_get_irq_chip(idata); + + if (chip->irq_mask_ack == omap_gpio_mask_ack_irq) { + /* It is a gpio. */ + struct gpio_bank *bank = irq_data_get_irq_chip_data(idata); + + if (ipd == &ipipe_root) { + bank->nonroot &= ~(1 << idata->hwirq); + if (bank->nonroot == 0) + omap2plus_pic_set_irq_prio(bank->irq, 0); + } else { + bank->nonroot |= (1 << idata->hwirq); + if (bank->nonroot == (1 << idata->hwirq)) + omap2plus_pic_set_irq_prio(bank->irq, 1); + } + } else + omap2plus_pic_set_irq_prio(irq, ipd != &ipipe_root); +} + +static void omap2plus_disable_irqdesc(struct ipipe_domain *ipd, unsigned irq) +{ + struct irq_desc *desc = irq_to_desc(irq); + struct irq_data *idata = irq_desc_get_irq_data(desc); + struct irq_chip *chip = irq_data_get_irq_chip(idata); + + if (chip->irq_mask_ack == omap_gpio_mask_ack_irq) { + /* It is a gpio. */ + struct gpio_bank *bank = irq_data_get_irq_chip_data(idata); + + if (ipd != &ipipe_root) { + bank->nonroot &= ~(1 << idata->hwirq); + if (bank->nonroot == 0) + omap2plus_pic_set_irq_prio(bank->irq, 0); + } + } else if (ipd != &ipipe_root) + omap2plus_pic_set_irq_prio(irq, 0); +} + +static inline void omap2plus_mute_gpio(void) +{ + struct gpio_bank *bank; + unsigned muted; + + list_for_each_entry(bank, &omap_gpio_list, node) { + if (bank->nonroot == 0) + continue; + + muted = ~bank->nonroot; + if (muted) + muted &= omap_get_gpio_irqbank_mask(bank); + bank->muted = muted; + if (muted) + omap_disable_gpio_irqbank(bank, muted); + } +} +static inline void omap2plus_unmute_gpio(void) +{ + struct gpio_bank *bank; + unsigned muted; + + list_for_each_entry(bank, &omap_gpio_list, node) { + if (bank->nonroot == 0) + continue; + + muted = bank->muted; + if (muted) + omap_enable_gpio_irqbank(bank, muted); + } +} + +#if defined(CONFIG_ARCH_OMAP3) || defined(CONFIG_SOC_AM33XX) +static void omap3_mute_pic(void) +{ + omap3_intc_mute(); + + omap2plus_mute_gpio(); +} + +static void omap3_unmute_pic(void) +{ + omap2plus_unmute_gpio(); + + omap3_intc_unmute(); +} + +void __init omap3_pic_muter_register(void) +{ + struct ipipe_mach_pic_muter muter = { + .enable_irqdesc = omap2plus_enable_irqdesc, + .disable_irqdesc = omap2plus_disable_irqdesc, + .mute = omap3_mute_pic, + .unmute = omap3_unmute_pic, + }; + + ipipe_pic_muter_register(&muter); + ipipe_mach_omap = 3; +} +#endif /* omap3 */ + +#ifdef CONFIG_ARM_GIC +static void omap4_mute_pic(void) +{ + gic_mute(); + + omap2plus_mute_gpio(); +} + +static void omap4_unmute_pic(void) +{ + omap2plus_unmute_gpio(); + + gic_unmute(); +} + +void __init omap4_pic_muter_register(void) +{ + struct ipipe_mach_pic_muter muter = { + .enable_irqdesc = omap2plus_enable_irqdesc, + .disable_irqdesc = omap2plus_disable_irqdesc, + .mute = omap4_mute_pic, + .unmute = omap4_unmute_pic, + }; + + ipipe_pic_muter_register(&muter); + ipipe_mach_omap = 4; +} +#endif /* GIC */ + +#endif /* CONFIG_IPIPE */ + #if defined(CONFIG_PM) static void omap_gpio_restore_context(struct gpio_bank *bank); diff --git a/drivers/gpio/gpio-pxa.c b/drivers/gpio/gpio-pxa.c index cdbbcf0..d620e6f 100644 --- a/drivers/gpio/gpio-pxa.c +++ b/drivers/gpio/gpio-pxa.c @@ -27,6 +27,7 @@ #include #include #include +#include /* * We handle the GPIOs by banks, each bank covers up to 32 GPIOs with @@ -103,7 +104,7 @@ struct pxa_gpio_id { int gpio_nums; }; -static DEFINE_SPINLOCK(gpio_lock); +static IPIPE_DEFINE_SPINLOCK(gpio_lock); static struct pxa_gpio_chip *pxa_gpio_chips; static enum pxa_gpio_type gpio_type; static void __iomem *gpio_reg_base; @@ -422,7 +423,7 @@ static void pxa_gpio_demux_handler(unsigned int irq, struct irq_desc *desc) for_each_set_bit(n, &gedr, BITS_PER_LONG) { loop = 1; - generic_handle_irq(gpio_to_irq(gpio_base + n)); + ipipe_handle_demuxed_irq(gpio_to_irq(gpio_base + n)); } } } while (loop); diff --git a/drivers/gpio/gpio-sa1100.c b/drivers/gpio/gpio-sa1100.c index bec397a6..c1b3877 100644 --- a/drivers/gpio/gpio-sa1100.c +++ b/drivers/gpio/gpio-sa1100.c @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -32,9 +33,9 @@ static int sa1100_direction_input(struct gpio_chip *chip, unsigned offset) { unsigned long flags; - local_irq_save(flags); + flags = hard_local_irq_save(); GPDR &= ~GPIO_GPIO(offset); - local_irq_restore(flags); + hard_local_irq_restore(flags); return 0; } @@ -42,10 +43,10 @@ static int sa1100_direction_output(struct gpio_chip *chip, unsigned offset, int { unsigned long flags; - local_irq_save(flags); + flags = hard_local_irq_save(); sa1100_gpio_set(chip, offset, value); GPDR |= GPIO_GPIO(offset); - local_irq_restore(flags); + hard_local_irq_restore(flags); return 0; } @@ -188,7 +189,7 @@ sa1100_gpio_handler(unsigned int irq, struct irq_desc *desc) irq = IRQ_GPIO0; do { if (mask & 1) - generic_handle_irq(irq); + ipipe_handle_demuxed_irq(irq); mask >>= 1; irq++; } while (mask); diff --git a/drivers/gpio/gpio-zynq.c b/drivers/gpio/gpio-zynq.c index 184c4b1..3c8bc4a 100644 --- a/drivers/gpio/gpio-zynq.c +++ b/drivers/gpio/gpio-zynq.c @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -99,6 +100,7 @@ struct zynq_gpio { static struct irq_chip zynq_gpio_level_irqchip; static struct irq_chip zynq_gpio_edge_irqchip; +static IPIPE_DEFINE_RAW_SPINLOCK(zynq_gpio_lock); /** * zynq_gpio_get_bank_pin - Get the bank number and pin number within that bank * for a given pin in the GPIO device @@ -221,6 +223,7 @@ static int zynq_gpio_dir_in(struct gpio_chip *chip, unsigned int pin) u32 reg; unsigned int bank_num, bank_pin_num; struct zynq_gpio *gpio = container_of(chip, struct zynq_gpio, chip); + unsigned long flags; zynq_gpio_get_bank_pin(pin, &bank_num, &bank_pin_num); @@ -228,10 +231,12 @@ static int zynq_gpio_dir_in(struct gpio_chip *chip, unsigned int pin) if (bank_num == 0 && (bank_pin_num == 7 || bank_pin_num == 8)) return -EINVAL; + raw_spin_lock_irqsave(&zynq_gpio_lock, flags); /* clear the bit in direction mode reg to set the pin as input */ reg = readl_relaxed(gpio->base_addr + ZYNQ_GPIO_DIRM_OFFSET(bank_num)); reg &= ~BIT(bank_pin_num); writel_relaxed(reg, gpio->base_addr + ZYNQ_GPIO_DIRM_OFFSET(bank_num)); + raw_spin_unlock_irqrestore(&zynq_gpio_lock, flags); return 0; } @@ -254,9 +259,11 @@ static int zynq_gpio_dir_out(struct gpio_chip *chip, unsigned int pin, u32 reg; unsigned int bank_num, bank_pin_num; struct zynq_gpio *gpio = container_of(chip, struct zynq_gpio, chip); + unsigned long flags; zynq_gpio_get_bank_pin(pin, &bank_num, &bank_pin_num); + raw_spin_lock_irqsave(&zynq_gpio_lock, flags); /* set the GPIO pin as output */ reg = readl_relaxed(gpio->base_addr + ZYNQ_GPIO_DIRM_OFFSET(bank_num)); reg |= BIT(bank_pin_num); @@ -266,6 +273,7 @@ static int zynq_gpio_dir_out(struct gpio_chip *chip, unsigned int pin, reg = readl_relaxed(gpio->base_addr + ZYNQ_GPIO_OUTEN_OFFSET(bank_num)); reg |= BIT(bank_pin_num); writel_relaxed(reg, gpio->base_addr + ZYNQ_GPIO_OUTEN_OFFSET(bank_num)); + raw_spin_unlock_irqrestore(&zynq_gpio_lock, flags); /* set the state of the pin */ zynq_gpio_set_value(chip, pin, state); @@ -284,11 +292,15 @@ static void zynq_gpio_irq_mask(struct irq_data *irq_data) { unsigned int device_pin_num, bank_num, bank_pin_num; struct zynq_gpio *gpio = irq_data_get_irq_chip_data(irq_data); + unsigned long flags; device_pin_num = irq_data->hwirq; zynq_gpio_get_bank_pin(device_pin_num, &bank_num, &bank_pin_num); + raw_spin_lock_irqsave(&zynq_gpio_lock, flags); + ipipe_lock_irq(irq_data->irq); writel_relaxed(BIT(bank_pin_num), gpio->base_addr + ZYNQ_GPIO_INTDIS_OFFSET(bank_num)); + raw_spin_unlock_irqrestore(&zynq_gpio_lock, flags); } /** @@ -304,11 +316,15 @@ static void zynq_gpio_irq_unmask(struct irq_data *irq_data) { unsigned int device_pin_num, bank_num, bank_pin_num; struct zynq_gpio *gpio = irq_data_get_irq_chip_data(irq_data); + unsigned long flags; device_pin_num = irq_data->hwirq; zynq_gpio_get_bank_pin(device_pin_num, &bank_num, &bank_pin_num); + raw_spin_lock_irqsave(&zynq_gpio_lock, flags); writel_relaxed(BIT(bank_pin_num), gpio->base_addr + ZYNQ_GPIO_INTEN_OFFSET(bank_num)); + ipipe_unlock_irq(irq_data->irq); + raw_spin_unlock_irqrestore(&zynq_gpio_lock, flags); } /** @@ -442,11 +458,43 @@ static int zynq_gpio_set_wake(struct irq_data *data, unsigned int on) return 0; } +#ifdef CONFIG_IPIPE + +static void zynq_gpio_hold_irq(struct irq_data *irq_data) +{ + unsigned int device_pin_num, bank_num, bank_pin_num; + struct zynq_gpio *gpio = irq_data_get_irq_chip_data(irq_data); + + device_pin_num = irq_data->hwirq; + zynq_gpio_get_bank_pin(device_pin_num, &bank_num, &bank_pin_num); + writel_relaxed(BIT(bank_pin_num), + gpio->base_addr + ZYNQ_GPIO_INTDIS_OFFSET(bank_num)); + writel_relaxed(BIT(bank_pin_num), + gpio->base_addr + ZYNQ_GPIO_INTSTS_OFFSET(bank_num)); +} + +static void zynq_gpio_release_irq(struct irq_data *irq_data) +{ + unsigned int device_pin_num, bank_num, bank_pin_num; + struct zynq_gpio *gpio = irq_data_get_irq_chip_data(irq_data); + + device_pin_num = irq_data->hwirq; + zynq_gpio_get_bank_pin(device_pin_num, &bank_num, &bank_pin_num); + writel_relaxed(BIT(bank_pin_num), + gpio->base_addr + ZYNQ_GPIO_INTEN_OFFSET(bank_num)); +} + +#endif /* CONFIG_IPIPE */ + /* irq chip descriptor */ static struct irq_chip zynq_gpio_level_irqchip = { - .name = DRIVER_NAME, + .name = DRIVER_NAME "-level", .irq_enable = zynq_gpio_irq_enable, .irq_eoi = zynq_gpio_irq_ack, +#ifdef CONFIG_IPIPE + .irq_hold = zynq_gpio_hold_irq, + .irq_release = zynq_gpio_release_irq, +#endif .irq_mask = zynq_gpio_irq_mask, .irq_unmask = zynq_gpio_irq_unmask, .irq_set_type = zynq_gpio_set_irq_type, @@ -456,9 +504,13 @@ static struct irq_chip zynq_gpio_level_irqchip = { }; static struct irq_chip zynq_gpio_edge_irqchip = { - .name = DRIVER_NAME, + .name = DRIVER_NAME "-edge", .irq_enable = zynq_gpio_irq_enable, +#ifdef CONFIG_IPIPE + .irq_mask_ack = zynq_gpio_hold_irq, +#else .irq_ack = zynq_gpio_irq_ack, +#endif .irq_mask = zynq_gpio_irq_mask, .irq_unmask = zynq_gpio_irq_unmask, .irq_set_type = zynq_gpio_set_irq_type, diff --git a/drivers/gpu/ipu-v3/ipu-common.c b/drivers/gpu/ipu-v3/ipu-common.c index 67bab5c..e9685fe 100644 --- a/drivers/gpu/ipu-v3/ipu-common.c +++ b/drivers/gpu/ipu-v3/ipu-common.c @@ -907,7 +907,7 @@ static void ipu_irq_handle(struct ipu_soc *ipu, const int *regs, int num_regs) irq = irq_linear_revmap(ipu->domain, regs[i] * 32 + bit); if (irq) - generic_handle_irq(irq); + ipipe_handle_demuxed_irq(irq); } } } diff --git a/drivers/gpu/ipu-v3/ipu-prv.h b/drivers/gpu/ipu-v3/ipu-prv.h index bfb1e8a..562b69a 100644 --- a/drivers/gpu/ipu-v3/ipu-prv.h +++ b/drivers/gpu/ipu-v3/ipu-prv.h @@ -146,7 +146,7 @@ struct ipu_soc { struct device *dev; const struct ipu_devtype *devtype; enum ipuv3_type ipu_type; - spinlock_t lock; + ipipe_spinlock_t lock; struct mutex channel_lock; void __iomem *cm_reg; diff --git a/drivers/irqchip/irq-atmel-aic.c b/drivers/irqchip/irq-atmel-aic.c index dae3604..6044bfd 100644 --- a/drivers/irqchip/irq-atmel-aic.c +++ b/drivers/irqchip/irq-atmel-aic.c @@ -71,17 +71,113 @@ aic_handle(struct pt_regs *regs) if (!irqstat) irq_reg_writel(gc, 0, AT91_AIC_EOICR); else - handle_domain_irq(aic_domain, irqnr, regs); + ipipe_handle_domain_irq(aic_domain, irqnr, regs); } +#ifdef CONFIG_IPIPE +static unsigned aic_root = ~0U; +static unsigned aic_muted; + +int at91_gpio_enable_irqdesc(struct ipipe_domain *ipd, unsigned irq); +int at91_gpio_disable_irqdesc(struct ipipe_domain *ipd, unsigned irq); +void at91_gpio_mute(void); +void at91_gpio_unmute(void); + +static void aic_hold(struct irq_data *d) +{ + struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); + + irq_gc_mask_disable_reg(d); + irq_reg_writel(gc, 0, AT91_AIC_EOICR); +} + +static void aic_release(struct irq_data *d) +{ + struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); + unsigned long flags; + + flags = irq_gc_lock(gc); + irq_gc_unmask_enable_reg(d); + irq_gc_unlock(gc, flags); +} + +static void at91_enable_irqdesc(struct ipipe_domain *ipd, unsigned irq) +{ + int err; + + err = at91_gpio_enable_irqdesc(ipd, irq); + if (err < 0) { + if (ipd != &ipipe_root) { + struct irq_desc *desc = irq_to_desc(irq); + struct irq_data *idata = irq_desc_get_irq_data(desc); + + aic_root &= ~(1 << idata->hwirq); + } + } else if (err) + aic_root &= ~(1 << err); +} + +static void at91_disable_irqdesc(struct ipipe_domain *ipd, unsigned irq) +{ + int err; + + err = at91_gpio_disable_irqdesc(ipd, irq); + if (err < 0) { + if (ipd != &ipipe_root) { + struct irq_desc *desc = irq_to_desc(irq); + struct irq_data *idata = irq_desc_get_irq_data(desc); + + aic_root |= (1 << idata->hwirq); + } + } else if (err) + aic_root |= (1 << err); +} + +static void at91_mute_pic(void) +{ + struct irq_domain_chip_generic *dgc = aic_domain->gc; + struct irq_chip_generic *gc = dgc->gc[0]; + unsigned long unmasked, muted; + + at91_gpio_mute(); + + unmasked = irq_reg_readl(gc, AT91_AIC_IMR); + aic_muted = muted = unmasked & aic_root; + irq_reg_writel(gc, muted, AT91_AIC_IDCR); +} + +static void at91_unmute_pic(void) +{ + struct irq_domain_chip_generic *dgc = aic_domain->gc; + struct irq_chip_generic *gc = dgc->gc[0]; + + irq_reg_writel(gc, aic_muted, AT91_AIC_IECR); + + at91_gpio_unmute(); +} + +static void at91_pic_muter_register(void) +{ + struct ipipe_mach_pic_muter at91_pic_muter = { + .enable_irqdesc = at91_enable_irqdesc, + .disable_irqdesc = at91_disable_irqdesc, + .mute = at91_mute_pic, + .unmute = at91_unmute_pic, + }; + + ipipe_pic_muter_register(&at91_pic_muter); +} +#endif + static int aic_retrigger(struct irq_data *d) { struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); + unsigned long flags; /* Enable interrupt on AIC5 */ - irq_gc_lock(gc); + flags = irq_gc_lock(gc); irq_reg_writel(gc, d->mask, AT91_AIC_ISCR); - irq_gc_unlock(gc); + irq_gc_unlock(gc, flags); return 0; } @@ -106,31 +202,34 @@ static int aic_set_type(struct irq_data *d, unsigned type) static void aic_suspend(struct irq_data *d) { struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); + unsigned long flags; - irq_gc_lock(gc); + flags = irq_gc_lock(gc); irq_reg_writel(gc, gc->mask_cache, AT91_AIC_IDCR); irq_reg_writel(gc, gc->wake_active, AT91_AIC_IECR); - irq_gc_unlock(gc); + irq_gc_unlock(gc, flags); } static void aic_resume(struct irq_data *d) { struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); + unsigned long flags; - irq_gc_lock(gc); + flags = irq_gc_lock(gc); irq_reg_writel(gc, gc->wake_active, AT91_AIC_IDCR); irq_reg_writel(gc, gc->mask_cache, AT91_AIC_IECR); - irq_gc_unlock(gc); + irq_gc_unlock(gc, flags); } static void aic_pm_shutdown(struct irq_data *d) { struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); + unsigned long flags; - irq_gc_lock(gc); + flags = irq_gc_lock(gc); irq_reg_writel(gc, 0xffffffff, AT91_AIC_IDCR); irq_reg_writel(gc, 0xffffffff, AT91_AIC_ICCR); - irq_gc_unlock(gc); + irq_gc_unlock(gc, flags); } #else #define aic_suspend NULL @@ -176,6 +275,7 @@ static int aic_irq_domain_xlate(struct irq_domain *d, { struct irq_domain_chip_generic *dgc = d->gc; struct irq_chip_generic *gc; + unsigned long flags; unsigned smr; int idx; int ret; @@ -194,12 +294,12 @@ static int aic_irq_domain_xlate(struct irq_domain *d, gc = dgc->gc[idx]; - irq_gc_lock(gc); + flags = irq_gc_lock(gc); smr = irq_reg_readl(gc, AT91_AIC_SMR(*out_hwirq)); ret = aic_common_set_priority(intspec[2], &smr); if (!ret) irq_reg_writel(gc, smr, AT91_AIC_SMR(*out_hwirq)); - irq_gc_unlock(gc); + irq_gc_unlock(gc, flags); return ret; } @@ -267,10 +367,16 @@ static int __init aic_of_init(struct device_node *node, gc->chip_types[0].chip.irq_suspend = aic_suspend; gc->chip_types[0].chip.irq_resume = aic_resume; gc->chip_types[0].chip.irq_pm_shutdown = aic_pm_shutdown; +#ifdef CONFIG_IPIPE + gc->chip_types[0].chip.irq_hold = aic_hold; + gc->chip_types[0].chip.irq_release = aic_release; + at91_pic_muter_register(); +#endif aic_hw_init(domain); set_handle_irq(aic_handle); + return 0; } IRQCHIP_DECLARE(at91rm9200_aic, "atmel,at91rm9200-aic", aic_of_init); diff --git a/drivers/irqchip/irq-atmel-aic5.c b/drivers/irqchip/irq-atmel-aic5.c index c2c578f..3d2f7ae 100644 --- a/drivers/irqchip/irq-atmel-aic5.c +++ b/drivers/irqchip/irq-atmel-aic5.c @@ -81,7 +81,14 @@ aic5_handle(struct pt_regs *regs) if (!irqstat) irq_reg_writel(gc, 0, AT91_AIC5_EOICR); else - handle_domain_irq(aic5_domain, irqnr, regs); + ipipe_handle_domain_irq(aic5_domain, irqnr, regs); +} + +static void aic5_hard_mask(struct irq_chip_generic *gc, struct irq_data *d) +{ + /* Disable interrupt on AIC5 */ + irq_reg_writel(gc, d->hwirq, AT91_AIC5_SSR); + irq_reg_writel(gc, 1, AT91_AIC5_IDCR); } static void aic5_mask(struct irq_data *d) @@ -90,16 +97,24 @@ static void aic5_mask(struct irq_data *d) struct irq_domain_chip_generic *dgc = domain->gc; struct irq_chip_generic *bgc = dgc->gc[0]; struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); + unsigned long flags; /* * Disable interrupt on AIC5. We always take the lock of the * first irq chip as all chips share the same registers. */ - irq_gc_lock(bgc); - irq_reg_writel(gc, d->hwirq, AT91_AIC5_SSR); - irq_reg_writel(gc, 1, AT91_AIC5_IDCR); + flags = irq_gc_lock(bgc); + ipipe_lock_irq(d->irq); + aic5_hard_mask(gc, d); gc->mask_cache &= ~d->mask; - irq_gc_unlock(bgc); + irq_gc_unlock(bgc, flags); +} + +static void aic5_hard_unmask(struct irq_chip_generic *gc, struct irq_data *d) +{ + /* Enable interrupt on AIC5 */ + irq_reg_writel(gc, d->hwirq, AT91_AIC5_SSR); + irq_reg_writel(gc, 1, AT91_AIC5_IECR); } static void aic5_unmask(struct irq_data *d) @@ -108,29 +123,92 @@ static void aic5_unmask(struct irq_data *d) struct irq_domain_chip_generic *dgc = domain->gc; struct irq_chip_generic *bgc = dgc->gc[0]; struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); + unsigned long flags; /* * Enable interrupt on AIC5. We always take the lock of the * first irq chip as all chips share the same registers. */ - irq_gc_lock(bgc); - irq_reg_writel(gc, d->hwirq, AT91_AIC5_SSR); - irq_reg_writel(gc, 1, AT91_AIC5_IECR); + flags = irq_gc_lock(bgc); + aic5_hard_unmask(gc, d); gc->mask_cache |= d->mask; - irq_gc_unlock(bgc); + ipipe_unlock_irq(d->irq); + irq_gc_unlock(bgc, flags); +} + +#ifdef CONFIG_IPIPE +int at91_gpio_enable_irqdesc(struct ipipe_domain *ipd, unsigned irq); +int at91_gpio_disable_irqdesc(struct ipipe_domain *ipd, unsigned irq); +void at91_gpio_mute(void); +void at91_gpio_unmute(void); + +static void aic5_hold(struct irq_data *d) +{ + struct irq_domain *domain = d->domain; + struct irq_domain_chip_generic *dgc = domain->gc; + struct irq_chip_generic *gc = dgc->gc[0]; + + aic5_hard_mask(gc, d); + irq_reg_writel(gc, 0, AT91_AIC5_EOICR); } +static void aic5_release(struct irq_data *d) +{ + struct irq_domain *domain = d->domain; + struct irq_domain_chip_generic *dgc = domain->gc; + struct irq_chip_generic *gc = dgc->gc[0]; + unsigned long flags; + + flags = irq_gc_lock(gc); + aic5_hard_unmask(gc, d); + irq_gc_unlock(gc, flags); +} + +static void at91_enable_irqdesc(struct ipipe_domain *ipd, unsigned irq) +{ + at91_gpio_enable_irqdesc(ipd, irq); +} + +static void at91_disable_irqdesc(struct ipipe_domain *ipd, unsigned irq) +{ + at91_gpio_disable_irqdesc(ipd, irq); +} + +static void at91_mute_pic(void) +{ + at91_gpio_mute(); +} + +static void at91_unmute_pic(void) +{ + at91_gpio_unmute(); +} + +static void at91_pic_muter_register(void) +{ + struct ipipe_mach_pic_muter at91_pic_muter = { + .enable_irqdesc = at91_enable_irqdesc, + .disable_irqdesc = at91_disable_irqdesc, + .mute = at91_mute_pic, + .unmute = at91_unmute_pic, + }; + + ipipe_pic_muter_register(&at91_pic_muter); +} +#endif + static int aic5_retrigger(struct irq_data *d) { struct irq_domain *domain = d->domain; struct irq_domain_chip_generic *dgc = domain->gc; struct irq_chip_generic *gc = dgc->gc[0]; + unsigned long flags; /* Enable interrupt on AIC5 */ - irq_gc_lock(gc); + flags = irq_gc_lock(gc); irq_reg_writel(gc, d->hwirq, AT91_AIC5_SSR); irq_reg_writel(gc, 1, AT91_AIC5_ISCR); - irq_gc_unlock(gc); + irq_gc_unlock(gc, flags); return 0; } @@ -140,16 +218,17 @@ static int aic5_set_type(struct irq_data *d, unsigned type) struct irq_domain *domain = d->domain; struct irq_domain_chip_generic *dgc = domain->gc; struct irq_chip_generic *gc = dgc->gc[0]; + unsigned long flags; unsigned int smr; int ret; - irq_gc_lock(gc); + flags = irq_gc_lock(gc); irq_reg_writel(gc, d->hwirq, AT91_AIC5_SSR); smr = irq_reg_readl(gc, AT91_AIC5_SMR); ret = aic_common_set_type(d, type, &smr); if (!ret) irq_reg_writel(gc, smr, AT91_AIC5_SMR); - irq_gc_unlock(gc); + irq_gc_unlock(gc, flags); return ret; } @@ -161,10 +240,11 @@ static void aic5_suspend(struct irq_data *d) struct irq_domain_chip_generic *dgc = domain->gc; struct irq_chip_generic *bgc = dgc->gc[0]; struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); + unsigned long flags; int i; u32 mask; - irq_gc_lock(bgc); + flags = irq_gc_lock(bgc); for (i = 0; i < dgc->irqs_per_chip; i++) { mask = 1 << i; if ((mask & gc->mask_cache) == (mask & gc->wake_active)) @@ -176,7 +256,7 @@ static void aic5_suspend(struct irq_data *d) else irq_reg_writel(bgc, 1, AT91_AIC5_IDCR); } - irq_gc_unlock(bgc); + irq_gc_unlock(bgc, flags); } static void aic5_resume(struct irq_data *d) @@ -185,10 +265,11 @@ static void aic5_resume(struct irq_data *d) struct irq_domain_chip_generic *dgc = domain->gc; struct irq_chip_generic *bgc = dgc->gc[0]; struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); + unsigned long flags; int i; u32 mask; - irq_gc_lock(bgc); + flags = irq_gc_lock(bgc); for (i = 0; i < dgc->irqs_per_chip; i++) { mask = 1 << i; if ((mask & gc->mask_cache) == (mask & gc->wake_active)) @@ -200,7 +281,7 @@ static void aic5_resume(struct irq_data *d) else irq_reg_writel(bgc, 1, AT91_AIC5_IDCR); } - irq_gc_unlock(bgc); + irq_gc_unlock(bgc, flags); } static void aic5_pm_shutdown(struct irq_data *d) @@ -209,15 +290,16 @@ static void aic5_pm_shutdown(struct irq_data *d) struct irq_domain_chip_generic *dgc = domain->gc; struct irq_chip_generic *bgc = dgc->gc[0]; struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); + unsigned long flags; int i; - irq_gc_lock(bgc); + flags = irq_gc_lock(bgc); for (i = 0; i < dgc->irqs_per_chip; i++) { irq_reg_writel(bgc, i + gc->irq_base, AT91_AIC5_SSR); irq_reg_writel(bgc, 1, AT91_AIC5_IDCR); irq_reg_writel(bgc, 1, AT91_AIC5_ICCR); } - irq_gc_unlock(bgc); + irq_gc_unlock(bgc, flags); } #else #define aic5_suspend NULL @@ -264,6 +346,7 @@ static int aic5_irq_domain_xlate(struct irq_domain *d, { struct irq_domain_chip_generic *dgc = d->gc; struct irq_chip_generic *gc; + unsigned long flags; unsigned smr; int ret; @@ -277,13 +360,13 @@ static int aic5_irq_domain_xlate(struct irq_domain *d, gc = dgc->gc[0]; - irq_gc_lock(gc); + flags = irq_gc_lock(gc); irq_reg_writel(gc, *out_hwirq, AT91_AIC5_SSR); smr = irq_reg_readl(gc, AT91_AIC5_SMR); ret = aic_common_set_priority(intspec[2], &smr); if (!ret) irq_reg_writel(gc, intspec[2] | smr, AT91_AIC5_SMR); - irq_gc_unlock(gc); + irq_gc_unlock(gc, flags); return ret; } @@ -339,11 +422,19 @@ static int __init aic5_of_init(struct device_node *node, gc->chip_types[0].chip.irq_suspend = aic5_suspend; gc->chip_types[0].chip.irq_resume = aic5_resume; gc->chip_types[0].chip.irq_pm_shutdown = aic5_pm_shutdown; +#ifdef CONFIG_IPIPE + gc->chip_types[0].chip.irq_hold = aic5_hold; + gc->chip_types[0].chip.irq_release = aic5_release; +#endif } aic5_hw_init(domain); set_handle_irq(aic5_handle); +#ifdef CONFIG_IPIPE + at91_pic_muter_register(); +#endif + return 0; } diff --git a/drivers/irqchip/irq-bcm7120-l2.c b/drivers/irqchip/irq-bcm7120-l2.c index 3ba5cc7..dc87b3b 100644 --- a/drivers/irqchip/irq-bcm7120-l2.c +++ b/drivers/irqchip/irq-bcm7120-l2.c @@ -56,6 +56,7 @@ static void bcm7120_l2_intc_irq_handle(unsigned int irq, struct irq_desc *desc) { struct bcm7120_l2_intc_data *b = irq_desc_get_handler_data(desc); struct irq_chip *chip = irq_desc_get_chip(desc); + unsigned long flags; unsigned int idx; chained_irq_enter(chip, desc); @@ -67,10 +68,10 @@ static void bcm7120_l2_intc_irq_handle(unsigned int irq, struct irq_desc *desc) unsigned long pending; int hwirq; - irq_gc_lock(gc); + flags = irq_gc_lock(gc); pending = irq_reg_readl(gc, b->stat_offset[idx]) & gc->mask_cache; - irq_gc_unlock(gc); + irq_gc_unlock(gc, flags); for_each_set_bit(hwirq, &pending, IRQS_PER_WORD) { generic_handle_irq(irq_find_mapping(b->domain, @@ -86,23 +87,25 @@ static void bcm7120_l2_intc_suspend(struct irq_data *d) struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); struct irq_chip_type *ct = irq_data_get_chip_type(d); struct bcm7120_l2_intc_data *b = gc->private; + unsigned long flags; - irq_gc_lock(gc); + flags = irq_gc_lock(gc); if (b->can_wake) irq_reg_writel(gc, gc->mask_cache | gc->wake_active, ct->regs.mask); - irq_gc_unlock(gc); + irq_gc_unlock(gc, flags); } static void bcm7120_l2_intc_resume(struct irq_data *d) { struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); struct irq_chip_type *ct = irq_data_get_chip_type(d); + unsigned long flags; /* Restore the saved mask */ - irq_gc_lock(gc); + flags = irq_gc_lock(gc); irq_reg_writel(gc, gc->mask_cache, ct->regs.mask); - irq_gc_unlock(gc); + irq_gc_unlock(gc, flags); } static int bcm7120_l2_intc_init_one(struct device_node *dn, diff --git a/drivers/irqchip/irq-brcmstb-l2.c b/drivers/irqchip/irq-brcmstb-l2.c index d6bcc6b..f3c04de 100644 --- a/drivers/irqchip/irq-brcmstb-l2.c +++ b/drivers/irqchip/irq-brcmstb-l2.c @@ -85,8 +85,9 @@ static void brcmstb_l2_intc_suspend(struct irq_data *d) { struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); struct brcmstb_l2_intc_data *b = gc->private; + unsigned long flags; - irq_gc_lock(gc); + flags = irq_gc_lock(gc); /* Save the current mask */ b->saved_mask = irq_reg_readl(gc, CPU_MASK_STATUS); @@ -95,22 +96,23 @@ static void brcmstb_l2_intc_suspend(struct irq_data *d) irq_reg_writel(gc, ~gc->wake_active, CPU_MASK_SET); irq_reg_writel(gc, gc->wake_active, CPU_MASK_CLEAR); } - irq_gc_unlock(gc); + irq_gc_unlock(gc, flags); } static void brcmstb_l2_intc_resume(struct irq_data *d) { struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); struct brcmstb_l2_intc_data *b = gc->private; + unsigned long flags; - irq_gc_lock(gc); + flags = irq_gc_lock(gc); /* Clear unmasked non-wakeup interrupts */ irq_reg_writel(gc, ~b->saved_mask & ~gc->wake_active, CPU_CLEAR); /* Restore the saved mask */ irq_reg_writel(gc, b->saved_mask, CPU_MASK_SET); irq_reg_writel(gc, ~b->saved_mask, CPU_MASK_CLEAR); - irq_gc_unlock(gc); + irq_gc_unlock(gc, flags); } int __init brcmstb_l2_intc_of_init(struct device_node *np, diff --git a/drivers/irqchip/irq-crossbar.c b/drivers/irqchip/irq-crossbar.c index c12bb93..8bb1af7 100644 --- a/drivers/irqchip/irq-crossbar.c +++ b/drivers/irqchip/irq-crossbar.c @@ -15,6 +15,7 @@ #include #include #include +#include #include "irqchip.h" @@ -74,6 +75,10 @@ static struct irq_chip crossbar_chip = { #ifdef CONFIG_SMP .irq_set_affinity = irq_chip_set_affinity_parent, #endif +#ifdef CONFIG_IPIPE + .irq_hold = irq_chip_hold_parent, + .irq_release = irq_chip_release_parent, +#endif }; static int allocate_gic_irq(struct irq_domain *domain, unsigned virq, diff --git a/drivers/irqchip/irq-dw-apb-ictl.c b/drivers/irqchip/irq-dw-apb-ictl.c index 53bb732..5ca21a2 100644 --- a/drivers/irqchip/irq-dw-apb-ictl.c +++ b/drivers/irqchip/irq-dw-apb-ictl.c @@ -55,11 +55,12 @@ static void dw_apb_ictl_resume(struct irq_data *d) { struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); struct irq_chip_type *ct = irq_data_get_chip_type(d); + unsigned long flags; - irq_gc_lock(gc); + flags = irq_gc_lock(gc); writel_relaxed(~0, gc->reg_base + ct->regs.enable); writel_relaxed(*ct->mask_cache, gc->reg_base + ct->regs.mask); - irq_gc_unlock(gc); + irq_gc_unlock(gc, flags); } #else #define dw_apb_ictl_resume NULL diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c index 49875ad..f5a54f6 100644 --- a/drivers/irqchip/irq-gic-v3.c +++ b/drivers/irqchip/irq-gic-v3.c @@ -343,7 +343,7 @@ static asmlinkage void __exception_irq_entry gic_handle_irq(struct pt_regs *regs if (likely(irqnr > 15 && irqnr < 1020) || irqnr >= 8192) { int err; - err = handle_domain_irq(gic_data.domain, irqnr, regs); + err = ipipe_handle_domain_irq(gic_data.domain, irqnr, regs); if (err) { WARN_ONCE(true, "Unexpected interrupt received!\n"); gic_write_eoir(irqnr); @@ -353,7 +353,7 @@ static asmlinkage void __exception_irq_entry gic_handle_irq(struct pt_regs *regs if (irqnr < 16) { gic_write_eoir(irqnr); #ifdef CONFIG_SMP - handle_IPI(irqnr, regs); + ipipe_handle_multi_ipi(irqnr, regs); #else WARN_ONCE(true, "Unexpected SGI received!\n"); #endif diff --git a/drivers/irqchip/irq-gic.c b/drivers/irqchip/irq-gic.c index 01999d7..d41e590 100644 --- a/drivers/irqchip/irq-gic.c +++ b/drivers/irqchip/irq-gic.c @@ -38,6 +38,7 @@ #include #include #include +#include #include #include #include @@ -72,7 +73,7 @@ struct gic_chip_data { #endif }; -static DEFINE_RAW_SPINLOCK(irq_controller_lock); +static IPIPE_DEFINE_RAW_SPINLOCK(irq_controller_lock); /* * The GIC mapping of CPU interfaces does not necessarily match @@ -154,12 +155,22 @@ static int gic_peek_irq(struct irq_data *d, u32 offset) static void gic_mask_irq(struct irq_data *d) { + unsigned long flags; + + raw_spin_lock_irqsave(&irq_controller_lock, flags); + ipipe_lock_irq(d->irq); gic_poke_irq(d, GIC_DIST_ENABLE_CLEAR); + raw_spin_unlock_irqrestore(&irq_controller_lock, flags); } static void gic_unmask_irq(struct irq_data *d) { + unsigned long flags; + + raw_spin_lock_irqsave(&irq_controller_lock, flags); gic_poke_irq(d, GIC_DIST_ENABLE_SET); + ipipe_unlock_irq(d->irq); + raw_spin_unlock_irqrestore(&irq_controller_lock, flags); } static void gic_eoi_irq(struct irq_data *d) @@ -167,6 +178,20 @@ static void gic_eoi_irq(struct irq_data *d) writel_relaxed(gic_irq(d), gic_cpu_base(d) + GIC_CPU_EOI); } +#ifdef CONFIG_IPIPE +static void gic_hold_irq(struct irq_data *d) +{ + gic_poke_irq(d, GIC_DIST_ENABLE_CLEAR); + gic_eoi_irq(d); +} + +static void gic_release_irq(struct irq_data *d) +{ + gic_poke_irq(d, GIC_DIST_ENABLE_SET); +} + +#endif /* CONFIG_IPIPE */ + static int gic_irq_set_irqchip_state(struct irq_data *d, enum irqchip_irq_state which, bool val) { @@ -272,13 +297,13 @@ static void __exception_irq_entry gic_handle_irq(struct pt_regs *regs) irqnr = irqstat & GICC_IAR_INT_ID_MASK; if (likely(irqnr > 15 && irqnr < 1021)) { - handle_domain_irq(gic->domain, irqnr, regs); + ipipe_handle_domain_irq(gic->domain, irqnr, regs); continue; } if (irqnr < 16) { writel_relaxed(irqstat, cpu_base + GIC_CPU_EOI); #ifdef CONFIG_SMP - handle_IPI(irqnr, regs); + ipipe_handle_multi_ipi(irqnr, regs); #endif continue; } @@ -291,13 +316,13 @@ static void gic_handle_cascade_irq(unsigned int irq, struct irq_desc *desc) struct gic_chip_data *chip_data = irq_get_handler_data(irq); struct irq_chip *chip = irq_get_chip(irq); unsigned int cascade_irq, gic_irq; - unsigned long status; + unsigned long status, flags; chained_irq_enter(chip, desc); - raw_spin_lock(&irq_controller_lock); + raw_spin_lock_irqsave_cond(&irq_controller_lock, flags); status = readl_relaxed(gic_data_cpu_base(chip_data) + GIC_CPU_INTACK); - raw_spin_unlock(&irq_controller_lock); + raw_spin_unlock_irqrestore_cond(&irq_controller_lock, flags); gic_irq = (status & GICC_IAR_INT_ID_MASK); if (gic_irq == GICC_INT_SPURIOUS) @@ -307,7 +332,7 @@ static void gic_handle_cascade_irq(unsigned int irq, struct irq_desc *desc) if (unlikely(gic_irq < 32 || gic_irq > 1020)) handle_bad_irq(cascade_irq, desc); else - generic_handle_irq(cascade_irq); + ipipe_handle_demuxed_irq(cascade_irq); out: chained_irq_exit(chip, desc); @@ -322,6 +347,10 @@ static struct irq_chip gic_chip = { #ifdef CONFIG_SMP .irq_set_affinity = gic_set_affinity, #endif +#ifdef CONFIG_IPIPE + .irq_hold = gic_hold_irq, + .irq_release = gic_release_irq, +#endif .irq_get_irqchip_state = gic_irq_get_irqchip_state, .irq_set_irqchip_state = gic_irq_set_irqchip_state, }; @@ -369,6 +398,39 @@ static void gic_cpu_if_up(void) } +#ifdef CONFIG_IPIPE + +void gic_mute(void) +{ + writel_relaxed(0x90, gic_data_cpu_base(&gic_data[0]) + GIC_CPU_PRIMASK); +} + +void gic_unmute(void) +{ + writel_relaxed(0xf0, gic_data_cpu_base(&gic_data[0]) + GIC_CPU_PRIMASK); +} + +void gic_set_irq_prio(int irq, int hi) +{ + void __iomem *dist_base; + unsigned gic_irqs; + + if (irq < 32) /* The IPIs always are high priority */ + return; + + dist_base = gic_data_dist_base(&gic_data[0]); + gic_irqs = readl_relaxed(dist_base + GIC_DIST_CTR) & 0x1f; + gic_irqs = (gic_irqs + 1) * 32; + if (gic_irqs > 1020) + gic_irqs = 1020; + if (irq >= gic_irqs) + return; + + writeb_relaxed(hi ? 0x10 : 0xa0, dist_base + GIC_DIST_PRI + irq); +} + +#endif /* CONFIG_IPIPE */ + static void __init gic_dist_init(struct gic_chip_data *gic) { unsigned int i; diff --git a/drivers/irqchip/irq-mxs.c b/drivers/irqchip/irq-mxs.c index e4acf1e..b53f641 100644 --- a/drivers/irqchip/irq-mxs.c +++ b/drivers/irqchip/irq-mxs.c @@ -60,6 +60,16 @@ static void icoll_mask_irq(struct irq_data *d) icoll_base + HW_ICOLL_INTERRUPTn_CLR(d->hwirq)); } +#ifdef CONFIG_IPIPE +static void icoll_mask_ack_irq(struct irq_data *d) +{ + __raw_writel(BM_ICOLL_INTERRUPTn_ENABLE, + icoll_base + HW_ICOLL_INTERRUPTn_CLR(d->hwirq)); + __raw_writel(BV_ICOLL_LEVELACK_IRQLEVELACK__LEVEL0, + icoll_base + HW_ICOLL_LEVELACK); +} +#endif + static void icoll_unmask_irq(struct irq_data *d) { __raw_writel(BM_ICOLL_INTERRUPTn_ENABLE, @@ -69,6 +79,9 @@ static void icoll_unmask_irq(struct irq_data *d) static struct irq_chip mxs_icoll_chip = { .irq_ack = icoll_ack_irq, .irq_mask = icoll_mask_irq, +#ifdef CONFIG_IPIPE + .irq_mask_ack = icoll_mask_ack_irq, +#endif /* CONFIG_IPIPE */ .irq_unmask = icoll_unmask_irq, }; @@ -78,7 +91,7 @@ asmlinkage void __exception_irq_entry icoll_handle_irq(struct pt_regs *regs) irqnr = __raw_readl(icoll_base + HW_ICOLL_STAT_OFFSET); __raw_writel(irqnr, icoll_base + HW_ICOLL_VECTOR); - handle_domain_irq(icoll_domain, irqnr, regs); + ipipe_handle_domain_irq(icoll_domain, irqnr, regs); } static int icoll_irq_domain_map(struct irq_domain *d, unsigned int virq, diff --git a/drivers/irqchip/irq-omap-intc.c b/drivers/irqchip/irq-omap-intc.c index a569c6d..9eba076 100644 --- a/drivers/irqchip/irq-omap-intc.c +++ b/drivers/irqchip/irq-omap-intc.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -42,6 +43,7 @@ #define INTC_MIR_CLEAR0 0x0088 #define INTC_MIR_SET0 0x008c #define INTC_PENDING_IRQ0 0x0098 +#define INTC_PRIO 0x0100 #define INTC_PENDING_IRQ1 0x00b8 #define INTC_PENDING_IRQ2 0x00d8 #define INTC_PENDING_IRQ3 0x00f8 @@ -51,6 +53,12 @@ #define INTCPS_NR_ILR_REGS 128 #define INTCPS_NR_MIR_REGS 4 +#if !defined(MULTI_OMAP1) && !defined(MULTI_OMAP2) +#define inline_single inline +#else +#define inline_single +#endif + #define INTC_IDLE_FUNCIDLE (1 << 0) #define INTC_IDLE_TURBO (1 << 1) @@ -71,12 +79,12 @@ static void __iomem *omap_irq_base; static int omap_nr_pending = 3; static int omap_nr_irqs = 96; -static void intc_writel(u32 reg, u32 val) +static inline_single void intc_writel(u32 reg, u32 val) { writel_relaxed(val, omap_irq_base + reg); } -static u32 intc_readl(u32 reg) +static inline_single u32 intc_readl(u32 reg) { return readl_relaxed(omap_irq_base + reg); } @@ -139,9 +147,10 @@ void omap3_intc_resume_idle(void) } /* XXX: FIQ and additional INTC support (only MPU at the moment) */ -static void omap_ack_irq(struct irq_data *d) +static inline_single void omap_ack_irq(struct irq_data *d) { intc_writel(INTC_CONTROL, 0x1); + dsb(); } static void omap_mask_ack_irq(struct irq_data *d) @@ -166,8 +175,14 @@ static void __init omap_irq_soft_reset(void) while (!(intc_readl(INTC_SYSSTATUS) & 0x1)) /* Wait for reset to complete */; +#ifndef CONFIG_IPIPE /* Enable autoidle */ intc_writel(INTC_SYSCONFIG, 1 << 0); +#else /* CONFIG_IPIPE */ + /* Disable autoidle */ + intc_writel(INTC_SYSCONFIG, 0); + intc_writel(INTC_IDLE, 0x1); +#endif /* CONFIG_IPIPE */ } int omap_irq_pending(void) @@ -234,6 +249,9 @@ static void __init omap_alloc_gc_legacy(void __iomem *base, ct = gc->chip_types; ct->chip.irq_ack = omap_mask_ack_irq; ct->chip.irq_mask = irq_gc_mask_disable_reg; +#ifdef CONFIG_IPIPE + ct->chip.irq_mask_ack = omap_mask_ack_irq; +#endif ct->chip.irq_unmask = irq_gc_unmask_enable_reg; ct->chip.flags |= IRQCHIP_SKIP_SET_WAKE; @@ -350,7 +368,7 @@ out: irqnr &= ACTIVEIRQ_MASK; if (irqnr) { - handle_domain_irq(domain, irqnr, regs); + ipipe_handle_domain_irq(domain, irqnr, regs); handled_irq = 1; } } while (irqnr); @@ -399,6 +417,28 @@ static int __init intc_of_init(struct device_node *node, return 0; } +#if defined(CONFIG_IPIPE) && defined(CONFIG_ARCH_OMAP2PLUS) +#if defined(CONFIG_ARCH_OMAP3) || defined(CONFIG_SOC_AM33XX) +void omap3_intc_mute(void) +{ + intc_writel(INTC_THRESHOLD, 0x1); + intc_writel(INTC_CONTROL, 0x1); +} + +void omap3_intc_unmute(void) +{ + intc_writel(INTC_THRESHOLD, 0xff); +} + +void omap3_intc_set_irq_prio(int irq, int hi) +{ + if (irq >= INTCPS_NR_MIR_REGS * 32) + return; + intc_writel(INTC_PRIO + 4 * irq, hi ? 0 : 0xfc); +} +#endif /* CONFIG_ARCH_OMAP3 */ +#endif /* CONFIG_IPIPE && ARCH_OMAP2PLUS */ + IRQCHIP_DECLARE(omap2_intc, "ti,omap2-intc", intc_of_init); IRQCHIP_DECLARE(omap3_intc, "ti,omap3-intc", intc_of_init); IRQCHIP_DECLARE(dm814x_intc, "ti,dm814-intc", intc_of_init); diff --git a/drivers/irqchip/irq-s3c24xx.c b/drivers/irqchip/irq-s3c24xx.c index c8d373f..1f81463 100644 --- a/drivers/irqchip/irq-s3c24xx.c +++ b/drivers/irqchip/irq-s3c24xx.c @@ -5,6 +5,8 @@ * Ben Dooks * Copyright (c) 2012 Heiko Stuebner * + * Copyright (C) 2006, 2007 Sebastian Smolorz , emlix GmbH + * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or @@ -24,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -328,7 +331,7 @@ static void s3c_irq_demux(unsigned int irq, struct irq_desc *desc) n = __ffs(src); src &= ~(1 << n); irq = irq_find_mapping(sub_intc->domain, offset + n); - generic_handle_irq(irq); + ipipe_handle_demuxed_irq(irq); } chained_irq_exit(chip, desc); diff --git a/drivers/irqchip/irq-sunxi-nmi.c b/drivers/irqchip/irq-sunxi-nmi.c index 6b2b582..0ff4c9e 100644 --- a/drivers/irqchip/irq-sunxi-nmi.c +++ b/drivers/irqchip/irq-sunxi-nmi.c @@ -77,8 +77,9 @@ static int sunxi_sc_nmi_set_type(struct irq_data *data, unsigned int flow_type) u32 ctrl_off = ct->regs.type; unsigned int src_type; unsigned int i; + unsigned long flags; - irq_gc_lock(gc); + flags = irq_gc_lock(gc); switch (flow_type & IRQF_TRIGGER_MASK) { case IRQ_TYPE_EDGE_FALLING: @@ -95,7 +96,7 @@ static int sunxi_sc_nmi_set_type(struct irq_data *data, unsigned int flow_type) src_type = SUNXI_SRC_TYPE_LEVEL_LOW; break; default: - irq_gc_unlock(gc); + irq_gc_unlock(gc, flags); pr_err("%s: Cannot assign multiple trigger modes to IRQ %d.\n", __func__, data->irq); return -EBADR; @@ -113,7 +114,7 @@ static int sunxi_sc_nmi_set_type(struct irq_data *data, unsigned int flow_type) src_type_reg |= src_type; sunxi_sc_nmi_write(gc, ctrl_off, src_type_reg); - irq_gc_unlock(gc); + irq_gc_unlock(gc, flags); return IRQ_SET_MASK_OK; } diff --git a/drivers/irqchip/irq-versatile-fpga.c b/drivers/irqchip/irq-versatile-fpga.c index 1ab4517..bd73658 100644 --- a/drivers/irqchip/irq-versatile-fpga.c +++ b/drivers/irqchip/irq-versatile-fpga.c @@ -79,7 +79,7 @@ static void fpga_irq_handle(unsigned int irq, struct irq_desc *desc) do { irq = ffs(status) - 1; status &= ~(1 << irq); - generic_handle_irq(irq_find_mapping(f->domain, irq)); + ipipe_handle_demuxed_irq(irq_find_mapping(f->domain, irq)); } while (status); } @@ -96,7 +96,7 @@ static int handle_one_fpga(struct fpga_irq_data *f, struct pt_regs *regs) while ((status = readl(f->base + IRQ_STATUS))) { irq = ffs(status) - 1; - handle_domain_irq(f->domain, irq, regs); + ipipe_handle_domain_irq(f->domain, irq, regs); handled = 1; } @@ -152,6 +152,9 @@ void __init fpga_irq_init(void __iomem *base, const char *name, int irq_start, f->chip.name = name; f->chip.irq_ack = fpga_irq_mask; f->chip.irq_mask = fpga_irq_mask; +#ifdef CONFIG_IPIPE + f->chip.irq_mask_ack = fpga_irq_mask; +#endif f->chip.irq_unmask = fpga_irq_unmask; f->valid = valid; diff --git a/drivers/irqchip/irq-vic.c b/drivers/irqchip/irq-vic.c index 54089de..bdb9395 100644 --- a/drivers/irqchip/irq-vic.c +++ b/drivers/irqchip/irq-vic.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include @@ -219,7 +220,7 @@ static int handle_one_vic(struct vic_device *vic, struct pt_regs *regs) while ((stat = readl_relaxed(vic->base + VIC_IRQ_STATUS))) { irq = ffs(stat) - 1; - handle_domain_irq(vic->domain, irq, regs); + ipipe_handle_domain_irq(vic->domain, irq, regs); handled = 1; } @@ -236,7 +237,7 @@ static void vic_handle_irq_cascaded(unsigned int irq, struct irq_desc *desc) while ((stat = readl_relaxed(vic->base + VIC_IRQ_STATUS))) { hwirq = ffs(stat) - 1; - generic_handle_irq(irq_find_mapping(vic->domain, hwirq)); + ipipe_handle_demuxed_irq(irq_find_mapping(vic->domain, hwirq)); } chained_irq_exit(host_chip, desc); @@ -340,7 +341,7 @@ static void vic_unmask_irq(struct irq_data *d) #if defined(CONFIG_PM) static struct vic_device *vic_from_irq(unsigned int irq) { - struct vic_device *v = vic_devices; + struct vic_device *v = vic_devices; unsigned int base_irq = irq & ~31; int id; @@ -379,6 +380,9 @@ static struct irq_chip vic_chip = { .name = "VIC", .irq_ack = vic_ack_irq, .irq_mask = vic_mask_irq, +#ifdef CONFIG_IPIPE + .irq_mask_ack = vic_ack_irq, +#endif /* CONFIG_IPIPE */ .irq_unmask = vic_unmask_irq, .irq_set_wake = vic_set_wake, }; diff --git a/drivers/irqchip/spear-shirq.c b/drivers/irqchip/spear-shirq.c index 9c145a7..928dbd6 100644 --- a/drivers/irqchip/spear-shirq.c +++ b/drivers/irqchip/spear-shirq.c @@ -54,7 +54,7 @@ struct spear_shirq { #define SPEAR300_INT_ENB_MASK_REG 0x54 #define SPEAR300_INT_STS_MASK_REG 0x58 -static DEFINE_RAW_SPINLOCK(shirq_lock); +static IPIPE_DEFINE_RAW_SPINLOCK(shirq_lock); static void shirq_irq_mask(struct irq_data *d) { @@ -83,6 +83,9 @@ static void shirq_irq_unmask(struct irq_data *d) static struct irq_chip shirq_chip = { .name = "spear-shirq", .irq_mask = shirq_irq_mask, +#ifdef CONFIG_IPIPE + .irq_mask_ack = shirq_irq_mask, +#endif /* CONFIG_IPIPE */ .irq_unmask = shirq_irq_unmask, }; @@ -195,7 +198,7 @@ static void shirq_handler(unsigned irq, struct irq_desc *desc) int irq = __ffs(pend); pend &= ~(0x1 << irq); - generic_handle_irq(shirq->virq_base + irq); + ipipe_handle_demuxed_irq(shirq->virq_base + irq); } } diff --git a/drivers/memory/omap-gpmc.c b/drivers/memory/omap-gpmc.c index c94ea0d..f812d82 100644 --- a/drivers/memory/omap-gpmc.c +++ b/drivers/memory/omap-gpmc.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -2238,7 +2239,7 @@ static irqreturn_t gpmc_handle_irq(int irq, void *dev) for (i = 0; i < GPMC_NR_IRQ; i++) if (regval & gpmc_client_irq[i].bitmask) - generic_handle_irq(gpmc_client_irq[i].irq); + ipipe_handle_demuxed_irq(gpmc_client_irq[i].irq); gpmc_write_reg(GPMC_IRQSTATUS, regval); diff --git a/drivers/mfd/twl4030-irq.c b/drivers/mfd/twl4030-irq.c index 1b772ef..3a0f00c 100644 --- a/drivers/mfd/twl4030-irq.c +++ b/drivers/mfd/twl4030-irq.c @@ -34,6 +34,7 @@ #include #include #include +#include #include "twl-core.h" diff --git a/drivers/mfd/twl6030-irq.c b/drivers/mfd/twl6030-irq.c index 2807e1a..c9f11a8 100644 --- a/drivers/mfd/twl6030-irq.c +++ b/drivers/mfd/twl6030-irq.c @@ -202,9 +202,14 @@ static irqreturn_t twl6030_irq_thread(int irq, void *data) int module_irq = irq_find_mapping(pdata->irq_domain, pdata->irq_mapping_tbl[i]); - if (module_irq) + if (module_irq) { +#ifdef CONFIG_IPIPE + struct irq_desc *d = irq_to_desc(module_irq); + d->ipipe_ack(module_irq, d); +#else handle_nested_irq(module_irq); - else +#endif + } else pr_err("twl6030_irq: Unmapped PIH ISR %u detected\n", i); pr_debug("twl6030_irq: PIH ISR %u, virq%u\n", @@ -483,4 +488,3 @@ int twl6030_exit_irq(void) } return 0; } - diff --git a/drivers/misc/Kconfig b/drivers/misc/Kconfig index 006242c..d5fb3f4 100644 --- a/drivers/misc/Kconfig +++ b/drivers/misc/Kconfig @@ -61,7 +61,7 @@ config ATMEL_TCLIB config ATMEL_TCB_CLKSRC bool "TC Block Clocksource" - depends on ATMEL_TCLIB + depends on ATMEL_TCLIB && !IPIPE default y help Select this to get a high precision clocksource based on a diff --git a/drivers/pinctrl/pinctrl-at91.c b/drivers/pinctrl/pinctrl-at91.c index 7747814..cd6e0e4 100644 --- a/drivers/pinctrl/pinctrl-at91.c +++ b/drivers/pinctrl/pinctrl-at91.c @@ -24,6 +24,7 @@ #include /* Since we request GPIOs from ourself */ #include +#include #include "pinctrl-at91.h" #include "core.h" @@ -43,6 +44,12 @@ struct at91_gpio_chip { void __iomem *regbase; /* PIO bank virtual address */ struct clk *clock; /* associated clock */ struct at91_pinctrl_mux_ops *ops; /* ops */ +#ifdef CONFIG_IPIPE + unsigned *nr_nonroot; + unsigned nr_nonroot_storage; + unsigned root; + unsigned muted; +#endif }; #define to_at91_gpio_chip(c) container_of(c, struct at91_gpio_chip, chip) @@ -1622,8 +1629,8 @@ static void gpio_irq_handler(unsigned irq, struct irq_desc *desc) } for_each_set_bit(n, &isr, BITS_PER_LONG) { - generic_handle_irq(irq_find_mapping( - gpio_chip->irqdomain, n)); + ipipe_handle_demuxed_irq(irq_find_mapping( + gpio_chip->irqdomain, n)); } } chained_irq_exit(chip, desc); @@ -1677,6 +1684,10 @@ static int at91_gpio_of_irq_setup(struct platform_device *pdev, } prev = container_of(gpiochip_prev, struct at91_gpio_chip, chip); +#ifdef CONFIG_IPIPE + if (prev->pioc_hwirq == at91_gpio->pioc_hwirq) + at91_gpio->nr_nonroot = prev->nr_nonroot; +#endif /* CONFIG_IPIPE */ /* we can only have 2 banks before */ for (i = 0; i < 2; i++) { @@ -1773,6 +1784,10 @@ static int at91_gpio_probe(struct platform_device *pdev) } at91_chip->chip = at91_gpio_template; +#ifdef CONFIG_IPIPE + at91_chip->nr_nonroot = &at91_chip->nr_nonroot_storage; + at91_chip->root = ~0U; +#endif /* CONFIG_IPIPE */ chip = &at91_chip->chip; chip->of_node = np; @@ -1838,6 +1853,96 @@ err: return ret; } +#ifdef CONFIG_IPIPE +int at91_gpio_enable_irqdesc(struct ipipe_domain *ipd, unsigned irq) +{ + struct at91_gpio_chip *at91_chip; + struct irq_data *idata; + struct irq_desc *desc; + struct irq_chip *chip; + + if (ipd == &ipipe_root) + return 0; + + desc = irq_to_desc(irq); + idata = irq_desc_get_irq_data(desc); + chip = irq_data_get_irq_chip(idata); + + if (chip != &gpio_irqchip) + return -EINVAL; + + at91_chip = irq_data_get_irq_chip_data(idata); + + at91_chip->root &= ~(1 << idata->hwirq); + + if (ipd != &ipipe_root && ++(*at91_chip->nr_nonroot) == 1) + return at91_chip->pioc_hwirq; + + return 0; +} + +int at91_gpio_disable_irqdesc(struct ipipe_domain *ipd, unsigned irq) +{ + struct at91_gpio_chip *at91_chip; + struct irq_data *idata; + struct irq_desc *desc; + struct irq_chip *chip; + + if (ipd == &ipipe_root) + return 0; + + desc = irq_to_desc(irq); + idata = irq_desc_get_irq_data(desc); + chip = irq_data_get_irq_chip(idata); + + if (chip != &gpio_irqchip) + return -EINVAL; + + at91_chip = irq_data_get_irq_chip_data(idata); + + at91_chip->root |= (1 << idata->hwirq); + + if (ipd != &ipipe_root && --(*at91_chip->nr_nonroot) == 0) + return at91_chip->pioc_hwirq; + + return 0; +} + +void at91_gpio_mute(void) +{ + struct at91_gpio_chip *prev, *chip = NULL; + unsigned long unmasked, muted; + unsigned i; + + for (i = 0; i < gpio_banks; i++) { + prev = chip; + chip = gpio_chips[i]; + if (!(*chip->nr_nonroot)) + continue; + + unmasked = __raw_readl(chip->regbase + PIO_IMR); + muted = unmasked & chip->root; + chip->muted = muted; + __raw_writel(muted, chip->regbase + PIO_IDR); + } +} + +void at91_gpio_unmute(void) +{ + struct at91_gpio_chip *prev, *chip = NULL; + unsigned i; + + for (i = 0; i < gpio_banks; i++) { + prev = chip; + chip = gpio_chips[i]; + if (!(*chip->nr_nonroot)) + continue; + + __raw_writel(chip->muted, chip->regbase + PIO_IER); + } +} +#endif /* CONFIG_IPIPE */ + static struct platform_driver at91_gpio_driver = { .driver = { .name = "gpio-at91", diff --git a/drivers/pinctrl/pinctrl-rockchip.c b/drivers/pinctrl/pinctrl-rockchip.c index dee7d5f..f4b5349 100644 --- a/drivers/pinctrl/pinctrl-rockchip.c +++ b/drivers/pinctrl/pinctrl-rockchip.c @@ -1463,7 +1463,7 @@ static int rockchip_irq_set_type(struct irq_data *d, unsigned int type) u32 polarity; u32 level; u32 data; - unsigned long flags; + unsigned long flags, flags2; int ret; /* make sure the pin is configured as gpio input */ @@ -1485,7 +1485,7 @@ static int rockchip_irq_set_type(struct irq_data *d, unsigned int type) __irq_set_handler_locked(d->irq, handle_level_irq); spin_lock_irqsave(&bank->slock, flags); - irq_gc_lock(gc); + flags2 = irq_gc_lock(gc); level = readl_relaxed(gc->reg_base + GPIO_INTTYPE_LEVEL); polarity = readl_relaxed(gc->reg_base + GPIO_INT_POLARITY); @@ -1526,7 +1526,7 @@ static int rockchip_irq_set_type(struct irq_data *d, unsigned int type) polarity &= ~mask; break; default: - irq_gc_unlock(gc); + irq_gc_unlock(gc, flags2); spin_unlock_irqrestore(&bank->slock, flags); return -EINVAL; } @@ -1534,7 +1534,7 @@ static int rockchip_irq_set_type(struct irq_data *d, unsigned int type) writel_relaxed(level, gc->reg_base + GPIO_INTTYPE_LEVEL); writel_relaxed(polarity, gc->reg_base + GPIO_INT_POLARITY); - irq_gc_unlock(gc); + irq_gc_unlock(gc, flags2); spin_unlock_irqrestore(&bank->slock, flags); return 0; diff --git a/drivers/pinctrl/pinctrl-single.c b/drivers/pinctrl/pinctrl-single.c index 13b45f2..637f923 100644 --- a/drivers/pinctrl/pinctrl-single.c +++ b/drivers/pinctrl/pinctrl-single.c @@ -16,6 +16,7 @@ #include #include #include +#include #include @@ -214,7 +215,11 @@ struct pcs_device { #define PCS_FEAT_IRQ (1 << 1) #define PCS_FEAT_PINCONF (1 << 0) struct pcs_soc_data socdata; +#ifdef CONFIG_IPIPE + ipipe_spinlock_t lock; +#else /* !IPIPE */ raw_spinlock_t lock; +#endif /* !IPIPE */ struct mutex mutex; unsigned width; unsigned fmask; @@ -1647,7 +1652,7 @@ static int pcs_irq_handle(struct pcs_soc_data *pcs_soc) mask = pcs->read(pcswi->reg); raw_spin_unlock(&pcs->lock); if (mask & pcs_soc->irq_status_mask) { - generic_handle_irq(irq_find_mapping(pcs->domain, + ipipe_handle_demuxed_irq(irq_find_mapping(pcs->domain, pcswi->hwirq)); count++; } @@ -1667,8 +1672,14 @@ static int pcs_irq_handle(struct pcs_soc_data *pcs_soc) static irqreturn_t pcs_irq_handler(int irq, void *d) { struct pcs_soc_data *pcs_soc = d; + unsigned long flags; + irqreturn_t ret; - return pcs_irq_handle(pcs_soc) ? IRQ_HANDLED : IRQ_NONE; + flags = hard_cond_local_irq_save(); + ret = pcs_irq_handle(pcs_soc) ? IRQ_HANDLED : IRQ_NONE; + hard_cond_local_irq_restore(flags); + + return ret; } /** diff --git a/drivers/tty/serial/xilinx_uartps.c b/drivers/tty/serial/xilinx_uartps.c index 3ddbac7..9ceb554 100644 --- a/drivers/tty/serial/xilinx_uartps.c +++ b/drivers/tty/serial/xilinx_uartps.c @@ -1137,6 +1137,33 @@ static void cdns_uart_console_write(struct console *co, const char *s, spin_unlock_irqrestore(&port->lock, flags); } +#ifdef CONFIG_RAW_PRINTK + +static void cdns_uart_console_write_raw(struct console *co, const char *s, + unsigned int count) +{ + struct uart_port *port = &cdns_uart_port[co->index]; + unsigned int imr, ctrl; + + imr = cdns_uart_readl(CDNS_UART_IMR_OFFSET); + cdns_uart_writel(imr, CDNS_UART_IDR_OFFSET); + + ctrl = cdns_uart_readl(CDNS_UART_CR_OFFSET); + cdns_uart_writel((ctrl & ~CDNS_UART_CR_TX_DIS) | CDNS_UART_CR_TX_EN, + CDNS_UART_CR_OFFSET); + + while (count-- > 0) { + if (*s == '\n') + cdns_uart_writel('\r', CDNS_UART_FIFO_OFFSET); + cdns_uart_writel(*s++, CDNS_UART_FIFO_OFFSET); + } + + cdns_uart_writel(ctrl, CDNS_UART_CR_OFFSET); + cdns_uart_writel(imr, CDNS_UART_IER_OFFSET); +} + +#endif + /** * cdns_uart_console_setup - Initialize the uart to default config * @co: Console handle @@ -1174,7 +1201,12 @@ static struct console cdns_uart_console = { .write = cdns_uart_console_write, .device = uart_console_device, .setup = cdns_uart_console_setup, +#ifdef CONFIG_RAW_PRINTK + .write_raw = cdns_uart_console_write_raw, + .flags = CON_PRINTBUFFER | CON_RAW, +#else .flags = CON_PRINTBUFFER, +#endif .index = -1, /* Specified on the cmdline (e.g. console=ttyPS ) */ .data = &cdns_uart_uart_driver, }; diff --git a/drivers/cpuidle/Kconfig b/drivers/cpuidle/Kconfig index 8c7930b..c7f7df0 100644 --- a/drivers/cpuidle/Kconfig +++ b/drivers/cpuidle/Kconfig @@ -3,6 +3,7 @@ menu "CPU Idle" config CPU_IDLE bool "CPU idle PM support" default y if ACPI || PPC_PSERIES + depends on !(ARCH_OMAP4 && IPIPE) select CPU_IDLE_GOV_LADDER if (!NO_HZ && !NO_HZ_IDLE) select CPU_IDLE_GOV_MENU if (NO_HZ || NO_HZ_IDLE) help diff --git a/drivers/iommu/irq_remapping.c b/drivers/iommu/irq_remapping.c index 390079e..51ca47e 100644 --- a/drivers/iommu/irq_remapping.c +++ b/drivers/iommu/irq_remapping.c @@ -334,12 +334,12 @@ void panic_if_irq_remap(const char *msg) static void ir_ack_apic_edge(struct irq_data *data) { - ack_APIC_irq(); + __ack_APIC_irq(); } static void ir_ack_apic_level(struct irq_data *data) { - ack_APIC_irq(); + __ack_APIC_irq(); eoi_ioapic_irq(data->irq, irqd_cfg(data)); } diff --git a/drivers/tty/serial/8250/8250_core.c b/drivers/tty/serial/8250/8250_core.c index b4fd8de..22ba96d 100644 --- a/drivers/tty/serial/8250/8250_core.c +++ b/drivers/tty/serial/8250/8250_core.c @@ -3440,6 +3440,48 @@ static void univ8250_console_write(struct console *co, const char *s, serial8250_console_write(up, s, count); } +#ifdef CONFIG_RAW_PRINTK + +static void raw_write_char(struct uart_8250_port *up, int c) +{ + unsigned int status, tmout = 10000; + + for (;;) { + status = serial_in(up, UART_LSR); + up->lsr_saved_flags |= status & LSR_SAVE_FLAGS; + if ((status & UART_LSR_THRE) == UART_LSR_THRE) + break; + if (--tmout == 0) + break; + cpu_relax(); + } + serial_port_out(&up->port, UART_TX, c); +} + +static void univ8250_console_write_raw(struct console *co, const char *s, + unsigned int count) +{ + struct uart_8250_port *up = &serial8250_ports[co->index]; + unsigned int ier; + + ier = serial_in(up, UART_IER); + + if (up->capabilities & UART_CAP_UUE) + serial_out(up, UART_IER, UART_IER_UUE); + else + serial_out(up, UART_IER, 0); + + while (count-- > 0) { + if (*s == '\n') + raw_write_char(up, '\r'); + raw_write_char(up, *s++); + } + + serial_out(up, UART_IER, ier); +} + +#endif + static unsigned int probe_baud(struct uart_port *port) { unsigned char lcr, dll, dlm; @@ -3549,7 +3591,12 @@ static struct console univ8250_console = { .device = uart_console_device, .setup = univ8250_console_setup, .match = univ8250_console_match, +#ifdef CONFIG_RAW_PRINTK + .write_raw = univ8250_console_write_raw, + .flags = CON_PRINTBUFFER | CON_ANYTIME | CON_RAW, +#else .flags = CON_PRINTBUFFER | CON_ANYTIME, +#endif .index = -1, .data = &serial8250_reg, }; diff --git a/fs/exec.c b/fs/exec.c index 1977c2a..55c8c66 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -838,6 +838,7 @@ static int exec_mmap(struct mm_struct *mm) { struct task_struct *tsk; struct mm_struct *old_mm, *active_mm; + unsigned long flags; /* Notify parent that we're no longer interested in the old VM */ tsk = current; @@ -861,8 +862,10 @@ static int exec_mmap(struct mm_struct *mm) task_lock(tsk); active_mm = tsk->active_mm; tsk->mm = mm; + ipipe_mm_switch_protect(flags); tsk->active_mm = mm; activate_mm(active_mm, mm); + ipipe_mm_switch_unprotect(flags); tsk->mm->vmacache_seqnum = 0; vmacache_flush(tsk); task_unlock(tsk); diff --git a/include/asm-generic/atomic.h b/include/asm-generic/atomic.h index 1973ad2..c3f2321 100644 --- a/include/asm-generic/atomic.h +++ b/include/asm-generic/atomic.h @@ -70,9 +70,9 @@ static inline void atomic_##op(int i, atomic_t *v) \ { \ unsigned long flags; \ \ - raw_local_irq_save(flags); \ + flags = hard_local_irq_save(); \ v->counter = v->counter c_op i; \ - raw_local_irq_restore(flags); \ + hard_local_irq_restore(flags); \ } #define ATOMIC_OP_RETURN(op, c_op) \ @@ -81,9 +81,9 @@ static inline int atomic_##op##_return(int i, atomic_t *v) \ unsigned long flags; \ int ret; \ \ - raw_local_irq_save(flags); \ + flags = hard_local_irq_save(); \ ret = (v->counter = v->counter c_op i); \ - raw_local_irq_restore(flags); \ + hard_local_irq_restore(flags); \ \ return ret; \ } diff --git a/include/asm-generic/bitops/atomic.h b/include/asm-generic/bitops/atomic.h index 4967351..ba3d54d 100644 --- a/include/asm-generic/bitops/atomic.h +++ b/include/asm-generic/bitops/atomic.h @@ -21,20 +21,20 @@ extern arch_spinlock_t __atomic_hash[ATOMIC_HASH_SIZE] __lock_aligned; * this is the substitute */ #define _atomic_spin_lock_irqsave(l,f) do { \ arch_spinlock_t *s = ATOMIC_HASH(l); \ - local_irq_save(f); \ + (f) = hard_local_irq_save(); \ arch_spin_lock(s); \ } while(0) #define _atomic_spin_unlock_irqrestore(l,f) do { \ arch_spinlock_t *s = ATOMIC_HASH(l); \ arch_spin_unlock(s); \ - local_irq_restore(f); \ + hard_local_irq_restore(f); \ } while(0) #else -# define _atomic_spin_lock_irqsave(l,f) do { local_irq_save(f); } while (0) -# define _atomic_spin_unlock_irqrestore(l,f) do { local_irq_restore(f); } while (0) +# define _atomic_spin_lock_irqsave(l,f) do { (f) = hard_local_irq_save(); } while (0) +# define _atomic_spin_unlock_irqrestore(l,f) do { hard_local_irq_restore(f); } while (0) #endif /* diff --git a/include/asm-generic/cmpxchg-local.h b/include/asm-generic/cmpxchg-local.h index 70bef78..145e071 100644 --- a/include/asm-generic/cmpxchg-local.h +++ b/include/asm-generic/cmpxchg-local.h @@ -22,7 +22,7 @@ static inline unsigned long __cmpxchg_local_generic(volatile void *ptr, if (size == 8 && sizeof(unsigned long) != 8) wrong_size_cmpxchg(ptr); - raw_local_irq_save(flags); + flags = hard_local_irq_save(); switch (size) { case 1: prev = *(u8 *)ptr; if (prev == old) @@ -43,7 +43,7 @@ static inline unsigned long __cmpxchg_local_generic(volatile void *ptr, default: wrong_size_cmpxchg(ptr); } - raw_local_irq_restore(flags); + hard_local_irq_restore(flags); return prev; } @@ -56,11 +56,11 @@ static inline u64 __cmpxchg64_local_generic(volatile void *ptr, u64 prev; unsigned long flags; - raw_local_irq_save(flags); + flags = hard_local_irq_save(); prev = *(u64 *)ptr; if (prev == old) *(u64 *)ptr = new; - raw_local_irq_restore(flags); + hard_local_irq_restore(flags); return prev; } diff --git a/include/asm-generic/percpu.h b/include/asm-generic/percpu.h index 4d9f233..f95a36d 100644 --- a/include/asm-generic/percpu.h +++ b/include/asm-generic/percpu.h @@ -43,11 +43,33 @@ extern unsigned long __per_cpu_offset[NR_CPUS]; #define arch_raw_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset) #endif +#ifdef CONFIG_IPIPE +#if defined(CONFIG_IPIPE_DEBUG_INTERNAL) && defined(CONFIG_SMP) +extern int __ipipe_check_percpu_access(void); +#define __ipipe_cpu_offset \ + ({ \ + WARN_ON_ONCE(__ipipe_check_percpu_access()); \ + __my_cpu_offset; \ + }) +#else +#define __ipipe_cpu_offset __my_cpu_offset +#endif +#ifndef __ipipe_raw_cpu_ptr +#define __ipipe_raw_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, __ipipe_cpu_offset) +#endif +#define __ipipe_raw_cpu_read(var) (*__ipipe_raw_cpu_ptr(&(var))) +#endif /* CONFIG_IPIPE */ + #ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA extern void setup_per_cpu_areas(void); #endif -#endif /* SMP */ +#else /* !SMP */ + +#define __ipipe_raw_cpu_ptr(ptr) VERIFY_PERCPU_PTR(ptr) +#define __ipipe_raw_cpu_read(var) (*__ipipe_raw_cpu_ptr(&(var))) + +#endif /* !SMP */ #ifndef PER_CPU_BASE_SECTION #ifdef CONFIG_SMP @@ -117,19 +139,19 @@ do { \ #define this_cpu_generic_to_op(pcp, val, op) \ do { \ unsigned long __flags; \ - raw_local_irq_save(__flags); \ + __flags = hard_local_irq_save(); \ *raw_cpu_ptr(&(pcp)) op val; \ - raw_local_irq_restore(__flags); \ + hard_local_irq_restore(__flags); \ } while (0) #define this_cpu_generic_add_return(pcp, val) \ ({ \ typeof(pcp) __ret; \ unsigned long __flags; \ - raw_local_irq_save(__flags); \ + __flags = hard_local_irq_save(); \ raw_cpu_add(pcp, val); \ __ret = raw_cpu_read(pcp); \ - raw_local_irq_restore(__flags); \ + hard_local_irq_restore(__flags); \ __ret; \ }) @@ -137,10 +159,10 @@ do { \ ({ \ typeof(pcp) __ret; \ unsigned long __flags; \ - raw_local_irq_save(__flags); \ + __flags = hard_local_irq_save(); \ __ret = raw_cpu_read(pcp); \ raw_cpu_write(pcp, nval); \ - raw_local_irq_restore(__flags); \ + hard_local_irq_restore(__flags); \ __ret; \ }) @@ -148,11 +170,11 @@ do { \ ({ \ typeof(pcp) __ret; \ unsigned long __flags; \ - raw_local_irq_save(__flags); \ + __flags = hard_local_irq_save(); \ __ret = raw_cpu_read(pcp); \ if (__ret == (oval)) \ raw_cpu_write(pcp, nval); \ - raw_local_irq_restore(__flags); \ + hard_local_irq_restore(__flags); \ __ret; \ }) @@ -160,10 +182,10 @@ do { \ ({ \ int __ret; \ unsigned long __flags; \ - raw_local_irq_save(__flags); \ + __flags = hard_local_irq_save(); \ __ret = raw_cpu_generic_cmpxchg_double(pcp1, pcp2, \ oval1, oval2, nval1, nval2); \ - raw_local_irq_restore(__flags); \ + hard_local_irq_restore(__flags); \ __ret; \ }) diff --git a/include/asm-generic/preempt.h b/include/asm-generic/preempt.h index b6a53e8..bf4f3a1 100644 --- a/include/asm-generic/preempt.h +++ b/include/asm-generic/preempt.h @@ -50,16 +50,19 @@ static __always_inline bool test_preempt_need_resched(void) static __always_inline void __preempt_count_add(int val) { + ipipe_preempt_root_only(); *preempt_count_ptr() += val; } static __always_inline void __preempt_count_sub(int val) { + ipipe_preempt_root_only(); *preempt_count_ptr() -= val; } static __always_inline bool __preempt_count_dec_and_test(void) { + ipipe_preempt_root_only(); /* * Because of load-store architectures cannot do per-cpu atomic * operations; we cannot use PREEMPT_NEED_RESCHED because it might get diff --git a/include/asm-generic/switch_to.h b/include/asm-generic/switch_to.h index 052c4ac..6472b80 100644 --- a/include/asm-generic/switch_to.h +++ b/include/asm-generic/switch_to.h @@ -21,10 +21,17 @@ */ extern struct task_struct *__switch_to(struct task_struct *, struct task_struct *); - +#ifdef CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH #define switch_to(prev, next, last) \ do { \ + hard_cond_local_irq_disable(); \ ((last) = __switch_to((prev), (next))); \ + hard_cond_local_irq_enable(); \ } while (0) - +#else /* !CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH */ +#define switch_to(prev, next, last) \ + do { \ + ((last) = __switch_to((prev), (next))); \ + } while (0) +#endif /* !CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH */ #endif /* __ASM_GENERIC_SWITCH_TO_H */ diff --git a/include/ipipe/setup.h b/include/ipipe/setup.h new file mode 100644 index 0000000..c2bc521 --- /dev/null +++ b/include/ipipe/setup.h @@ -0,0 +1,10 @@ +#ifndef _IPIPE_SETUP_H +#define _IPIPE_SETUP_H + +/* + * Placeholders for setup hooks defined by client domains. + */ + +static inline void __ipipe_early_client_setup(void) { } + +#endif /* !_IPIPE_SETUP_H */ diff --git a/include/ipipe/thread_info.h b/include/ipipe/thread_info.h new file mode 100644 index 0000000..1f6e9c3 --- /dev/null +++ b/include/ipipe/thread_info.h @@ -0,0 +1,14 @@ +#ifndef _IPIPE_THREAD_INFO_H +#define _IPIPE_THREAD_INFO_H + +/* + * Placeholder for private thread information defined by client + * domains. + */ + +struct ipipe_threadinfo { +}; + +static inline void __ipipe_init_threadinfo(struct ipipe_threadinfo *p) { } + +#endif /* !_IPIPE_THREAD_INFO_H */ diff --git a/include/linux/basic_mmio_gpio.h b/include/linux/basic_mmio_gpio.h index 0e97856..b5d7e60 100644 --- a/include/linux/basic_mmio_gpio.h +++ b/include/linux/basic_mmio_gpio.h @@ -50,7 +50,7 @@ struct bgpio_chip { * Used to lock bgpio_chip->data. Also, this is needed to keep * shadowed and real data registers writes together. */ - spinlock_t lock; + ipipe_spinlock_t lock; /* Shadowed data register to clear/set bits safely. */ unsigned long data; diff --git a/include/linux/clockchips.h b/include/linux/clockchips.h index 96c280b..0baa8f1 100644 --- a/include/linux/clockchips.h +++ b/include/linux/clockchips.h @@ -142,6 +142,15 @@ struct clock_event_device { const struct cpumask *cpumask; struct list_head list; struct module *owner; + +#ifdef CONFIG_IPIPE + struct ipipe_timer *ipipe_timer; + unsigned ipipe_stolen; + +#define clockevent_ipipe_stolen(evt) ((evt)->ipipe_stolen) +#else +#define clockevent_ipipe_stolen(evt) (0) +#endif /* !CONFIG_IPIPE */ } ____cacheline_aligned; /* diff --git a/include/linux/clocksource.h b/include/linux/clocksource.h index d27d015..d673882 100644 --- a/include/linux/clocksource.h +++ b/include/linux/clocksource.h @@ -95,6 +95,10 @@ struct clocksource { cycle_t wd_last; #endif struct module *owner; +#ifdef CONFIG_IPIPE_WANT_CLOCKSOURCE + cycle_t (*ipipe_read)(struct clocksource *cs); +#endif /* CONFIG_IPIPE_WANT_CLOCKSOURCE */ + } ____cacheline_aligned; /* diff --git a/include/linux/compiler-gcc.h b/include/linux/compiler-gcc.h index 371e560..484ef6e 100644 --- a/include/linux/compiler-gcc.h +++ b/include/linux/compiler-gcc.h @@ -115,10 +115,122 @@ #define __maybe_unused __attribute__((unused)) #define __always_unused __attribute__((unused)) -#define __gcc_header(x) #x -#define _gcc_header(x) __gcc_header(linux/compiler-gcc##x.h) -#define gcc_header(x) _gcc_header(x) -#include gcc_header(__GNUC__) +/* gcc version specific checks */ + +#if GCC_VERSION < 30200 +# error Sorry, your compiler is too old - please upgrade it. +#endif + +#if GCC_VERSION < 30300 +# define __used __attribute__((__unused__)) +#else +# define __used __attribute__((__used__)) +#endif + +#ifdef CONFIG_GCOV_KERNEL +# if GCC_VERSION < 30400 +# error "GCOV profiling support for gcc versions below 3.4 not included" +# endif /* __GNUC_MINOR__ */ +#endif /* CONFIG_GCOV_KERNEL */ + +#if GCC_VERSION >= 30400 +#define __must_check __attribute__((warn_unused_result)) +#endif + +#if GCC_VERSION >= 40000 + +/* GCC 4.1.[01] miscompiles __weak */ +#ifdef __KERNEL__ +# if GCC_VERSION >= 40100 && GCC_VERSION <= 40101 +# error Your version of gcc miscompiles the __weak directive +# endif +#endif + +#define __used __attribute__((__used__)) +#define __compiler_offsetof(a, b) \ + __builtin_offsetof(a, b) + +#if GCC_VERSION >= 40100 && GCC_VERSION < 40600 +# define __compiletime_object_size(obj) __builtin_object_size(obj, 0) +#endif + +#if GCC_VERSION >= 40300 +/* Mark functions as cold. gcc will assume any path leading to a call + * to them will be unlikely. This means a lot of manual unlikely()s + * are unnecessary now for any paths leading to the usual suspects + * like BUG(), printk(), panic() etc. [but let's keep them for now for + * older compilers] + * + * Early snapshots of gcc 4.3 don't support this and we can't detect this + * in the preprocessor, but we can live with this because they're unreleased. + * Maketime probing would be overkill here. + * + * gcc also has a __attribute__((__hot__)) to move hot functions into + * a special section, but I don't see any sense in this right now in + * the kernel context + */ +#define __cold __attribute__((__cold__)) + +#define __UNIQUE_ID(prefix) __PASTE(__PASTE(__UNIQUE_ID_, prefix), __COUNTER__) + +#ifndef __CHECKER__ +# define __compiletime_warning(message) __attribute__((warning(message))) +# define __compiletime_error(message) __attribute__((error(message))) +#endif /* __CHECKER__ */ +#endif /* GCC_VERSION >= 40300 */ + +#if GCC_VERSION >= 40500 +/* + * Mark a position in code as unreachable. This can be used to + * suppress control flow warnings after asm blocks that transfer + * control elsewhere. + * + * Early snapshots of gcc 4.5 don't support this and we can't detect + * this in the preprocessor, but we can live with this because they're + * unreleased. Really, we need to have autoconf for the kernel. + */ +#define unreachable() __builtin_unreachable() + +/* Mark a function definition as prohibited from being cloned. */ +#define __noclone __attribute__((__noclone__)) + +#endif /* GCC_VERSION >= 40500 */ + +#if GCC_VERSION >= 40600 +/* + * Tell the optimizer that something else uses this function or variable. + */ +#define __visible __attribute__((externally_visible)) +#endif + +/* + * GCC 'asm goto' miscompiles certain code sequences: + * + * http://gcc.gnu.org/bugzilla/show_bug.cgi?id=58670 + * + * Work it around via a compiler barrier quirk suggested by Jakub Jelinek. + * + * (asm goto is automatically volatile - the naming reflects this.) + */ +#define asm_volatile_goto(x...) do { asm goto(x); asm (""); } while (0) + +#ifdef CONFIG_ARCH_USE_BUILTIN_BSWAP +#if GCC_VERSION >= 40400 +#define __HAVE_BUILTIN_BSWAP32__ +#define __HAVE_BUILTIN_BSWAP64__ +#endif +#if GCC_VERSION >= 40800 || (defined(__powerpc__) && GCC_VERSION >= 40600) +#define __HAVE_BUILTIN_BSWAP16__ +#endif +#endif /* CONFIG_ARCH_USE_BUILTIN_BSWAP */ + +#if GCC_VERSION >= 50000 +#define KASAN_ABI_VERSION 4 +#elif GCC_VERSION >= 40902 +#define KASAN_ABI_VERSION 3 +#endif + +#endif /* gcc version >= 40000 specific checks */ #if !defined(__noclone) #define __noclone /* not needed */ diff --git a/include/linux/compiler-gcc3.h b/include/linux/compiler-gcc3.h deleted file mode 100644 index 7d89feb..0000000 --- a/include/linux/compiler-gcc3.h +++ /dev/null @@ -1,23 +0,0 @@ -#ifndef __LINUX_COMPILER_H -#error "Please don't include directly, include instead." -#endif - -#if GCC_VERSION < 30200 -# error Sorry, your compiler is too old - please upgrade it. -#endif - -#if GCC_VERSION >= 30300 -# define __used __attribute__((__used__)) -#else -# define __used __attribute__((__unused__)) -#endif - -#if GCC_VERSION >= 30400 -#define __must_check __attribute__((warn_unused_result)) -#endif - -#ifdef CONFIG_GCOV_KERNEL -# if GCC_VERSION < 30400 -# error "GCOV profiling support for gcc versions below 3.4 not included" -# endif /* __GNUC_MINOR__ */ -#endif /* CONFIG_GCOV_KERNEL */ diff --git a/include/linux/compiler-gcc4.h b/include/linux/compiler-gcc4.h deleted file mode 100644 index 769e198..0000000 --- a/include/linux/compiler-gcc4.h +++ /dev/null @@ -1,91 +0,0 @@ -#ifndef __LINUX_COMPILER_H -#error "Please don't include directly, include instead." -#endif - -/* GCC 4.1.[01] miscompiles __weak */ -#ifdef __KERNEL__ -# if GCC_VERSION >= 40100 && GCC_VERSION <= 40101 -# error Your version of gcc miscompiles the __weak directive -# endif -#endif - -#define __used __attribute__((__used__)) -#define __must_check __attribute__((warn_unused_result)) -#define __compiler_offsetof(a,b) __builtin_offsetof(a,b) - -#if GCC_VERSION >= 40100 && GCC_VERSION < 40600 -# define __compiletime_object_size(obj) __builtin_object_size(obj, 0) -#endif - -#if GCC_VERSION >= 40300 -/* Mark functions as cold. gcc will assume any path leading to a call - to them will be unlikely. This means a lot of manual unlikely()s - are unnecessary now for any paths leading to the usual suspects - like BUG(), printk(), panic() etc. [but let's keep them for now for - older compilers] - - Early snapshots of gcc 4.3 don't support this and we can't detect this - in the preprocessor, but we can live with this because they're unreleased. - Maketime probing would be overkill here. - - gcc also has a __attribute__((__hot__)) to move hot functions into - a special section, but I don't see any sense in this right now in - the kernel context */ -#define __cold __attribute__((__cold__)) - -#define __UNIQUE_ID(prefix) __PASTE(__PASTE(__UNIQUE_ID_, prefix), __COUNTER__) - -#ifndef __CHECKER__ -# define __compiletime_warning(message) __attribute__((warning(message))) -# define __compiletime_error(message) __attribute__((error(message))) -#endif /* __CHECKER__ */ -#endif /* GCC_VERSION >= 40300 */ - -#if GCC_VERSION >= 40500 -/* - * Mark a position in code as unreachable. This can be used to - * suppress control flow warnings after asm blocks that transfer - * control elsewhere. - * - * Early snapshots of gcc 4.5 don't support this and we can't detect - * this in the preprocessor, but we can live with this because they're - * unreleased. Really, we need to have autoconf for the kernel. - */ -#define unreachable() __builtin_unreachable() - -/* Mark a function definition as prohibited from being cloned. */ -#define __noclone __attribute__((__noclone__)) - -#endif /* GCC_VERSION >= 40500 */ - -#if GCC_VERSION >= 40600 -/* - * Tell the optimizer that something else uses this function or variable. - */ -#define __visible __attribute__((externally_visible)) -#endif - -/* - * GCC 'asm goto' miscompiles certain code sequences: - * - * http://gcc.gnu.org/bugzilla/show_bug.cgi?id=58670 - * - * Work it around via a compiler barrier quirk suggested by Jakub Jelinek. - * - * (asm goto is automatically volatile - the naming reflects this.) - */ -#define asm_volatile_goto(x...) do { asm goto(x); asm (""); } while (0) - -#ifdef CONFIG_ARCH_USE_BUILTIN_BSWAP -#if GCC_VERSION >= 40400 -#define __HAVE_BUILTIN_BSWAP32__ -#define __HAVE_BUILTIN_BSWAP64__ -#endif -#if GCC_VERSION >= 40800 || (defined(__powerpc__) && GCC_VERSION >= 40600) -#define __HAVE_BUILTIN_BSWAP16__ -#endif -#endif /* CONFIG_ARCH_USE_BUILTIN_BSWAP */ - -#if GCC_VERSION >= 40902 -#define KASAN_ABI_VERSION 3 -#endif diff --git a/include/linux/compiler-gcc5.h b/include/linux/compiler-gcc5.h deleted file mode 100644 index efee493..0000000 --- a/include/linux/compiler-gcc5.h +++ /dev/null @@ -1,67 +0,0 @@ -#ifndef __LINUX_COMPILER_H -#error "Please don't include directly, include instead." -#endif - -#define __used __attribute__((__used__)) -#define __must_check __attribute__((warn_unused_result)) -#define __compiler_offsetof(a, b) __builtin_offsetof(a, b) - -/* Mark functions as cold. gcc will assume any path leading to a call - to them will be unlikely. This means a lot of manual unlikely()s - are unnecessary now for any paths leading to the usual suspects - like BUG(), printk(), panic() etc. [but let's keep them for now for - older compilers] - - Early snapshots of gcc 4.3 don't support this and we can't detect this - in the preprocessor, but we can live with this because they're unreleased. - Maketime probing would be overkill here. - - gcc also has a __attribute__((__hot__)) to move hot functions into - a special section, but I don't see any sense in this right now in - the kernel context */ -#define __cold __attribute__((__cold__)) - -#define __UNIQUE_ID(prefix) __PASTE(__PASTE(__UNIQUE_ID_, prefix), __COUNTER__) - -#ifndef __CHECKER__ -# define __compiletime_warning(message) __attribute__((warning(message))) -# define __compiletime_error(message) __attribute__((error(message))) -#endif /* __CHECKER__ */ - -/* - * Mark a position in code as unreachable. This can be used to - * suppress control flow warnings after asm blocks that transfer - * control elsewhere. - * - * Early snapshots of gcc 4.5 don't support this and we can't detect - * this in the preprocessor, but we can live with this because they're - * unreleased. Really, we need to have autoconf for the kernel. - */ -#define unreachable() __builtin_unreachable() - -/* Mark a function definition as prohibited from being cloned. */ -#define __noclone __attribute__((__noclone__)) - -/* - * Tell the optimizer that something else uses this function or variable. - */ -#define __visible __attribute__((externally_visible)) - -/* - * GCC 'asm goto' miscompiles certain code sequences: - * - * http://gcc.gnu.org/bugzilla/show_bug.cgi?id=58670 - * - * Work it around via a compiler barrier quirk suggested by Jakub Jelinek. - * - * (asm goto is automatically volatile - the naming reflects this.) - */ -#define asm_volatile_goto(x...) do { asm goto(x); asm (""); } while (0) - -#ifdef CONFIG_ARCH_USE_BUILTIN_BSWAP -#define __HAVE_BUILTIN_BSWAP32__ -#define __HAVE_BUILTIN_BSWAP64__ -#define __HAVE_BUILTIN_BSWAP16__ -#endif /* CONFIG_ARCH_USE_BUILTIN_BSWAP */ - -#define KASAN_ABI_VERSION 4 diff --git a/include/linux/console.h b/include/linux/console.h index 901555a..1ba117c 100644 --- a/include/linux/console.h +++ b/include/linux/console.h @@ -115,10 +115,12 @@ static inline int con_debug_leave(void) #define CON_BOOT (8) #define CON_ANYTIME (16) /* Safe to call when cpu is offline */ #define CON_BRL (32) /* Used for a braille device */ +#define CON_RAW (64) /* Supports raw write mode */ struct console { char name[16]; void (*write)(struct console *, const char *, unsigned); + void (*write_raw)(struct console *, const char *, unsigned); int (*read)(struct console *, char *, unsigned); struct tty_driver *(*device)(struct console *, int *); void (*unblank)(void); diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h index 6cd8c0e..0951ab4 100644 --- a/include/linux/ftrace.h +++ b/include/linux/ftrace.h @@ -134,6 +134,7 @@ enum { FTRACE_OPS_FL_ALLOC_TRAMP = 1 << 12, FTRACE_OPS_FL_IPMODIFY = 1 << 13, FTRACE_OPS_FL_PID = 1 << 14, + FTRACE_OPS_FL_IPIPE_EXCLUSIVE = 1 << 15, }; #ifdef CONFIG_DYNAMIC_FTRACE diff --git a/include/linux/hardirq.h b/include/linux/hardirq.h index f4af034..fa16b8c 100644 --- a/include/linux/hardirq.h +++ b/include/linux/hardirq.h @@ -61,6 +61,7 @@ extern void irq_exit(void); #define nmi_enter() \ do { \ + __ipipe_nmi_enter(); \ lockdep_off(); \ ftrace_nmi_enter(); \ BUG_ON(in_nmi()); \ @@ -77,6 +78,7 @@ extern void irq_exit(void); preempt_count_sub(NMI_OFFSET + HARDIRQ_OFFSET); \ ftrace_nmi_exit(); \ lockdep_on(); \ + __ipipe_nmi_exit(); \ } while (0) #endif /* LINUX_HARDIRQ_H */ diff --git a/include/linux/i8253.h b/include/linux/i8253.h index e6bb36a..898a91a 100644 --- a/include/linux/i8253.h +++ b/include/linux/i8253.h @@ -12,6 +12,7 @@ #include #include #include +#include /* i8253A PIT registers */ #define PIT_MODE 0x43 @@ -20,7 +21,7 @@ #define PIT_LATCH ((PIT_TICK_RATE + HZ/2) / HZ) -extern raw_spinlock_t i8253_lock; +IPIPE_DECLARE_RAW_SPINLOCK(i8253_lock); extern struct clock_event_device i8253_clockevent; extern void clockevent_i8253_init(bool oneshot); diff --git a/include/linux/ipipe.h b/include/linux/ipipe.h new file mode 100644 index 0000000..0a9b5b6 --- /dev/null +++ b/include/linux/ipipe.h @@ -0,0 +1,461 @@ +/* -*- linux-c -*- + * include/linux/ipipe.h + * + * Copyright (C) 2002-2014 Philippe Gerum. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139, + * USA; either version 2 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ + +#ifndef __LINUX_IPIPE_H +#define __LINUX_IPIPE_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef CONFIG_IPIPE + +#include + +/* ipipe_set_hooks(..., enables) */ +#define IPIPE_SYSCALL __IPIPE_SYSCALL_E +#define IPIPE_TRAP __IPIPE_TRAP_E +#define IPIPE_KEVENT __IPIPE_KEVENT_E + +struct ipipe_sysinfo { + int sys_nr_cpus; /* Number of CPUs on board */ + int sys_hrtimer_irq; /* hrtimer device IRQ */ + u64 sys_hrtimer_freq; /* hrtimer device frequency */ + u64 sys_hrclock_freq; /* hrclock device frequency */ + u64 sys_cpu_freq; /* CPU frequency (Hz) */ + struct ipipe_arch_sysinfo arch; +}; + +struct ipipe_work_header { + size_t size; + void (*handler)(struct ipipe_work_header *work); +}; + +extern unsigned int __ipipe_printk_virq; + +void __ipipe_set_irq_pending(struct ipipe_domain *ipd, unsigned int irq); + +void __ipipe_complete_domain_migration(void); + +int __ipipe_switch_tail(void); + +void __ipipe_share_current(int flags); + +void __ipipe_arch_share_current(int flags); + +int __ipipe_migrate_head(void); + +void __ipipe_reenter_root(void); + +int __ipipe_disable_ondemand_mappings(struct task_struct *p); + +int __ipipe_pin_vma(struct mm_struct *mm, struct vm_area_struct *vma); + +#ifdef CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH + +#define prepare_arch_switch(next) \ + do { \ + hard_local_irq_enable(); \ + __ipipe_report_schedule(current, next); \ + } while(0) + +#ifndef ipipe_get_active_mm +static inline struct mm_struct *ipipe_get_active_mm(void) +{ + return __this_cpu_read(ipipe_percpu.active_mm); +} +#define ipipe_get_active_mm ipipe_get_active_mm +#endif + +#else /* !CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH */ + +#define prepare_arch_switch(next) \ + do { \ + __ipipe_report_schedule(current, next); \ + hard_local_irq_disable(); \ + } while(0) + +#ifndef ipipe_get_active_mm +#define ipipe_get_active_mm() (current->active_mm) +#endif + +#endif /* !CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH */ + +#ifdef CONFIG_IPIPE_WANT_CLOCKSOURCE + +extern unsigned long long __ipipe_cs_freq; + +extern struct clocksource *__ipipe_cs; + +#endif /* CONFIG_IPIPE_WANT_CLOCKSOURCE */ + +static inline bool __ipipe_hrclock_ok(void) +{ + return __ipipe_hrclock_freq != 0; +} + +static inline void __ipipe_nmi_enter(void) +{ + __this_cpu_write(ipipe_percpu.nmi_state, __ipipe_root_status); + __set_bit(IPIPE_STALL_FLAG, &__ipipe_root_status); + ipipe_save_context_nmi(); +} + +static inline void __ipipe_nmi_exit(void) +{ + ipipe_restore_context_nmi(); + if (!test_bit(IPIPE_STALL_FLAG, raw_cpu_ptr(&ipipe_percpu.nmi_state))) + __clear_bit(IPIPE_STALL_FLAG, &__ipipe_root_status); +} + +/* KVM-side calls, hw IRQs off. */ +static inline void __ipipe_enter_vm(struct ipipe_vm_notifier *vmf) +{ + struct ipipe_percpu_data *p; + + p = raw_cpu_ptr(&ipipe_percpu); + p->vm_notifier = vmf; + barrier(); +} + +static inline void __ipipe_exit_vm(void) +{ + struct ipipe_percpu_data *p; + + p = raw_cpu_ptr(&ipipe_percpu); + p->vm_notifier = NULL; + barrier(); +} + +/* Client-side call, hw IRQs off. */ +void __ipipe_notify_vm_preemption(void); + +static inline void __ipipe_sync_pipeline(struct ipipe_domain *top) +{ + if (__ipipe_current_domain != top) { + __ipipe_do_sync_pipeline(top); + return; + } + if (!test_bit(IPIPE_STALL_FLAG, &ipipe_this_cpu_context(top)->status)) + __ipipe_sync_stage(); +} + +void ipipe_register_head(struct ipipe_domain *ipd, + const char *name); + +void ipipe_unregister_head(struct ipipe_domain *ipd); + +int ipipe_request_irq(struct ipipe_domain *ipd, + unsigned int irq, + ipipe_irq_handler_t handler, + void *cookie, + ipipe_irq_ackfn_t ackfn); + +void ipipe_free_irq(struct ipipe_domain *ipd, + unsigned int irq); + +void ipipe_raise_irq(unsigned int irq); + +void ipipe_set_hooks(struct ipipe_domain *ipd, + int enables); + +unsigned int ipipe_alloc_virq(void); + +void ipipe_free_virq(unsigned int virq); + +static inline void ipipe_post_irq_head(unsigned int irq) +{ + __ipipe_set_irq_pending(ipipe_head_domain, irq); +} + +static inline void ipipe_post_irq_root(unsigned int irq) +{ + __ipipe_set_irq_pending(&ipipe_root, irq); +} + +static inline void ipipe_stall_head(void) +{ + hard_local_irq_disable(); + __set_bit(IPIPE_STALL_FLAG, &__ipipe_head_status); +} + +static inline unsigned long ipipe_test_and_stall_head(void) +{ + hard_local_irq_disable(); + return __test_and_set_bit(IPIPE_STALL_FLAG, &__ipipe_head_status); +} + +static inline unsigned long ipipe_test_head(void) +{ + unsigned long flags, ret; + + flags = hard_smp_local_irq_save(); + ret = test_bit(IPIPE_STALL_FLAG, &__ipipe_head_status); + hard_smp_local_irq_restore(flags); + + return ret; +} + +void ipipe_unstall_head(void); + +void __ipipe_restore_head(unsigned long x); + +static inline void ipipe_restore_head(unsigned long x) +{ + ipipe_check_irqoff(); + if ((x ^ test_bit(IPIPE_STALL_FLAG, &__ipipe_head_status)) & 1) + __ipipe_restore_head(x); +} + +void __ipipe_post_work_root(struct ipipe_work_header *work); + +#define ipipe_post_work_root(p, header) \ + do { \ + void header_not_at_start(void); \ + if (offsetof(typeof(*(p)), header)) { \ + header_not_at_start(); \ + } \ + __ipipe_post_work_root(&(p)->header); \ + } while (0) + +int ipipe_get_sysinfo(struct ipipe_sysinfo *sysinfo); + +unsigned long ipipe_critical_enter(void (*syncfn)(void)); + +void ipipe_critical_exit(unsigned long flags); + +void ipipe_prepare_panic(void); + +#ifdef CONFIG_SMP +#ifndef ipipe_smp_p +#define ipipe_smp_p (1) +#endif +void ipipe_set_irq_affinity(unsigned int irq, cpumask_t cpumask); +void ipipe_send_ipi(unsigned int ipi, cpumask_t cpumask); +#else /* !CONFIG_SMP */ +#define ipipe_smp_p (0) +static inline +void ipipe_set_irq_affinity(unsigned int irq, cpumask_t cpumask) { } +static inline void ipipe_send_ipi(unsigned int ipi, cpumask_t cpumask) { } +static inline void ipipe_disable_smp(void) { } +#endif /* CONFIG_SMP */ + +static inline void ipipe_restore_root_nosync(unsigned long x) +{ + unsigned long flags; + + flags = hard_smp_local_irq_save(); + __ipipe_restore_root_nosync(x); + hard_smp_local_irq_restore(flags); +} + +/* Must be called hw IRQs off. */ +static inline void ipipe_lock_irq(unsigned int irq) +{ + struct ipipe_domain *ipd = __ipipe_current_domain; + if (ipd == ipipe_root_domain) + __ipipe_lock_irq(irq); +} + +/* Must be called hw IRQs off. */ +static inline void ipipe_unlock_irq(unsigned int irq) +{ + struct ipipe_domain *ipd = __ipipe_current_domain; + if (ipd == ipipe_root_domain) + __ipipe_unlock_irq(irq); +} + +static inline struct ipipe_threadinfo *ipipe_current_threadinfo(void) +{ + return ¤t_thread_info()->ipipe_data; +} + +#define ipipe_task_threadinfo(p) (&task_thread_info(p)->ipipe_data) + +void ipipe_enable_irq(unsigned int irq); + +static inline void ipipe_disable_irq(unsigned int irq) +{ + struct irq_desc *desc; + struct irq_chip *chip; + + desc = irq_to_desc(irq); + if (desc == NULL) + return; + + chip = irq_desc_get_chip(desc); + + if (WARN_ON_ONCE(chip->irq_disable == NULL && chip->irq_mask == NULL)) + return; + + if (chip->irq_disable) + chip->irq_disable(&desc->irq_data); + else + chip->irq_mask(&desc->irq_data); +} + +static inline void ipipe_end_irq(unsigned int irq) +{ + struct irq_desc *desc = irq_to_desc(irq); + + if (desc) + desc->ipipe_end(irq, desc); +} + +static inline int ipipe_chained_irq_p(struct irq_desc *desc) +{ + void __ipipe_chained_irq(unsigned irq, struct irq_desc *desc); + + return desc->handle_irq == __ipipe_chained_irq; +} + +static inline void ipipe_handle_demuxed_irq(unsigned int cascade_irq) +{ + ipipe_trace_irq_entry(cascade_irq); + __ipipe_dispatch_irq(cascade_irq, IPIPE_IRQF_NOSYNC); + ipipe_trace_irq_exit(cascade_irq); +} + +static inline void __ipipe_init_threadflags(struct thread_info *ti) +{ + ti->ipipe_flags = 0; +} + +static inline +void ipipe_set_ti_thread_flag(struct thread_info *ti, int flag) +{ + set_bit(flag, &ti->ipipe_flags); +} + +static inline +void ipipe_clear_ti_thread_flag(struct thread_info *ti, int flag) +{ + clear_bit(flag, &ti->ipipe_flags); +} + +static inline +void ipipe_test_and_clear_ti_thread_flag(struct thread_info *ti, int flag) +{ + test_and_clear_bit(flag, &ti->ipipe_flags); +} + +static inline +int ipipe_test_ti_thread_flag(struct thread_info *ti, int flag) +{ + return test_bit(flag, &ti->ipipe_flags); +} + +#define ipipe_set_thread_flag(flag) \ + ipipe_set_ti_thread_flag(current_thread_info(), flag) + +#define ipipe_clear_thread_flag(flag) \ + ipipe_clear_ti_thread_flag(current_thread_info(), flag) + +#define ipipe_test_and_clear_thread_flag(flag) \ + ipipe_test_and_clear_ti_thread_flag(current_thread_info(), flag) + +#define ipipe_test_thread_flag(flag) \ + ipipe_test_ti_thread_flag(current_thread_info(), flag) + +#define ipipe_enable_notifier(p) \ + ipipe_set_ti_thread_flag(task_thread_info(p), TIP_NOTIFY) + +#define ipipe_disable_notifier(p) \ + do { \ + struct thread_info *ti = task_thread_info(p); \ + ipipe_clear_ti_thread_flag(ti, TIP_NOTIFY); \ + ipipe_clear_ti_thread_flag(ti, TIP_MAYDAY); \ + } while (0) + +#define ipipe_notifier_enabled_p(p) \ + ipipe_test_ti_thread_flag(task_thread_info(p), TIP_NOTIFY) + +#define ipipe_raise_mayday(p) \ + do { \ + struct thread_info *ti = task_thread_info(p); \ + ipipe_check_irqoff(); \ + if (ipipe_test_ti_thread_flag(ti, TIP_NOTIFY)) \ + ipipe_set_ti_thread_flag(ti, TIP_MAYDAY); \ + } while (0) + +extern bool __ipipe_probe_access; + +long ipipe_probe_kernel_read(void *dst, void *src, size_t size); +long ipipe_probe_kernel_write(void *dst, void *src, size_t size); + +#if defined(CONFIG_DEBUG_ATOMIC_SLEEP) || defined(CONFIG_PROVE_LOCKING) || \ + defined(CONFIG_PREEMPT_VOLUNTARY) || defined(CONFIG_IPIPE_DEBUG_CONTEXT) +extern void __ipipe_uaccess_might_fault(void); +#else +#define __ipipe_uaccess_might_fault() might_fault() +#endif + +#ifdef CONFIG_IPIPE_TRACE +void __ipipe_tracer_hrclock_initialized(void); +#else /* !CONFIG_IPIPE_TRACE */ +#define __ipipe_tracer_hrclock_initialized() do { } while(0) +#endif /* !CONFIG_IPIPE_TRACE */ + +#include + +#else /* !CONFIG_IPIPE */ + +#define __ipipe_root_p 1 +#define ipipe_root_p 1 + +static inline void __ipipe_init_threadflags(struct thread_info *ti) { } + +static inline void __ipipe_complete_domain_migration(void) { } + +static inline int __ipipe_switch_tail(void) +{ + return 0; +} + +static inline void __ipipe_nmi_enter(void) { } + +static inline void __ipipe_nmi_exit(void) { } + +#define ipipe_safe_current() current +#define ipipe_processor_id() smp_processor_id() + +static inline int ipipe_test_foreign_stack(void) +{ + return 0; +} + +static inline void ipipe_lock_irq(unsigned int irq) { } + +static inline void ipipe_unlock_irq(unsigned int irq) { } + +#define ipipe_probe_kernel_read(d, s, sz) probe_kernel_read(d, s, sz) +#define ipipe_probe_kernel_write(d, s, sz) probe_kernel_write(d, s, sz) +#define __ipipe_uaccess_might_fault() might_fault() + +#endif /* !CONFIG_IPIPE */ + +#endif /* !__LINUX_IPIPE_H */ diff --git a/include/linux/ipipe_base.h b/include/linux/ipipe_base.h new file mode 100644 index 0000000..42b368a --- /dev/null +++ b/include/linux/ipipe_base.h @@ -0,0 +1,365 @@ +/* -*- linux-c -*- + * include/linux/ipipe_base.h + * + * Copyright (C) 2002-2014 Philippe Gerum. + * 2007 Jan Kiszka. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139, + * USA; either version 2 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ + +#ifndef __LINUX_IPIPE_BASE_H +#define __LINUX_IPIPE_BASE_H + +struct kvm_vcpu; +struct ipipe_vm_notifier; +struct irq_desc; + +#ifdef CONFIG_IPIPE + +#define IPIPE_CORE_APIREV CONFIG_IPIPE_CORE_APIREV + +#ifdef CONFIG_IPIPE_DEBUG_CONTEXT +void ipipe_root_only(void); +#else /* !CONFIG_IPIPE_DEBUG_CONTEXT */ +static inline void ipipe_root_only(void) { } +#endif /* !CONFIG_IPIPE_DEBUG_CONTEXT */ + +typedef void (*ipipe_irq_handler_t)(unsigned int irq, + void *cookie); + +void ipipe_unstall_root(void); + +void ipipe_restore_root(unsigned long x); + +#include +#include +#include + +#ifndef IPIPE_NR_ROOT_IRQS +#define IPIPE_NR_ROOT_IRQS NR_IRQS +#endif /* !IPIPE_NR_ROOT_IRQS */ + +#define __bpl_up(x) (((x)+(BITS_PER_LONG-1)) & ~(BITS_PER_LONG-1)) +/* Number of virtual IRQs (must be a multiple of BITS_PER_LONG) */ +#define IPIPE_NR_VIRQS BITS_PER_LONG +/* First virtual IRQ # (must be aligned on BITS_PER_LONG) */ +#define IPIPE_VIRQ_BASE __bpl_up(IPIPE_NR_XIRQS) +/* Total number of IRQ slots */ +#define IPIPE_NR_IRQS (IPIPE_VIRQ_BASE+IPIPE_NR_VIRQS) + +static inline int ipipe_virtual_irq_p(unsigned int irq) +{ + return irq >= IPIPE_VIRQ_BASE && irq < IPIPE_NR_IRQS; +} + +#define IPIPE_IRQ_LOMAPSZ (IPIPE_NR_IRQS / BITS_PER_LONG) +#if IPIPE_IRQ_LOMAPSZ > BITS_PER_LONG +/* + * We need a 3-level mapping. This allows us to handle up to 32k IRQ + * vectors on 32bit machines, 256k on 64bit ones. + */ +#define __IPIPE_3LEVEL_IRQMAP 1 +#define IPIPE_IRQ_MDMAPSZ (__bpl_up(IPIPE_IRQ_LOMAPSZ) / BITS_PER_LONG) +#else +/* + * 2-level mapping is enough. This allows us to handle up to 1024 IRQ + * vectors on 32bit machines, 4096 on 64bit ones. + */ +#define __IPIPE_2LEVEL_IRQMAP 1 +#endif + +/* Per-cpu pipeline status */ +#define IPIPE_STALL_FLAG 0 /* interrupts (virtually) disabled. */ +#define IPIPE_STALL_MASK (1L << IPIPE_STALL_FLAG) + +/* Interrupt control bits */ +#define IPIPE_HANDLE_FLAG 0 +#define IPIPE_STICKY_FLAG 1 +#define IPIPE_LOCK_FLAG 2 +#define IPIPE_HANDLE_MASK (1 << IPIPE_HANDLE_FLAG) +#define IPIPE_STICKY_MASK (1 << IPIPE_STICKY_FLAG) +#define IPIPE_LOCK_MASK (1 << IPIPE_LOCK_FLAG) + +struct pt_regs; +struct ipipe_domain; + +struct ipipe_trap_data { + int exception; + struct pt_regs *regs; +}; + +#define IPIPE_KEVT_SCHEDULE 0 +#define IPIPE_KEVT_SIGWAKE 1 +#define IPIPE_KEVT_SETSCHED 2 +#define IPIPE_KEVT_SETAFFINITY 3 +#define IPIPE_KEVT_EXIT 4 +#define IPIPE_KEVT_CLEANUP 5 +#define IPIPE_KEVT_HOSTRT 6 +#define IPIPE_KEVT_CLOCKFREQ 7 + +struct ipipe_vm_notifier { + void (*handler)(struct ipipe_vm_notifier *nfy); +}; + +void __ipipe_init_early(void); + +void __ipipe_init(void); + +#ifdef CONFIG_PROC_FS +void __ipipe_init_proc(void); +#ifdef CONFIG_IPIPE_TRACE +void __ipipe_init_tracer(void); +#else /* !CONFIG_IPIPE_TRACE */ +static inline void __ipipe_init_tracer(void) { } +#endif /* CONFIG_IPIPE_TRACE */ +#else /* !CONFIG_PROC_FS */ +static inline void __ipipe_init_proc(void) { } +#endif /* CONFIG_PROC_FS */ + +void __ipipe_restore_root_nosync(unsigned long x); + +#define IPIPE_IRQF_NOACK 0x1 +#define IPIPE_IRQF_NOSYNC 0x2 + +void __ipipe_dispatch_irq(unsigned int irq, int flags); + +void __ipipe_do_sync_stage(void); + +void __ipipe_do_sync_pipeline(struct ipipe_domain *top); + +void __ipipe_lock_irq(unsigned int irq); + +void __ipipe_unlock_irq(unsigned int irq); + +void __ipipe_do_critical_sync(unsigned int irq, void *cookie); + +void __ipipe_ack_edge_irq(unsigned int irq, struct irq_desc *desc); + +void __ipipe_nop_irq(unsigned int irq, struct irq_desc *desc); + +static inline void __ipipe_idle(void) +{ + ipipe_unstall_root(); +} + +#ifndef __ipipe_sync_check +#define __ipipe_sync_check 1 +#endif + +static inline void __ipipe_sync_stage(void) +{ + if (likely(__ipipe_sync_check)) + __ipipe_do_sync_stage(); +} + +#ifndef __ipipe_check_root_resched +#ifdef CONFIG_PREEMPT +#define __ipipe_check_root_resched() \ + (preempt_count() == 0 && need_resched()) +#else +#define __ipipe_check_root_resched() 0 +#endif +#endif + +#ifndef __ipipe_run_irqtail +#define __ipipe_run_irqtail(irq) do { } while(0) +#endif + +void __ipipe_flush_printk(unsigned int irq, void *cookie); + +#define __ipipe_get_cpu(flags) ({ (flags) = hard_preempt_disable(); ipipe_processor_id(); }) +#define __ipipe_put_cpu(flags) hard_preempt_enable(flags) + +int __ipipe_notify_syscall(struct pt_regs *regs); + +int __ipipe_notify_trap(int exception, struct pt_regs *regs); + +int __ipipe_notify_kevent(int event, void *data); + +#define __ipipe_report_trap(exception, regs) \ + __ipipe_notify_trap(exception, regs) + +#define __ipipe_report_sigwake(p) \ + do { \ + if (ipipe_notifier_enabled_p(p)) \ + __ipipe_notify_kevent(IPIPE_KEVT_SIGWAKE, p); \ + } while (0) + +struct ipipe_cpu_migration_data { + struct task_struct *task; + int dest_cpu; +}; + +#define __ipipe_report_setaffinity(__p, __dest_cpu) \ + do { \ + struct ipipe_cpu_migration_data d = { \ + .task = (__p), \ + .dest_cpu = (__dest_cpu), \ + }; \ + if (ipipe_notifier_enabled_p(__p)) \ + __ipipe_notify_kevent(IPIPE_KEVT_SETAFFINITY, &d); \ + } while (0) + +#define __ipipe_report_exit(p) \ + do { \ + if (ipipe_notifier_enabled_p(p)) \ + __ipipe_notify_kevent(IPIPE_KEVT_EXIT, p); \ + } while (0) + +#define __ipipe_report_setsched(p) \ + do { \ + if (ipipe_notifier_enabled_p(p)) \ + __ipipe_notify_kevent(IPIPE_KEVT_SETSCHED, p); \ + } while (0) + +#define __ipipe_report_schedule(prev, next) \ +do { \ + if (ipipe_notifier_enabled_p(next) || \ + ipipe_notifier_enabled_p(prev)) { \ + __this_cpu_write(ipipe_percpu.rqlock_owner, prev); \ + __ipipe_notify_kevent(IPIPE_KEVT_SCHEDULE, next); \ + } \ +} while (0) + +#define __ipipe_report_cleanup(mm) \ + __ipipe_notify_kevent(IPIPE_KEVT_CLEANUP, mm) + +#define __ipipe_report_clockfreq_update(freq) \ + __ipipe_notify_kevent(IPIPE_KEVT_CLOCKFREQ, &(freq)) + +void __ipipe_notify_vm_preemption(void); + +void __ipipe_call_mayday(struct pt_regs *regs); + +#define hard_cond_local_irq_enable() hard_local_irq_enable() +#define hard_cond_local_irq_disable() hard_local_irq_disable() +#define hard_cond_local_irq_save() hard_local_irq_save() +#define hard_cond_local_irq_restore(flags) hard_local_irq_restore(flags) + +#ifdef CONFIG_IPIPE_LEGACY + +#define IPIPE_FIRST_EVENT IPIPE_NR_FAULTS +#define IPIPE_EVENT_SCHEDULE IPIPE_FIRST_EVENT +#define IPIPE_EVENT_SIGWAKE (IPIPE_FIRST_EVENT + 1) +#define IPIPE_EVENT_SETSCHED (IPIPE_FIRST_EVENT + 2) +#define IPIPE_EVENT_SETAFFINITY (IPIPE_FIRST_EVENT + 3) +#define IPIPE_EVENT_EXIT (IPIPE_FIRST_EVENT + 4) +#define IPIPE_EVENT_CLEANUP (IPIPE_FIRST_EVENT + 5) +#define IPIPE_EVENT_HOSTRT (IPIPE_FIRST_EVENT + 6) +#define IPIPE_EVENT_CLOCKFREQ (IPIPE_FIRST_EVENT + 7) +#define IPIPE_EVENT_SYSCALL (IPIPE_FIRST_EVENT + 8) +#define IPIPE_LAST_EVENT IPIPE_EVENT_SYSCALL +#define IPIPE_NR_EVENTS (IPIPE_LAST_EVENT + 1) + +typedef int (*ipipe_event_handler_t)(unsigned int event, + struct ipipe_domain *from, + void *data); +struct ipipe_legacy_context { + unsigned int domid; + int priority; + void *pdd; + ipipe_event_handler_t handlers[IPIPE_NR_EVENTS]; +}; + +#define __ipipe_init_taskinfo(p) \ + do { \ + memset(p->ptd, 0, sizeof(p->ptd)); \ + } while (0) + +#else /* !CONFIG_IPIPE_LEGACY */ + +struct ipipe_legacy_context { +}; + +static inline void __ipipe_init_taskinfo(struct task_struct *p) { } + +#endif /* !CONFIG_IPIPE_LEGACY */ + +#define __ipipe_serial_debug(__fmt, __args...) raw_printk(__fmt, ##__args) + +#else /* !CONFIG_IPIPE */ + +struct task_struct; +struct mm_struct; + +static inline void __ipipe_init_early(void) { } + +static inline void __ipipe_init(void) { } + +static inline void __ipipe_init_proc(void) { } + +static inline void __ipipe_idle(void) { } + +static inline void __ipipe_report_sigwake(struct task_struct *p) { } + +static inline void __ipipe_report_setaffinity(struct task_struct *p, + int dest_cpu) { } + +static inline void __ipipe_report_setsched(struct task_struct *p) { } + +static inline void __ipipe_report_exit(struct task_struct *p) { } + +static inline void __ipipe_report_cleanup(struct mm_struct *mm) { } + +#define __ipipe_report_trap(exception, regs) 0 + +static inline void __ipipe_init_taskinfo(struct task_struct *p) { } + +#define hard_preempt_disable() ({ preempt_disable(); 0; }) +#define hard_preempt_enable(flags) ({ preempt_enable(); (void)(flags); }) + +#define __ipipe_get_cpu(flags) ({ (void)(flags); get_cpu(); }) +#define __ipipe_put_cpu(flags) \ + do { \ + (void)(flags); \ + put_cpu(); \ + } while (0) + +#define __ipipe_root_tick_p(regs) 1 + +#define ipipe_handle_demuxed_irq(irq) generic_handle_irq(irq) + +#define __ipipe_enter_vm(vmf) do { } while (0) + +static inline void __ipipe_exit_vm(void) { } + +static inline void __ipipe_notify_vm_preemption(void) { } + +static inline void ipipe_root_only(void) { } + +#define __ipipe_serial_debug(__fmt, __args...) do { } while (0) + +#endif /* !CONFIG_IPIPE */ + +#ifdef CONFIG_IPIPE_WANT_PTE_PINNING +void __ipipe_pin_mapping_globally(unsigned long start, + unsigned long end); +#else +static inline void __ipipe_pin_mapping_globally(unsigned long start, + unsigned long end) +{ } +#endif + +static inline void ipipe_preempt_root_only(void) +{ +#if defined(CONFIG_IPIPE_DEBUG_CONTEXT) && \ + defined(CONFIG_IPIPE_LEGACY) && \ + !defined(CONFIG_IPIPE_HAVE_SAFE_THREAD_INFO) + ipipe_root_only(); +#endif +} + +#endif /* !__LINUX_IPIPE_BASE_H */ diff --git a/include/linux/ipipe_compat.h b/include/linux/ipipe_compat.h new file mode 100644 index 0000000..6521852 --- /dev/null +++ b/include/linux/ipipe_compat.h @@ -0,0 +1,319 @@ +/* -*- linux-c -*- + * include/linux/ipipe_compat.h + * + * Copyright (C) 2012 Philippe Gerum. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139, + * USA; either version 2 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ + +#ifndef __LINUX_IPIPE_COMPAT_H +#define __LINUX_IPIPE_COMPAT_H + +#ifndef __LINUX_IPIPE_H +#error "Do not include this file directly, use linux/ipipe.h instead" +#endif + +#ifdef CONFIG_IPIPE_LEGACY + +#define IPIPE_HEAD_PRIORITY (-1) +#define IPIPE_ROOT_PRIO 100 +#define IPIPE_ROOT_ID 0 +#define IPIPE_ROOT_NPTDKEYS 4 + +/* Legacy pipeline status bit */ +#define IPIPE_NOSTACK_FLAG 1 /* running on foreign stack. */ +#define IPIPE_NOSTACK_MASK (1L << IPIPE_NOSTACK_FLAG) + +/* Legacy interrupt control bits */ +#define IPIPE_DUMMY_FLAG 31 +#define IPIPE_WIRED_FLAG IPIPE_HANDLE_FLAG +#define IPIPE_WIRED_MASK (1 << IPIPE_WIRED_FLAG) +#define IPIPE_PASS_FLAG IPIPE_DUMMY_FLAG +#define IPIPE_PASS_MASK (1 << IPIPE_PASS_FLAG) +#define IPIPE_DYNAMIC_FLAG IPIPE_HANDLE_FLAG +#define IPIPE_DYNAMIC_MASK (1 << IPIPE_DYNAMIC_FLAG) +#define IPIPE_SYSTEM_FLAG IPIPE_DUMMY_FLAG +#define IPIPE_SYSTEM_MASK (1 << IPIPE_SYSTEM_FLAG) +#define IPIPE_EXCLUSIVE_FLAG IPIPE_DUMMY_FLAG +#define IPIPE_EXCLUSIVE_MASK (1 << IPIPE_EXCLUSIVE_FLAG) + +#define IPIPE_NR_CPUS NR_CPUS + +#define IPIPE_EVENT_SELF 0x80000000 +#define IPIPE_EVENT_RETURN IPIPE_TRAP_MAYDAY + +#define TASK_ATOMICSWITCH TASK_HARDENING + +struct ipipe_domain_attr { + unsigned int domid; + const char *name; + int priority; + void (*entry) (void); + void *pdd; +}; + +void ipipe_init_attr(struct ipipe_domain_attr *attr); + +int ipipe_register_domain(struct ipipe_domain *ipd, + struct ipipe_domain_attr *attr); + +int ipipe_unregister_domain(struct ipipe_domain *ipd); + +int ipipe_alloc_ptdkey(void); + +int ipipe_free_ptdkey(int key); + +int ipipe_set_ptd(int key, void *value); + +void *ipipe_get_ptd(int key); + +int ipipe_virtualize_irq(struct ipipe_domain *ipd, + unsigned int irq, + ipipe_irq_handler_t handler, + void *cookie, + ipipe_irq_ackfn_t ackfn, + unsigned int modemask); + +ipipe_event_handler_t ipipe_catch_event(struct ipipe_domain *ipd, + unsigned int event, + ipipe_event_handler_t handler); + +int ipipe_setscheduler_root(struct task_struct *p, + int policy, + int prio); + +static inline void ipipe_check_context(struct ipipe_domain *border_ipd) +{ + ipipe_root_only(); +} + +static inline void ipipe_set_printk_sync(struct ipipe_domain *ipd) +{ + ipipe_prepare_panic(); +} + +static inline void __ipipe_propagate_irq(unsigned int irq) +{ + ipipe_post_irq_root(irq); +} + +static inline void __ipipe_schedule_irq_head(unsigned int irq) +{ + ipipe_post_irq_head(irq); +} + +static inline void __ipipe_schedule_irq_root(unsigned int irq) +{ + ipipe_post_irq_root(irq); +} + +static inline int ipipe_trigger_irq(unsigned int irq) +{ + ipipe_raise_irq(irq); + return 1; +} + +static inline void ipipe_stall_pipeline_from(struct ipipe_domain *ipd) +{ + if (ipd != ipipe_root_domain) + ipipe_stall_head(); + else + ipipe_stall_root(); +} + +static inline +unsigned long ipipe_test_and_stall_pipeline_from(struct ipipe_domain *ipd) +{ + if (ipd != ipipe_root_domain) + return ipipe_test_and_stall_head(); + + return ipipe_test_and_stall_root(); +} + +static inline +void ipipe_unstall_pipeline_from(struct ipipe_domain *ipd) +{ + if (ipd != ipipe_root_domain) + ipipe_unstall_head(); + else + ipipe_unstall_root(); +} + +static inline +void ipipe_restore_pipeline_from(struct ipipe_domain *ipd, + unsigned long x) +{ + if (ipd != ipipe_root_domain) + ipipe_restore_head(x); + else + ipipe_restore_root(x); +} + +static inline +unsigned long ipipe_test_pipeline_from(struct ipipe_domain *ipd) +{ + return test_bit(IPIPE_STALL_FLAG, &ipipe_this_cpu_context(ipd)->status); +} + +static inline void ipipe_stall_pipeline_head(void) +{ + ipipe_stall_head(); +} + +static inline unsigned long ipipe_test_and_stall_pipeline_head(void) +{ + return ipipe_test_and_stall_head(); +} + +static inline void ipipe_unstall_pipeline_head(void) +{ + ipipe_unstall_head(); +} + +static inline void ipipe_restore_pipeline_head(unsigned long x) +{ + ipipe_restore_head(x); +} + +static inline int ipipe_disable_ondemand_mappings(struct task_struct *p) +{ + return __ipipe_disable_ondemand_mappings(p); +} + +static inline int ipipe_reenter_root(struct task_struct *prev, + int policy, + int prio) +{ + __ipipe_reenter_root(); + return 0; +} + +static inline void ipipe_root_preempt_notify(void) +{ + ipipe_notify_root_preemption(); +} + +#define ipipe_return_notify(p) ipipe_raise_mayday(p) + +/* + * Keep the following as a macro, so that client code could check for + * the support of the invariant pipeline head optimization. + */ +#define __ipipe_pipeline_head() ipipe_head_domain + +static inline int irqs_disabled_hw(void) +{ + return hard_irqs_disabled(); +} + +static inline void local_irq_disable_hw(void) +{ + hard_local_irq_disable(); +} + +static inline void local_irq_enable_hw(void) +{ + hard_local_irq_enable(); +} + +#define local_irq_save_hw(flags) \ + do { \ + (flags) = hard_local_irq_save(); \ + } while (0) + +static inline void local_irq_restore_hw(unsigned long flags) +{ + hard_local_irq_restore(flags); +} + +#define local_save_flags_hw(flags) \ + do { \ + (flags) = hard_local_save_flags(); \ + } while (0) + +#define local_irq_save_hw_smp(flags) \ + do { \ + (flags) = hard_smp_local_irq_save(); \ + } while (0) +#define local_irq_restore_hw_smp(flags) hard_smp_local_irq_restore(flags) + +#define local_irq_save_hw_cond(flags) \ + do { \ + (flags) = hard_cond_local_irq_save(); \ + } while (0) +#define local_irq_restore_hw_cond(flags) hard_cond_local_irq_restore(flags) + +static inline void ipipe_set_foreign_stack(struct ipipe_domain *ipd) +{ + /* Must be called hw interrupts off. */ + __set_bit(IPIPE_NOSTACK_FLAG, &ipipe_this_cpu_context(ipd)->status); +} + +static inline void ipipe_clear_foreign_stack(struct ipipe_domain *ipd) +{ + /* Must be called hw interrupts off. */ + __clear_bit(IPIPE_NOSTACK_FLAG, &ipipe_this_cpu_context(ipd)->status); +} + +static inline int ipipe_test_foreign_stack(void) +{ + /* Must be called hw interrupts off. */ + return test_bit(IPIPE_NOSTACK_FLAG, &__ipipe_current_context->status); +} + +#ifndef ipipe_safe_current +#define ipipe_safe_current() \ + ({ \ + struct task_struct *__p__; \ + unsigned long __flags__; \ + __flags__ = hard_smp_local_irq_save(); \ + __p__ = ipipe_test_foreign_stack() ? &init_task : current; \ + hard_smp_local_irq_restore(__flags__); \ + __p__; \ + }) +#endif + +void __ipipe_legacy_init_stage(struct ipipe_domain *ipd); + +/* + * These values have no real meaning from a versioning POV, however + * they are guaranteed to look more recent than any legacy patch + * release ever published in the past. + */ +#define IPIPE_MAJOR_NUMBER 3 +#define IPIPE_MINOR_NUMBER 0 +#define IPIPE_PATCH_NUMBER 0 + +#define __IPIPE_FEATURE_REQUEST_TICKDEV 1 +#define __IPIPE_FEATURE_FASTPEND_IRQ 1 +#define __IPIPE_FEATURE_TRACE_EVENT 1 +#define __IPIPE_FEATURE_ENABLE_NOTIFIER 1 +#define __IPIPE_FEATURE_PREPARE_PANIC 1 +#define __IPIPE_FEATURE_SYSINFO_V2 1 +#define __IPIPE_FEATURE_PIC_MUTE 1 +#ifdef CONFIG_IPIPE_HAVE_VM_NOTIFIER +#define __IPIPE_FEATURE_ROOTPREEMPT_NOTIFIER 1 +#endif + +#else /* !CONFIG_IPIPE_LEGACY */ + +static inline void __ipipe_legacy_init_stage(struct ipipe_domain *ipd) +{ +} + +#endif /* !CONFIG_IPIPE_LEGACY */ + +#endif /* !__LINUX_IPIPE_COMPAT_H */ diff --git a/include/linux/ipipe_debug.h b/include/linux/ipipe_debug.h new file mode 100644 index 0000000..5d7efef --- /dev/null +++ b/include/linux/ipipe_debug.h @@ -0,0 +1,100 @@ +/* -*- linux-c -*- + * include/linux/ipipe_debug.h + * + * Copyright (C) 2012 Philippe Gerum . + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139, + * USA; either version 2 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ + +#ifndef __LINUX_IPIPE_DEBUG_H +#define __LINUX_IPIPE_DEBUG_H + +#include + +#ifdef CONFIG_IPIPE_DEBUG_CONTEXT + +#include + +static inline int ipipe_disable_context_check(void) +{ + return xchg(raw_cpu_ptr(&ipipe_percpu.context_check), 0); +} + +static inline void ipipe_restore_context_check(int old_state) +{ + __this_cpu_write(ipipe_percpu.context_check, old_state); +} + +static inline void ipipe_context_check_off(void) +{ + int cpu; + for_each_online_cpu(cpu) + per_cpu(ipipe_percpu, cpu).context_check = 0; +} + +static inline void ipipe_save_context_nmi(void) +{ + int state = ipipe_disable_context_check(); + __this_cpu_write(ipipe_percpu.context_check_saved, state); +} + +static inline void ipipe_restore_context_nmi(void) +{ + ipipe_restore_context_check(__this_cpu_read(ipipe_percpu.context_check_saved)); +} + +#else /* !CONFIG_IPIPE_DEBUG_CONTEXT */ + +static inline int ipipe_disable_context_check(void) +{ + return 0; +} + +static inline void ipipe_restore_context_check(int old_state) { } + +static inline void ipipe_context_check_off(void) { } + +static inline void ipipe_save_context_nmi(void) { } + +static inline void ipipe_restore_context_nmi(void) { } + +#endif /* !CONFIG_IPIPE_DEBUG_CONTEXT */ + +#ifdef CONFIG_IPIPE_DEBUG + +#define ipipe_check_irqoff() \ + do { \ + if (WARN_ON_ONCE(!hard_irqs_disabled())) \ + hard_local_irq_disable(); \ + } while (0) + +#else /* !CONFIG_IPIPE_DEBUG */ + +static inline void ipipe_check_irqoff(void) { } + +#endif /* !CONFIG_IPIPE_DEBUG */ + +#ifdef CONFIG_IPIPE_DEBUG_INTERNAL +#define IPIPE_WARN(c) WARN_ON(c) +#define IPIPE_WARN_ONCE(c) WARN_ON_ONCE(c) +#define IPIPE_BUG_ON(c) BUG_ON(c) +#else +#define IPIPE_WARN(c) do { (void)(c); } while (0) +#define IPIPE_WARN_ONCE(c) do { (void)(c); } while (0) +#define IPIPE_BUG_ON(c) do { (void)(c); } while (0) +#endif + +#endif /* !__LINUX_IPIPE_DEBUG_H */ diff --git a/include/linux/ipipe_domain.h b/include/linux/ipipe_domain.h new file mode 100644 index 0000000..d00c56b --- /dev/null +++ b/include/linux/ipipe_domain.h @@ -0,0 +1,309 @@ +/* -*- linux-c -*- + * include/linux/ipipe_domain.h + * + * Copyright (C) 2007-2012 Philippe Gerum. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139, + * USA; either version 2 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ + +#ifndef __LINUX_IPIPE_DOMAIN_H +#define __LINUX_IPIPE_DOMAIN_H + +#ifdef CONFIG_IPIPE + +#include +#include +#include + +struct task_struct; +struct mm_struct; +struct irq_desc; +struct ipipe_vm_notifier; + +#define __IPIPE_SYSCALL_P 0 +#define __IPIPE_TRAP_P 1 +#define __IPIPE_KEVENT_P 2 +#define __IPIPE_SYSCALL_E (1 << __IPIPE_SYSCALL_P) +#define __IPIPE_TRAP_E (1 << __IPIPE_TRAP_P) +#define __IPIPE_KEVENT_E (1 << __IPIPE_KEVENT_P) +#define __IPIPE_ALL_E 0x7 +#define __IPIPE_SYSCALL_R (8 << __IPIPE_SYSCALL_P) +#define __IPIPE_TRAP_R (8 << __IPIPE_TRAP_P) +#define __IPIPE_KEVENT_R (8 << __IPIPE_KEVENT_P) +#define __IPIPE_SHIFT_R 3 +#define __IPIPE_ALL_R (__IPIPE_ALL_E << __IPIPE_SHIFT_R) + +typedef void (*ipipe_irq_ackfn_t)(unsigned int irq, struct irq_desc *desc); + +struct ipipe_domain { + int context_offset; + struct ipipe_irqdesc { + unsigned long control; + ipipe_irq_ackfn_t ackfn; + ipipe_irq_handler_t handler; + void *cookie; + } ____cacheline_aligned irqs[IPIPE_NR_IRQS]; + const char *name; + struct mutex mutex; + struct ipipe_legacy_context legacy; +}; + +static inline void * +__ipipe_irq_cookie(struct ipipe_domain *ipd, unsigned int irq) +{ + return ipd->irqs[irq].cookie; +} + +static inline ipipe_irq_handler_t +__ipipe_irq_handler(struct ipipe_domain *ipd, unsigned int irq) +{ + return ipd->irqs[irq].handler; +} + +extern struct ipipe_domain ipipe_root; + +#define ipipe_root_domain (&ipipe_root) + +extern struct ipipe_domain *ipipe_head_domain; + +struct ipipe_percpu_domain_data { + unsigned long status; /* <= Must be first in struct. */ + unsigned long irqpend_himap; +#ifdef __IPIPE_3LEVEL_IRQMAP + unsigned long irqpend_mdmap[IPIPE_IRQ_MDMAPSZ]; +#endif + unsigned long irqpend_lomap[IPIPE_IRQ_LOMAPSZ]; + unsigned long irqheld_map[IPIPE_IRQ_LOMAPSZ]; + unsigned long irqall[IPIPE_NR_IRQS]; + struct ipipe_domain *domain; + int coflags; +}; + +struct ipipe_percpu_data { + struct ipipe_percpu_domain_data root; + struct ipipe_percpu_domain_data head; + struct ipipe_percpu_domain_data *curr; + struct pt_regs tick_regs; + int hrtimer_irq; + struct task_struct *task_hijacked; + struct task_struct *rqlock_owner; + struct ipipe_vm_notifier *vm_notifier; + unsigned long nmi_state; + struct mm_struct *active_mm; +#ifdef CONFIG_IPIPE_DEBUG_CONTEXT + int context_check; + int context_check_saved; +#endif +}; + +/* + * CAREFUL: all accessors based on __ipipe_raw_cpu_ptr() you may find + * in this file should be used only while hw interrupts are off, to + * prevent from CPU migration regardless of the running domain. + */ +DECLARE_PER_CPU(struct ipipe_percpu_data, ipipe_percpu); + +static inline struct ipipe_percpu_domain_data * +__context_of(struct ipipe_percpu_data *p, struct ipipe_domain *ipd) +{ + return (void *)p + ipd->context_offset; +} + +/** + * ipipe_percpu_context - return the address of the pipeline context + * data for a domain on a given CPU. + * + * NOTE: this is the slowest accessor, use it carefully. Prefer + * ipipe_this_cpu_context() for requests targeted at the current + * CPU. Additionally, if the target domain is known at build time, + * consider ipipe_this_cpu_{root, head}_context(). + */ +static inline struct ipipe_percpu_domain_data * +ipipe_percpu_context(struct ipipe_domain *ipd, int cpu) +{ + return __context_of(&per_cpu(ipipe_percpu, cpu), ipd); +} + +/** + * ipipe_this_cpu_context - return the address of the pipeline context + * data for a domain on the current CPU. hw IRQs must be off. + * + * NOTE: this accessor is a bit faster, but since we don't know which + * one of "root" or "head" ipd refers to, we still need to compute the + * context address from its offset. + */ +static inline struct ipipe_percpu_domain_data * +ipipe_this_cpu_context(struct ipipe_domain *ipd) +{ + return __context_of(__ipipe_raw_cpu_ptr(&ipipe_percpu), ipd); +} + +/** + * ipipe_this_cpu_root_context - return the address of the pipeline + * context data for the root domain on the current CPU. hw IRQs must + * be off. + * + * NOTE: this accessor is recommended when the domain we refer to is + * known at build time to be the root one. + */ +static inline struct ipipe_percpu_domain_data * +ipipe_this_cpu_root_context(void) +{ + return __ipipe_raw_cpu_ptr(&ipipe_percpu.root); +} + +/** + * ipipe_this_cpu_head_context - return the address of the pipeline + * context data for the registered head domain on the current CPU. hw + * IRQs must be off. + * + * NOTE: this accessor is recommended when the domain we refer to is + * known at build time to be the registered head domain. This address + * is always different from the context data of the root domain in + * absence of registered head domain. To get the address of the + * context data for the domain leading the pipeline at the time of the + * call (which may be root in absence of registered head domain), use + * ipipe_this_cpu_leading_context() instead. + */ +static inline struct ipipe_percpu_domain_data * +ipipe_this_cpu_head_context(void) +{ + return __ipipe_raw_cpu_ptr(&ipipe_percpu.head); +} + +/** + * ipipe_this_cpu_leading_context - return the address of the pipeline + * context data for the domain leading the pipeline on the current + * CPU. hw IRQs must be off. + * + * NOTE: this accessor is required when either root or a registered + * head domain may be the final target of this call, depending on + * whether the high priority domain was installed via + * ipipe_register_head(). + */ +static inline struct ipipe_percpu_domain_data * +ipipe_this_cpu_leading_context(void) +{ + return ipipe_this_cpu_context(ipipe_head_domain); +} + +/** + * __ipipe_get_current_context() - return the address of the pipeline + * context data of the domain running on the current CPU. hw IRQs must + * be off. + */ +static inline struct ipipe_percpu_domain_data *__ipipe_get_current_context(void) +{ + return __ipipe_raw_cpu_read(ipipe_percpu.curr); +} + +#define __ipipe_current_context __ipipe_get_current_context() + +/** + * __ipipe_set_current_context() - switch the current CPU to the + * specified domain context. hw IRQs must be off. + * + * NOTE: this is the only way to change the current domain for the + * current CPU. Don't bypass. + */ +static inline +void __ipipe_set_current_context(struct ipipe_percpu_domain_data *pd) +{ + struct ipipe_percpu_data *p; + p = __ipipe_raw_cpu_ptr(&ipipe_percpu); + p->curr = pd; +} + +/** + * __ipipe_set_current_domain() - switch the current CPU to the + * specified domain. This is equivalent to calling + * __ipipe_set_current_context() with the context data of that + * domain. hw IRQs must be off. + */ +static inline void __ipipe_set_current_domain(struct ipipe_domain *ipd) +{ + struct ipipe_percpu_data *p; + p = __ipipe_raw_cpu_ptr(&ipipe_percpu); + p->curr = __context_of(p, ipd); +} + +static inline struct ipipe_percpu_domain_data *ipipe_current_context(void) +{ + struct ipipe_percpu_domain_data *pd; + unsigned long flags; + + flags = hard_smp_local_irq_save(); + pd = __ipipe_get_current_context(); + hard_smp_local_irq_restore(flags); + + return pd; +} + +static inline struct ipipe_domain *__ipipe_get_current_domain(void) +{ + return __ipipe_get_current_context()->domain; +} + +#define __ipipe_current_domain __ipipe_get_current_domain() + +/** + * __ipipe_get_current_domain() - return the address of the pipeline + * domain running on the current CPU. hw IRQs must be off. + */ +static inline struct ipipe_domain *ipipe_get_current_domain(void) +{ + struct ipipe_domain *ipd; + unsigned long flags; + + flags = hard_smp_local_irq_save(); + ipd = __ipipe_get_current_domain(); + hard_smp_local_irq_restore(flags); + + return ipd; +} + +#define ipipe_current_domain ipipe_get_current_domain() + +#define __ipipe_root_p (__ipipe_current_domain == ipipe_root_domain) +#define ipipe_root_p (ipipe_current_domain == ipipe_root_domain) + +#ifdef CONFIG_SMP +#define __ipipe_root_status (ipipe_this_cpu_root_context()->status) +#else +extern unsigned long __ipipe_root_status; +#endif + +#define __ipipe_head_status (ipipe_this_cpu_head_context()->status) + +/** + * __ipipe_ipending_p() - Whether we have interrupts pending + * (i.e. logged) for the given domain context on the current CPU. hw + * IRQs must be off. + */ +static inline int __ipipe_ipending_p(struct ipipe_percpu_domain_data *pd) +{ + return pd->irqpend_himap != 0; +} + +static inline unsigned long +__ipipe_cpudata_irq_hits(struct ipipe_domain *ipd, int cpu, unsigned int irq) +{ + return ipipe_percpu_context(ipd, cpu)->irqall[irq]; +} + +#endif /* CONFIG_IPIPE */ + +#endif /* !__LINUX_IPIPE_DOMAIN_H */ diff --git a/include/linux/ipipe_lock.h b/include/linux/ipipe_lock.h new file mode 100644 index 0000000..a108278 --- /dev/null +++ b/include/linux/ipipe_lock.h @@ -0,0 +1,327 @@ +/* -*- linux-c -*- + * include/linux/ipipe_lock.h + * + * Copyright (C) 2009 Philippe Gerum. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139, + * USA; either version 2 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ + +#ifndef __LINUX_IPIPE_LOCK_H +#define __LINUX_IPIPE_LOCK_H + +typedef struct { + arch_spinlock_t arch_lock; +} __ipipe_spinlock_t; + +#define ipipe_spinlock(lock) ((__ipipe_spinlock_t *)(lock)) +#define ipipe_spinlock_p(lock) \ + __builtin_types_compatible_p(typeof(lock), __ipipe_spinlock_t *) || \ + __builtin_types_compatible_p(typeof(lock), __ipipe_spinlock_t []) + +#define std_spinlock_raw(lock) ((raw_spinlock_t *)(lock)) +#define std_spinlock_raw_p(lock) \ + __builtin_types_compatible_p(typeof(lock), raw_spinlock_t *) || \ + __builtin_types_compatible_p(typeof(lock), raw_spinlock_t []) + +#ifdef CONFIG_PREEMPT_RT_FULL + +#define PICK_SPINLOCK_IRQSAVE(lock, flags) \ + do { \ + if (ipipe_spinlock_p(lock)) \ + (flags) = __ipipe_spin_lock_irqsave(ipipe_spinlock(lock)); \ + else if (std_spinlock_raw_p(lock)) \ + __real_raw_spin_lock_irqsave(std_spinlock_raw(lock), flags); \ + else __bad_lock_type(); \ + } while (0) + +#define PICK_SPINTRYLOCK_IRQSAVE(lock, flags) \ + ({ \ + int __ret__; \ + if (ipipe_spinlock_p(lock)) \ + __ret__ = __ipipe_spin_trylock_irqsave(ipipe_spinlock(lock), &(flags)); \ + else if (std_spinlock_raw_p(lock)) \ + __ret__ = __real_raw_spin_trylock_irqsave(std_spinlock_raw(lock), flags); \ + else __bad_lock_type(); \ + __ret__; \ + }) + +#define PICK_SPINTRYLOCK_IRQ(lock) \ + ({ \ + int __ret__; \ + if (ipipe_spinlock_p(lock)) \ + __ret__ = __ipipe_spin_trylock_irq(ipipe_spinlock(lock)); \ + else if (std_spinlock_raw_p(lock)) \ + __ret__ = __real_raw_spin_trylock_irq(std_spinlock_raw(lock)); \ + else __bad_lock_type(); \ + __ret__; \ + }) + +#define PICK_SPINUNLOCK_IRQRESTORE(lock, flags) \ + do { \ + if (ipipe_spinlock_p(lock)) \ + __ipipe_spin_unlock_irqrestore(ipipe_spinlock(lock), flags); \ + else if (std_spinlock_raw_p(lock)) { \ + __ipipe_spin_unlock_debug(flags); \ + __real_raw_spin_unlock_irqrestore(std_spinlock_raw(lock), flags); \ + } else __bad_lock_type(); \ + } while (0) + +#define PICK_SPINOP(op, lock) \ + ({ \ + if (ipipe_spinlock_p(lock)) \ + arch_spin##op(&ipipe_spinlock(lock)->arch_lock); \ + else if (std_spinlock_raw_p(lock)) \ + __real_raw_spin##op(std_spinlock_raw(lock)); \ + else __bad_lock_type(); \ + (void)0; \ + }) + +#define PICK_SPINOP_RET(op, lock, type) \ + ({ \ + type __ret__; \ + if (ipipe_spinlock_p(lock)) \ + __ret__ = arch_spin##op(&ipipe_spinlock(lock)->arch_lock); \ + else if (std_spinlock_raw_p(lock)) \ + __ret__ = __real_raw_spin##op(std_spinlock_raw(lock)); \ + else { __ret__ = -1; __bad_lock_type(); } \ + __ret__; \ + }) + +#else /* !CONFIG_PREEMPT_RT_FULL */ + +#define std_spinlock(lock) ((spinlock_t *)(lock)) +#define std_spinlock_p(lock) \ + __builtin_types_compatible_p(typeof(lock), spinlock_t *) || \ + __builtin_types_compatible_p(typeof(lock), spinlock_t []) + +#define PICK_SPINLOCK_IRQSAVE(lock, flags) \ + do { \ + if (ipipe_spinlock_p(lock)) \ + (flags) = __ipipe_spin_lock_irqsave(ipipe_spinlock(lock)); \ + else if (std_spinlock_raw_p(lock)) \ + __real_raw_spin_lock_irqsave(std_spinlock_raw(lock), flags); \ + else if (std_spinlock_p(lock)) \ + __real_raw_spin_lock_irqsave(&std_spinlock(lock)->rlock, flags); \ + else __bad_lock_type(); \ + } while (0) + +#define PICK_SPINTRYLOCK_IRQSAVE(lock, flags) \ + ({ \ + int __ret__; \ + if (ipipe_spinlock_p(lock)) \ + __ret__ = __ipipe_spin_trylock_irqsave(ipipe_spinlock(lock), &(flags)); \ + else if (std_spinlock_raw_p(lock)) \ + __ret__ = __real_raw_spin_trylock_irqsave(std_spinlock_raw(lock), flags); \ + else if (std_spinlock_p(lock)) \ + __ret__ = __real_raw_spin_trylock_irqsave(&std_spinlock(lock)->rlock, flags); \ + else __bad_lock_type(); \ + __ret__; \ + }) + +#define PICK_SPINTRYLOCK_IRQ(lock) \ + ({ \ + int __ret__; \ + if (ipipe_spinlock_p(lock)) \ + __ret__ = __ipipe_spin_trylock_irq(ipipe_spinlock(lock)); \ + else if (std_spinlock_raw_p(lock)) \ + __ret__ = __real_raw_spin_trylock_irq(std_spinlock_raw(lock)); \ + else if (std_spinlock_p(lock)) \ + __ret__ = __real_raw_spin_trylock_irq(&std_spinlock(lock)->rlock); \ + else __bad_lock_type(); \ + __ret__; \ + }) + +#define PICK_SPINUNLOCK_IRQRESTORE(lock, flags) \ + do { \ + if (ipipe_spinlock_p(lock)) \ + __ipipe_spin_unlock_irqrestore(ipipe_spinlock(lock), flags); \ + else { \ + __ipipe_spin_unlock_debug(flags); \ + if (std_spinlock_raw_p(lock)) \ + __real_raw_spin_unlock_irqrestore(std_spinlock_raw(lock), flags); \ + else if (std_spinlock_p(lock)) \ + __real_raw_spin_unlock_irqrestore(&std_spinlock(lock)->rlock, flags); \ + } \ + } while (0) + +#define PICK_SPINOP(op, lock) \ + ({ \ + if (ipipe_spinlock_p(lock)) \ + arch_spin##op(&ipipe_spinlock(lock)->arch_lock); \ + else if (std_spinlock_raw_p(lock)) \ + __real_raw_spin##op(std_spinlock_raw(lock)); \ + else if (std_spinlock_p(lock)) \ + __real_raw_spin##op(&std_spinlock(lock)->rlock); \ + else __bad_lock_type(); \ + (void)0; \ + }) + +#define PICK_SPINOP_RET(op, lock, type) \ + ({ \ + type __ret__; \ + if (ipipe_spinlock_p(lock)) \ + __ret__ = arch_spin##op(&ipipe_spinlock(lock)->arch_lock); \ + else if (std_spinlock_raw_p(lock)) \ + __ret__ = __real_raw_spin##op(std_spinlock_raw(lock)); \ + else if (std_spinlock_p(lock)) \ + __ret__ = __real_raw_spin##op(&std_spinlock(lock)->rlock); \ + else { __ret__ = -1; __bad_lock_type(); } \ + __ret__; \ + }) + +#endif /* !CONFIG_PREEMPT_RT_FULL */ + +#define arch_spin_lock_init(lock) \ + do { \ + IPIPE_DEFINE_SPINLOCK(__lock__); \ + *((ipipe_spinlock_t *)lock) = __lock__; \ + } while (0) + +#define arch_spin_lock_irq(lock) \ + do { \ + hard_local_irq_disable(); \ + arch_spin_lock(lock); \ + } while (0) + +#define arch_spin_unlock_irq(lock) \ + do { \ + arch_spin_unlock(lock); \ + hard_local_irq_enable(); \ + } while (0) + +typedef struct { + arch_rwlock_t arch_lock; +} __ipipe_rwlock_t; + +#define ipipe_rwlock_p(lock) \ + __builtin_types_compatible_p(typeof(lock), __ipipe_rwlock_t *) + +#define std_rwlock_p(lock) \ + __builtin_types_compatible_p(typeof(lock), rwlock_t *) + +#define ipipe_rwlock(lock) ((__ipipe_rwlock_t *)(lock)) +#define std_rwlock(lock) ((rwlock_t *)(lock)) + +#define PICK_RWOP(op, lock) \ + do { \ + if (ipipe_rwlock_p(lock)) \ + arch##op(&ipipe_rwlock(lock)->arch_lock); \ + else if (std_rwlock_p(lock)) \ + _raw##op(std_rwlock(lock)); \ + else __bad_lock_type(); \ + } while (0) + +extern int __bad_lock_type(void); + +#ifdef CONFIG_IPIPE + +#define ipipe_spinlock_t __ipipe_spinlock_t +#define IPIPE_DEFINE_RAW_SPINLOCK(x) ipipe_spinlock_t x = IPIPE_SPIN_LOCK_UNLOCKED +#define IPIPE_DECLARE_RAW_SPINLOCK(x) extern ipipe_spinlock_t x +#define IPIPE_DEFINE_SPINLOCK(x) IPIPE_DEFINE_RAW_SPINLOCK(x) +#define IPIPE_DECLARE_SPINLOCK(x) IPIPE_DECLARE_RAW_SPINLOCK(x) + +#define IPIPE_SPIN_LOCK_UNLOCKED \ + (__ipipe_spinlock_t) { .arch_lock = __ARCH_SPIN_LOCK_UNLOCKED } + +#define spin_lock_irqsave_cond(lock, flags) \ + spin_lock_irqsave(lock, flags) + +#define spin_unlock_irqrestore_cond(lock, flags) \ + spin_unlock_irqrestore(lock, flags) + +#define raw_spin_lock_irqsave_cond(lock, flags) \ + raw_spin_lock_irqsave(lock, flags) + +#define raw_spin_unlock_irqrestore_cond(lock, flags) \ + raw_spin_unlock_irqrestore(lock, flags) + +void __ipipe_spin_lock_irq(ipipe_spinlock_t *lock); + +int __ipipe_spin_trylock_irq(ipipe_spinlock_t *lock); + +void __ipipe_spin_unlock_irq(ipipe_spinlock_t *lock); + +unsigned long __ipipe_spin_lock_irqsave(ipipe_spinlock_t *lock); + +int __ipipe_spin_trylock_irqsave(ipipe_spinlock_t *lock, + unsigned long *x); + +void __ipipe_spin_unlock_irqrestore(ipipe_spinlock_t *lock, + unsigned long x); + +void __ipipe_spin_unlock_irqbegin(ipipe_spinlock_t *lock); + +void __ipipe_spin_unlock_irqcomplete(unsigned long x); + +#if defined(CONFIG_IPIPE_DEBUG_INTERNAL) && defined(CONFIG_SMP) +void __ipipe_spin_unlock_debug(unsigned long flags); +#else +#define __ipipe_spin_unlock_debug(flags) do { } while (0) +#endif + +#define ipipe_rwlock_t __ipipe_rwlock_t +#define IPIPE_DEFINE_RWLOCK(x) ipipe_rwlock_t x = IPIPE_RW_LOCK_UNLOCKED +#define IPIPE_DECLARE_RWLOCK(x) extern ipipe_rwlock_t x + +#define IPIPE_RW_LOCK_UNLOCKED \ + (__ipipe_rwlock_t) { .arch_lock = __ARCH_RW_LOCK_UNLOCKED } + +#else /* !CONFIG_IPIPE */ + +#define ipipe_spinlock_t spinlock_t +#define IPIPE_DEFINE_SPINLOCK(x) DEFINE_SPINLOCK(x) +#define IPIPE_DECLARE_SPINLOCK(x) extern spinlock_t x +#define IPIPE_SPIN_LOCK_UNLOCKED __SPIN_LOCK_UNLOCKED(unknown) +#define IPIPE_DEFINE_RAW_SPINLOCK(x) DEFINE_RAW_SPINLOCK(x) +#define IPIPE_DECLARE_RAW_SPINLOCK(x) extern raw_spinlock_t x + +#define spin_lock_irqsave_cond(lock, flags) \ + do { \ + (void)(flags); \ + spin_lock(lock); \ + } while(0) + +#define spin_unlock_irqrestore_cond(lock, flags) \ + spin_unlock(lock) + +#define raw_spin_lock_irqsave_cond(lock, flags) \ + do { \ + (void)(flags); \ + raw_spin_lock(lock); \ + } while(0) + +#define raw_spin_unlock_irqrestore_cond(lock, flags) \ + raw_spin_unlock(lock) + +#define __ipipe_spin_lock_irq(lock) do { } while (0) +#define __ipipe_spin_unlock_irq(lock) do { } while (0) +#define __ipipe_spin_lock_irqsave(lock) 0 +#define __ipipe_spin_trylock_irq(lock) 1 +#define __ipipe_spin_trylock_irqsave(lock, x) ({ (void)(x); 1; }) +#define __ipipe_spin_unlock_irqrestore(lock, x) do { (void)(x); } while (0) +#define __ipipe_spin_unlock_irqbegin(lock) spin_unlock(lock) +#define __ipipe_spin_unlock_irqcomplete(x) do { (void)(x); } while (0) +#define __ipipe_spin_unlock_debug(flags) do { } while (0) + +#define ipipe_rwlock_t rwlock_t +#define IPIPE_DEFINE_RWLOCK(x) DEFINE_RWLOCK(x) +#define IPIPE_DECLARE_RWLOCK(x) extern rwlock_t x +#define IPIPE_RW_LOCK_UNLOCKED RW_LOCK_UNLOCKED + +#endif /* !CONFIG_IPIPE */ + +#endif /* !__LINUX_IPIPE_LOCK_H */ diff --git a/include/linux/ipipe_tickdev.h b/include/linux/ipipe_tickdev.h new file mode 100644 index 0000000..2f065ad --- /dev/null +++ b/include/linux/ipipe_tickdev.h @@ -0,0 +1,148 @@ +/* -*- linux-c -*- + * include/linux/ipipe_tickdev.h + * + * Copyright (C) 2007 Philippe Gerum. + * Copyright (C) 2012 Gilles Chanteperdrix + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139, + * USA; either version 2 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ + +#ifndef __LINUX_IPIPE_TICKDEV_H +#define __LINUX_IPIPE_TICKDEV_H + +#include +#include +#include +#include +#include +#include + +#ifdef CONFIG_IPIPE + +enum clock_event_mode; +struct clock_event_device; + +struct ipipe_hostrt_data { + short live; + seqcount_t seqcount; + time_t wall_time_sec; + u32 wall_time_nsec; + struct timespec wall_to_monotonic; + cycle_t cycle_last; + cycle_t mask; + u32 mult; + u32 shift; +}; + +struct ipipe_timer { + int irq; + void (*request)(struct ipipe_timer *timer, int steal); + int (*set)(unsigned long ticks, void *timer); + void (*ack)(void); + void (*release)(struct ipipe_timer *timer); + + /* Only if registering a timer directly */ + const char *name; + unsigned rating; + unsigned long freq; + unsigned min_delay_ticks; + const struct cpumask *cpumask; + + /* For internal use */ + void *timer_set; /* pointer passed to ->set() callback */ + struct clock_event_device *host_timer; + struct list_head link; + + /* Conversions between clock frequency and timer frequency */ + unsigned c2t_integ; + unsigned c2t_frac; + + /* For clockevent interception */ + u32 real_mult; + u32 real_shift; + void (*real_set_mode)(enum clock_event_mode mode, + struct clock_event_device *cdev); + int (*real_set_next_event)(unsigned long evt, + struct clock_event_device *cdev); + unsigned int (*refresh_freq)(void); +}; + +#define __ipipe_hrtimer_irq __ipipe_raw_cpu_read(ipipe_percpu.hrtimer_irq) + +extern unsigned long __ipipe_hrtimer_freq; + +/* + * Called by clockevents_register_device, to register a piggybacked + * ipipe timer, if there is one + */ +void ipipe_host_timer_register(struct clock_event_device *clkevt); + +/* + * Register a standalone ipipe timer + */ +void ipipe_timer_register(struct ipipe_timer *timer); + +/* + * Chooses the best timer for each cpu. Take over its handling. + */ +int ipipe_select_timers(const struct cpumask *mask); + +/* + * Release the per-cpu timers + */ +void ipipe_timers_release(void); + +/* + * Start handling the per-cpu timer irq, and intercepting the linux clockevent + * device callbacks. + */ +int ipipe_timer_start(void (*tick_handler)(void), + void (*emumode)(enum clock_event_mode mode, + struct clock_event_device *cdev), + int (*emutick)(unsigned long evt, + struct clock_event_device *cdev), + unsigned cpu); + +/* + * Stop handling a per-cpu timer + */ +void ipipe_timer_stop(unsigned cpu); + +/* + * Program the timer + */ +void ipipe_timer_set(unsigned long delay); + +const char *ipipe_timer_name(void); + +unsigned ipipe_timer_ns2ticks(struct ipipe_timer *timer, unsigned ns); + +void __ipipe_timer_refresh_freq(unsigned int hrclock_freq); + +#else /* !CONFIG_IPIPE */ + +#define ipipe_host_timer_register(clkevt) do { } while (0) + +#endif /* !CONFIG_IPIPE */ + +#ifdef CONFIG_IPIPE_HAVE_HOSTRT +void ipipe_update_hostrt(struct timekeeper *tk); +#else +static inline void +ipipe_update_hostrt(struct timekeeper *tk) {} +#endif + +#endif /* __LINUX_IPIPE_TICKDEV_H */ diff --git a/include/linux/ipipe_trace.h b/include/linux/ipipe_trace.h new file mode 100644 index 0000000..deb0a47 --- /dev/null +++ b/include/linux/ipipe_trace.h @@ -0,0 +1,77 @@ +/* -*- linux-c -*- + * include/linux/ipipe_trace.h + * + * Copyright (C) 2005 Luotao Fu. + * 2005-2007 Jan Kiszka. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139, + * USA; either version 2 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ + +#ifndef _LINUX_IPIPE_TRACE_H +#define _LINUX_IPIPE_TRACE_H + +#ifdef CONFIG_IPIPE_TRACE + +#include + +#ifndef BROKEN_BUILTIN_RETURN_ADDRESS +#define __BUILTIN_RETURN_ADDRESS0 ((unsigned long)__builtin_return_address(0)) +#define __BUILTIN_RETURN_ADDRESS1 ((unsigned long)__builtin_return_address(1)) +#endif /* !BUILTIN_RETURN_ADDRESS */ + +void ipipe_trace_begin(unsigned long v); +void ipipe_trace_end(unsigned long v); +void ipipe_trace_freeze(unsigned long v); +void ipipe_trace_special(unsigned char special_id, unsigned long v); +void ipipe_trace_pid(pid_t pid, short prio); +void ipipe_trace_event(unsigned char id, unsigned long delay_tsc); +int ipipe_trace_max_reset(void); +int ipipe_trace_frozen_reset(void); + +#else /* !CONFIG_IPIPE_TRACE */ + +#define ipipe_trace_begin(v) do { (void)(v); } while(0) +#define ipipe_trace_end(v) do { (void)(v); } while(0) +#define ipipe_trace_freeze(v) do { (void)(v); } while(0) +#define ipipe_trace_special(id, v) do { (void)(id); (void)(v); } while(0) +#define ipipe_trace_pid(pid, prio) do { (void)(pid); (void)(prio); } while(0) +#define ipipe_trace_event(id, delay_tsc) do { (void)(id); (void)(delay_tsc); } while(0) +#define ipipe_trace_max_reset() ({ 0; }) +#define ipipe_trace_frozen_reset() ({ 0; }) + +#endif /* !CONFIG_IPIPE_TRACE */ + +#ifdef CONFIG_IPIPE_TRACE_PANIC +void ipipe_trace_panic_freeze(void); +void ipipe_trace_panic_dump(void); +#else +static inline void ipipe_trace_panic_freeze(void) { } +static inline void ipipe_trace_panic_dump(void) { } +#endif + +#ifdef CONFIG_IPIPE_TRACE_IRQSOFF +#define ipipe_trace_irq_entry(irq) ipipe_trace_begin(irq) +#define ipipe_trace_irq_exit(irq) ipipe_trace_end(irq) +#define ipipe_trace_irqsoff() ipipe_trace_begin(0x80000000UL) +#define ipipe_trace_irqson() ipipe_trace_end(0x80000000UL) +#else +#define ipipe_trace_irq_entry(irq) do { (void)(irq);} while(0) +#define ipipe_trace_irq_exit(irq) do { (void)(irq);} while(0) +#define ipipe_trace_irqsoff() do { } while(0) +#define ipipe_trace_irqson() do { } while(0) +#endif + +#endif /* !__LINUX_IPIPE_TRACE_H */ diff --git a/include/linux/irq.h b/include/linux/irq.h index 3532dca..16a9933 100644 --- a/include/linux/irq.h +++ b/include/linux/irq.h @@ -349,6 +349,11 @@ struct irq_chip { void (*irq_bus_lock)(struct irq_data *data); void (*irq_bus_sync_unlock)(struct irq_data *data); +#ifdef CONFIG_IPIPE + void (*irq_move)(struct irq_data *data); + void (*irq_hold)(struct irq_data *data); + void (*irq_release)(struct irq_data *data); +#endif /* CONFIG_IPIPE */ void (*irq_cpu_online)(struct irq_data *data); void (*irq_cpu_offline)(struct irq_data *data); @@ -463,6 +468,11 @@ extern int irq_chip_retrigger_hierarchy(struct irq_data *data); extern void irq_chip_mask_parent(struct irq_data *data); extern void irq_chip_unmask_parent(struct irq_data *data); extern void irq_chip_eoi_parent(struct irq_data *data); +#ifdef CONFIG_IPIPE +extern void irq_chip_hold_parent(struct irq_data *data); +extern void irq_chip_release_parent(struct irq_data *data); +#endif + extern int irq_chip_set_affinity_parent(struct irq_data *data, const struct cpumask *dest, bool force); @@ -573,7 +583,14 @@ extern int irq_set_irq_type(unsigned int irq, unsigned int type); extern int irq_set_msi_desc(unsigned int irq, struct msi_desc *entry); extern int irq_set_msi_desc_off(unsigned int irq_base, unsigned int irq_offset, struct msi_desc *entry); -extern struct irq_data *irq_get_irq_data(unsigned int irq); + +static inline __attribute__((const)) struct irq_data * +irq_get_irq_data(unsigned int irq) +{ + struct irq_desc *desc = irq_to_desc(irq); + + return desc ? &desc->irq_data : NULL; +} static inline struct irq_chip *irq_get_chip(unsigned int irq) { @@ -741,7 +758,11 @@ struct irq_chip_type { * different flow mechanisms (level/edge) for it. */ struct irq_chip_generic { +#ifdef CONFIG_IPIPE + ipipe_spinlock_t lock; +#else raw_spinlock_t lock; +#endif void __iomem *reg_base; u32 (*reg_readl)(void __iomem *addr); void (*reg_writel)(u32 val, void __iomem *addr); @@ -838,18 +859,28 @@ static inline struct irq_chip_type *irq_data_get_chip_type(struct irq_data *d) #define IRQ_MSK(n) (u32)((n) < 32 ? ((1 << (n)) - 1) : UINT_MAX) #ifdef CONFIG_SMP -static inline void irq_gc_lock(struct irq_chip_generic *gc) +static inline unsigned long irq_gc_lock(struct irq_chip_generic *gc) { - raw_spin_lock(&gc->lock); + unsigned long flags = 0; + raw_spin_lock_irqsave_cond(&gc->lock, flags); + return flags; } -static inline void irq_gc_unlock(struct irq_chip_generic *gc) +static inline void +irq_gc_unlock(struct irq_chip_generic *gc, unsigned long flags) { - raw_spin_unlock(&gc->lock); + raw_spin_unlock_irqrestore_cond(&gc->lock, flags); } #else -static inline void irq_gc_lock(struct irq_chip_generic *gc) { } -static inline void irq_gc_unlock(struct irq_chip_generic *gc) { } +static inline unsigned long irq_gc_lock(struct irq_chip_generic *gc) +{ + return hard_cond_local_irq_save(); +} +static inline void +irq_gc_unlock(struct irq_chip_generic *gc, unsigned long flags) +{ + hard_cond_local_irq_restore(flags); +} #endif static inline void irq_reg_writel(struct irq_chip_generic *gc, diff --git a/include/linux/irqchip/arm-gic.h b/include/linux/irqchip/arm-gic.h index 9de976b..ba4aa5d 100644 --- a/include/linux/irqchip/arm-gic.h +++ b/include/linux/irqchip/arm-gic.h @@ -49,7 +49,11 @@ #define GICD_INT_EN_CLR_X32 0xffffffff #define GICD_INT_EN_SET_SGI 0x0000ffff #define GICD_INT_EN_CLR_PPI 0xffff0000 +#ifndef CONFIG_IPIPE #define GICD_INT_DEF_PRI 0xa0 +#else +#define GICD_INT_DEF_PRI 0x10 +#endif #define GICD_INT_DEF_PRI_X4 ((GICD_INT_DEF_PRI << 24) |\ (GICD_INT_DEF_PRI << 16) |\ (GICD_INT_DEF_PRI << 8) |\ diff --git a/include/linux/irqdesc.h b/include/linux/irqdesc.h index dd1109f..8056567 100644 --- a/include/linux/irqdesc.h +++ b/include/linux/irqdesc.h @@ -49,6 +49,12 @@ struct pt_regs; struct irq_desc { struct irq_data irq_data; unsigned int __percpu *kstat_irqs; +#ifdef CONFIG_IPIPE + void (*ipipe_ack)(unsigned int irq, + struct irq_desc *desc); + void (*ipipe_end)(unsigned int irq, + struct irq_desc *desc); +#endif /* CONFIG_IPIPE */ irq_flow_handler_t handle_irq; #ifdef CONFIG_IRQ_PREFLOW_FASTEOI irq_preflow_handler_t preflow_handler; @@ -155,6 +161,10 @@ static inline int irq_has_action(unsigned int irq) return desc->action != NULL; } +irq_flow_handler_t +__fixup_irq_handler(struct irq_desc *desc, irq_flow_handler_t handle, + int is_chained); + /* caller has locked the irq_desc and both params are valid */ static inline void __irq_set_handler_locked(unsigned int irq, irq_flow_handler_t handler) @@ -162,6 +172,7 @@ static inline void __irq_set_handler_locked(unsigned int irq, struct irq_desc *desc; desc = irq_to_desc(irq); + handler = __fixup_irq_handler(desc, handler, 0); desc->handle_irq = handler; } diff --git a/include/linux/irqnr.h b/include/linux/irqnr.h index fdd5cc1..be145be 100644 --- a/include/linux/irqnr.h +++ b/include/linux/irqnr.h @@ -5,7 +5,11 @@ extern int nr_irqs; +#if !defined(CONFIG_IPIPE) || defined(CONFIG_SPARSE_IRQ) extern struct irq_desc *irq_to_desc(unsigned int irq); +#else +#define irq_to_desc(irq) ({ ipipe_virtual_irq_p(irq) ? NULL : &irq_desc[irq]; }) +#endif unsigned int irq_get_next_irq(unsigned int offset); # define for_each_irq_desc(irq, desc) \ diff --git a/include/linux/kernel.h b/include/linux/kernel.h index 3a5b48e..7ca5a3e 100644 --- a/include/linux/kernel.h +++ b/include/linux/kernel.h @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -168,9 +169,12 @@ struct user; #ifdef CONFIG_PREEMPT_VOLUNTARY extern int _cond_resched(void); -# define might_resched() _cond_resched() +# define might_resched() do { \ + ipipe_root_only(); \ + _cond_resched(); \ + } while (0) #else -# define might_resched() do { } while (0) +# define might_resched() ipipe_root_only() #endif #ifdef CONFIG_DEBUG_ATOMIC_SLEEP diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 29a57a5..eb44d33 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -218,6 +218,9 @@ struct kvm_vcpu { #ifdef CONFIG_PREEMPT_NOTIFIERS struct preempt_notifier preempt_notifier; #endif +#ifdef CONFIG_IPIPE + struct ipipe_vm_notifier ipipe_notifier; +#endif int cpu; int vcpu_id; int srcu_idx; diff --git a/include/linux/preempt.h b/include/linux/preempt.h index 8cd6725..b9b549c 100644 --- a/include/linux/preempt.h +++ b/include/linux/preempt.h @@ -8,6 +8,7 @@ #include #include +#include /* * We use the MSB mostly because its available; see for @@ -125,7 +126,27 @@ do { \ #endif /* CONFIG_PREEMPT_COUNT */ -#ifdef MODULE +#ifdef CONFIG_IPIPE +#define hard_preempt_disable() \ + ({ \ + unsigned long __flags__; \ + __flags__ = hard_local_irq_save(); \ + if (__ipipe_root_p) \ + preempt_disable(); \ + __flags__; \ + }) + +#define hard_preempt_enable(__flags__) \ + do { \ + if (__ipipe_root_p) { \ + preempt_enable_no_resched(); \ + hard_local_irq_restore(__flags__); \ + preempt_check_resched(); \ + } else \ + hard_local_irq_restore(__flags__); \ + } while (0) + +#elif defined(MODULE) /* * Modules have no business playing preemption tricks. */ @@ -133,7 +154,7 @@ do { \ #undef preempt_enable_no_resched #undef preempt_enable_no_resched_notrace #undef preempt_check_resched -#endif +#endif /* !IPIPE && MODULE */ #define preempt_set_need_resched() \ do { \ diff --git a/include/linux/printk.h b/include/linux/printk.h index 9b30871..fbe8688 100644 --- a/include/linux/printk.h +++ b/include/linux/printk.h @@ -120,6 +120,17 @@ static inline __printf(1, 2) __cold void early_printk(const char *s, ...) { } #endif +#ifdef CONFIG_RAW_PRINTK +void raw_vprintk(const char *fmt, va_list ap); +asmlinkage __printf(1, 2) +void raw_printk(const char *fmt, ...); +#else +static inline __cold +void raw_vprintk(const char *s, va_list ap) { } +static inline __printf(1, 2) __cold +void raw_printk(const char *s, ...) { } +#endif + typedef int(*printk_func_t)(const char *fmt, va_list args); #ifdef CONFIG_PRINTK diff --git a/include/linux/rwlock.h b/include/linux/rwlock.h index bc2994e..5e2da8d 100644 --- a/include/linux/rwlock.h +++ b/include/linux/rwlock.h @@ -61,8 +61,8 @@ do { \ #define read_trylock(lock) __cond_lock(lock, _raw_read_trylock(lock)) #define write_trylock(lock) __cond_lock(lock, _raw_write_trylock(lock)) -#define write_lock(lock) _raw_write_lock(lock) -#define read_lock(lock) _raw_read_lock(lock) +#define write_lock(lock) PICK_RWOP(_write_lock, lock) +#define read_lock(lock) PICK_RWOP(_read_lock, lock) #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK) @@ -96,8 +96,8 @@ do { \ #define read_lock_bh(lock) _raw_read_lock_bh(lock) #define write_lock_irq(lock) _raw_write_lock_irq(lock) #define write_lock_bh(lock) _raw_write_lock_bh(lock) -#define read_unlock(lock) _raw_read_unlock(lock) -#define write_unlock(lock) _raw_write_unlock(lock) +#define read_unlock(lock) PICK_RWOP(_read_unlock, lock) +#define write_unlock(lock) PICK_RWOP(_write_unlock, lock) #define read_unlock_irq(lock) _raw_read_unlock_irq(lock) #define write_unlock_irq(lock) _raw_write_unlock_irq(lock) diff --git a/include/linux/rwlock_api_smp.h b/include/linux/rwlock_api_smp.h index 5b9b84b..6c8bb4d 100644 --- a/include/linux/rwlock_api_smp.h +++ b/include/linux/rwlock_api_smp.h @@ -141,7 +141,9 @@ static inline int __raw_write_trylock(rwlock_t *lock) * even on CONFIG_PREEMPT, because lockdep assumes that interrupts are * not re-enabled during lock-acquire (which the preempt-spin-ops do): */ -#if !defined(CONFIG_GENERIC_LOCKBREAK) || defined(CONFIG_DEBUG_LOCK_ALLOC) +#if !defined(CONFIG_GENERIC_LOCKBREAK) || \ + defined(CONFIG_DEBUG_LOCK_ALLOC) || \ + defined(CONFIG_IPIPE) static inline void __raw_read_lock(rwlock_t *lock) { diff --git a/include/linux/sched.h b/include/linux/sched.h index 9128b4e..0b94416 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -26,6 +26,7 @@ struct sched_param { #include #include #include +#include #include #include @@ -213,9 +214,17 @@ print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq); #define TASK_WAKEKILL 128 #define TASK_WAKING 256 #define TASK_PARKED 512 +#ifdef CONFIG_IPIPE +#define TASK_HARDENING 1024 +#define TASK_NOWAKEUP 2048 +#define TASK_STATE_MAX 4096 +#define TASK_STATE_TO_CHAR_STR "RSDTtZXxKWPHN" +#else /* !CONFIG_IPIPE */ +#define TASK_HARDENING 0 +#define TASK_NOWAKEUP 0 #define TASK_STATE_MAX 1024 - -#define TASK_STATE_TO_CHAR_STR "RSDTtXZxKWP" +#define TASK_STATE_TO_CHAR_STR "RSDTtZXxKWP" +#endif /* CONFIG_IPIPE */ extern char ___assert_task_state[1 - 2*!!( sizeof(TASK_STATE_TO_CHAR_STR)-1 != ilog2(TASK_STATE_MAX)+1)]; @@ -369,6 +378,15 @@ extern void trap_init(void); extern void update_process_times(int user); extern void scheduler_tick(void); +#ifdef CONFIG_IPIPE +void update_root_process_times(struct pt_regs *regs); +#else /* !CONFIG_IPIPE */ +static inline void update_root_process_times(struct pt_regs *regs) +{ + update_process_times(user_mode(regs)); +} +#endif /* CONFIG_IPIPE */ + extern void sched_show_task(struct task_struct *p); #ifdef CONFIG_LOCKUP_DETECTOR @@ -496,6 +514,9 @@ static inline int get_dumpable(struct mm_struct *mm) #define MMF_VM_MERGEABLE 16 /* KSM may merge identical pages */ #define MMF_VM_HUGEPAGE 17 /* set when VM_HUGEPAGE is set on vma */ #define MMF_EXE_FILE_CHANGED 18 /* see prctl_set_mm_exe_file() */ +#ifdef CONFIG_IPIPE +#define MMF_VM_PINNED 31 /* ondemand load up and COW disabled */ +#endif #define MMF_HAS_UPROBES 19 /* has uprobes */ #define MMF_RECALC_UPROBES 20 /* MMF_HAS_UPROBES can be wrong */ @@ -1650,6 +1671,9 @@ struct task_struct { #endif /* CONFIG_NUMA_BALANCING */ struct rcu_head rcu; +#ifdef CONFIG_IPIPE_LEGACY + void *ptd[IPIPE_ROOT_NPTDKEYS]; +#endif /* * cache last used pipe for splice diff --git a/include/linux/spinlock.h b/include/linux/spinlock.h index 3e18379..673a6bb 100644 --- a/include/linux/spinlock.h +++ b/include/linux/spinlock.h @@ -89,10 +89,12 @@ # include #endif +#include + #ifdef CONFIG_DEBUG_SPINLOCK extern void __raw_spin_lock_init(raw_spinlock_t *lock, const char *name, struct lock_class_key *key); -# define raw_spin_lock_init(lock) \ +# define __real_raw_spin_lock_init(lock) \ do { \ static struct lock_class_key __key; \ \ @@ -100,11 +102,14 @@ do { \ } while (0) #else -# define raw_spin_lock_init(lock) \ +# define __real_raw_spin_lock_init(lock) \ do { *(lock) = __RAW_SPIN_LOCK_UNLOCKED(lock); } while (0) #endif +#define raw_spin_lock_init(lock) PICK_SPINOP(_lock_init, lock) -#define raw_spin_is_locked(lock) arch_spin_is_locked(&(lock)->raw_lock) +#define __real_raw_spin_is_locked(lock) \ + arch_spin_is_locked(&(lock)->raw_lock) +#define raw_spin_is_locked(lock) PICK_SPINOP_RET(_is_locked, lock, int) #ifdef CONFIG_GENERIC_LOCKBREAK #define raw_spin_is_contended(lock) ((lock)->break_lock) @@ -183,9 +188,11 @@ static inline void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock) * various methods are defined as nops in the case they are not * required. */ -#define raw_spin_trylock(lock) __cond_lock(lock, _raw_spin_trylock(lock)) +#define __real_raw_spin_trylock(lock) __cond_lock(lock, _raw_spin_trylock(lock)) +#define raw_spin_trylock(lock) PICK_SPINOP_RET(_trylock, lock, int) -#define raw_spin_lock(lock) _raw_spin_lock(lock) +#define __real_raw_spin_lock(lock) _raw_spin_lock(lock) +#define raw_spin_lock(lock) PICK_SPINOP(_lock, lock) #ifdef CONFIG_DEBUG_LOCK_ALLOC # define raw_spin_lock_nested(lock, subclass) \ @@ -212,7 +219,7 @@ static inline void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock) #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK) -#define raw_spin_lock_irqsave(lock, flags) \ +#define __real_raw_spin_lock_irqsave(lock, flags) \ do { \ typecheck(unsigned long, flags); \ flags = _raw_spin_lock_irqsave(lock); \ @@ -234,7 +241,7 @@ static inline void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock) #else -#define raw_spin_lock_irqsave(lock, flags) \ +#define __real_raw_spin_lock_irqsave(lock, flags) \ do { \ typecheck(unsigned long, flags); \ _raw_spin_lock_irqsave(lock, flags); \ @@ -245,34 +252,46 @@ static inline void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock) #endif -#define raw_spin_lock_irq(lock) _raw_spin_lock_irq(lock) +#define raw_spin_lock_irqsave(lock, flags) \ + PICK_SPINLOCK_IRQSAVE(lock, flags) + +#define __real_raw_spin_lock_irq(lock) _raw_spin_lock_irq(lock) +#define raw_spin_lock_irq(lock) PICK_SPINOP(_lock_irq, lock) #define raw_spin_lock_bh(lock) _raw_spin_lock_bh(lock) -#define raw_spin_unlock(lock) _raw_spin_unlock(lock) -#define raw_spin_unlock_irq(lock) _raw_spin_unlock_irq(lock) +#define __real_raw_spin_unlock(lock) _raw_spin_unlock(lock) +#define raw_spin_unlock(lock) PICK_SPINOP(_unlock, lock) +#define __real_raw_spin_unlock_irq(lock) _raw_spin_unlock_irq(lock) +#define raw_spin_unlock_irq(lock) PICK_SPINOP(_unlock_irq, lock) -#define raw_spin_unlock_irqrestore(lock, flags) \ +#define __real_raw_spin_unlock_irqrestore(lock, flags) \ do { \ typecheck(unsigned long, flags); \ _raw_spin_unlock_irqrestore(lock, flags); \ } while (0) +#define raw_spin_unlock_irqrestore(lock, flags) \ + PICK_SPINUNLOCK_IRQRESTORE(lock, flags) + #define raw_spin_unlock_bh(lock) _raw_spin_unlock_bh(lock) #define raw_spin_trylock_bh(lock) \ __cond_lock(lock, _raw_spin_trylock_bh(lock)) -#define raw_spin_trylock_irq(lock) \ +#define __real_raw_spin_trylock_irq(lock) \ ({ \ local_irq_disable(); \ - raw_spin_trylock(lock) ? \ + __real_raw_spin_trylock(lock) ? \ 1 : ({ local_irq_enable(); 0; }); \ }) +#define raw_spin_trylock_irq(lock) PICK_SPINTRYLOCK_IRQ(lock) -#define raw_spin_trylock_irqsave(lock, flags) \ +#define __real_raw_spin_trylock_irqsave(lock, flags) \ ({ \ local_irq_save(flags); \ raw_spin_trylock(lock) ? \ 1 : ({ local_irq_restore(flags); 0; }); \ }) +#define raw_spin_trylock_irqsave(lock, flags) \ + PICK_SPINTRYLOCK_IRQSAVE(lock, flags) /** * raw_spin_can_lock - would raw_spin_trylock() succeed? @@ -303,24 +322,17 @@ static inline raw_spinlock_t *spinlock_check(spinlock_t *lock) #define spin_lock_init(_lock) \ do { \ - spinlock_check(_lock); \ - raw_spin_lock_init(&(_lock)->rlock); \ + raw_spin_lock_init(_lock); \ } while (0) -static inline void spin_lock(spinlock_t *lock) -{ - raw_spin_lock(&lock->rlock); -} +#define spin_lock(lock) raw_spin_lock(lock) static inline void spin_lock_bh(spinlock_t *lock) { raw_spin_lock_bh(&lock->rlock); } -static inline int spin_trylock(spinlock_t *lock) -{ - return raw_spin_trylock(&lock->rlock); -} +#define spin_trylock(lock) raw_spin_trylock(lock) #define spin_lock_nested(lock, subclass) \ do { \ @@ -337,14 +349,11 @@ do { \ raw_spin_lock_nest_lock(spinlock_check(lock), nest_lock); \ } while (0) -static inline void spin_lock_irq(spinlock_t *lock) -{ - raw_spin_lock_irq(&lock->rlock); -} +#define spin_lock_irq(lock) raw_spin_lock_irq(lock) #define spin_lock_irqsave(lock, flags) \ do { \ - raw_spin_lock_irqsave(spinlock_check(lock), flags); \ + raw_spin_lock_irqsave(lock, flags); \ } while (0) #define spin_lock_irqsave_nested(lock, flags, subclass) \ @@ -352,39 +361,28 @@ do { \ raw_spin_lock_irqsave_nested(spinlock_check(lock), flags, subclass); \ } while (0) -static inline void spin_unlock(spinlock_t *lock) -{ - raw_spin_unlock(&lock->rlock); -} +#define spin_unlock(lock) raw_spin_unlock(lock) static inline void spin_unlock_bh(spinlock_t *lock) { raw_spin_unlock_bh(&lock->rlock); } -static inline void spin_unlock_irq(spinlock_t *lock) -{ - raw_spin_unlock_irq(&lock->rlock); -} +#define spin_unlock_irq(lock) raw_spin_unlock_irq(lock) -static inline void spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags) -{ - raw_spin_unlock_irqrestore(&lock->rlock, flags); -} +#define spin_unlock_irqrestore(lock, flags) \ + raw_spin_unlock_irqrestore(lock, flags) static inline int spin_trylock_bh(spinlock_t *lock) { return raw_spin_trylock_bh(&lock->rlock); } -static inline int spin_trylock_irq(spinlock_t *lock) -{ - return raw_spin_trylock_irq(&lock->rlock); -} +#define spin_trylock_irq(lock) raw_spin_trylock_irq(lock) #define spin_trylock_irqsave(lock, flags) \ ({ \ - raw_spin_trylock_irqsave(spinlock_check(lock), flags); \ + raw_spin_trylock_irqsave(lock, flags); \ }) static inline void spin_unlock_wait(spinlock_t *lock) diff --git a/include/linux/spinlock_api_smp.h b/include/linux/spinlock_api_smp.h index 5344268..5026fea 100644 --- a/include/linux/spinlock_api_smp.h +++ b/include/linux/spinlock_api_smp.h @@ -101,7 +101,9 @@ static inline int __raw_spin_trylock(raw_spinlock_t *lock) * even on CONFIG_PREEMPT, because lockdep assumes that interrupts are * not re-enabled during lock-acquire (which the preempt-spin-ops do): */ -#if !defined(CONFIG_GENERIC_LOCKBREAK) || defined(CONFIG_DEBUG_LOCK_ALLOC) +#if !defined(CONFIG_GENERIC_LOCKBREAK) || \ + defined(CONFIG_DEBUG_LOCK_ALLOC) || \ + defined(CONFIG_IPIPE) static inline unsigned long __raw_spin_lock_irqsave(raw_spinlock_t *lock) { @@ -115,7 +117,7 @@ static inline unsigned long __raw_spin_lock_irqsave(raw_spinlock_t *lock) * do_raw_spin_lock_flags() code, because lockdep assumes * that interrupts are not re-enabled during lock-acquire: */ -#ifdef CONFIG_LOCKDEP +#if defined(CONFIG_LOCKDEP) || defined(CONFIG_IPIPE) LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock); #else do_raw_spin_lock_flags(lock, &flags); diff --git a/include/linux/spinlock_up.h b/include/linux/spinlock_up.h index 8b3ac0d..cf14140 100644 --- a/include/linux/spinlock_up.h +++ b/include/linux/spinlock_up.h @@ -55,16 +55,6 @@ static inline void arch_spin_unlock(arch_spinlock_t *lock) lock->slock = 1; } -/* - * Read-write spinlocks. No debug version. - */ -#define arch_read_lock(lock) do { barrier(); (void)(lock); } while (0) -#define arch_write_lock(lock) do { barrier(); (void)(lock); } while (0) -#define arch_read_trylock(lock) ({ barrier(); (void)(lock); 1; }) -#define arch_write_trylock(lock) ({ barrier(); (void)(lock); 1; }) -#define arch_read_unlock(lock) do { barrier(); (void)(lock); } while (0) -#define arch_write_unlock(lock) do { barrier(); (void)(lock); } while (0) - #else /* DEBUG_SPINLOCK */ #define arch_spin_is_locked(lock) ((void)(lock), 0) /* for sched/core.c and kernel_lock.c: */ @@ -74,6 +64,13 @@ static inline void arch_spin_unlock(arch_spinlock_t *lock) # define arch_spin_trylock(lock) ({ barrier(); (void)(lock); 1; }) #endif /* DEBUG_SPINLOCK */ +#define arch_read_lock(lock) do { barrier(); (void)(lock); } while (0) +#define arch_write_lock(lock) do { barrier(); (void)(lock); } while (0) +#define arch_read_trylock(lock) ({ barrier(); (void)(lock); 1; }) +#define arch_write_trylock(lock) ({ barrier(); (void)(lock); 1; }) +#define arch_read_unlock(lock) do { barrier(); (void)(lock); } while (0) +#define arch_write_unlock(lock) do { barrier(); (void)(lock); } while (0) + #define arch_spin_is_contended(lock) (((void)(lock), 0)) #define arch_read_can_lock(lock) (((void)(lock), 1)) diff --git a/include/linux/timekeeper_internal.h b/include/linux/timekeeper_internal.h index fb86963..39f2968 100644 --- a/include/linux/timekeeper_internal.h +++ b/include/linux/timekeeper_internal.h @@ -114,7 +114,7 @@ extern void update_vsyscall_tz(void); #elif defined(CONFIG_GENERIC_TIME_VSYSCALL_OLD) extern void update_vsyscall_old(struct timespec *ts, struct timespec *wtm, - struct clocksource *c, u32 mult, + struct clocksource *c, u32 mult, u32 shift, cycle_t cycle_last); extern void update_vsyscall_tz(void); diff --git a/include/uapi/asm-generic/mman-common.h b/include/uapi/asm-generic/mman-common.h index ddc3b36..a9dd00a 100644 --- a/include/uapi/asm-generic/mman-common.h +++ b/include/uapi/asm-generic/mman-common.h @@ -19,6 +19,9 @@ #define MAP_TYPE 0x0f /* Mask for type of mapping */ #define MAP_FIXED 0x10 /* Interpret addr exactly */ #define MAP_ANONYMOUS 0x20 /* don't use a file */ +#ifndef MAP_BRK +# define MAP_BRK 0 +#endif #ifdef CONFIG_MMAP_ALLOW_UNINITIALIZED # define MAP_UNINITIALIZED 0x4000000 /* For anonymous mmap, memory could be uninitialized */ #else diff --git a/include/uapi/asm-generic/resource.h b/include/uapi/asm-generic/resource.h index c6d10af..83cd38b 100644 --- a/include/uapi/asm-generic/resource.h +++ b/include/uapi/asm-generic/resource.h @@ -57,5 +57,13 @@ # define RLIM_INFINITY (~0UL) #endif +/* + * Limit the stack by to some sane default: root can always + * increase this limit if needed.. 8MB seems reasonable. + */ +#ifndef _STK_LIM +# define _STK_LIM (8*1024*1024) +#endif + #endif /* _UAPI_ASM_GENERIC_RESOURCE_H */ diff --git a/include/uapi/linux/resource.h b/include/uapi/linux/resource.h index 36fb3b5..3b5e2dc6 100644 --- a/include/uapi/linux/resource.h +++ b/include/uapi/linux/resource.h @@ -59,12 +59,6 @@ struct rlimit64 { #define PRIO_USER 2 /* - * Limit the stack by to some sane default: root can always - * increase this limit if needed.. 8MB seems reasonable. - */ -#define _STK_LIM (8*1024*1024) - -/* * GPG2 wants 64kB of mlocked memory, to make sure pass phrases * and other sensitive information are never written to disk. */ diff --git a/init/Kconfig b/init/Kconfig index dc24dec..461fe81 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -69,6 +69,7 @@ config COMPILE_TEST config LOCALVERSION string "Local version - append to kernel release" + default "-ipipe" help Append an extra string to the end of your kernel version. This will show up when you type uname, for example. @@ -1445,6 +1446,19 @@ config PRINTK very difficult to diagnose system problems, saying N here is strongly discouraged. +config RAW_PRINTK + bool "Enable support for raw printk" + default n + select DEBUG_LL if ARM + help + This option enables a printk variant called raw_printk() for + writing all output unmodified to a raw console channel + immediately, without any header or preparation whatsoever, + usable from any context. + + Unlike early_printk() console devices, raw_printk() devices + can live past the boot sequence. + config BUG bool "BUG() support" if EXPERT default y diff --git a/init/main.c b/init/main.c index 2a89545..cdb089b 100644 --- a/init/main.c +++ b/init/main.c @@ -510,7 +510,7 @@ asmlinkage __visible void __init start_kernel(void) cgroup_init_early(); - local_irq_disable(); + hard_local_irq_disable(); early_boot_irqs_disabled = true; /* @@ -550,6 +550,7 @@ asmlinkage __visible void __init start_kernel(void) pidhash_init(); vfs_caches_init_early(); sort_main_extable(); + __ipipe_init_early(); trap_init(); mm_init(); @@ -585,6 +586,11 @@ asmlinkage __visible void __init start_kernel(void) softirq_init(); timekeeping_init(); time_init(); + /* + * We need to wait for the interrupt and time subsystems to be + * initialized before enabling the pipeline. + */ + __ipipe_init(); sched_clock_postinit(); perf_event_init(); profile_init(); @@ -876,6 +882,7 @@ static void __init do_basic_setup(void) shmem_init(); driver_init(); init_irq_proc(); + __ipipe_init_proc(); do_ctors(); usermodehelper_enable(); do_initcalls(); diff --git a/kernel/Makefile b/kernel/Makefile index 60c302c..1ddea04 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -73,6 +73,7 @@ obj-$(CONFIG_DETECT_HUNG_TASK) += hung_task.o obj-$(CONFIG_LOCKUP_DETECTOR) += watchdog.o obj-$(CONFIG_SECCOMP) += seccomp.o obj-$(CONFIG_RELAY) += relay.o +obj-$(CONFIG_IPIPE) += ipipe/ obj-$(CONFIG_SYSCTL) += utsname_sysctl.o obj-$(CONFIG_TASK_DELAY_ACCT) += delayacct.o obj-$(CONFIG_TASKSTATS) += taskstats.o tsacct.o diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c index 72d59a1..81a153a 100644 --- a/kernel/context_tracking.c +++ b/kernel/context_tracking.c @@ -68,7 +68,7 @@ void context_tracking_enter(enum ctx_state state) * helpers are enough to protect RCU uses inside the exception. So * just return immediately if we detect we are in an IRQ. */ - if (in_interrupt()) + if (ipipe_root_p == 0 || in_interrupt()) return; /* Kernel threads aren't supposed to go to userspace */ @@ -132,7 +132,7 @@ void context_tracking_exit(enum ctx_state state) { unsigned long flags; - if (!context_tracking_is_enabled()) + if (!ipipe_root_p || !context_tracking_is_enabled()) return; if (in_interrupt()) diff --git a/kernel/debug/debug_core.c b/kernel/debug/debug_core.c index 0874e2e..e9a17d9 100644 --- a/kernel/debug/debug_core.c +++ b/kernel/debug/debug_core.c @@ -118,8 +118,8 @@ static struct kgdb_bkpt kgdb_break[KGDB_MAX_BREAKPOINTS] = { */ atomic_t kgdb_active = ATOMIC_INIT(-1); EXPORT_SYMBOL_GPL(kgdb_active); -static DEFINE_RAW_SPINLOCK(dbg_master_lock); -static DEFINE_RAW_SPINLOCK(dbg_slave_lock); +static IPIPE_DEFINE_RAW_SPINLOCK(dbg_master_lock); +static IPIPE_DEFINE_RAW_SPINLOCK(dbg_slave_lock); /* * We use NR_CPUs not PERCPU, in case kgdb is used to debug early @@ -169,19 +169,21 @@ int __weak kgdb_arch_set_breakpoint(struct kgdb_bkpt *bpt) { int err; - err = probe_kernel_read(bpt->saved_instr, (char *)bpt->bpt_addr, - BREAK_INSTR_SIZE); + err = ipipe_probe_kernel_read(bpt->saved_instr, (char *)bpt->bpt_addr, + BREAK_INSTR_SIZE); if (err) return err; - err = probe_kernel_write((char *)bpt->bpt_addr, - arch_kgdb_ops.gdb_bpt_instr, BREAK_INSTR_SIZE); + err = ipipe_probe_kernel_write((char *)bpt->bpt_addr, + arch_kgdb_ops.gdb_bpt_instr, + BREAK_INSTR_SIZE); return err; } int __weak kgdb_arch_remove_breakpoint(struct kgdb_bkpt *bpt) { - return probe_kernel_write((char *)bpt->bpt_addr, - (char *)bpt->saved_instr, BREAK_INSTR_SIZE); + return ipipe_probe_kernel_write((char *)bpt->bpt_addr, + (char *)bpt->saved_instr, + BREAK_INSTR_SIZE); } int __weak kgdb_validate_break_address(unsigned long addr) @@ -460,7 +462,9 @@ static int kgdb_reenter_check(struct kgdb_state *ks) static void dbg_touch_watchdogs(void) { touch_softlockup_watchdog_sync(); +#ifndef CONFIG_IPIPE clocksource_touch_watchdog(); +#endif rcu_cpu_stall_reset(); } @@ -491,7 +495,7 @@ acquirelock: * Interrupts will be restored by the 'trap return' code, except when * single stepping. */ - local_irq_save(flags); + flags = hard_local_irq_save(); cpu = ks->cpu; kgdb_info[cpu].debuggerinfo = regs; @@ -540,7 +544,7 @@ return_normal: smp_mb__before_atomic(); atomic_dec(&slaves_in_kgdb); dbg_touch_watchdogs(); - local_irq_restore(flags); + hard_local_irq_restore(flags); return 0; } cpu_relax(); @@ -558,7 +562,7 @@ return_normal: atomic_set(&kgdb_active, -1); raw_spin_unlock(&dbg_master_lock); dbg_touch_watchdogs(); - local_irq_restore(flags); + hard_local_irq_restore(flags); goto acquirelock; } @@ -675,7 +679,7 @@ kgdb_restore: atomic_set(&kgdb_active, -1); raw_spin_unlock(&dbg_master_lock); dbg_touch_watchdogs(); - local_irq_restore(flags); + hard_local_irq_restore(flags); return kgdb_info[cpu].ret_state; } @@ -794,9 +798,9 @@ static void kgdb_console_write(struct console *co, const char *s, if (!kgdb_connected || atomic_read(&kgdb_active) != -1 || dbg_kdb_mode) return; - local_irq_save(flags); + flags = hard_local_irq_save(); gdbstub_msg_write(s, count); - local_irq_restore(flags); + hard_local_irq_restore(flags); } static struct console kgdbcons = { diff --git a/kernel/debug/gdbstub.c b/kernel/debug/gdbstub.c index 19d9a57..4e99085 100644 --- a/kernel/debug/gdbstub.c +++ b/kernel/debug/gdbstub.c @@ -246,7 +246,7 @@ char *kgdb_mem2hex(char *mem, char *buf, int count) */ tmp = buf + count; - err = probe_kernel_read(tmp, mem, count); + err = ipipe_probe_kernel_read(tmp, mem, count); if (err) return NULL; while (count > 0) { @@ -282,7 +282,7 @@ int kgdb_hex2mem(char *buf, char *mem, int count) *tmp_raw |= hex_to_bin(*tmp_hex--) << 4; } - return probe_kernel_write(mem, tmp_raw, count); + return ipipe_probe_kernel_write(mem, tmp_raw, count); } /* @@ -334,7 +334,7 @@ static int kgdb_ebin2mem(char *buf, char *mem, int count) size++; } - return probe_kernel_write(mem, c, size); + return ipipe_probe_kernel_write(mem, c, size); } #if DBG_MAX_REG_NUM > 0 diff --git a/kernel/exit.c b/kernel/exit.c index 22fcc05..7c93c8a 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -705,6 +705,7 @@ void do_exit(long code) */ smp_mb(); raw_spin_unlock_wait(&tsk->pi_lock); + __ipipe_report_exit(tsk); if (unlikely(in_atomic())) pr_info("note: %s[%d] exited with preempt_count %d\n", diff --git a/kernel/fork.c b/kernel/fork.c index 8209fa2..c4afece 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -356,6 +356,8 @@ static struct task_struct *dup_task_struct(struct task_struct *orig) #endif setup_thread_stack(tsk, orig); + __ipipe_init_threadflags(ti); + __ipipe_init_threadinfo(&ti->ipipe_data); clear_user_return_notifier(tsk); clear_tsk_need_resched(tsk); set_task_stack_end_magic(tsk); @@ -692,6 +694,7 @@ void mmput(struct mm_struct *mm) exit_aio(mm); ksm_exit(mm); khugepaged_exit(mm); /* must run before exit_mmap */ + __ipipe_report_cleanup(mm); exit_mmap(mm); set_mm_exe_file(mm, NULL); if (!list_empty(&mm->mmlist)) { @@ -1596,6 +1599,7 @@ static struct task_struct *copy_process(unsigned long clone_flags, cgroup_post_fork(p); if (clone_flags & CLONE_THREAD) threadgroup_change_end(current); + __ipipe_init_taskinfo(p); perf_event_fork(p); trace_task_newtask(p, clone_flags); diff --git a/kernel/ipipe/Kconfig b/kernel/ipipe/Kconfig new file mode 100644 index 0000000..218f51da --- /dev/null +++ b/kernel/ipipe/Kconfig @@ -0,0 +1,65 @@ +config IPIPE + bool "Interrupt pipeline" + default y + ---help--- + Activate this option if you want the interrupt pipeline to be + compiled in. + +config IPIPE_LEGACY + bool "I-pipe legacy interface" + depends on IPIPE + ---help--- + Activate this option if you want to control the interrupt + pipeline via the legacy interface. + +config IPIPE_CORE + def_bool y if IPIPE + +config IPIPE_WANT_CLOCKSOURCE + bool + +config IPIPE_WANT_PTE_PINNING + bool + +config IPIPE_CORE_APIREV + int + depends on IPIPE + default 2 + ---help--- + The API revision level we implement. + +config IPIPE_WANT_APIREV_1 + bool + +config IPIPE_WANT_APIREV_2 + bool + +config IPIPE_TARGET_APIREV + int + depends on IPIPE + default 1 if IPIPE_WANT_APIREV_1 + default 2 if IPIPE_WANT_APIREV_2 + default 1 if IPIPE_LEGACY + default IPIPE_CORE_APIREV + ---help--- + The API revision level the we want (must be <= + IPIPE_CORE_APIREV). + +config IPIPE_HAVE_HOSTRT + bool + +config IPIPE_HAVE_PIC_MUTE + bool + +config HAVE_IPIPE_HOSTRT + depends on IPIPE_LEGACY + bool + +config IPIPE_DELAYED_ATOMICSW + def_bool y if IPIPE_LEGACY + +config IPIPE_HAVE_SAFE_THREAD_INFO + bool + +config IPIPE_HAVE_VM_NOTIFIER + bool diff --git a/kernel/ipipe/Kconfig.debug b/kernel/ipipe/Kconfig.debug new file mode 100644 index 0000000..cee7fab --- /dev/null +++ b/kernel/ipipe/Kconfig.debug @@ -0,0 +1,96 @@ +config IPIPE_DEBUG + bool "I-pipe debugging" + depends on IPIPE + select RAW_PRINTK + +config IPIPE_DEBUG_CONTEXT + bool "Check for illicit cross-domain calls" + depends on IPIPE_DEBUG + default y + ---help--- + Enable this feature to arm checkpoints in the kernel that + verify the correct invocation context. On entry of critical + Linux services a warning is issued if the caller is not + running over the root domain. + +config IPIPE_DEBUG_INTERNAL + bool "Enable internal debug checks" + depends on IPIPE_DEBUG + default y + ---help--- + When this feature is enabled, I-pipe will perform internal + consistency checks of its subsystems, e.g. on per-cpu variable + access. + +config IPIPE_TRACE + bool "Latency tracing" + depends on IPIPE_DEBUG + select CONFIG_FTRACE + select CONFIG_FUNCTION_TRACER + select KALLSYMS + select PROC_FS + ---help--- + Activate this option if you want to use per-function tracing of + the kernel. The tracer will collect data via instrumentation + features like the one below or with the help of explicite calls + of ipipe_trace_xxx(). See include/linux/ipipe_trace.h for the + in-kernel tracing API. The collected data and runtime control + is available via /proc/ipipe/trace/*. + +if IPIPE_TRACE + +config IPIPE_TRACE_ENABLE + bool "Enable tracing on boot" + default y + ---help--- + Disable this option if you want to arm the tracer after booting + manually ("echo 1 > /proc/ipipe/tracer/enable"). This can reduce + boot time on slow embedded devices due to the tracer overhead. + +config IPIPE_TRACE_MCOUNT + bool "Instrument function entries" + default y + select FTRACE + select FUNCTION_TRACER + ---help--- + When enabled, records every kernel function entry in the tracer + log. While this slows down the system noticeably, it provides + the highest level of information about the flow of events. + However, it can be switch off in order to record only explicit + I-pipe trace points. + +config IPIPE_TRACE_IRQSOFF + bool "Trace IRQs-off times" + default y + ---help--- + Activate this option if I-pipe shall trace the longest path + with hard-IRQs switched off. + +config IPIPE_TRACE_SHIFT + int "Depth of trace log (14 => 16Kpoints, 15 => 32Kpoints)" + range 10 18 + default 14 + ---help--- + The number of trace points to hold tracing data for each + trace path, as a power of 2. + +config IPIPE_TRACE_VMALLOC + bool "Use vmalloc'ed trace buffer" + default y if EMBEDDED + ---help--- + Instead of reserving static kernel data, the required buffer + is allocated via vmalloc during boot-up when this option is + enabled. This can help to start systems that are low on memory, + but it slightly degrades overall performance. Try this option + when a traced kernel hangs unexpectedly at boot time. + +config IPIPE_TRACE_PANIC + bool "Enable panic back traces" + default y + ---help--- + Provides services to freeze and dump a back trace on panic + situations. This is used on IPIPE_DEBUG_CONTEXT exceptions + as well as ordinary kernel oopses. You can control the number + of printed back trace points via /proc/ipipe/trace. + +endif diff --git a/kernel/ipipe/Makefile b/kernel/ipipe/Makefile new file mode 100644 index 0000000..c3ffe63 --- /dev/null +++ b/kernel/ipipe/Makefile @@ -0,0 +1,3 @@ +obj-$(CONFIG_IPIPE) += core.o timer.o +obj-$(CONFIG_IPIPE_TRACE) += tracer.o +obj-$(CONFIG_IPIPE_LEGACY) += compat.o diff --git a/kernel/ipipe/compat.c b/kernel/ipipe/compat.c new file mode 100644 index 0000000..797a849 --- /dev/null +++ b/kernel/ipipe/compat.c @@ -0,0 +1,273 @@ +/* -*- linux-c -*- + * linux/kernel/ipipe/compat.c + * + * Copyright (C) 2012 Philippe Gerum. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139, + * USA; either version 2 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * + * I-pipe legacy interface. + */ +#include +#include +#include +#include +#include +#include + +static int ptd_key_count; + +static unsigned long ptd_key_map; + +IPIPE_DECLARE_SPINLOCK(__ipipe_lock); + +void ipipe_init_attr(struct ipipe_domain_attr *attr) +{ + attr->name = "anon"; + attr->domid = 1; + attr->entry = NULL; + attr->priority = IPIPE_ROOT_PRIO; + attr->pdd = NULL; +} +EXPORT_SYMBOL_GPL(ipipe_init_attr); + +int ipipe_register_domain(struct ipipe_domain *ipd, + struct ipipe_domain_attr *attr) +{ + struct ipipe_percpu_domain_data *p; + unsigned long flags; + + BUG_ON(attr->priority != IPIPE_HEAD_PRIORITY); + + ipipe_register_head(ipd, attr->name); + ipd->legacy.domid = attr->domid; + ipd->legacy.pdd = attr->pdd; + ipd->legacy.priority = INT_MAX; + + if (attr->entry == NULL) + return 0; + + flags = hard_smp_local_irq_save(); + __ipipe_set_current_domain(ipd); + hard_smp_local_irq_restore(flags); + + attr->entry(); + + flags = hard_local_irq_save(); + __ipipe_set_current_domain(ipipe_root_domain); + p = ipipe_this_cpu_root_context(); + if (__ipipe_ipending_p(p) && + !test_bit(IPIPE_STALL_FLAG, &p->status)) + __ipipe_sync_stage(); + hard_local_irq_restore(flags); + + return 0; +} +EXPORT_SYMBOL_GPL(ipipe_register_domain); + +int ipipe_unregister_domain(struct ipipe_domain *ipd) +{ + ipipe_unregister_head(ipd); + + return 0; +} +EXPORT_SYMBOL_GPL(ipipe_unregister_domain); + +int ipipe_alloc_ptdkey(void) +{ + unsigned long flags; + int key = -1; + + spin_lock_irqsave(&__ipipe_lock,flags); + + if (ptd_key_count < IPIPE_ROOT_NPTDKEYS) { + key = ffz(ptd_key_map); + set_bit(key,&ptd_key_map); + ptd_key_count++; + } + + spin_unlock_irqrestore(&__ipipe_lock,flags); + + return key; +} +EXPORT_SYMBOL_GPL(ipipe_alloc_ptdkey); + +int ipipe_free_ptdkey(int key) +{ + unsigned long flags; + + if (key < 0 || key >= IPIPE_ROOT_NPTDKEYS) + return -EINVAL; + + spin_lock_irqsave(&__ipipe_lock,flags); + + if (test_and_clear_bit(key,&ptd_key_map)) + ptd_key_count--; + + spin_unlock_irqrestore(&__ipipe_lock,flags); + + return 0; +} +EXPORT_SYMBOL_GPL(ipipe_free_ptdkey); + +int ipipe_set_ptd(int key, void *value) +{ + if (key < 0 || key >= IPIPE_ROOT_NPTDKEYS) + return -EINVAL; + + current->ptd[key] = value; + + return 0; +} +EXPORT_SYMBOL_GPL(ipipe_set_ptd); + +void *ipipe_get_ptd(int key) +{ + if (key < 0 || key >= IPIPE_ROOT_NPTDKEYS) + return NULL; + + return current->ptd[key]; +} +EXPORT_SYMBOL_GPL(ipipe_get_ptd); + +int ipipe_virtualize_irq(struct ipipe_domain *ipd, + unsigned int irq, + ipipe_irq_handler_t handler, + void *cookie, + ipipe_irq_ackfn_t ackfn, + unsigned int modemask) +{ + if (handler == NULL) { + ipipe_free_irq(ipd, irq); + return 0; + } + + return ipipe_request_irq(ipd, irq, handler, cookie, ackfn); +} +EXPORT_SYMBOL_GPL(ipipe_virtualize_irq); + +static int null_handler(unsigned int event, + struct ipipe_domain *from, void *data) +{ + /* + * Legacy mode users will trap all events, at worst most + * frequent ones. Therefore it is actually faster to run a + * dummy handler once in a while rather than testing for a + * null handler pointer each time an event is fired. + */ + return 0; +} + +ipipe_event_handler_t ipipe_catch_event(struct ipipe_domain *ipd, + unsigned int event, + ipipe_event_handler_t handler) +{ + ipipe_event_handler_t oldhandler; + int n, enables = 0; + + if (event & IPIPE_EVENT_SELF) { + event &= ~IPIPE_EVENT_SELF; + IPIPE_WARN(event >= IPIPE_NR_FAULTS); + } + + if (event >= IPIPE_NR_EVENTS) + return NULL; + + /* + * It makes no sense to run a SETSCHED notification handler + * over the head domain, this introduces a useless domain + * switch for doing work which ought to be root specific. + * Unfortunately, some client domains using the legacy + * interface still ask for this, so we silently fix their + * request. This prevents ipipe_set_hooks() from yelling at us + * because of an attempt to enable kernel event notifications + * for the head domain. + */ + if (event == IPIPE_EVENT_SETSCHED) + ipd = ipipe_root_domain; + + oldhandler = ipd->legacy.handlers[event]; + ipd->legacy.handlers[event] = handler ?: null_handler; + + for (n = 0; n < IPIPE_NR_FAULTS; n++) { + if (ipd->legacy.handlers[n] != null_handler) { + enables |= __IPIPE_TRAP_E; + break; + } + } + + for (n = IPIPE_FIRST_EVENT; n < IPIPE_LAST_EVENT; n++) { + if (ipd->legacy.handlers[n] != null_handler) { + enables |= __IPIPE_KEVENT_E; + break; + } + } + + if (ipd->legacy.handlers[IPIPE_EVENT_SYSCALL] != null_handler) + enables |= __IPIPE_SYSCALL_E; + + ipipe_set_hooks(ipd, enables); + + return oldhandler == null_handler ? NULL : oldhandler; +} +EXPORT_SYMBOL_GPL(ipipe_catch_event); + +int ipipe_setscheduler_root(struct task_struct *p, int policy, int prio) +{ + struct sched_param param = { .sched_priority = prio }; + return sched_setscheduler_nocheck(p, policy, ¶m); +} +EXPORT_SYMBOL_GPL(ipipe_setscheduler_root); + +int ipipe_syscall_hook(struct ipipe_domain *ipd, struct pt_regs *regs) +{ + const int event = IPIPE_EVENT_SYSCALL; + return ipipe_current_domain->legacy.handlers[event](event, ipd, regs); +} + +int ipipe_trap_hook(struct ipipe_trap_data *data) +{ + struct ipipe_domain *ipd = ipipe_head_domain; + struct pt_regs *regs = data->regs; + int ex = data->exception; + + return ipd->legacy.handlers[ex](ex, ipd, regs); +} + +int ipipe_kevent_hook(int kevent, void *data) +{ + unsigned int event = IPIPE_FIRST_EVENT + kevent; + struct ipipe_domain *ipd = ipipe_root_domain; + + return ipd->legacy.handlers[event](event, ipd, data); +} + +void __ipipe_legacy_init_stage(struct ipipe_domain *ipd) +{ + int n; + + for (n = 0; n < IPIPE_NR_EVENTS; n++) + ipd->legacy.handlers[n] = null_handler; + + if (ipd == &ipipe_root) { + ipd->legacy.domid = IPIPE_ROOT_ID; + ipd->legacy.priority = IPIPE_ROOT_PRIO; + } +} + +notrace asmlinkage int __ipipe_check_root(void) /* hw IRQs off */ +{ + return __ipipe_root_p; +} diff --git a/kernel/ipipe/core.c b/kernel/ipipe/core.c new file mode 100644 index 0000000..339fb95 --- /dev/null +++ b/kernel/ipipe/core.c @@ -0,0 +1,1916 @@ +/* -*- linux-c -*- + * linux/kernel/ipipe/core.c + * + * Copyright (C) 2002-2012 Philippe Gerum. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139, + * USA; either version 2 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * + * Architecture-independent I-PIPE core support. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef CONFIG_PROC_FS +#include +#include +#endif /* CONFIG_PROC_FS */ +#include +#include +#include + +struct ipipe_domain ipipe_root; +EXPORT_SYMBOL_GPL(ipipe_root); + +struct ipipe_domain *ipipe_head_domain = &ipipe_root; +EXPORT_SYMBOL_GPL(ipipe_head_domain); + +#ifdef CONFIG_SMP +static __initdata struct ipipe_percpu_domain_data bootup_context = { + .status = IPIPE_STALL_MASK, + .domain = &ipipe_root, +}; +#else +#define bootup_context ipipe_percpu.root +#endif /* !CONFIG_SMP */ + +DEFINE_PER_CPU(struct ipipe_percpu_data, ipipe_percpu) = { + .root = { + .status = IPIPE_STALL_MASK, + .domain = &ipipe_root, + }, + .curr = &bootup_context, + .hrtimer_irq = -1, +#ifdef CONFIG_IPIPE_DEBUG_CONTEXT + .context_check = 1, +#endif +}; +EXPORT_PER_CPU_SYMBOL(ipipe_percpu); + +/* Up to 2k of pending work data per CPU. */ +#define WORKBUF_SIZE 2048 +static DEFINE_PER_CPU_ALIGNED(unsigned char[WORKBUF_SIZE], work_buf); +static DEFINE_PER_CPU(void *, work_tail); +static unsigned int __ipipe_work_virq; + +static void __ipipe_do_work(unsigned int virq, void *cookie); + +#ifdef CONFIG_SMP + +#define IPIPE_CRITICAL_TIMEOUT 1000000 +static cpumask_t __ipipe_cpu_sync_map; +static cpumask_t __ipipe_cpu_lock_map; +static cpumask_t __ipipe_cpu_pass_map; +static unsigned long __ipipe_critical_lock; +static IPIPE_DEFINE_SPINLOCK(__ipipe_cpu_barrier); +static atomic_t __ipipe_critical_count = ATOMIC_INIT(0); +static void (*__ipipe_cpu_sync) (void); + +#else /* !CONFIG_SMP */ +/* + * Create an alias to the unique root status, so that arch-dep code + * may get fast access to this percpu variable including from + * assembly. A hard-coded assumption is that root.status appears at + * offset #0 of the ipipe_percpu struct. + */ +extern unsigned long __ipipe_root_status +__attribute__((alias(__stringify(ipipe_percpu)))); +EXPORT_SYMBOL(__ipipe_root_status); + +#endif /* !CONFIG_SMP */ + +IPIPE_DEFINE_SPINLOCK(__ipipe_lock); + +static unsigned long __ipipe_virtual_irq_map; + +#ifdef CONFIG_PRINTK +unsigned int __ipipe_printk_virq; +int __ipipe_printk_bypass; +#endif /* CONFIG_PRINTK */ + +#ifdef CONFIG_PROC_FS + +struct proc_dir_entry *ipipe_proc_root; + +static int __ipipe_version_info_show(struct seq_file *p, void *data) +{ + seq_printf(p, "%d\n", IPIPE_CORE_RELEASE); + return 0; +} + +static int __ipipe_version_info_open(struct inode *inode, struct file *file) +{ + return single_open(file, __ipipe_version_info_show, NULL); +} + +static const struct file_operations __ipipe_version_proc_ops = { + .open = __ipipe_version_info_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static int __ipipe_common_info_show(struct seq_file *p, void *data) +{ + struct ipipe_domain *ipd = (struct ipipe_domain *)p->private; + char handling, lockbit, virtuality; + unsigned long ctlbits; + unsigned int irq; + + seq_printf(p, " +--- Handled\n"); + seq_printf(p, " |+-- Locked\n"); + seq_printf(p, " ||+- Virtual\n"); + seq_printf(p, " [IRQ] ||| Handler\n"); + + mutex_lock(&ipd->mutex); + + for (irq = 0; irq < IPIPE_NR_IRQS; irq++) { + ctlbits = ipd->irqs[irq].control; + /* + * There might be a hole between the last external IRQ + * and the first virtual one; skip it. + */ + if (irq >= IPIPE_NR_XIRQS && !ipipe_virtual_irq_p(irq)) + continue; + + if (ipipe_virtual_irq_p(irq) + && !test_bit(irq - IPIPE_VIRQ_BASE, &__ipipe_virtual_irq_map)) + /* Non-allocated virtual IRQ; skip it. */ + continue; + + if (ctlbits & IPIPE_HANDLE_MASK) + handling = 'H'; + else + handling = '.'; + + if (ctlbits & IPIPE_LOCK_MASK) + lockbit = 'L'; + else + lockbit = '.'; + + if (ipipe_virtual_irq_p(irq)) + virtuality = 'V'; + else + virtuality = '.'; + + if (ctlbits & IPIPE_HANDLE_MASK) + seq_printf(p, " %4u: %c%c%c %pf\n", + irq, handling, lockbit, virtuality, + ipd->irqs[irq].handler); + else + seq_printf(p, " %4u: %c%c%c\n", + irq, handling, lockbit, virtuality); + } + + mutex_unlock(&ipd->mutex); + + return 0; +} + +static int __ipipe_common_info_open(struct inode *inode, struct file *file) +{ + return single_open(file, __ipipe_common_info_show, PDE_DATA(inode)); +} + +static const struct file_operations __ipipe_info_proc_ops = { + .owner = THIS_MODULE, + .open = __ipipe_common_info_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +void add_domain_proc(struct ipipe_domain *ipd) +{ + proc_create_data(ipd->name, 0444, ipipe_proc_root, + &__ipipe_info_proc_ops, ipd); +} + +void remove_domain_proc(struct ipipe_domain *ipd) +{ + remove_proc_entry(ipd->name, ipipe_proc_root); +} + +void __init __ipipe_init_proc(void) +{ + ipipe_proc_root = proc_mkdir("ipipe", NULL); + proc_create("version", 0444, ipipe_proc_root, + &__ipipe_version_proc_ops); + add_domain_proc(ipipe_root_domain); + + __ipipe_init_tracer(); +} + +#else + +static inline void add_domain_proc(struct ipipe_domain *ipd) +{ +} + +static inline void remove_domain_proc(struct ipipe_domain *ipd) +{ +} + +#endif /* CONFIG_PROC_FS */ + +static void init_stage(struct ipipe_domain *ipd) +{ + memset(&ipd->irqs, 0, sizeof(ipd->irqs)); + mutex_init(&ipd->mutex); + __ipipe_legacy_init_stage(ipd); + __ipipe_hook_critical_ipi(ipd); +} + +static inline int root_context_offset(void) +{ + void root_context_not_at_start_of_ipipe_percpu(void); + + /* ipipe_percpu.root must be found at offset #0. */ + + if (offsetof(struct ipipe_percpu_data, root)) + root_context_not_at_start_of_ipipe_percpu(); + + return 0; +} + +#ifdef CONFIG_SMP + +static inline void fixup_percpu_data(void) +{ + struct ipipe_percpu_data *p; + int cpu; + + /* + * ipipe_percpu.curr cannot be assigned statically to + * &ipipe_percpu.root, due to the dynamic nature of percpu + * data. So we make ipipe_percpu.curr refer to a temporary + * boot up context in static memory, until we can fixup all + * context pointers in this routine, after per-cpu areas have + * been eventually set up. The temporary context data is + * copied to per_cpu(ipipe_percpu, 0).root in the same move. + * + * Obviously, this code must run over the boot CPU, before SMP + * operations start. + */ + BUG_ON(smp_processor_id() || !irqs_disabled()); + + per_cpu(ipipe_percpu, 0).root = bootup_context; + + for_each_possible_cpu(cpu) { + p = &per_cpu(ipipe_percpu, cpu); + p->curr = &p->root; + } +} + +#else /* !CONFIG_SMP */ + +static inline void fixup_percpu_data(void) { } + +#endif /* CONFIG_SMP */ + +void __init __ipipe_init_early(void) +{ + struct ipipe_domain *ipd = &ipipe_root; + int cpu; + + fixup_percpu_data(); + + /* + * A lightweight registration code for the root domain. We are + * running on the boot CPU, hw interrupts are off, and + * secondary CPUs are still lost in space. + */ + ipd->name = "Linux"; + ipd->context_offset = root_context_offset(); + init_stage(ipd); + + /* + * Do the early init stuff. First we do the per-arch pipeline + * core setup, then we run the per-client setup code. At this + * point, the kernel does not provide much services yet: be + * careful. + */ + __ipipe_early_core_setup(); + __ipipe_early_client_setup(); + +#ifdef CONFIG_PRINTK + __ipipe_printk_virq = ipipe_alloc_virq(); + ipd->irqs[__ipipe_printk_virq].handler = __ipipe_flush_printk; + ipd->irqs[__ipipe_printk_virq].cookie = NULL; + ipd->irqs[__ipipe_printk_virq].ackfn = NULL; + ipd->irqs[__ipipe_printk_virq].control = IPIPE_HANDLE_MASK; +#endif /* CONFIG_PRINTK */ + + __ipipe_work_virq = ipipe_alloc_virq(); + ipd->irqs[__ipipe_work_virq].handler = __ipipe_do_work; + ipd->irqs[__ipipe_work_virq].cookie = NULL; + ipd->irqs[__ipipe_work_virq].ackfn = NULL; + ipd->irqs[__ipipe_work_virq].control = IPIPE_HANDLE_MASK; + + for_each_possible_cpu(cpu) + per_cpu(work_tail, cpu) = per_cpu(work_buf, cpu); +} + +void __init __ipipe_init(void) +{ + /* Now we may engage the pipeline. */ + __ipipe_enable_pipeline(); + + pr_info("Interrupt pipeline (release #%d)\n", IPIPE_CORE_RELEASE); +} + +static inline void init_head_stage(struct ipipe_domain *ipd) +{ + struct ipipe_percpu_domain_data *p; + int cpu; + + /* Must be set first, used in ipipe_percpu_context(). */ + ipd->context_offset = offsetof(struct ipipe_percpu_data, head); + + for_each_online_cpu(cpu) { + p = ipipe_percpu_context(ipd, cpu); + memset(p, 0, sizeof(*p)); + p->domain = ipd; + } + + init_stage(ipd); +} + +void ipipe_register_head(struct ipipe_domain *ipd, const char *name) +{ + BUG_ON(!ipipe_root_p || ipd == &ipipe_root); + + ipd->name = name; + init_head_stage(ipd); + barrier(); + ipipe_head_domain = ipd; + add_domain_proc(ipd); + + pr_info("I-pipe: head domain %s registered.\n", name); +} +EXPORT_SYMBOL_GPL(ipipe_register_head); + +void ipipe_unregister_head(struct ipipe_domain *ipd) +{ + BUG_ON(!ipipe_root_p || ipd != ipipe_head_domain); + + ipipe_head_domain = &ipipe_root; + smp_mb(); + mutex_lock(&ipd->mutex); + remove_domain_proc(ipd); + mutex_unlock(&ipd->mutex); + + pr_info("I-pipe: head domain %s unregistered.\n", ipd->name); +} +EXPORT_SYMBOL_GPL(ipipe_unregister_head); + +void ipipe_unstall_root(void) +{ + struct ipipe_percpu_domain_data *p; + + hard_local_irq_disable(); + + /* This helps catching bad usage from assembly call sites. */ + ipipe_root_only(); + + p = ipipe_this_cpu_root_context(); + + __clear_bit(IPIPE_STALL_FLAG, &p->status); + + if (unlikely(__ipipe_ipending_p(p))) + __ipipe_sync_stage(); + + hard_local_irq_enable(); +} +EXPORT_SYMBOL(ipipe_unstall_root); + +void ipipe_restore_root(unsigned long x) +{ + ipipe_root_only(); + + if (x) + ipipe_stall_root(); + else + ipipe_unstall_root(); +} +EXPORT_SYMBOL(ipipe_restore_root); + +void __ipipe_restore_root_nosync(unsigned long x) +{ + struct ipipe_percpu_domain_data *p = ipipe_this_cpu_root_context(); + + if (raw_irqs_disabled_flags(x)) { + __set_bit(IPIPE_STALL_FLAG, &p->status); + trace_hardirqs_off(); + } else { + trace_hardirqs_on(); + __clear_bit(IPIPE_STALL_FLAG, &p->status); + } +} +EXPORT_SYMBOL_GPL(__ipipe_restore_root_nosync); + +void ipipe_unstall_head(void) +{ + struct ipipe_percpu_domain_data *p = ipipe_this_cpu_head_context(); + + hard_local_irq_disable(); + + __clear_bit(IPIPE_STALL_FLAG, &p->status); + + if (unlikely(__ipipe_ipending_p(p))) + __ipipe_sync_pipeline(ipipe_head_domain); + + hard_local_irq_enable(); +} +EXPORT_SYMBOL_GPL(ipipe_unstall_head); + +void __ipipe_restore_head(unsigned long x) /* hw interrupt off */ +{ + struct ipipe_percpu_domain_data *p = ipipe_this_cpu_head_context(); + + if (x) { +#ifdef CONFIG_DEBUG_KERNEL + static int warned; + if (!warned && + __test_and_set_bit(IPIPE_STALL_FLAG, &p->status)) { + /* + * Already stalled albeit ipipe_restore_head() + * should have detected it? Send a warning once. + */ + hard_local_irq_enable(); + warned = 1; + pr_warning("I-pipe: ipipe_restore_head() " + "optimization failed.\n"); + dump_stack(); + hard_local_irq_disable(); + } +#else /* !CONFIG_DEBUG_KERNEL */ + __set_bit(IPIPE_STALL_FLAG, &p->status); +#endif /* CONFIG_DEBUG_KERNEL */ + } else { + __clear_bit(IPIPE_STALL_FLAG, &p->status); + if (unlikely(__ipipe_ipending_p(p))) + __ipipe_sync_pipeline(ipipe_head_domain); + hard_local_irq_enable(); + } +} +EXPORT_SYMBOL_GPL(__ipipe_restore_head); + +void __ipipe_spin_lock_irq(ipipe_spinlock_t *lock) +{ + hard_local_irq_disable(); + if (ipipe_smp_p) + arch_spin_lock(&lock->arch_lock); + __set_bit(IPIPE_STALL_FLAG, &__ipipe_current_context->status); +} +EXPORT_SYMBOL_GPL(__ipipe_spin_lock_irq); + +void __ipipe_spin_unlock_irq(ipipe_spinlock_t *lock) +{ + if (ipipe_smp_p) + arch_spin_unlock(&lock->arch_lock); + __clear_bit(IPIPE_STALL_FLAG, &__ipipe_current_context->status); + hard_local_irq_enable(); +} +EXPORT_SYMBOL_GPL(__ipipe_spin_unlock_irq); + +unsigned long __ipipe_spin_lock_irqsave(ipipe_spinlock_t *lock) +{ + unsigned long flags; + int s; + + flags = hard_local_irq_save(); + if (ipipe_smp_p) + arch_spin_lock(&lock->arch_lock); + s = __test_and_set_bit(IPIPE_STALL_FLAG, &__ipipe_current_context->status); + + return arch_mangle_irq_bits(s, flags); +} +EXPORT_SYMBOL_GPL(__ipipe_spin_lock_irqsave); + +int __ipipe_spin_trylock_irqsave(ipipe_spinlock_t *lock, + unsigned long *x) +{ + unsigned long flags; + int s; + + flags = hard_local_irq_save(); + if (ipipe_smp_p && !arch_spin_trylock(&lock->arch_lock)) { + hard_local_irq_restore(flags); + return 0; + } + s = __test_and_set_bit(IPIPE_STALL_FLAG, &__ipipe_current_context->status); + *x = arch_mangle_irq_bits(s, flags); + + return 1; +} +EXPORT_SYMBOL_GPL(__ipipe_spin_trylock_irqsave); + +void __ipipe_spin_unlock_irqrestore(ipipe_spinlock_t *lock, + unsigned long x) +{ + if (ipipe_smp_p) + arch_spin_unlock(&lock->arch_lock); + if (!arch_demangle_irq_bits(&x)) + __clear_bit(IPIPE_STALL_FLAG, &__ipipe_current_context->status); + hard_local_irq_restore(x); +} +EXPORT_SYMBOL_GPL(__ipipe_spin_unlock_irqrestore); + +int __ipipe_spin_trylock_irq(ipipe_spinlock_t *lock) +{ + unsigned long flags; + + flags = hard_local_irq_save(); + if (ipipe_smp_p && !arch_spin_trylock(&lock->arch_lock)) { + hard_local_irq_restore(flags); + return 0; + } + __set_bit(IPIPE_STALL_FLAG, &__ipipe_current_context->status); + + return 1; +} +EXPORT_SYMBOL_GPL(__ipipe_spin_trylock_irq); + +void __ipipe_spin_unlock_irqbegin(ipipe_spinlock_t *lock) +{ + if (ipipe_smp_p) + arch_spin_unlock(&lock->arch_lock); +} + +void __ipipe_spin_unlock_irqcomplete(unsigned long x) +{ + if (!arch_demangle_irq_bits(&x)) + __clear_bit(IPIPE_STALL_FLAG, &__ipipe_current_context->status); + hard_local_irq_restore(x); +} + +#ifdef __IPIPE_3LEVEL_IRQMAP + +/* Must be called hw IRQs off. */ +static inline void __ipipe_set_irq_held(struct ipipe_percpu_domain_data *p, + unsigned int irq) +{ + __set_bit(irq, p->irqheld_map); + p->irqall[irq]++; +} + +/* Must be called hw IRQs off. */ +void __ipipe_set_irq_pending(struct ipipe_domain *ipd, unsigned int irq) +{ + struct ipipe_percpu_domain_data *p = ipipe_this_cpu_context(ipd); + int l0b, l1b; + + IPIPE_WARN_ONCE(!hard_irqs_disabled()); + + l0b = irq / (BITS_PER_LONG * BITS_PER_LONG); + l1b = irq / BITS_PER_LONG; + + if (likely(!test_bit(IPIPE_LOCK_FLAG, &ipd->irqs[irq].control))) { + __set_bit(irq, p->irqpend_lomap); + __set_bit(l1b, p->irqpend_mdmap); + __set_bit(l0b, &p->irqpend_himap); + } else + __set_bit(irq, p->irqheld_map); + + p->irqall[irq]++; +} +EXPORT_SYMBOL_GPL(__ipipe_set_irq_pending); + +/* Must be called hw IRQs off. */ +void __ipipe_lock_irq(unsigned int irq) +{ + struct ipipe_domain *ipd = ipipe_root_domain; + struct ipipe_percpu_domain_data *p; + int l0b, l1b; + + IPIPE_WARN_ONCE(!hard_irqs_disabled()); + + /* + * Interrupts requested by a registered head domain cannot be + * locked, since this would make no sense: interrupts are + * globally masked at CPU level when the head domain is + * stalled, so there is no way we could encounter the + * situation IRQ locks are handling. + */ + if (test_and_set_bit(IPIPE_LOCK_FLAG, &ipd->irqs[irq].control)) + return; + + l0b = irq / (BITS_PER_LONG * BITS_PER_LONG); + l1b = irq / BITS_PER_LONG; + + p = ipipe_this_cpu_context(ipd); + if (__test_and_clear_bit(irq, p->irqpend_lomap)) { + __set_bit(irq, p->irqheld_map); + if (p->irqpend_lomap[l1b] == 0) { + __clear_bit(l1b, p->irqpend_mdmap); + if (p->irqpend_mdmap[l0b] == 0) + __clear_bit(l0b, &p->irqpend_himap); + } + } +} +EXPORT_SYMBOL_GPL(__ipipe_lock_irq); + +/* Must be called hw IRQs off. */ +void __ipipe_unlock_irq(unsigned int irq) +{ + struct ipipe_domain *ipd = ipipe_root_domain; + struct ipipe_percpu_domain_data *p; + int l0b, l1b, cpu; + + IPIPE_WARN_ONCE(!hard_irqs_disabled()); + + if (!test_and_clear_bit(IPIPE_LOCK_FLAG, &ipd->irqs[irq].control)) + return; + + l0b = irq / (BITS_PER_LONG * BITS_PER_LONG); + l1b = irq / BITS_PER_LONG; + + for_each_online_cpu(cpu) { + p = ipipe_this_cpu_root_context(); + if (test_and_clear_bit(irq, p->irqheld_map)) { + /* We need atomic ops here: */ + set_bit(irq, p->irqpend_lomap); + set_bit(l1b, p->irqpend_mdmap); + set_bit(l0b, &p->irqpend_himap); + } + } +} +EXPORT_SYMBOL_GPL(__ipipe_unlock_irq); + +static inline int __ipipe_next_irq(struct ipipe_percpu_domain_data *p) +{ + int l0b, l1b, l2b; + unsigned long l0m, l1m, l2m; + unsigned int irq; + + l0m = p->irqpend_himap; + if (unlikely(l0m == 0)) + return -1; + + l0b = __ipipe_ffnz(l0m); + l1m = p->irqpend_mdmap[l0b]; + if (unlikely(l1m == 0)) + return -1; + + l1b = __ipipe_ffnz(l1m) + l0b * BITS_PER_LONG; + l2m = p->irqpend_lomap[l1b]; + if (unlikely(l2m == 0)) + return -1; + + l2b = __ipipe_ffnz(l2m); + irq = l1b * BITS_PER_LONG + l2b; + + __clear_bit(irq, p->irqpend_lomap); + if (p->irqpend_lomap[l1b] == 0) { + __clear_bit(l1b, p->irqpend_mdmap); + if (p->irqpend_mdmap[l0b] == 0) + __clear_bit(l0b, &p->irqpend_himap); + } + + return irq; +} + +#else /* __IPIPE_2LEVEL_IRQMAP */ + +/* Must be called hw IRQs off. */ +static inline void __ipipe_set_irq_held(struct ipipe_percpu_domain_data *p, + unsigned int irq) +{ + __set_bit(irq, p->irqheld_map); + p->irqall[irq]++; +} + +/* Must be called hw IRQs off. */ +void __ipipe_set_irq_pending(struct ipipe_domain *ipd, unsigned int irq) +{ + struct ipipe_percpu_domain_data *p = ipipe_this_cpu_context(ipd); + int l0b = irq / BITS_PER_LONG; + + IPIPE_WARN_ONCE(!hard_irqs_disabled()); + + if (likely(!test_bit(IPIPE_LOCK_FLAG, &ipd->irqs[irq].control))) { + __set_bit(irq, p->irqpend_lomap); + __set_bit(l0b, &p->irqpend_himap); + } else + __set_bit(irq, p->irqheld_map); + + p->irqall[irq]++; +} +EXPORT_SYMBOL_GPL(__ipipe_set_irq_pending); + +/* Must be called hw IRQs off. */ +void __ipipe_lock_irq(unsigned int irq) +{ + struct ipipe_percpu_domain_data *p; + int l0b = irq / BITS_PER_LONG; + + IPIPE_WARN_ONCE(!hard_irqs_disabled()); + + if (test_and_set_bit(IPIPE_LOCK_FLAG, + &ipipe_root_domain->irqs[irq].control)) + return; + + p = ipipe_this_cpu_root_context(); + if (__test_and_clear_bit(irq, p->irqpend_lomap)) { + __set_bit(irq, p->irqheld_map); + if (p->irqpend_lomap[l0b] == 0) + __clear_bit(l0b, &p->irqpend_himap); + } +} +EXPORT_SYMBOL_GPL(__ipipe_lock_irq); + +/* Must be called hw IRQs off. */ +void __ipipe_unlock_irq(unsigned int irq) +{ + struct ipipe_domain *ipd = ipipe_root_domain; + struct ipipe_percpu_domain_data *p; + int l0b = irq / BITS_PER_LONG, cpu; + + IPIPE_WARN_ONCE(!hard_irqs_disabled()); + + if (!test_and_clear_bit(IPIPE_LOCK_FLAG, &ipd->irqs[irq].control)) + return; + + for_each_online_cpu(cpu) { + p = ipipe_percpu_context(ipd, cpu); + if (test_and_clear_bit(irq, p->irqheld_map)) { + /* We need atomic ops here: */ + set_bit(irq, p->irqpend_lomap); + set_bit(l0b, &p->irqpend_himap); + } + } +} +EXPORT_SYMBOL_GPL(__ipipe_unlock_irq); + +static inline int __ipipe_next_irq(struct ipipe_percpu_domain_data *p) +{ + unsigned long l0m, l1m; + int l0b, l1b; + + l0m = p->irqpend_himap; + if (unlikely(l0m == 0)) + return -1; + + l0b = __ipipe_ffnz(l0m); + l1m = p->irqpend_lomap[l0b]; + if (unlikely(l1m == 0)) + return -1; + + l1b = __ipipe_ffnz(l1m); + __clear_bit(l1b, &p->irqpend_lomap[l0b]); + if (p->irqpend_lomap[l0b] == 0) + __clear_bit(l0b, &p->irqpend_himap); + + return l0b * BITS_PER_LONG + l1b; +} + +#endif /* __IPIPE_2LEVEL_IRQMAP */ + +void __ipipe_do_sync_pipeline(struct ipipe_domain *top) +{ + struct ipipe_percpu_domain_data *p; + struct ipipe_domain *ipd; + + /* We must enter over the root domain. */ + IPIPE_WARN_ONCE(__ipipe_current_domain != ipipe_root_domain); + ipd = top; +next: + p = ipipe_this_cpu_context(ipd); + if (test_bit(IPIPE_STALL_FLAG, &p->status)) + return; + + if (__ipipe_ipending_p(p)) { + if (ipd == ipipe_root_domain) + __ipipe_sync_stage(); + else { + /* Switching to head. */ + p->coflags &= ~__IPIPE_ALL_R; + __ipipe_set_current_context(p); + __ipipe_sync_stage(); + __ipipe_set_current_domain(ipipe_root_domain); + } + } + + if (ipd != ipipe_root_domain) { + ipd = ipipe_root_domain; + goto next; + } +} +EXPORT_SYMBOL_GPL(__ipipe_do_sync_pipeline); + +unsigned int ipipe_alloc_virq(void) +{ + unsigned long flags, irq = 0; + int ipos; + + raw_spin_lock_irqsave(&__ipipe_lock, flags); + + if (__ipipe_virtual_irq_map != ~0) { + ipos = ffz(__ipipe_virtual_irq_map); + set_bit(ipos, &__ipipe_virtual_irq_map); + irq = ipos + IPIPE_VIRQ_BASE; + } + + raw_spin_unlock_irqrestore(&__ipipe_lock, flags); + + return irq; +} +EXPORT_SYMBOL_GPL(ipipe_alloc_virq); + +void ipipe_free_virq(unsigned int virq) +{ + clear_bit(virq - IPIPE_VIRQ_BASE, &__ipipe_virtual_irq_map); + smp_mb__after_atomic(); +} +EXPORT_SYMBOL_GPL(ipipe_free_virq); + +int ipipe_request_irq(struct ipipe_domain *ipd, + unsigned int irq, + ipipe_irq_handler_t handler, + void *cookie, + ipipe_irq_ackfn_t ackfn) +{ + unsigned long flags; + int ret = 0; + +#ifndef CONFIG_IPIPE_LEGACY + ipipe_root_only(); +#endif /* CONFIG_IPIPE_LEGACY */ + + if (handler == NULL || + (irq >= IPIPE_NR_XIRQS && !ipipe_virtual_irq_p(irq))) + return -EINVAL; + + raw_spin_lock_irqsave(&__ipipe_lock, flags); + + if (ipd->irqs[irq].handler) { + ret = -EBUSY; + goto out; + } + + if (ackfn == NULL) + ackfn = ipipe_root_domain->irqs[irq].ackfn; + + ipd->irqs[irq].handler = handler; + ipd->irqs[irq].cookie = cookie; + ipd->irqs[irq].ackfn = ackfn; + ipd->irqs[irq].control = IPIPE_HANDLE_MASK; + + if (irq < IPIPE_NR_ROOT_IRQS) + __ipipe_enable_irqdesc(ipd, irq); +out: + raw_spin_unlock_irqrestore(&__ipipe_lock, flags); + + return ret; +} +EXPORT_SYMBOL_GPL(ipipe_request_irq); + +void ipipe_free_irq(struct ipipe_domain *ipd, + unsigned int irq) +{ + unsigned long flags; + +#ifndef CONFIG_IPIPE_LEGACY + ipipe_root_only(); +#endif /* CONFIG_IPIPE_LEGACY */ + + raw_spin_lock_irqsave(&__ipipe_lock, flags); + + if (ipd->irqs[irq].handler == NULL) + goto out; + + ipd->irqs[irq].handler = NULL; + ipd->irqs[irq].cookie = NULL; + ipd->irqs[irq].ackfn = NULL; + ipd->irqs[irq].control = 0; + + if (irq < IPIPE_NR_ROOT_IRQS) + __ipipe_disable_irqdesc(ipd, irq); +out: + raw_spin_unlock_irqrestore(&__ipipe_lock, flags); +} +EXPORT_SYMBOL_GPL(ipipe_free_irq); + +void ipipe_set_hooks(struct ipipe_domain *ipd, int enables) +{ + struct ipipe_percpu_domain_data *p; + unsigned long flags; + int cpu, wait; + + if (ipd == ipipe_root_domain) { + IPIPE_WARN(enables & __IPIPE_TRAP_E); + enables &= ~__IPIPE_TRAP_E; + } else { + IPIPE_WARN(enables & __IPIPE_KEVENT_E); + enables &= ~__IPIPE_KEVENT_E; + } + + flags = ipipe_critical_enter(NULL); + + for_each_online_cpu(cpu) { + p = ipipe_percpu_context(ipd, cpu); + p->coflags &= ~__IPIPE_ALL_E; + p->coflags |= enables; + } + + wait = (enables ^ __IPIPE_ALL_E) << __IPIPE_SHIFT_R; + if (wait == 0 || !__ipipe_root_p) { + ipipe_critical_exit(flags); + return; + } + + ipipe_this_cpu_context(ipd)->coflags &= ~wait; + + ipipe_critical_exit(flags); + + /* + * In case we cleared some hooks over the root domain, we have + * to wait for any ongoing execution to finish, since our + * caller might subsequently unmap the target domain code. + * + * We synchronize with the relevant __ipipe_notify_*() + * helpers, disabling all hooks before we start waiting for + * completion on all CPUs. + */ + for_each_online_cpu(cpu) { + while (ipipe_percpu_context(ipd, cpu)->coflags & wait) + schedule_timeout_interruptible(HZ / 50); + } +} +EXPORT_SYMBOL_GPL(ipipe_set_hooks); + +int __weak ipipe_fastcall_hook(struct pt_regs *regs) +{ + return -1; /* i.e. fall back to slow path. */ +} + +int __weak ipipe_syscall_hook(struct ipipe_domain *ipd, struct pt_regs *regs) +{ + return 0; +} + +void __ipipe_root_sync(void) +{ + struct ipipe_percpu_domain_data *p; + unsigned long flags; + + flags = hard_local_irq_save(); + + p = ipipe_this_cpu_root_context(); + if (__ipipe_ipending_p(p)) + __ipipe_sync_stage(); + + hard_local_irq_restore(flags); +} + +int __ipipe_notify_syscall(struct pt_regs *regs) +{ + struct ipipe_domain *caller_domain, *this_domain, *ipd; + struct ipipe_percpu_domain_data *p; + unsigned long flags; + int ret = 0; + + /* + * We should definitely not pipeline a syscall with IRQs off. + */ + IPIPE_WARN_ONCE(hard_irqs_disabled()); + + flags = hard_local_irq_save(); + caller_domain = this_domain = __ipipe_current_domain; + ipd = ipipe_head_domain; +next: + p = ipipe_this_cpu_context(ipd); + if (likely(p->coflags & __IPIPE_SYSCALL_E)) { + __ipipe_set_current_context(p); + p->coflags |= __IPIPE_SYSCALL_R; + hard_local_irq_restore(flags); + ret = ipipe_syscall_hook(caller_domain, regs); + flags = hard_local_irq_save(); + p->coflags &= ~__IPIPE_SYSCALL_R; + if (__ipipe_current_domain != ipd) + /* Account for domain migration. */ + this_domain = __ipipe_current_domain; + else + __ipipe_set_current_domain(this_domain); + } + + if (this_domain == ipipe_root_domain) { + if (ipd != ipipe_root_domain && ret == 0) { + ipd = ipipe_root_domain; + goto next; + } + /* + * Careful: we may have migrated from head->root, so p + * would be ipipe_this_cpu_context(head). + */ + p = ipipe_this_cpu_root_context(); + if (__ipipe_ipending_p(p)) + __ipipe_sync_stage(); + } else if (ipipe_test_thread_flag(TIP_MAYDAY)) + __ipipe_call_mayday(regs); + + hard_local_irq_restore(flags); + + return ret; +} + +int __weak ipipe_trap_hook(struct ipipe_trap_data *data) +{ + return 0; +} + +int __ipipe_notify_trap(int exception, struct pt_regs *regs) +{ + struct ipipe_percpu_domain_data *p; + struct ipipe_trap_data data; + unsigned long flags; + int ret = 0; + + flags = hard_local_irq_save(); + + /* + * We send a notification about all traps raised over a + * registered head domain only. + */ + if (__ipipe_root_p) + goto out; + + p = ipipe_this_cpu_head_context(); + if (likely(p->coflags & __IPIPE_TRAP_E)) { + p->coflags |= __IPIPE_TRAP_R; + hard_local_irq_restore(flags); + data.exception = exception; + data.regs = regs; + ret = ipipe_trap_hook(&data); + flags = hard_local_irq_save(); + p->coflags &= ~__IPIPE_TRAP_R; + } +out: + hard_local_irq_restore(flags); + + return ret; +} + +int __weak ipipe_kevent_hook(int kevent, void *data) +{ + return 0; +} + +int __ipipe_notify_kevent(int kevent, void *data) +{ + struct ipipe_percpu_domain_data *p; + unsigned long flags; + int ret = 0; + + ipipe_root_only(); + + flags = hard_local_irq_save(); + + p = ipipe_this_cpu_root_context(); + if (likely(p->coflags & __IPIPE_KEVENT_E)) { + p->coflags |= __IPIPE_KEVENT_R; + hard_local_irq_restore(flags); + ret = ipipe_kevent_hook(kevent, data); + flags = hard_local_irq_save(); + p->coflags &= ~__IPIPE_KEVENT_R; + } + + hard_local_irq_restore(flags); + + return ret; +} + +void __weak ipipe_migration_hook(struct task_struct *p) +{ +} + +#ifdef CONFIG_IPIPE_LEGACY + +static inline void complete_domain_migration(void) /* hw IRQs off */ +{ + if (current->state & TASK_HARDENING) { + current->state &= ~TASK_HARDENING; + ipipe_set_thread_flag(TIP_HEAD); + } +} + +#else /* !CONFIG_IPIPE_LEGACY */ + +static void complete_domain_migration(void) /* hw IRQs off */ +{ + struct ipipe_percpu_domain_data *p; + struct ipipe_percpu_data *pd; + struct task_struct *t; + + ipipe_root_only(); + pd = raw_cpu_ptr(&ipipe_percpu); + t = pd->task_hijacked; + if (t == NULL) + return; + + pd->task_hijacked = NULL; + t->state &= ~TASK_HARDENING; + if (t->state != TASK_INTERRUPTIBLE) + /* Migration aborted (by signal). */ + return; + + ipipe_set_ti_thread_flag(task_thread_info(t), TIP_HEAD); + p = ipipe_this_cpu_head_context(); + IPIPE_WARN_ONCE(test_bit(IPIPE_STALL_FLAG, &p->status)); + /* + * hw IRQs are disabled, but the completion hook assumes the + * head domain is logically stalled: fix it up. + */ + __set_bit(IPIPE_STALL_FLAG, &p->status); + ipipe_migration_hook(t); + __clear_bit(IPIPE_STALL_FLAG, &p->status); + if (__ipipe_ipending_p(p)) + __ipipe_sync_pipeline(p->domain); +} + +#endif /* !CONFIG_IPIPE_LEGACY */ + +void __ipipe_complete_domain_migration(void) +{ + unsigned long flags; + + flags = hard_local_irq_save(); + complete_domain_migration(); + hard_local_irq_restore(flags); +} +EXPORT_SYMBOL_GPL(__ipipe_complete_domain_migration); + +int __ipipe_switch_tail(void) +{ + int x; + +#ifdef CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH + hard_local_irq_disable(); +#endif + x = __ipipe_root_p; +#ifndef CONFIG_IPIPE_LEGACY + if (x) +#endif + complete_domain_migration(); + +#ifndef CONFIG_IPIPE_WANT_PREEMPTIBLE_SWITCH + if (x) +#endif + hard_local_irq_enable(); + + return !x; +} + +#ifdef CONFIG_IPIPE_HAVE_VM_NOTIFIER +void __ipipe_notify_vm_preemption(void) +{ + struct ipipe_vm_notifier *vmf; + struct ipipe_percpu_data *p; + + ipipe_check_irqoff(); + p = __ipipe_raw_cpu_ptr(&ipipe_percpu); + vmf = p->vm_notifier; + if (unlikely(vmf)) + vmf->handler(vmf); +} +EXPORT_SYMBOL_GPL(__ipipe_notify_vm_preemption); +#endif /* CONFIG_IPIPE_HAVE_VM_NOTIFIER */ + +static void dispatch_irq_head(unsigned int irq) /* hw interrupts off */ +{ + struct ipipe_percpu_domain_data *p = ipipe_this_cpu_head_context(), *old; + struct ipipe_domain *head = p->domain; + + if (unlikely(test_bit(IPIPE_STALL_FLAG, &p->status))) { + __ipipe_set_irq_pending(head, irq); + return; + } + + /* Switch to the head domain if not current. */ + old = __ipipe_current_context; + if (old != p) + __ipipe_set_current_context(p); + + p->irqall[irq]++; + __set_bit(IPIPE_STALL_FLAG, &p->status); + barrier(); + head->irqs[irq].handler(irq, head->irqs[irq].cookie); + __ipipe_run_irqtail(irq); + hard_local_irq_disable(); + p = ipipe_this_cpu_head_context(); + __clear_bit(IPIPE_STALL_FLAG, &p->status); + + /* Are we still running in the head domain? */ + if (likely(__ipipe_current_context == p)) { + /* Did we enter this code over the head domain? */ + if (old->domain == head) { + /* Yes, do immediate synchronization. */ + if (__ipipe_ipending_p(p)) + __ipipe_sync_stage(); + return; + } + __ipipe_set_current_context(ipipe_this_cpu_root_context()); + } + + /* + * We must be running over the root domain, synchronize + * the pipeline for high priority IRQs (slow path). + */ + __ipipe_do_sync_pipeline(head); +} + +void __ipipe_dispatch_irq(unsigned int irq, int flags) /* hw interrupts off */ +{ + struct ipipe_domain *ipd; + struct irq_desc *desc; + unsigned long control; + int chained_irq; + + /* + * Survival kit when reading this code: + * + * - we have two main situations, leading to three cases for + * handling interrupts: + * + * a) the root domain is alone, no registered head domain + * => all interrupts go through the interrupt log + * b) a head domain is registered + * => head domain IRQs go through the fast dispatcher + * => root domain IRQs go through the interrupt log + * + * - when no head domain is registered, ipipe_head_domain == + * ipipe_root_domain == &ipipe_root. + * + * - the caller tells us whether we should acknowledge this + * IRQ. Even virtual IRQs may require acknowledge on some + * platforms (e.g. arm/SMP). + * + * - the caller tells us whether we may try to run the IRQ log + * syncer. Typically, demuxed IRQs won't be synced + * immediately. + * + * - multiplex IRQs most likely have a valid acknowledge + * handler and we may not be called with IPIPE_IRQF_NOACK + * for them. The ack handler for the multiplex IRQ actually + * decodes the demuxed interrupts. + */ + +#ifdef CONFIG_IPIPE_DEBUG + if (unlikely(irq >= IPIPE_NR_IRQS) || + (irq < IPIPE_NR_ROOT_IRQS && irq_to_desc(irq) == NULL)) { + pr_err("I-pipe: spurious interrupt %u\n", irq); + return; + } +#endif + /* + * CAUTION: on some archs, virtual IRQs may have acknowledge + * handlers. Multiplex IRQs should have one too. + */ + if (unlikely(irq >= IPIPE_NR_ROOT_IRQS)) { + desc = NULL; + chained_irq = 0; + } else { + desc = irq_to_desc(irq); + chained_irq = desc ? ipipe_chained_irq_p(desc) : 0; + } + if (flags & IPIPE_IRQF_NOACK) + IPIPE_WARN_ONCE(chained_irq); + else { + ipd = ipipe_head_domain; + control = ipd->irqs[irq].control; + if ((control & IPIPE_HANDLE_MASK) == 0) + ipd = ipipe_root_domain; + if (ipd->irqs[irq].ackfn) + ipd->irqs[irq].ackfn(irq, desc); + if (chained_irq) { + if ((flags & IPIPE_IRQF_NOSYNC) == 0) + /* Run demuxed IRQ handlers. */ + goto sync; + return; + } + } + + /* + * Sticky interrupts must be handled early and separately, so + * that we always process them on the current domain. + */ + ipd = __ipipe_current_domain; + control = ipd->irqs[irq].control; + if (control & IPIPE_STICKY_MASK) + goto log; + + /* + * In case we have no registered head domain + * (i.e. ipipe_head_domain == &ipipe_root), we always go + * through the interrupt log, and leave the dispatching work + * ultimately to __ipipe_sync_pipeline(). + */ + ipd = ipipe_head_domain; + control = ipd->irqs[irq].control; + if (ipd == ipipe_root_domain) + /* + * The root domain must handle all interrupts, so + * testing the HANDLE bit would be pointless. + */ + goto log; + + if (control & IPIPE_HANDLE_MASK) { + if (unlikely(flags & IPIPE_IRQF_NOSYNC)) + __ipipe_set_irq_pending(ipd, irq); + else + dispatch_irq_head(irq); + return; + } + + ipd = ipipe_root_domain; +log: + __ipipe_set_irq_pending(ipd, irq); + + if (flags & IPIPE_IRQF_NOSYNC) + return; + + /* + * Optimize if we preempted a registered high priority head + * domain: we don't need to synchronize the pipeline unless + * there is a pending interrupt for it. + */ + if (!__ipipe_root_p && + !__ipipe_ipending_p(ipipe_this_cpu_head_context())) + return; +sync: + __ipipe_sync_pipeline(ipipe_head_domain); +} + +void ipipe_raise_irq(unsigned int irq) +{ + struct ipipe_domain *ipd = ipipe_head_domain; + unsigned long flags, control; + + flags = hard_local_irq_save(); + + /* + * Fast path: raising a virtual IRQ handled by the head + * domain. + */ + if (likely(ipipe_virtual_irq_p(irq) && ipd != ipipe_root_domain)) { + control = ipd->irqs[irq].control; + if (likely(control & IPIPE_HANDLE_MASK)) { + dispatch_irq_head(irq); + goto out; + } + } + + /* Emulate regular device IRQ receipt. */ + __ipipe_dispatch_irq(irq, IPIPE_IRQF_NOACK); +out: + hard_local_irq_restore(flags); + +} +EXPORT_SYMBOL_GPL(ipipe_raise_irq); + +#ifdef CONFIG_PREEMPT + +void preempt_schedule_irq(void); + +void __sched __ipipe_preempt_schedule_irq(void) +{ + struct ipipe_percpu_domain_data *p; + unsigned long flags; + + BUG_ON(!hard_irqs_disabled()); + local_irq_save(flags); + hard_local_irq_enable(); + preempt_schedule_irq(); /* Ok, may reschedule now. */ + hard_local_irq_disable(); + + /* + * Flush any pending interrupt that may have been logged after + * preempt_schedule_irq() stalled the root stage before + * returning to us, and now. + */ + p = ipipe_this_cpu_root_context(); + if (unlikely(__ipipe_ipending_p(p))) { + __preempt_count_add(PREEMPT_ACTIVE); + trace_hardirqs_on(); + __clear_bit(IPIPE_STALL_FLAG, &p->status); + __ipipe_sync_stage(); + __preempt_count_sub(PREEMPT_ACTIVE); + } + + __ipipe_restore_root_nosync(flags); +} + +#else /* !CONFIG_PREEMPT */ + +#define __ipipe_preempt_schedule_irq() do { } while (0) + +#endif /* !CONFIG_PREEMPT */ + +#ifdef CONFIG_TRACE_IRQFLAGS +#define root_stall_after_handler() local_irq_disable() +#else +#define root_stall_after_handler() do { } while (0) +#endif + +/* + * __ipipe_do_sync_stage() -- Flush the pending IRQs for the current + * domain (and processor). This routine flushes the interrupt log (see + * "Optimistic interrupt protection" from D. Stodolsky et al. for more + * on the deferred interrupt scheme). Every interrupt that occurred + * while the pipeline was stalled gets played. + * + * WARNING: CPU migration may occur over this routine. + */ +void __ipipe_do_sync_stage(void) +{ + struct ipipe_percpu_domain_data *p; + struct ipipe_domain *ipd; + int irq; + + p = __ipipe_current_context; +respin: + ipd = p->domain; + + __set_bit(IPIPE_STALL_FLAG, &p->status); + smp_wmb(); + + if (ipd == ipipe_root_domain) + trace_hardirqs_off(); + + for (;;) { + irq = __ipipe_next_irq(p); + if (irq < 0) + break; + /* + * Make sure the compiler does not reorder wrongly, so + * that all updates to maps are done before the + * handler gets called. + */ + barrier(); + + if (test_bit(IPIPE_LOCK_FLAG, &ipd->irqs[irq].control)) + continue; + + if (ipd != ipipe_head_domain) + hard_local_irq_enable(); + + if (likely(ipd != ipipe_root_domain)) { + ipd->irqs[irq].handler(irq, ipd->irqs[irq].cookie); + __ipipe_run_irqtail(irq); + hard_local_irq_disable(); + } else if (ipipe_virtual_irq_p(irq)) { + irq_enter(); + ipd->irqs[irq].handler(irq, ipd->irqs[irq].cookie); + irq_exit(); + root_stall_after_handler(); + hard_local_irq_disable(); + while (__ipipe_check_root_resched()) + __ipipe_preempt_schedule_irq(); + } else { + ipd->irqs[irq].handler(irq, ipd->irqs[irq].cookie); + root_stall_after_handler(); + hard_local_irq_disable(); + } + + /* + * We may have migrated to a different CPU (1) upon + * return from the handler, or downgraded from the + * head domain to the root one (2), the opposite way + * is NOT allowed though. + * + * (1) reload the current per-cpu context pointer, so + * that we further pull pending interrupts from the + * proper per-cpu log. + * + * (2) check the stall bit to know whether we may + * dispatch any interrupt pending for the root domain, + * and respin the entire dispatch loop if + * so. Otherwise, immediately return to the caller, + * _without_ affecting the stall state for the root + * domain, since we do not own it at this stage. This + * case is basically reflecting what may happen in + * dispatch_irq_head() for the fast path. + */ + p = __ipipe_current_context; + if (p->domain != ipd) { + IPIPE_BUG_ON(ipd == ipipe_root_domain); + if (test_bit(IPIPE_STALL_FLAG, &p->status)) + return; + goto respin; + } + } + + if (ipd == ipipe_root_domain) + trace_hardirqs_on(); + + __clear_bit(IPIPE_STALL_FLAG, &p->status); +} + +void __ipipe_call_mayday(struct pt_regs *regs) +{ + unsigned long flags; + + ipipe_clear_thread_flag(TIP_MAYDAY); + flags = hard_local_irq_save(); + __ipipe_notify_trap(IPIPE_TRAP_MAYDAY, regs); + hard_local_irq_restore(flags); +} + +#ifdef CONFIG_SMP + +/* Always called with hw interrupts off. */ +void __ipipe_do_critical_sync(unsigned int irq, void *cookie) +{ + int cpu = ipipe_processor_id(); + + cpumask_set_cpu(cpu, &__ipipe_cpu_sync_map); + + /* + * Now we are in sync with the lock requestor running on + * another CPU. Enter a spinning wait until he releases the + * global lock. + */ + raw_spin_lock(&__ipipe_cpu_barrier); + + /* Got it. Now get out. */ + + /* Call the sync routine if any. */ + if (__ipipe_cpu_sync) + __ipipe_cpu_sync(); + + cpumask_set_cpu(cpu, &__ipipe_cpu_pass_map); + + raw_spin_unlock(&__ipipe_cpu_barrier); + + cpumask_clear_cpu(cpu, &__ipipe_cpu_sync_map); +} +#endif /* CONFIG_SMP */ + +unsigned long ipipe_critical_enter(void (*syncfn)(void)) +{ + cpumask_t allbutself __maybe_unused, online __maybe_unused; + int cpu __maybe_unused, n __maybe_unused; + unsigned long flags, loops __maybe_unused; + + flags = hard_local_irq_save(); + + if (num_online_cpus() == 1) + return flags; + +#ifdef CONFIG_SMP + + cpu = ipipe_processor_id(); + if (!cpumask_test_and_set_cpu(cpu, &__ipipe_cpu_lock_map)) { + while (test_and_set_bit(0, &__ipipe_critical_lock)) { + n = 0; + hard_local_irq_enable(); + + do + cpu_relax(); + while (++n < cpu); + + hard_local_irq_disable(); + } +restart: + online = *cpu_online_mask; + raw_spin_lock(&__ipipe_cpu_barrier); + + __ipipe_cpu_sync = syncfn; + + cpumask_clear(&__ipipe_cpu_pass_map); + cpumask_set_cpu(cpu, &__ipipe_cpu_pass_map); + + /* + * Send the sync IPI to all processors but the current + * one. + */ + cpumask_andnot(&allbutself, &online, &__ipipe_cpu_pass_map); + ipipe_send_ipi(IPIPE_CRITICAL_IPI, allbutself); + loops = IPIPE_CRITICAL_TIMEOUT; + + while (!cpumask_equal(&__ipipe_cpu_sync_map, &allbutself)) { + if (--loops > 0) { + cpu_relax(); + continue; + } + /* + * We ran into a deadlock due to a contended + * rwlock. Cancel this round and retry. + */ + __ipipe_cpu_sync = NULL; + + raw_spin_unlock(&__ipipe_cpu_barrier); + /* + * Ensure all CPUs consumed the IPI to avoid + * running __ipipe_cpu_sync prematurely. This + * usually resolves the deadlock reason too. + */ + while (!cpumask_equal(&online, &__ipipe_cpu_pass_map)) + cpu_relax(); + + goto restart; + } + } + + atomic_inc(&__ipipe_critical_count); + +#endif /* CONFIG_SMP */ + + return flags; +} +EXPORT_SYMBOL_GPL(ipipe_critical_enter); + +void ipipe_critical_exit(unsigned long flags) +{ + if (num_online_cpus() == 1) { + hard_local_irq_restore(flags); + return; + } + +#ifdef CONFIG_SMP + if (atomic_dec_and_test(&__ipipe_critical_count)) { + raw_spin_unlock(&__ipipe_cpu_barrier); + while (!cpumask_empty(&__ipipe_cpu_sync_map)) + cpu_relax(); + cpumask_clear_cpu(ipipe_processor_id(), &__ipipe_cpu_lock_map); + clear_bit(0, &__ipipe_critical_lock); + smp_mb__after_atomic(); + } +#endif /* CONFIG_SMP */ + + hard_local_irq_restore(flags); +} +EXPORT_SYMBOL_GPL(ipipe_critical_exit); + +#ifdef CONFIG_IPIPE_DEBUG_CONTEXT + +void ipipe_root_only(void) +{ + struct ipipe_domain *this_domain; + unsigned long flags; + + flags = hard_smp_local_irq_save(); + + this_domain = __ipipe_current_domain; + if (likely(this_domain == ipipe_root_domain && + !test_bit(IPIPE_STALL_FLAG, &__ipipe_head_status))) { + hard_smp_local_irq_restore(flags); + return; + } + + if (!__this_cpu_read(ipipe_percpu.context_check)) { + hard_smp_local_irq_restore(flags); + return; + } + + hard_smp_local_irq_restore(flags); + + ipipe_prepare_panic(); + ipipe_trace_panic_freeze(); + + if (this_domain != ipipe_root_domain) + pr_err("I-pipe: Detected illicit call from head domain '%s'\n" + " into a regular Linux service\n", + this_domain->name); + else + pr_err("I-pipe: Detected stalled head domain, " + "probably caused by a bug.\n" + " A critical section may have been " + "left unterminated.\n"); + dump_stack(); + ipipe_trace_panic_dump(); +} +EXPORT_SYMBOL(ipipe_root_only); + +#endif /* CONFIG_IPIPE_DEBUG_CONTEXT */ + +#if defined(CONFIG_IPIPE_DEBUG_INTERNAL) && defined(CONFIG_SMP) + +int notrace __ipipe_check_percpu_access(void) +{ + struct ipipe_percpu_domain_data *p; + struct ipipe_domain *this_domain; + unsigned long flags; + int ret = 0; + + flags = hard_local_irq_save_notrace(); + + /* + * Don't use __ipipe_current_domain here, this would recurse + * indefinitely. + */ + this_domain = raw_cpu_read(ipipe_percpu.curr)->domain; + + /* + * Only the root domain may implement preemptive CPU migration + * of tasks, so anything above in the pipeline should be fine. + */ + if (this_domain != ipipe_root_domain) + goto out; + + if (raw_irqs_disabled_flags(flags)) + goto out; + + /* + * Last chance: hw interrupts were enabled on entry while + * running over the root domain, but the root stage might be + * currently stalled, in which case preemption would be + * disabled, and no migration could occur. + */ + + p = raw_cpu_ptr(&ipipe_percpu.root); + if (!preemptible()) + goto out; + /* + * Our caller may end up accessing the wrong per-cpu variable + * instance due to CPU migration; tell it to complain about + * this. + */ + ret = 1; +out: + hard_local_irq_restore_notrace(flags); + + return ret; +} +EXPORT_SYMBOL_GPL(__ipipe_check_percpu_access); + +void __ipipe_spin_unlock_debug(unsigned long flags) +{ + /* + * We catch a nasty issue where spin_unlock_irqrestore() on a + * regular kernel spinlock is about to re-enable hw interrupts + * in a section entered with hw irqs off. This is clearly the + * sign of a massive breakage coming. Usual suspect is a + * regular spinlock which was overlooked, used within a + * section which must run with hw irqs disabled. + */ + IPIPE_WARN_ONCE(!raw_irqs_disabled_flags(flags) && hard_irqs_disabled()); +} +EXPORT_SYMBOL(__ipipe_spin_unlock_debug); + +#endif /* CONFIG_IPIPE_DEBUG_INTERNAL && CONFIG_SMP */ + +void ipipe_prepare_panic(void) +{ +#ifdef CONFIG_PRINTK + __ipipe_printk_bypass = 1; +#endif + ipipe_context_check_off(); +} +EXPORT_SYMBOL_GPL(ipipe_prepare_panic); + +static void __ipipe_do_work(unsigned int virq, void *cookie) +{ + struct ipipe_work_header *work; + unsigned long flags; + void *curr, *tail; + int cpu; + + /* + * Work is dispatched in enqueuing order. This interrupt + * context can't migrate to another CPU. + */ + cpu = smp_processor_id(); + curr = per_cpu(work_buf, cpu); + + for (;;) { + flags = hard_local_irq_save(); + tail = per_cpu(work_tail, cpu); + if (curr == tail) { + per_cpu(work_tail, cpu) = per_cpu(work_buf, cpu); + hard_local_irq_restore(flags); + return; + } + work = curr; + curr += work->size; + hard_local_irq_restore(flags); + work->handler(work); + } +} + +void __ipipe_post_work_root(struct ipipe_work_header *work) +{ + unsigned long flags; + void *tail; + int cpu; + + /* + * Subtle: we want to use the head stall/unstall operators, + * not the hard_* routines to protect against races. This way, + * we ensure that a root-based caller will trigger the virq + * handling immediately when unstalling the head stage, as a + * result of calling __ipipe_sync_pipeline() under the hood. + */ + flags = ipipe_test_and_stall_head(); + cpu = ipipe_processor_id(); + tail = per_cpu(work_tail, cpu); + + if (WARN_ON_ONCE((unsigned char *)tail + work->size >= + per_cpu(work_buf, cpu) + WORKBUF_SIZE)) + goto out; + + /* Work handling is deferred, so data has to be copied. */ + memcpy(tail, work, work->size); + per_cpu(work_tail, cpu) = tail + work->size; + ipipe_post_irq_root(__ipipe_work_virq); +out: + ipipe_restore_head(flags); +} +EXPORT_SYMBOL_GPL(__ipipe_post_work_root); + +void __weak __ipipe_arch_share_current(int flags) +{ +} + +void __ipipe_share_current(int flags) +{ + ipipe_root_only(); + + __ipipe_arch_share_current(flags); +} +EXPORT_SYMBOL_GPL(__ipipe_share_current); + +#ifdef CONFIG_KGDB +bool __ipipe_probe_access; + +long ipipe_probe_kernel_read(void *dst, void *src, size_t size) +{ + long ret; + mm_segment_t old_fs = get_fs(); + + set_fs(KERNEL_DS); + __ipipe_probe_access = true; + barrier(); + ret = __copy_from_user_inatomic(dst, + (__force const void __user *)src, size); + barrier(); + __ipipe_probe_access = false; + set_fs(old_fs); + + return ret ? -EFAULT : 0; +} + +long ipipe_probe_kernel_write(void *dst, void *src, size_t size) +{ + long ret; + mm_segment_t old_fs = get_fs(); + + set_fs(KERNEL_DS); + __ipipe_probe_access = true; + barrier(); + ret = __copy_to_user_inatomic((__force void __user *)dst, src, size); + barrier(); + __ipipe_probe_access = false; + set_fs(old_fs); + + return ret ? -EFAULT : 0; +} +#endif /* CONFIG_KGDB */ + +#if defined(CONFIG_DEBUG_ATOMIC_SLEEP) || defined(CONFIG_PROVE_LOCKING) || \ + defined(CONFIG_PREEMPT_VOLUNTARY) || defined(CONFIG_IPIPE_DEBUG_CONTEXT) +void __ipipe_uaccess_might_fault(void) +{ + struct ipipe_percpu_domain_data *pdd; + struct ipipe_domain *ipd; + unsigned long flags; + + flags = hard_local_irq_save(); + ipd = __ipipe_current_domain; + if (ipd == ipipe_root_domain) { + hard_local_irq_restore(flags); + might_fault(); + return; + } + +#ifdef CONFIG_IPIPE_DEBUG_CONTEXT + pdd = ipipe_this_cpu_context(ipd); + WARN_ON_ONCE(hard_irqs_disabled_flags(flags) + || test_bit(IPIPE_STALL_FLAG, &pdd->status)); +#else /* !CONFIG_IPIPE_DEBUG_CONTEXT */ + (void)pdd; +#endif /* !CONFIG_IPIPE_DEBUG_CONTEXT */ + hard_local_irq_restore(flags); + +} +EXPORT_SYMBOL_GPL(__ipipe_uaccess_might_fault); +#endif diff --git a/kernel/ipipe/timer.c b/kernel/ipipe/timer.c new file mode 100644 index 0000000..a9917f4 --- /dev/null +++ b/kernel/ipipe/timer.c @@ -0,0 +1,520 @@ +/* -*- linux-c -*- + * linux/kernel/ipipe/timer.c + * + * Copyright (C) 2012 Gilles Chanteperdrix + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139, + * USA; either version 2 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * + * I-pipe timer request interface. + */ +#include +#include +#include +#include +#include +#include +#include +#include + +unsigned long __ipipe_hrtimer_freq; + +static LIST_HEAD(timers); +static IPIPE_DEFINE_SPINLOCK(lock); + +static DEFINE_PER_CPU(struct ipipe_timer *, percpu_timer); + +#ifdef CONFIG_GENERIC_CLOCKEVENTS +/* + * Default request method: switch to oneshot mode if supported. + */ +static void ipipe_timer_default_request(struct ipipe_timer *timer, int steal) +{ + struct clock_event_device *evtdev = timer->host_timer; + + if (!(evtdev->features & CLOCK_EVT_FEAT_ONESHOT)) + return; + + if (evtdev->mode != CLOCK_EVT_MODE_ONESHOT) { + evtdev->set_mode(CLOCK_EVT_MODE_ONESHOT, evtdev); + evtdev->set_next_event(timer->freq / HZ, evtdev); + } +} + +/* + * Default release method: return the timer to the mode it had when + * starting. + */ +static void ipipe_timer_default_release(struct ipipe_timer *timer) +{ + struct clock_event_device *evtdev = timer->host_timer; + + evtdev->set_mode(evtdev->mode, evtdev); + if (evtdev->mode == CLOCK_EVT_MODE_ONESHOT) + evtdev->set_next_event(timer->freq / HZ, evtdev); +} + +void ipipe_host_timer_register(struct clock_event_device *evtdev) +{ + struct ipipe_timer *timer = evtdev->ipipe_timer; + + if (timer == NULL) + return; + + if (timer->request == NULL) + timer->request = ipipe_timer_default_request; + + /* + * By default, use the same method as linux timer, on ARM at + * least, most set_next_event methods are safe to be called + * from Xenomai domain anyway. + */ + if (timer->set == NULL) { + timer->timer_set = evtdev; + timer->set = (typeof(timer->set))evtdev->set_next_event; + } + + if (timer->release == NULL) + timer->release = ipipe_timer_default_release; + + if (timer->name == NULL) + timer->name = evtdev->name; + + if (timer->rating == 0) + timer->rating = evtdev->rating; + + timer->freq = (1000000000ULL * evtdev->mult) >> evtdev->shift; + + if (timer->min_delay_ticks == 0) + timer->min_delay_ticks = + (evtdev->min_delta_ns * evtdev->mult) >> evtdev->shift; + + if (timer->cpumask == NULL) + timer->cpumask = evtdev->cpumask; + + timer->host_timer = evtdev; + + ipipe_timer_register(timer); +} +#endif /* CONFIG_GENERIC_CLOCKEVENTS */ + +/* + * register a timer: maintain them in a list sorted by rating + */ +void ipipe_timer_register(struct ipipe_timer *timer) +{ + struct ipipe_timer *t; + unsigned long flags; + + if (timer->timer_set == NULL) + timer->timer_set = timer; + + if (timer->cpumask == NULL) + timer->cpumask = cpumask_of(smp_processor_id()); + + raw_spin_lock_irqsave(&lock, flags); + + list_for_each_entry(t, &timers, link) { + if (t->rating <= timer->rating) { + __list_add(&timer->link, t->link.prev, &t->link); + goto done; + } + } + list_add_tail(&timer->link, &timers); + done: + raw_spin_unlock_irqrestore(&lock, flags); +} + +static void ipipe_timer_request_sync(void) +{ + struct ipipe_timer *timer = __ipipe_raw_cpu_read(percpu_timer); + struct clock_event_device *evtdev; + int steal; + + if (!timer) + return; + + evtdev = timer->host_timer; + +#ifdef CONFIG_GENERIC_CLOCKEVENTS + steal = evtdev != NULL && evtdev->mode != CLOCK_EVT_MODE_UNUSED; +#else /* !CONFIG_GENERIC_CLOCKEVENTS */ + steal = 1; +#endif /* !CONFIG_GENERIC_CLOCKEVENTS */ + + timer->request(timer, steal); +} + +static void config_pcpu_timer(struct ipipe_timer *t, unsigned hrclock_freq) +{ + unsigned long long tmp; + unsigned hrtimer_freq; + + if (__ipipe_hrtimer_freq != t->freq) + __ipipe_hrtimer_freq = t->freq; + + hrtimer_freq = t->freq; + if (__ipipe_hrclock_freq > UINT_MAX) + hrtimer_freq /= 1000; + + t->c2t_integ = hrtimer_freq / hrclock_freq; + tmp = (((unsigned long long) + (hrtimer_freq % hrclock_freq)) << 32) + + hrclock_freq - 1; + do_div(tmp, hrclock_freq); + t->c2t_frac = tmp; +} + +/* Set up a timer as per-cpu timer for ipipe */ +static void install_pcpu_timer(unsigned cpu, unsigned hrclock_freq, + struct ipipe_timer *t) +{ + per_cpu(ipipe_percpu.hrtimer_irq, cpu) = t->irq; + per_cpu(percpu_timer, cpu) = t; + config_pcpu_timer(t, hrclock_freq); +} + +static void select_root_only_timer(unsigned cpu, unsigned hrclock_khz, + const struct cpumask *mask, + struct ipipe_timer *t) { + unsigned icpu; + struct clock_event_device *evtdev; + + /* + * If no ipipe-supported CPU shares an interrupt with the + * timer, we do not need to care about it. + */ + for_each_cpu(icpu, mask) { + if (t->irq == per_cpu(ipipe_percpu.hrtimer_irq, icpu)) { +#ifdef CONFIG_GENERIC_CLOCKEVENTS + evtdev = t->host_timer; + if (evtdev && evtdev->mode == CLOCK_EVT_MODE_SHUTDOWN) + continue; +#endif /* CONFIG_GENERIC_CLOCKEVENTS */ + goto found; + } + } + + return; + +found: + install_pcpu_timer(cpu, hrclock_khz, t); +} + +/* + * Choose per-cpu timers with the highest rating by traversing the + * rating-sorted list for each CPU. + */ +int ipipe_select_timers(const struct cpumask *mask) +{ + unsigned hrclock_freq; + unsigned long long tmp; + struct ipipe_timer *t; + struct clock_event_device *evtdev; + unsigned long flags; + unsigned cpu; + cpumask_t fixup; + + if (!__ipipe_hrclock_ok()) { + printk("I-pipe: high-resolution clock not working\n"); + return -ENODEV; + } + + if (__ipipe_hrclock_freq > UINT_MAX) { + tmp = __ipipe_hrclock_freq; + do_div(tmp, 1000); + hrclock_freq = tmp; + } else + hrclock_freq = __ipipe_hrclock_freq; + + raw_spin_lock_irqsave(&lock, flags); + + /* First, choose timers for the CPUs handled by ipipe */ + for_each_cpu(cpu, mask) { + list_for_each_entry(t, &timers, link) { + if (!cpumask_test_cpu(cpu, t->cpumask)) + continue; + +#ifdef CONFIG_GENERIC_CLOCKEVENTS + evtdev = t->host_timer; + if (evtdev && evtdev->mode == CLOCK_EVT_MODE_SHUTDOWN) + continue; +#endif /* CONFIG_GENERIC_CLOCKEVENTS */ + goto found; + } + + printk("I-pipe: could not find timer for cpu #%d\n", + cpu); + goto err_remove_all; +found: + install_pcpu_timer(cpu, hrclock_freq, t); + } + + /* + * Second, check if we need to fix up any CPUs not supported + * by ipipe (but by Linux) whose interrupt may need to be + * forwarded because they have the same IRQ as an ipipe-enabled + * timer. + */ + cpumask_andnot(&fixup, cpu_online_mask, mask); + + for_each_cpu(cpu, &fixup) { + list_for_each_entry(t, &timers, link) { + if (!cpumask_test_cpu(cpu, t->cpumask)) + continue; + + select_root_only_timer(cpu, hrclock_freq, mask, t); + } + } + + raw_spin_unlock_irqrestore(&lock, flags); + + flags = ipipe_critical_enter(ipipe_timer_request_sync); + ipipe_timer_request_sync(); + ipipe_critical_exit(flags); + + return 0; + +err_remove_all: + raw_spin_unlock_irqrestore(&lock, flags); + + for_each_cpu(cpu, mask) { + per_cpu(ipipe_percpu.hrtimer_irq, cpu) = -1; + per_cpu(percpu_timer, cpu) = NULL; + } + __ipipe_hrtimer_freq = 0; + + return -ENODEV; +} + +static void ipipe_timer_release_sync(void) +{ + struct ipipe_timer *timer = __ipipe_raw_cpu_read(percpu_timer); + + if (timer) + timer->release(timer); +} + +void ipipe_timers_release(void) +{ + unsigned long flags; + unsigned cpu; + + flags = ipipe_critical_enter(ipipe_timer_release_sync); + ipipe_timer_release_sync(); + ipipe_critical_exit(flags); + + for_each_online_cpu(cpu) { + per_cpu(ipipe_percpu.hrtimer_irq, cpu) = -1; + per_cpu(percpu_timer, cpu) = NULL; + __ipipe_hrtimer_freq = 0; + } +} + +static void __ipipe_ack_hrtimer_irq(unsigned int irq, struct irq_desc *desc) +{ + struct ipipe_timer *timer = __ipipe_raw_cpu_read(percpu_timer); + + if (desc) + desc->ipipe_ack(irq, desc); + if (timer->ack) + timer->ack(); + if (desc) + desc->ipipe_end(irq, desc); +} + +int ipipe_timer_start(void (*tick_handler)(void), + void (*emumode)(enum clock_event_mode mode, + struct clock_event_device *cdev), + int (*emutick)(unsigned long evt, + struct clock_event_device *cdev), + unsigned cpu) +{ + struct clock_event_device *evtdev; + struct ipipe_timer *timer; + struct irq_desc *desc; + unsigned long flags; + int steal, ret; + + timer = per_cpu(percpu_timer, cpu); + evtdev = timer->host_timer; + + flags = ipipe_critical_enter(NULL); + + ret = ipipe_request_irq(ipipe_head_domain, timer->irq, + (ipipe_irq_handler_t)tick_handler, + NULL, __ipipe_ack_hrtimer_irq); + if (ret < 0 && ret != -EBUSY) { + ipipe_critical_exit(flags); + return ret; + } + +#ifdef CONFIG_GENERIC_CLOCKEVENTS + steal = evtdev != NULL && evtdev->mode != CLOCK_EVT_MODE_UNUSED; + if (steal && evtdev->ipipe_stolen == 0) { + timer->real_mult = evtdev->mult; + timer->real_shift = evtdev->shift; + timer->real_set_mode = evtdev->set_mode; + timer->real_set_next_event = evtdev->set_next_event; + evtdev->mult = 1; + evtdev->shift = 0; + evtdev->max_delta_ns = UINT_MAX; + evtdev->set_mode = emumode; + evtdev->set_next_event = emutick; + evtdev->ipipe_stolen = 1; + } + + ret = evtdev ? evtdev->mode : CLOCK_EVT_MODE_UNUSED; +#else /* CONFIG_GENERIC_CLOCKEVENTS */ + steal = 1; + ret = 0; +#endif /* CONFIG_GENERIC_CLOCKEVENTS */ + + ipipe_critical_exit(flags); + + desc = irq_to_desc(timer->irq); + if (desc && irqd_irq_disabled(&desc->irq_data)) + ipipe_enable_irq(timer->irq); + + return ret; +} + +void ipipe_timer_stop(unsigned cpu) +{ + unsigned long __maybe_unused flags; + struct clock_event_device *evtdev; + struct ipipe_timer *timer; + struct irq_desc *desc; + + timer = per_cpu(percpu_timer, cpu); + evtdev = timer->host_timer; + + desc = irq_to_desc(timer->irq); + if (desc && irqd_irq_disabled(&desc->irq_data)) + ipipe_disable_irq(timer->irq); + +#ifdef CONFIG_GENERIC_CLOCKEVENTS + if (evtdev) { + flags = ipipe_critical_enter(NULL); + + if (evtdev->ipipe_stolen) { + evtdev->mult = timer->real_mult; + evtdev->shift = timer->real_shift; + evtdev->set_mode = timer->real_set_mode; + evtdev->set_next_event = timer->real_set_next_event; + timer->real_mult = timer->real_shift = 0; + timer->real_set_mode = NULL; + timer->real_set_next_event = NULL; + evtdev->ipipe_stolen = 0; + } + + ipipe_critical_exit(flags); + } +#endif /* CONFIG_GENERIC_CLOCKEVENTS */ + + ipipe_free_irq(ipipe_head_domain, timer->irq); +} + +void ipipe_timer_set(unsigned long cdelay) +{ + unsigned long tdelay; + struct ipipe_timer *t; + + t = __ipipe_raw_cpu_read(percpu_timer); + + /* + * Even though some architectures may use a 64 bits delay + * here, we voluntarily limit to 32 bits, 4 billions ticks + * should be enough for now. Would a timer needs more, an + * extra call to the tick handler would simply occur after 4 + * billions ticks. + */ + if (cdelay > UINT_MAX) + cdelay = UINT_MAX; + + tdelay = cdelay; + if (t->c2t_integ != 1) + tdelay *= t->c2t_integ; + if (t->c2t_frac) + tdelay += ((unsigned long long)cdelay * t->c2t_frac) >> 32; + + if (tdelay < t->min_delay_ticks + || t->set(tdelay, t->timer_set) < 0) + ipipe_raise_irq(t->irq); +} +EXPORT_SYMBOL_GPL(ipipe_timer_set); + +const char *ipipe_timer_name(void) +{ + return per_cpu(percpu_timer, 0)->name; +} +EXPORT_SYMBOL_GPL(ipipe_timer_name); + +unsigned ipipe_timer_ns2ticks(struct ipipe_timer *timer, unsigned ns) +{ + unsigned long long tmp; + BUG_ON(!timer->freq); + tmp = (unsigned long long)ns * timer->freq; + do_div(tmp, 1000000000); + return tmp; +} + +#ifdef CONFIG_IPIPE_HAVE_HOSTRT +/* + * NOTE: The architecture specific code must only call this function + * when a clocksource suitable for CLOCK_HOST_REALTIME is enabled. + * The event receiver is responsible for providing proper locking. + */ +void ipipe_update_hostrt(struct timekeeper *tk) +{ + struct tk_read_base *tkr = &tk->tkr_mono; + struct clocksource *clock = tkr->clock; + struct ipipe_hostrt_data data; + struct timespec xt; + + xt.tv_sec = tk->xtime_sec; + xt.tv_nsec = (long)(tkr->xtime_nsec >> tkr->shift); + ipipe_root_only(); + data.live = 1; + data.cycle_last = tkr->cycle_last; + data.mask = clock->mask; + data.mult = tkr->mult; + data.shift = tkr->shift; + data.wall_time_sec = xt.tv_sec; + data.wall_time_nsec = xt.tv_nsec; + data.wall_to_monotonic.tv_sec = tk->wall_to_monotonic.tv_sec; + data.wall_to_monotonic.tv_nsec = tk->wall_to_monotonic.tv_nsec; + __ipipe_notify_kevent(IPIPE_KEVT_HOSTRT, &data); +} + +#endif /* CONFIG_IPIPE_HAVE_HOSTRT */ + +int clockevents_program_event(struct clock_event_device *dev, ktime_t expires, + bool force); + +void __ipipe_timer_refresh_freq(unsigned int hrclock_freq) +{ + struct ipipe_timer *t = __ipipe_raw_cpu_read(percpu_timer); + unsigned long flags; + + if (t && t->refresh_freq) { + t->freq = t->refresh_freq(); + flags = hard_local_irq_save(); + config_pcpu_timer(t, hrclock_freq); + hard_local_irq_restore(flags); + clockevents_program_event(t->host_timer, + t->host_timer->next_event, false); + } +} diff --git a/kernel/ipipe/tracer.c b/kernel/ipipe/tracer.c new file mode 100644 index 0000000..1ae7bc2 --- /dev/null +++ b/kernel/ipipe/tracer.c @@ -0,0 +1,1468 @@ +/* -*- linux-c -*- + * kernel/ipipe/tracer.c + * + * Copyright (C) 2005 Luotao Fu. + * 2005-2008 Jan Kiszka. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139, + * USA; either version 2 of the License, or (at your option) any later + * version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define IPIPE_TRACE_PATHS 4 /* Do not lower below 3 */ +#define IPIPE_DEFAULT_ACTIVE 0 +#define IPIPE_DEFAULT_MAX 1 +#define IPIPE_DEFAULT_FROZEN 2 + +#define IPIPE_TRACE_POINTS (1 << CONFIG_IPIPE_TRACE_SHIFT) +#define WRAP_POINT_NO(point) ((point) & (IPIPE_TRACE_POINTS-1)) + +#define IPIPE_DEFAULT_PRE_TRACE 10 +#define IPIPE_DEFAULT_POST_TRACE 10 +#define IPIPE_DEFAULT_BACK_TRACE 100 + +#define IPIPE_DELAY_NOTE 1000 /* in nanoseconds */ +#define IPIPE_DELAY_WARN 10000 /* in nanoseconds */ + +#define IPIPE_TFLG_NMI_LOCK 0x0001 +#define IPIPE_TFLG_NMI_HIT 0x0002 +#define IPIPE_TFLG_NMI_FREEZE_REQ 0x0004 + +#define IPIPE_TFLG_HWIRQ_OFF 0x0100 +#define IPIPE_TFLG_FREEZING 0x0200 +#define IPIPE_TFLG_CURRDOM_SHIFT 10 /* bits 10..11: current domain */ +#define IPIPE_TFLG_CURRDOM_MASK 0x0C00 +#define IPIPE_TFLG_DOMSTATE_SHIFT 12 /* bits 12..15: domain stalled? */ +#define IPIPE_TFLG_DOMSTATE_BITS 1 + +#define IPIPE_TFLG_DOMAIN_STALLED(point, n) \ + (point->flags & (1 << (n + IPIPE_TFLG_DOMSTATE_SHIFT))) +#define IPIPE_TFLG_CURRENT_DOMAIN(point) \ + ((point->flags & IPIPE_TFLG_CURRDOM_MASK) >> IPIPE_TFLG_CURRDOM_SHIFT) + +struct ipipe_trace_point { + short type; + short flags; + unsigned long eip; + unsigned long parent_eip; + unsigned long v; + unsigned long long timestamp; +}; + +struct ipipe_trace_path { + volatile int flags; + int dump_lock; /* separated from flags due to cross-cpu access */ + int trace_pos; /* next point to fill */ + int begin, end; /* finalised path begin and end */ + int post_trace; /* non-zero when in post-trace phase */ + unsigned long long length; /* max path length in cycles */ + unsigned long nmi_saved_eip; /* for deferred requests from NMIs */ + unsigned long nmi_saved_parent_eip; + unsigned long nmi_saved_v; + struct ipipe_trace_point point[IPIPE_TRACE_POINTS]; +} ____cacheline_aligned_in_smp; + +enum ipipe_trace_type +{ + IPIPE_TRACE_FUNC = 0, + IPIPE_TRACE_BEGIN, + IPIPE_TRACE_END, + IPIPE_TRACE_FREEZE, + IPIPE_TRACE_SPECIAL, + IPIPE_TRACE_PID, + IPIPE_TRACE_EVENT, +}; + +#define IPIPE_TYPE_MASK 0x0007 +#define IPIPE_TYPE_BITS 3 + +#ifdef CONFIG_IPIPE_TRACE_VMALLOC +static DEFINE_PER_CPU(struct ipipe_trace_path *, trace_path); +#else /* !CONFIG_IPIPE_TRACE_VMALLOC */ +static DEFINE_PER_CPU(struct ipipe_trace_path, trace_path[IPIPE_TRACE_PATHS]) = + { [0 ... IPIPE_TRACE_PATHS-1] = { .begin = -1, .end = -1 } }; +#endif /* CONFIG_IPIPE_TRACE_VMALLOC */ + +int ipipe_trace_enable = 0; + +static DEFINE_PER_CPU(int, active_path) = { IPIPE_DEFAULT_ACTIVE }; +static DEFINE_PER_CPU(int, max_path) = { IPIPE_DEFAULT_MAX }; +static DEFINE_PER_CPU(int, frozen_path) = { IPIPE_DEFAULT_FROZEN }; +static IPIPE_DEFINE_SPINLOCK(global_path_lock); +static int pre_trace = IPIPE_DEFAULT_PRE_TRACE; +static int post_trace = IPIPE_DEFAULT_POST_TRACE; +static int back_trace = IPIPE_DEFAULT_BACK_TRACE; +static int verbose_trace = 1; +static unsigned long trace_overhead; + +static unsigned long trigger_begin; +static unsigned long trigger_end; + +static DEFINE_MUTEX(out_mutex); +static struct ipipe_trace_path *print_path; +#ifdef CONFIG_IPIPE_TRACE_PANIC +static struct ipipe_trace_path *panic_path; +#endif /* CONFIG_IPIPE_TRACE_PANIC */ +static int print_pre_trace; +static int print_post_trace; + + +static long __ipipe_signed_tsc2us(long long tsc); +static void +__ipipe_trace_point_type(char *buf, struct ipipe_trace_point *point); +static void __ipipe_print_symname(struct seq_file *m, unsigned long eip); + +static inline void store_states(struct ipipe_domain *ipd, + struct ipipe_trace_point *point, int pos) +{ + if (test_bit(IPIPE_STALL_FLAG, &ipipe_this_cpu_context(ipd)->status)) + point->flags |= 1 << (pos + IPIPE_TFLG_DOMSTATE_SHIFT); + + if (ipd == __ipipe_current_domain) + point->flags |= pos << IPIPE_TFLG_CURRDOM_SHIFT; +} + +static notrace void +__ipipe_store_domain_states(struct ipipe_trace_point *point) +{ + store_states(ipipe_root_domain, point, 0); + if (ipipe_head_domain != ipipe_root_domain) + store_states(ipipe_head_domain, point, 1); +} + +static notrace int __ipipe_get_free_trace_path(int old, int cpu) +{ + int new_active = old; + struct ipipe_trace_path *tp; + + do { + if (++new_active == IPIPE_TRACE_PATHS) + new_active = 0; + tp = &per_cpu(trace_path, cpu)[new_active]; + } while (new_active == per_cpu(max_path, cpu) || + new_active == per_cpu(frozen_path, cpu) || + tp->dump_lock); + + return new_active; +} + +static notrace void +__ipipe_migrate_pre_trace(struct ipipe_trace_path *new_tp, + struct ipipe_trace_path *old_tp, int old_pos) +{ + int i; + + new_tp->trace_pos = pre_trace+1; + + for (i = new_tp->trace_pos; i > 0; i--) + memcpy(&new_tp->point[WRAP_POINT_NO(new_tp->trace_pos-i)], + &old_tp->point[WRAP_POINT_NO(old_pos-i)], + sizeof(struct ipipe_trace_point)); + + /* mark the end (i.e. the point before point[0]) invalid */ + new_tp->point[IPIPE_TRACE_POINTS-1].eip = 0; +} + +static notrace struct ipipe_trace_path * +__ipipe_trace_end(int cpu, struct ipipe_trace_path *tp, int pos) +{ + struct ipipe_trace_path *old_tp = tp; + long active = per_cpu(active_path, cpu); + unsigned long long length; + + /* do we have a new worst case? */ + length = tp->point[tp->end].timestamp - + tp->point[tp->begin].timestamp; + if (length > per_cpu(trace_path, cpu)[per_cpu(max_path, cpu)].length) { + /* we need protection here against other cpus trying + to start a proc dump */ + raw_spin_lock(&global_path_lock); + + /* active path holds new worst case */ + tp->length = length; + per_cpu(max_path, cpu) = active; + + /* find next unused trace path */ + active = __ipipe_get_free_trace_path(active, cpu); + + raw_spin_unlock(&global_path_lock); + + tp = &per_cpu(trace_path, cpu)[active]; + + /* migrate last entries for pre-tracing */ + __ipipe_migrate_pre_trace(tp, old_tp, pos); + } + + return tp; +} + +static notrace struct ipipe_trace_path * +__ipipe_trace_freeze(int cpu, struct ipipe_trace_path *tp, int pos) +{ + struct ipipe_trace_path *old_tp = tp; + long active = per_cpu(active_path, cpu); + int n; + + /* frozen paths have no core (begin=end) */ + tp->begin = tp->end; + + /* we need protection here against other cpus trying + * to set their frozen path or to start a proc dump */ + raw_spin_lock(&global_path_lock); + + per_cpu(frozen_path, cpu) = active; + + /* find next unused trace path */ + active = __ipipe_get_free_trace_path(active, cpu); + + /* check if this is the first frozen path */ + for_each_possible_cpu(n) { + if (n != cpu && + per_cpu(trace_path, n)[per_cpu(frozen_path, n)].end >= 0) + tp->end = -1; + } + + raw_spin_unlock(&global_path_lock); + + tp = &per_cpu(trace_path, cpu)[active]; + + /* migrate last entries for pre-tracing */ + __ipipe_migrate_pre_trace(tp, old_tp, pos); + + return tp; +} + +void notrace +__ipipe_trace(enum ipipe_trace_type type, unsigned long eip, + unsigned long parent_eip, unsigned long v) +{ + struct ipipe_trace_path *tp, *old_tp; + int pos, next_pos, begin; + struct ipipe_trace_point *point; + unsigned long flags; + int cpu; + + flags = hard_local_irq_save_notrace(); + + cpu = ipipe_processor_id(); + restart: + tp = old_tp = &per_cpu(trace_path, cpu)[per_cpu(active_path, cpu)]; + + /* here starts a race window with NMIs - catched below */ + + /* check for NMI recursion */ + if (unlikely(tp->flags & IPIPE_TFLG_NMI_LOCK)) { + tp->flags |= IPIPE_TFLG_NMI_HIT; + + /* first freeze request from NMI context? */ + if ((type == IPIPE_TRACE_FREEZE) && + !(tp->flags & IPIPE_TFLG_NMI_FREEZE_REQ)) { + /* save arguments and mark deferred freezing */ + tp->flags |= IPIPE_TFLG_NMI_FREEZE_REQ; + tp->nmi_saved_eip = eip; + tp->nmi_saved_parent_eip = parent_eip; + tp->nmi_saved_v = v; + } + return; /* no need for restoring flags inside IRQ */ + } + + /* clear NMI events and set lock (atomically per cpu) */ + tp->flags = (tp->flags & ~(IPIPE_TFLG_NMI_HIT | + IPIPE_TFLG_NMI_FREEZE_REQ)) + | IPIPE_TFLG_NMI_LOCK; + + /* check active_path again - some nasty NMI may have switched + * it meanwhile */ + if (unlikely(tp != + &per_cpu(trace_path, cpu)[per_cpu(active_path, cpu)])) { + /* release lock on wrong path and restart */ + tp->flags &= ~IPIPE_TFLG_NMI_LOCK; + + /* there is no chance that the NMI got deferred + * => no need to check for pending freeze requests */ + goto restart; + } + + /* get the point buffer */ + pos = tp->trace_pos; + point = &tp->point[pos]; + + /* store all trace point data */ + point->type = type; + point->flags = hard_irqs_disabled_flags(flags) ? IPIPE_TFLG_HWIRQ_OFF : 0; + point->eip = eip; + point->parent_eip = parent_eip; + point->v = v; + ipipe_read_tsc(point->timestamp); + + __ipipe_store_domain_states(point); + + /* forward to next point buffer */ + next_pos = WRAP_POINT_NO(pos+1); + tp->trace_pos = next_pos; + + /* only mark beginning if we haven't started yet */ + begin = tp->begin; + if (unlikely(type == IPIPE_TRACE_BEGIN) && (begin < 0)) + tp->begin = pos; + + /* end of critical path, start post-trace if not already started */ + if (unlikely(type == IPIPE_TRACE_END) && + (begin >= 0) && !tp->post_trace) + tp->post_trace = post_trace + 1; + + /* freeze only if the slot is free and we are not already freezing */ + if ((unlikely(type == IPIPE_TRACE_FREEZE) || + (unlikely(eip >= trigger_begin && eip <= trigger_end) && + type == IPIPE_TRACE_FUNC)) && + per_cpu(trace_path, cpu)[per_cpu(frozen_path, cpu)].begin < 0 && + !(tp->flags & IPIPE_TFLG_FREEZING)) { + tp->post_trace = post_trace + 1; + tp->flags |= IPIPE_TFLG_FREEZING; + } + + /* enforce end of trace in case of overflow */ + if (unlikely(WRAP_POINT_NO(next_pos + 1) == begin)) { + tp->end = pos; + goto enforce_end; + } + + /* stop tracing this path if we are in post-trace and + * a) that phase is over now or + * b) a new TRACE_BEGIN came in but we are not freezing this path */ + if (unlikely((tp->post_trace > 0) && ((--tp->post_trace == 0) || + ((type == IPIPE_TRACE_BEGIN) && + !(tp->flags & IPIPE_TFLG_FREEZING))))) { + /* store the path's end (i.e. excluding post-trace) */ + tp->end = WRAP_POINT_NO(pos - post_trace + tp->post_trace); + + enforce_end: + if (tp->flags & IPIPE_TFLG_FREEZING) + tp = __ipipe_trace_freeze(cpu, tp, pos); + else + tp = __ipipe_trace_end(cpu, tp, pos); + + /* reset the active path, maybe already start a new one */ + tp->begin = (type == IPIPE_TRACE_BEGIN) ? + WRAP_POINT_NO(tp->trace_pos - 1) : -1; + tp->end = -1; + tp->post_trace = 0; + tp->flags = 0; + + /* update active_path not earlier to avoid races with NMIs */ + per_cpu(active_path, cpu) = tp - per_cpu(trace_path, cpu); + } + + /* we still have old_tp and point, + * let's reset NMI lock and check for catches */ + old_tp->flags &= ~IPIPE_TFLG_NMI_LOCK; + if (unlikely(old_tp->flags & IPIPE_TFLG_NMI_HIT)) { + /* well, this late tagging may not immediately be visible for + * other cpus already dumping this path - a minor issue */ + point->flags |= IPIPE_TFLG_NMI_HIT; + + /* handle deferred freezing from NMI context */ + if (old_tp->flags & IPIPE_TFLG_NMI_FREEZE_REQ) + __ipipe_trace(IPIPE_TRACE_FREEZE, old_tp->nmi_saved_eip, + old_tp->nmi_saved_parent_eip, + old_tp->nmi_saved_v); + } + + hard_local_irq_restore_notrace(flags); +} + +static unsigned long __ipipe_global_path_lock(void) +{ + unsigned long flags; + int cpu; + struct ipipe_trace_path *tp; + + raw_spin_lock_irqsave(&global_path_lock, flags); + + cpu = ipipe_processor_id(); + restart: + tp = &per_cpu(trace_path, cpu)[per_cpu(active_path, cpu)]; + + /* here is small race window with NMIs - catched below */ + + /* clear NMI events and set lock (atomically per cpu) */ + tp->flags = (tp->flags & ~(IPIPE_TFLG_NMI_HIT | + IPIPE_TFLG_NMI_FREEZE_REQ)) + | IPIPE_TFLG_NMI_LOCK; + + /* check active_path again - some nasty NMI may have switched + * it meanwhile */ + if (tp != &per_cpu(trace_path, cpu)[per_cpu(active_path, cpu)]) { + /* release lock on wrong path and restart */ + tp->flags &= ~IPIPE_TFLG_NMI_LOCK; + + /* there is no chance that the NMI got deferred + * => no need to check for pending freeze requests */ + goto restart; + } + + return flags; +} + +static void __ipipe_global_path_unlock(unsigned long flags) +{ + int cpu; + struct ipipe_trace_path *tp; + + /* release spinlock first - it's not involved in the NMI issue */ + __ipipe_spin_unlock_irqbegin(&global_path_lock); + + cpu = ipipe_processor_id(); + tp = &per_cpu(trace_path, cpu)[per_cpu(active_path, cpu)]; + + tp->flags &= ~IPIPE_TFLG_NMI_LOCK; + + /* handle deferred freezing from NMI context */ + if (tp->flags & IPIPE_TFLG_NMI_FREEZE_REQ) + __ipipe_trace(IPIPE_TRACE_FREEZE, tp->nmi_saved_eip, + tp->nmi_saved_parent_eip, tp->nmi_saved_v); + + /* See __ipipe_spin_lock_irqsave() and friends. */ + __ipipe_spin_unlock_irqcomplete(flags); +} + +void notrace asmlinkage +ipipe_trace_asm(enum ipipe_trace_type type, unsigned long eip, + unsigned long parent_eip, unsigned long v) +{ + if (!ipipe_trace_enable) + return; + __ipipe_trace(type, eip, parent_eip, v); +} + +void notrace ipipe_trace_begin(unsigned long v) +{ + if (!ipipe_trace_enable) + return; + __ipipe_trace(IPIPE_TRACE_BEGIN, __BUILTIN_RETURN_ADDRESS0, + __BUILTIN_RETURN_ADDRESS1, v); +} +EXPORT_SYMBOL_GPL(ipipe_trace_begin); + +void notrace ipipe_trace_end(unsigned long v) +{ + if (!ipipe_trace_enable) + return; + __ipipe_trace(IPIPE_TRACE_END, __BUILTIN_RETURN_ADDRESS0, + __BUILTIN_RETURN_ADDRESS1, v); +} +EXPORT_SYMBOL_GPL(ipipe_trace_end); + +void notrace ipipe_trace_freeze(unsigned long v) +{ + if (!ipipe_trace_enable) + return; + __ipipe_trace(IPIPE_TRACE_FREEZE, __BUILTIN_RETURN_ADDRESS0, + __BUILTIN_RETURN_ADDRESS1, v); +} +EXPORT_SYMBOL_GPL(ipipe_trace_freeze); + +void notrace ipipe_trace_special(unsigned char id, unsigned long v) +{ + if (!ipipe_trace_enable) + return; + __ipipe_trace(IPIPE_TRACE_SPECIAL | (id << IPIPE_TYPE_BITS), + __BUILTIN_RETURN_ADDRESS0, + __BUILTIN_RETURN_ADDRESS1, v); +} +EXPORT_SYMBOL_GPL(ipipe_trace_special); + +void notrace ipipe_trace_pid(pid_t pid, short prio) +{ + if (!ipipe_trace_enable) + return; + __ipipe_trace(IPIPE_TRACE_PID | (prio << IPIPE_TYPE_BITS), + __BUILTIN_RETURN_ADDRESS0, + __BUILTIN_RETURN_ADDRESS1, pid); +} +EXPORT_SYMBOL_GPL(ipipe_trace_pid); + +void notrace ipipe_trace_event(unsigned char id, unsigned long delay_tsc) +{ + if (!ipipe_trace_enable) + return; + __ipipe_trace(IPIPE_TRACE_EVENT | (id << IPIPE_TYPE_BITS), + __BUILTIN_RETURN_ADDRESS0, + __BUILTIN_RETURN_ADDRESS1, delay_tsc); +} +EXPORT_SYMBOL_GPL(ipipe_trace_event); + +int ipipe_trace_max_reset(void) +{ + int cpu; + unsigned long flags; + struct ipipe_trace_path *path; + int ret = 0; + + flags = __ipipe_global_path_lock(); + + for_each_possible_cpu(cpu) { + path = &per_cpu(trace_path, cpu)[per_cpu(max_path, cpu)]; + + if (path->dump_lock) { + ret = -EBUSY; + break; + } + + path->begin = -1; + path->end = -1; + path->trace_pos = 0; + path->length = 0; + } + + __ipipe_global_path_unlock(flags); + + return ret; +} +EXPORT_SYMBOL_GPL(ipipe_trace_max_reset); + +int ipipe_trace_frozen_reset(void) +{ + int cpu; + unsigned long flags; + struct ipipe_trace_path *path; + int ret = 0; + + flags = __ipipe_global_path_lock(); + + for_each_online_cpu(cpu) { + path = &per_cpu(trace_path, cpu)[per_cpu(frozen_path, cpu)]; + + if (path->dump_lock) { + ret = -EBUSY; + break; + } + + path->begin = -1; + path->end = -1; + path->trace_pos = 0; + path->length = 0; + } + + __ipipe_global_path_unlock(flags); + + return ret; +} +EXPORT_SYMBOL_GPL(ipipe_trace_frozen_reset); + +static void +__ipipe_get_task_info(char *task_info, struct ipipe_trace_point *point, + int trylock) +{ + struct task_struct *task = NULL; + char buf[8]; + int i; + int locked = 1; + + if (trylock) { + if (!read_trylock(&tasklist_lock)) + locked = 0; + } else + read_lock(&tasklist_lock); + + if (locked) + task = find_task_by_pid_ns((pid_t)point->v, &init_pid_ns); + + if (task) + strncpy(task_info, task->comm, 11); + else + strcpy(task_info, "--"); + + if (locked) + read_unlock(&tasklist_lock); + + for (i = strlen(task_info); i < 11; i++) + task_info[i] = ' '; + + sprintf(buf, " %d ", point->type >> IPIPE_TYPE_BITS); + strcpy(task_info + (11 - strlen(buf)), buf); +} + +static void +__ipipe_get_event_date(char *buf,struct ipipe_trace_path *path, + struct ipipe_trace_point *point) +{ + long time; + int type; + + time = __ipipe_signed_tsc2us(point->timestamp - + path->point[path->begin].timestamp + point->v); + type = point->type >> IPIPE_TYPE_BITS; + + if (type == 0) + /* + * Event type #0 is predefined, stands for the next + * timer tick. + */ + sprintf(buf, "tick@%-6ld", time); + else + sprintf(buf, "%3d@%-7ld", type, time); +} + +#ifdef CONFIG_IPIPE_TRACE_PANIC + +void ipipe_trace_panic_freeze(void) +{ + unsigned long flags; + int cpu; + + if (!ipipe_trace_enable) + return; + + ipipe_trace_enable = 0; + flags = hard_local_irq_save_notrace(); + + cpu = ipipe_processor_id(); + + panic_path = &per_cpu(trace_path, cpu)[per_cpu(active_path, cpu)]; + + hard_local_irq_restore(flags); +} +EXPORT_SYMBOL_GPL(ipipe_trace_panic_freeze); + +void ipipe_trace_panic_dump(void) +{ + int cnt = back_trace; + int start, pos; + char buf[16]; + + if (!panic_path) + return; + + ipipe_context_check_off(); + + printk("I-pipe tracer log (%d points):\n", cnt); + + start = pos = WRAP_POINT_NO(panic_path->trace_pos-1); + + while (cnt-- > 0) { + struct ipipe_trace_point *point = &panic_path->point[pos]; + long time; + char info[16]; + int i; + + printk(" %c", + (point->flags & IPIPE_TFLG_HWIRQ_OFF) ? '|' : ' '); + + for (i = IPIPE_TFLG_DOMSTATE_BITS; i >= 0; i--) + printk("%c", + (IPIPE_TFLG_CURRENT_DOMAIN(point) == i) ? + (IPIPE_TFLG_DOMAIN_STALLED(point, i) ? + '#' : '+') : + (IPIPE_TFLG_DOMAIN_STALLED(point, i) ? + '*' : ' ')); + + if (!point->eip) + printk("--\n"); + else { + __ipipe_trace_point_type(buf, point); + printk("%s", buf); + + switch (point->type & IPIPE_TYPE_MASK) { + case IPIPE_TRACE_FUNC: + printk(" "); + break; + + case IPIPE_TRACE_PID: + __ipipe_get_task_info(info, + point, 1); + printk("%s", info); + break; + + case IPIPE_TRACE_EVENT: + __ipipe_get_event_date(info, + panic_path, point); + printk("%s", info); + break; + + default: + printk("0x%08lx ", point->v); + } + + time = __ipipe_signed_tsc2us(point->timestamp - + panic_path->point[start].timestamp); + printk(" %5ld ", time); + + __ipipe_print_symname(NULL, point->eip); + printk(" ("); + __ipipe_print_symname(NULL, point->parent_eip); + printk(")\n"); + } + pos = WRAP_POINT_NO(pos - 1); + } + + panic_path = NULL; +} +EXPORT_SYMBOL_GPL(ipipe_trace_panic_dump); + +#endif /* CONFIG_IPIPE_TRACE_PANIC */ + + +/* --- /proc output --- */ + +static notrace int __ipipe_in_critical_trpath(long point_no) +{ + return ((WRAP_POINT_NO(point_no-print_path->begin) < + WRAP_POINT_NO(print_path->end-print_path->begin)) || + ((print_path->end == print_path->begin) && + (WRAP_POINT_NO(point_no-print_path->end) > + print_post_trace))); +} + +static long __ipipe_signed_tsc2us(long long tsc) +{ + unsigned long long abs_tsc; + long us; + + if (!__ipipe_hrclock_ok()) + return 0; + + /* ipipe_tsc2us works on unsigned => handle sign separately */ + abs_tsc = (tsc >= 0) ? tsc : -tsc; + us = ipipe_tsc2us(abs_tsc); + if (tsc < 0) + return -us; + else + return us; +} + +static void +__ipipe_trace_point_type(char *buf, struct ipipe_trace_point *point) +{ + switch (point->type & IPIPE_TYPE_MASK) { + case IPIPE_TRACE_FUNC: + strcpy(buf, "func "); + break; + + case IPIPE_TRACE_BEGIN: + strcpy(buf, "begin "); + break; + + case IPIPE_TRACE_END: + strcpy(buf, "end "); + break; + + case IPIPE_TRACE_FREEZE: + strcpy(buf, "freeze "); + break; + + case IPIPE_TRACE_SPECIAL: + sprintf(buf, "(0x%02x) ", + point->type >> IPIPE_TYPE_BITS); + break; + + case IPIPE_TRACE_PID: + sprintf(buf, "[%5d] ", (pid_t)point->v); + break; + + case IPIPE_TRACE_EVENT: + sprintf(buf, "event "); + break; + } +} + +static void +__ipipe_print_pathmark(struct seq_file *m, struct ipipe_trace_point *point) +{ + char mark = ' '; + int point_no = point - print_path->point; + int i; + + if (print_path->end == point_no) + mark = '<'; + else if (print_path->begin == point_no) + mark = '>'; + else if (__ipipe_in_critical_trpath(point_no)) + mark = ':'; + seq_printf(m, "%c%c", mark, + (point->flags & IPIPE_TFLG_HWIRQ_OFF) ? '|' : ' '); + + if (!verbose_trace) + return; + + for (i = IPIPE_TFLG_DOMSTATE_BITS; i >= 0; i--) + seq_printf(m, "%c", + (IPIPE_TFLG_CURRENT_DOMAIN(point) == i) ? + (IPIPE_TFLG_DOMAIN_STALLED(point, i) ? + '#' : '+') : + (IPIPE_TFLG_DOMAIN_STALLED(point, i) ? '*' : ' ')); +} + +static void +__ipipe_print_delay(struct seq_file *m, struct ipipe_trace_point *point) +{ + unsigned long delay = 0; + int next; + char *mark = " "; + + next = WRAP_POINT_NO(point+1 - print_path->point); + + if (next != print_path->trace_pos) + delay = ipipe_tsc2ns(print_path->point[next].timestamp - + point->timestamp); + + if (__ipipe_in_critical_trpath(point - print_path->point)) { + if (delay > IPIPE_DELAY_WARN) + mark = "! "; + else if (delay > IPIPE_DELAY_NOTE) + mark = "+ "; + } + seq_puts(m, mark); + + if (verbose_trace) + seq_printf(m, "%3lu.%03lu%c ", delay/1000, delay%1000, + (point->flags & IPIPE_TFLG_NMI_HIT) ? 'N' : ' '); + else + seq_puts(m, " "); +} + +static void __ipipe_print_symname(struct seq_file *m, unsigned long eip) +{ + char namebuf[KSYM_NAME_LEN+1]; + unsigned long size, offset; + const char *sym_name; + char *modname; + + sym_name = kallsyms_lookup(eip, &size, &offset, &modname, namebuf); + +#ifdef CONFIG_IPIPE_TRACE_PANIC + if (!m) { + /* panic dump */ + if (sym_name) { + printk("%s+0x%lx", sym_name, offset); + if (modname) + printk(" [%s]", modname); + } else + printk("<%08lx>", eip); + } else +#endif /* CONFIG_IPIPE_TRACE_PANIC */ + { + if (sym_name) { + if (verbose_trace) { + seq_printf(m, "%s+0x%lx", sym_name, offset); + if (modname) + seq_printf(m, " [%s]", modname); + } else + seq_puts(m, sym_name); + } else + seq_printf(m, "<%08lx>", eip); + } +} + +static void __ipipe_print_headline(struct seq_file *m) +{ + const char *name[2]; + + seq_printf(m, "Calibrated minimum trace-point overhead: %lu.%03lu " + "us\n\n", trace_overhead/1000, trace_overhead%1000); + + if (verbose_trace) { + name[0] = ipipe_root_domain->name; + if (ipipe_head_domain != ipipe_root_domain) + name[1] = ipipe_head_domain->name; + else + name[1] = ""; + + seq_printf(m, + " +----- Hard IRQs ('|': locked)\n" + " |+-- %s\n" + " ||+- %s%s\n" + " ||| +---------- " + "Delay flag ('+': > %d us, '!': > %d us)\n" + " ||| | +- " + "NMI noise ('N')\n" + " ||| | |\n" + " Type User Val. Time Delay Function " + "(Parent)\n", + name[1], name[0], + " ('*': domain stalled, '+': current, " + "'#': current+stalled)", + IPIPE_DELAY_NOTE/1000, IPIPE_DELAY_WARN/1000); + } else + seq_printf(m, + " +--------------- Hard IRQs ('|': locked)\n" + " | +- Delay flag " + "('+': > %d us, '!': > %d us)\n" + " | |\n" + " Type Time Function (Parent)\n", + IPIPE_DELAY_NOTE/1000, IPIPE_DELAY_WARN/1000); +} + +static void *__ipipe_max_prtrace_start(struct seq_file *m, loff_t *pos) +{ + loff_t n = *pos; + + mutex_lock(&out_mutex); + + if (!n) { + struct ipipe_trace_path *tp; + unsigned long length_usecs; + int points, cpu; + unsigned long flags; + + /* protect against max_path/frozen_path updates while we + * haven't locked our target path, also avoid recursively + * taking global_path_lock from NMI context */ + flags = __ipipe_global_path_lock(); + + /* find the longest of all per-cpu paths */ + print_path = NULL; + for_each_online_cpu(cpu) { + tp = &per_cpu(trace_path, cpu)[per_cpu(max_path, cpu)]; + if ((print_path == NULL) || + (tp->length > print_path->length)) { + print_path = tp; + break; + } + } + print_path->dump_lock = 1; + + __ipipe_global_path_unlock(flags); + + if (!__ipipe_hrclock_ok()) { + seq_printf(m, "No hrclock available, dumping traces disabled\n"); + return NULL; + } + + /* does this path actually contain data? */ + if (print_path->end == print_path->begin) + return NULL; + + /* number of points inside the critical path */ + points = WRAP_POINT_NO(print_path->end-print_path->begin+1); + + /* pre- and post-tracing length, post-trace length was frozen + in __ipipe_trace, pre-trace may have to be reduced due to + buffer overrun */ + print_pre_trace = pre_trace; + print_post_trace = WRAP_POINT_NO(print_path->trace_pos - + print_path->end - 1); + if (points+pre_trace+print_post_trace > IPIPE_TRACE_POINTS - 1) + print_pre_trace = IPIPE_TRACE_POINTS - 1 - points - + print_post_trace; + + length_usecs = ipipe_tsc2us(print_path->length); + seq_printf(m, "I-pipe worst-case tracing service on %s/ipipe release #%d\n" + "-------------------------------------------------------------\n", + UTS_RELEASE, IPIPE_CORE_RELEASE); + seq_printf(m, "CPU: %d, Begin: %lld cycles, Trace Points: " + "%d (-%d/+%d), Length: %lu us\n", + cpu, print_path->point[print_path->begin].timestamp, + points, print_pre_trace, print_post_trace, length_usecs); + __ipipe_print_headline(m); + } + + /* check if we are inside the trace range */ + if (n >= WRAP_POINT_NO(print_path->end - print_path->begin + 1 + + print_pre_trace + print_post_trace)) + return NULL; + + /* return the next point to be shown */ + return &print_path->point[WRAP_POINT_NO(print_path->begin - + print_pre_trace + n)]; +} + +static void *__ipipe_prtrace_next(struct seq_file *m, void *p, loff_t *pos) +{ + loff_t n = ++*pos; + + /* check if we are inside the trace range with the next entry */ + if (n >= WRAP_POINT_NO(print_path->end - print_path->begin + 1 + + print_pre_trace + print_post_trace)) + return NULL; + + /* return the next point to be shown */ + return &print_path->point[WRAP_POINT_NO(print_path->begin - + print_pre_trace + *pos)]; +} + +static void __ipipe_prtrace_stop(struct seq_file *m, void *p) +{ + if (print_path) + print_path->dump_lock = 0; + mutex_unlock(&out_mutex); +} + +static int __ipipe_prtrace_show(struct seq_file *m, void *p) +{ + long time; + struct ipipe_trace_point *point = p; + char buf[16]; + + if (!point->eip) { + seq_puts(m, "--\n"); + return 0; + } + + __ipipe_print_pathmark(m, point); + __ipipe_trace_point_type(buf, point); + seq_puts(m, buf); + if (verbose_trace) + switch (point->type & IPIPE_TYPE_MASK) { + case IPIPE_TRACE_FUNC: + seq_puts(m, " "); + break; + + case IPIPE_TRACE_PID: + __ipipe_get_task_info(buf, point, 0); + seq_puts(m, buf); + break; + + case IPIPE_TRACE_EVENT: + __ipipe_get_event_date(buf, print_path, point); + seq_puts(m, buf); + break; + + default: + seq_printf(m, "0x%08lx ", point->v); + } + + time = __ipipe_signed_tsc2us(point->timestamp - + print_path->point[print_path->begin].timestamp); + seq_printf(m, "%5ld", time); + + __ipipe_print_delay(m, point); + __ipipe_print_symname(m, point->eip); + seq_puts(m, " ("); + __ipipe_print_symname(m, point->parent_eip); + seq_puts(m, ")\n"); + + return 0; +} + +static struct seq_operations __ipipe_max_ptrace_ops = { + .start = __ipipe_max_prtrace_start, + .next = __ipipe_prtrace_next, + .stop = __ipipe_prtrace_stop, + .show = __ipipe_prtrace_show +}; + +static int __ipipe_max_prtrace_open(struct inode *inode, struct file *file) +{ + return seq_open(file, &__ipipe_max_ptrace_ops); +} + +static ssize_t +__ipipe_max_reset(struct file *file, const char __user *pbuffer, + size_t count, loff_t *data) +{ + mutex_lock(&out_mutex); + ipipe_trace_max_reset(); + mutex_unlock(&out_mutex); + + return count; +} + +static const struct file_operations __ipipe_max_prtrace_fops = { + .open = __ipipe_max_prtrace_open, + .read = seq_read, + .write = __ipipe_max_reset, + .llseek = seq_lseek, + .release = seq_release, +}; + +static void *__ipipe_frozen_prtrace_start(struct seq_file *m, loff_t *pos) +{ + loff_t n = *pos; + + mutex_lock(&out_mutex); + + if (!n) { + struct ipipe_trace_path *tp; + int cpu; + unsigned long flags; + + /* protect against max_path/frozen_path updates while we + * haven't locked our target path, also avoid recursively + * taking global_path_lock from NMI context */ + flags = __ipipe_global_path_lock(); + + /* find the first of all per-cpu frozen paths */ + print_path = NULL; + for_each_online_cpu(cpu) { + tp = &per_cpu(trace_path, cpu)[per_cpu(frozen_path, cpu)]; + if (tp->end >= 0) { + print_path = tp; + break; + } + } + if (print_path) + print_path->dump_lock = 1; + + __ipipe_global_path_unlock(flags); + + if (!print_path) + return NULL; + + if (!__ipipe_hrclock_ok()) { + seq_printf(m, "No hrclock available, dumping traces disabled\n"); + return NULL; + } + + /* back- and post-tracing length, post-trace length was frozen + in __ipipe_trace, back-trace may have to be reduced due to + buffer overrun */ + print_pre_trace = back_trace-1; /* substract freeze point */ + print_post_trace = WRAP_POINT_NO(print_path->trace_pos - + print_path->end - 1); + if (1+pre_trace+print_post_trace > IPIPE_TRACE_POINTS - 1) + print_pre_trace = IPIPE_TRACE_POINTS - 2 - + print_post_trace; + + seq_printf(m, "I-pipe frozen back-tracing service on %s/ipipe release #%d\n" + "------------------------------------------------------------\n", + UTS_RELEASE, IPIPE_CORE_RELEASE); + seq_printf(m, "CPU: %d, Freeze: %lld cycles, Trace Points: %d (+%d)\n", + cpu, print_path->point[print_path->begin].timestamp, + print_pre_trace+1, print_post_trace); + __ipipe_print_headline(m); + } + + /* check if we are inside the trace range */ + if (n >= print_pre_trace + 1 + print_post_trace) + return NULL; + + /* return the next point to be shown */ + return &print_path->point[WRAP_POINT_NO(print_path->begin- + print_pre_trace+n)]; +} + +static struct seq_operations __ipipe_frozen_ptrace_ops = { + .start = __ipipe_frozen_prtrace_start, + .next = __ipipe_prtrace_next, + .stop = __ipipe_prtrace_stop, + .show = __ipipe_prtrace_show +}; + +static int __ipipe_frozen_prtrace_open(struct inode *inode, struct file *file) +{ + return seq_open(file, &__ipipe_frozen_ptrace_ops); +} + +static ssize_t +__ipipe_frozen_ctrl(struct file *file, const char __user *pbuffer, + size_t count, loff_t *data) +{ + char *end, buf[16]; + int val; + int n; + + n = (count > sizeof(buf) - 1) ? sizeof(buf) - 1 : count; + + if (copy_from_user(buf, pbuffer, n)) + return -EFAULT; + + buf[n] = '\0'; + val = simple_strtol(buf, &end, 0); + + if (((*end != '\0') && !isspace(*end)) || (val < 0)) + return -EINVAL; + + mutex_lock(&out_mutex); + ipipe_trace_frozen_reset(); + if (val > 0) + ipipe_trace_freeze(-1); + mutex_unlock(&out_mutex); + + return count; +} + +static const struct file_operations __ipipe_frozen_prtrace_fops = { + .open = __ipipe_frozen_prtrace_open, + .read = seq_read, + .write = __ipipe_frozen_ctrl, + .llseek = seq_lseek, + .release = seq_release, +}; + +static int __ipipe_rd_proc_val(struct seq_file *p, void *data) +{ + seq_printf(p, "%u\n", *(int *)p->private); + return 0; +} + +static ssize_t +__ipipe_wr_proc_val(struct file *file, const char __user *buffer, + size_t count, loff_t *data) +{ + struct seq_file *p = file->private_data; + char *end, buf[16]; + int val; + int n; + + n = (count > sizeof(buf) - 1) ? sizeof(buf) - 1 : count; + + if (copy_from_user(buf, buffer, n)) + return -EFAULT; + + buf[n] = '\0'; + val = simple_strtol(buf, &end, 0); + + if (((*end != '\0') && !isspace(*end)) || (val < 0)) + return -EINVAL; + + mutex_lock(&out_mutex); + *(int *)p->private = val; + mutex_unlock(&out_mutex); + + return count; +} + +static int __ipipe_rw_proc_val_open(struct inode *inode, struct file *file) +{ + return single_open(file, __ipipe_rd_proc_val, PDE_DATA(inode)); +} + +static const struct file_operations __ipipe_rw_proc_val_ops = { + .open = __ipipe_rw_proc_val_open, + .read = seq_read, + .write = __ipipe_wr_proc_val, + .llseek = seq_lseek, + .release = single_release, +}; + +static void __init +__ipipe_create_trace_proc_val(struct proc_dir_entry *trace_dir, + const char *name, int *value_ptr) +{ + proc_create_data(name, 0644, trace_dir, &__ipipe_rw_proc_val_ops, + value_ptr); +} + +static int __ipipe_rd_trigger(struct seq_file *p, void *data) +{ + char str[KSYM_SYMBOL_LEN]; + + if (trigger_begin) { + sprint_symbol(str, trigger_begin); + seq_printf(p, "%s\n", str); + } + return 0; +} + +static ssize_t +__ipipe_wr_trigger(struct file *file, const char __user *buffer, + size_t count, loff_t *data) +{ + char buf[KSYM_SYMBOL_LEN]; + unsigned long begin, end; + + if (count > sizeof(buf) - 1) + count = sizeof(buf) - 1; + if (copy_from_user(buf, buffer, count)) + return -EFAULT; + buf[count] = 0; + if (buf[count-1] == '\n') + buf[count-1] = 0; + + begin = kallsyms_lookup_name(buf); + if (!begin || !kallsyms_lookup_size_offset(begin, &end, NULL)) + return -ENOENT; + end += begin - 1; + + mutex_lock(&out_mutex); + /* invalidate the current range before setting a new one */ + trigger_end = 0; + wmb(); + ipipe_trace_frozen_reset(); + + /* set new range */ + trigger_begin = begin; + wmb(); + trigger_end = end; + mutex_unlock(&out_mutex); + + return count; +} + +static int __ipipe_rw_trigger_open(struct inode *inode, struct file *file) +{ + return single_open(file, __ipipe_rd_trigger, NULL); +} + +static const struct file_operations __ipipe_rw_trigger_ops = { + .open = __ipipe_rw_trigger_open, + .read = seq_read, + .write = __ipipe_wr_trigger, + .llseek = seq_lseek, + .release = single_release, +}; + + +#ifdef CONFIG_IPIPE_TRACE_MCOUNT +static void notrace +ipipe_trace_function(unsigned long ip, unsigned long parent_ip, + struct ftrace_ops *op, struct pt_regs *regs) +{ + if (!ipipe_trace_enable) + return; + __ipipe_trace(IPIPE_TRACE_FUNC, ip, parent_ip, 0); +} + +static struct ftrace_ops ipipe_trace_ops = { + .func = ipipe_trace_function, + .flags = FTRACE_OPS_FL_IPIPE_EXCLUSIVE, +}; + +static ssize_t __ipipe_wr_enable(struct file *file, const char __user *buffer, + size_t count, loff_t *data) +{ + char *end, buf[16]; + int val; + int n; + + n = (count > sizeof(buf) - 1) ? sizeof(buf) - 1 : count; + + if (copy_from_user(buf, buffer, n)) + return -EFAULT; + + buf[n] = '\0'; + val = simple_strtol(buf, &end, 0); + + if (((*end != '\0') && !isspace(*end)) || (val < 0)) + return -EINVAL; + + mutex_lock(&out_mutex); + + if (ipipe_trace_enable) { + if (!val) + unregister_ftrace_function(&ipipe_trace_ops); + } else if (val) + register_ftrace_function(&ipipe_trace_ops); + + ipipe_trace_enable = val; + + mutex_unlock(&out_mutex); + + return count; +} + +static const struct file_operations __ipipe_rw_enable_ops = { + .open = __ipipe_rw_proc_val_open, + .read = seq_read, + .write = __ipipe_wr_enable, + .llseek = seq_lseek, + .release = single_release, +}; +#endif /* CONFIG_IPIPE_TRACE_MCOUNT */ + +extern struct proc_dir_entry *ipipe_proc_root; + +void __init __ipipe_tracer_hrclock_initialized(void) +{ + unsigned long long start, end, min = ULLONG_MAX; + int i; + +#ifdef CONFIG_IPIPE_TRACE_VMALLOC + if (!per_cpu(trace_path, 0)) + return; +#endif + /* Calculate minimum overhead of __ipipe_trace() */ + hard_local_irq_disable(); + for (i = 0; i < 100; i++) { + ipipe_read_tsc(start); + __ipipe_trace(IPIPE_TRACE_FUNC, __BUILTIN_RETURN_ADDRESS0, + __BUILTIN_RETURN_ADDRESS1, 0); + ipipe_read_tsc(end); + + end -= start; + if (end < min) + min = end; + } + hard_local_irq_enable(); + trace_overhead = ipipe_tsc2ns(min); +} + +void __init __ipipe_init_tracer(void) +{ + struct proc_dir_entry *trace_dir; +#ifdef CONFIG_IPIPE_TRACE_VMALLOC + int cpu, path; +#endif /* CONFIG_IPIPE_TRACE_VMALLOC */ + +#ifdef CONFIG_IPIPE_TRACE_VMALLOC + for_each_possible_cpu(cpu) { + struct ipipe_trace_path *tp_buf; + + tp_buf = vmalloc_node(sizeof(struct ipipe_trace_path) * + IPIPE_TRACE_PATHS, cpu_to_node(cpu)); + if (!tp_buf) { + pr_err("I-pipe: " + "insufficient memory for trace buffer.\n"); + return; + } + memset(tp_buf, 0, + sizeof(struct ipipe_trace_path) * IPIPE_TRACE_PATHS); + for (path = 0; path < IPIPE_TRACE_PATHS; path++) { + tp_buf[path].begin = -1; + tp_buf[path].end = -1; + } + per_cpu(trace_path, cpu) = tp_buf; + } +#endif /* CONFIG_IPIPE_TRACE_VMALLOC */ + + if (__ipipe_hrclock_ok() && !trace_overhead) + __ipipe_tracer_hrclock_initialized(); + +#ifdef CONFIG_IPIPE_TRACE_ENABLE + ipipe_trace_enable = 1; +#ifdef CONFIG_IPIPE_TRACE_MCOUNT + ftrace_enabled = 1; + register_ftrace_function(&ipipe_trace_ops); +#endif /* CONFIG_IPIPE_TRACE_MCOUNT */ +#endif /* CONFIG_IPIPE_TRACE_ENABLE */ + + trace_dir = proc_mkdir("trace", ipipe_proc_root); + + proc_create("max", 0644, trace_dir, &__ipipe_max_prtrace_fops); + proc_create("frozen", 0644, trace_dir, &__ipipe_frozen_prtrace_fops); + + proc_create("trigger", 0644, trace_dir, &__ipipe_rw_trigger_ops); + + __ipipe_create_trace_proc_val(trace_dir, "pre_trace_points", + &pre_trace); + __ipipe_create_trace_proc_val(trace_dir, "post_trace_points", + &post_trace); + __ipipe_create_trace_proc_val(trace_dir, "back_trace_points", + &back_trace); + __ipipe_create_trace_proc_val(trace_dir, "verbose", + &verbose_trace); +#ifdef CONFIG_IPIPE_TRACE_MCOUNT + proc_create_data("enable", 0644, trace_dir, &__ipipe_rw_enable_ops, + &ipipe_trace_enable); +#else /* !CONFIG_IPIPE_TRACE_MCOUNT */ + __ipipe_create_trace_proc_val(trace_dir, "enable", + &ipipe_trace_enable); +#endif /* !CONFIG_IPIPE_TRACE_MCOUNT */ +} diff --git a/kernel/irq/chip.c b/kernel/irq/chip.c index 94bbd8f..04b01a0 100644 --- a/kernel/irq/chip.c +++ b/kernel/irq/chip.c @@ -16,6 +16,7 @@ #include #include #include +#include #include @@ -144,14 +145,6 @@ int irq_set_chip_data(unsigned int irq, void *data) } EXPORT_SYMBOL(irq_set_chip_data); -struct irq_data *irq_get_irq_data(unsigned int irq) -{ - struct irq_desc *desc = irq_to_desc(irq); - - return desc ? &desc->irq_data : NULL; -} -EXPORT_SYMBOL_GPL(irq_get_irq_data); - static void irq_state_clr_disabled(struct irq_desc *desc) { irqd_clear(&desc->irq_data, IRQD_IRQ_DISABLED); @@ -181,8 +174,13 @@ int irq_startup(struct irq_desc *desc, bool resend) irq_domain_activate_irq(&desc->irq_data); if (desc->irq_data.chip->irq_startup) { + unsigned long flags = hard_cond_local_irq_save(); ret = desc->irq_data.chip->irq_startup(&desc->irq_data); irq_state_clr_masked(desc); + hard_cond_local_irq_restore(flags); +#ifdef CONFIG_IPIPE + desc->istate &= ~IPIPE_IRQS_NEEDS_STARTUP; +#endif } else { irq_enable(desc); } @@ -195,9 +193,12 @@ void irq_shutdown(struct irq_desc *desc) { irq_state_set_disabled(desc); desc->depth = 1; - if (desc->irq_data.chip->irq_shutdown) + if (desc->irq_data.chip->irq_shutdown) { desc->irq_data.chip->irq_shutdown(&desc->irq_data); - else if (desc->irq_data.chip->irq_disable) +#ifdef CONFIG_IPIPE + desc->istate |= IPIPE_IRQS_NEEDS_STARTUP; +#endif + } else if (desc->irq_data.chip->irq_disable) desc->irq_data.chip->irq_disable(&desc->irq_data); else desc->irq_data.chip->irq_mask(&desc->irq_data); @@ -207,12 +208,14 @@ void irq_shutdown(struct irq_desc *desc) void irq_enable(struct irq_desc *desc) { + unsigned long flags = hard_cond_local_irq_save(); irq_state_clr_disabled(desc); if (desc->irq_data.chip->irq_enable) desc->irq_data.chip->irq_enable(&desc->irq_data); else desc->irq_data.chip->irq_unmask(&desc->irq_data); irq_state_clr_masked(desc); + hard_cond_local_irq_restore(flags); } /** @@ -239,11 +242,13 @@ void irq_disable(struct irq_desc *desc) void irq_percpu_enable(struct irq_desc *desc, unsigned int cpu) { + unsigned long flags = hard_cond_local_irq_save(); if (desc->irq_data.chip->irq_enable) desc->irq_data.chip->irq_enable(&desc->irq_data); else desc->irq_data.chip->irq_unmask(&desc->irq_data); cpumask_set_cpu(cpu, desc->percpu_enabled); + hard_cond_local_irq_restore(flags); } void irq_percpu_disable(struct irq_desc *desc, unsigned int cpu) @@ -277,9 +282,13 @@ void mask_irq(struct irq_desc *desc) void unmask_irq(struct irq_desc *desc) { + unsigned long flags; + if (desc->irq_data.chip->irq_unmask) { + flags = hard_cond_local_irq_save(); desc->irq_data.chip->irq_unmask(&desc->irq_data); irq_state_clr_masked(desc); + hard_cond_local_irq_restore(flags); } } @@ -437,7 +446,9 @@ void handle_level_irq(unsigned int irq, struct irq_desc *desc) { raw_spin_lock(&desc->lock); +#ifndef CONFIG_IPIPE mask_ack_irq(desc); +#endif if (!irq_may_run(desc)) goto out_unlock; @@ -473,7 +484,17 @@ static inline void preflow_handler(struct irq_desc *desc) static inline void preflow_handler(struct irq_desc *desc) { } #endif -static void cond_unmask_eoi_irq(struct irq_desc *desc, struct irq_chip *chip) +#ifdef CONFIG_IPIPE +static void cond_release_fasteoi_irq(struct irq_desc *desc, + struct irq_chip *chip) +{ + if (chip->irq_release && + !irqd_irq_disabled(&desc->irq_data) && !desc->threads_oneshot) + chip->irq_release(&desc->irq_data); +} +#endif /* CONFIG_IPIPE */ + +static inline void cond_unmask_eoi_irq(struct irq_desc *desc, struct irq_chip *chip) { if (!(desc->istate & IRQS_ONESHOT)) { chip->irq_eoi(&desc->irq_data); @@ -527,14 +548,23 @@ handle_fasteoi_irq(unsigned int irq, struct irq_desc *desc) goto out; } +#ifndef CONFIG_IPIPE if (desc->istate & IRQS_ONESHOT) mask_irq(desc); +#endif preflow_handler(desc); handle_irq_event(desc); +#ifdef CONFIG_IPIPE + /* + * IRQCHIP_EOI_IF_HANDLED is ignored as the I-pipe always + * sends EOI. + */ + cond_release_fasteoi_irq(desc, chip); +#else /* !CONFIG_IPIPE */ cond_unmask_eoi_irq(desc, chip); - +#endif /* !CONFIG_IPIPE */ raw_spin_unlock(&desc->lock); return; out: @@ -586,7 +616,9 @@ handle_edge_irq(unsigned int irq, struct irq_desc *desc) kstat_incr_irqs_this_cpu(irq, desc); /* Start handling the irq */ +#ifndef CONFIG_IPIPE desc->irq_data.chip->irq_ack(&desc->irq_data); +#endif do { if (unlikely(!desc->action)) { @@ -677,6 +709,18 @@ handle_percpu_irq(unsigned int irq, struct irq_desc *desc) kstat_incr_irqs_this_cpu(irq, desc); +#ifdef CONFIG_IPIPE + (void)chip; + + handle_irq_event_percpu(desc, desc->action); + + if ((desc->percpu_enabled == NULL || + cpumask_test_cpu(smp_processor_id(), desc->percpu_enabled)) && + !irqd_irq_masked(&desc->irq_data) && + !desc->threads_oneshot && + desc->ipipe_end) + desc->ipipe_end(desc->irq_data.irq, desc); +#else if (chip->irq_ack) chip->irq_ack(&desc->irq_data); @@ -684,6 +728,7 @@ handle_percpu_irq(unsigned int irq, struct irq_desc *desc) if (chip->irq_eoi) chip->irq_eoi(&desc->irq_data); +#endif } /** @@ -707,17 +752,180 @@ void handle_percpu_devid_irq(unsigned int irq, struct irq_desc *desc) kstat_incr_irqs_this_cpu(irq, desc); +#ifndef CONFIG_IPIPE if (chip->irq_ack) chip->irq_ack(&desc->irq_data); +#else + (void)chip; +#endif trace_irq_handler_entry(irq, action); res = action->handler(irq, dev_id); trace_irq_handler_exit(irq, action, res); +#ifndef CONFIG_IPIPE if (chip->irq_eoi) chip->irq_eoi(&desc->irq_data); +#else + if ((desc->percpu_enabled == NULL || + cpumask_test_cpu(smp_processor_id(), desc->percpu_enabled)) && + !irqd_irq_masked(&desc->irq_data) && + !desc->threads_oneshot && + desc->ipipe_end) + desc->ipipe_end(desc->irq_data.irq, desc); +#endif +} + +#ifdef CONFIG_IPIPE + +void __ipipe_ack_level_irq(unsigned irq, struct irq_desc *desc) +{ + mask_ack_irq(desc); +} + +void __ipipe_end_level_irq(unsigned irq, struct irq_desc *desc) +{ + desc->irq_data.chip->irq_unmask(&desc->irq_data); +} + +void __ipipe_ack_fasteoi_irq(unsigned irq, struct irq_desc *desc) +{ + desc->irq_data.chip->irq_hold(&desc->irq_data); +} + +void __ipipe_end_fasteoi_irq(unsigned irq, struct irq_desc *desc) +{ + if (desc->irq_data.chip->irq_release) + desc->irq_data.chip->irq_release(&desc->irq_data); +} + +void __ipipe_ack_edge_irq(unsigned irq, struct irq_desc *desc) +{ + desc->irq_data.chip->irq_ack(&desc->irq_data); +} + +void __ipipe_ack_percpu_irq(unsigned irq, struct irq_desc *desc) +{ + if (desc->irq_data.chip->irq_ack) + desc->irq_data.chip->irq_ack(&desc->irq_data); + + if (desc->irq_data.chip->irq_eoi) + desc->irq_data.chip->irq_eoi(&desc->irq_data); +} + +void __ipipe_nop_irq(unsigned irq, struct irq_desc *desc) +{ +} + +void __ipipe_chained_irq(unsigned irq, struct irq_desc *desc) +{ + /* + * XXX: Do NOT fold this into __ipipe_nop_irq(), see + * ipipe_chained_irq_p(). + */ +} + +static void __ipipe_ack_bad_irq(unsigned irq, struct irq_desc *desc) +{ + handle_bad_irq(irq, desc); + WARN_ON_ONCE(1); } +irq_flow_handler_t +__fixup_irq_handler(struct irq_desc *desc, irq_flow_handler_t handle, int is_chained) +{ + if (unlikely(handle == NULL)) { + desc->ipipe_ack = __ipipe_ack_bad_irq; + desc->ipipe_end = __ipipe_nop_irq; + } else { + if (is_chained) { + desc->ipipe_ack = handle; + desc->ipipe_end = __ipipe_nop_irq; + handle = __ipipe_chained_irq; + } else if (handle == handle_simple_irq) { + desc->ipipe_ack = __ipipe_nop_irq; + desc->ipipe_end = __ipipe_nop_irq; + } else if (handle == handle_level_irq) { + desc->ipipe_ack = __ipipe_ack_level_irq; + desc->ipipe_end = __ipipe_end_level_irq; + } else if (handle == handle_edge_irq) { + desc->ipipe_ack = __ipipe_ack_edge_irq; + desc->ipipe_end = __ipipe_nop_irq; + } else if (handle == handle_fasteoi_irq) { + desc->ipipe_ack = __ipipe_ack_fasteoi_irq; + desc->ipipe_end = __ipipe_end_fasteoi_irq; + } else if (handle == handle_percpu_irq || + handle == handle_percpu_devid_irq) { + if (irq_desc_get_chip(desc) && + irq_desc_get_chip(desc)->irq_hold) { + desc->ipipe_ack = __ipipe_ack_fasteoi_irq; + desc->ipipe_end = __ipipe_end_fasteoi_irq; + } else { + desc->ipipe_ack = __ipipe_ack_percpu_irq; + desc->ipipe_end = __ipipe_nop_irq; + } + } else if (irq_desc_get_chip(desc) == &no_irq_chip) { + desc->ipipe_ack = __ipipe_nop_irq; + desc->ipipe_end = __ipipe_nop_irq; + } else { + desc->ipipe_ack = __ipipe_ack_bad_irq; + desc->ipipe_end = __ipipe_nop_irq; + } + } + + /* Suppress intermediate trampoline routine. */ + ipipe_root_domain->irqs[desc->irq_data.irq].ackfn = desc->ipipe_ack; + + return handle; +} + +void ipipe_enable_irq(unsigned int irq) +{ + struct irq_desc *desc; + struct irq_chip *chip; + unsigned long flags; + + desc = irq_to_desc(irq); + if (desc == NULL) + return; + + chip = irq_desc_get_chip(desc); + + if (chip->irq_startup && (desc->istate & IPIPE_IRQS_NEEDS_STARTUP)) { + + ipipe_root_only(); + + raw_spin_lock_irqsave(&desc->lock, flags); + if (desc->istate & IPIPE_IRQS_NEEDS_STARTUP) { + desc->istate &= ~IPIPE_IRQS_NEEDS_STARTUP; + chip->irq_startup(&desc->irq_data); + } + raw_spin_unlock_irqrestore(&desc->lock, flags); + + return; + } + + if (WARN_ON_ONCE(chip->irq_enable == NULL && chip->irq_unmask == NULL)) + return; + + if (chip->irq_enable) + chip->irq_enable(&desc->irq_data); + else + chip->irq_unmask(&desc->irq_data); +} +EXPORT_SYMBOL_GPL(ipipe_enable_irq); + +#else /* !CONFIG_IPIPE */ + +irq_flow_handler_t +__fixup_irq_handler(struct irq_desc *desc, irq_flow_handler_t handle, int is_chained) +{ + return handle; +} + +#endif /* !CONFIG_IPIPE */ +EXPORT_SYMBOL_GPL(__fixup_irq_handler); + void __irq_set_handler(unsigned int irq, irq_flow_handler_t handle, int is_chained, const char *name) @@ -758,6 +966,8 @@ __irq_set_handler(unsigned int irq, irq_flow_handler_t handle, int is_chained, goto out; } + handle = __fixup_irq_handler(desc, handle, is_chained); + /* Uninstall? */ if (handle == handle_bad_irq) { if (desc->irq_data.chip != &no_irq_chip) @@ -915,6 +1125,20 @@ void irq_chip_eoi_parent(struct irq_data *data) data->chip->irq_eoi(data); } +#ifdef CONFIG_IPIPE +void irq_chip_hold_parent(struct irq_data *data) +{ + data = data->parent_data; + data->chip->irq_hold(data); +} + +void irq_chip_release_parent(struct irq_data *data) +{ + data = data->parent_data; + data->chip->irq_release(data); +} +#endif + /** * irq_chip_set_affinity_parent - Set affinity on the parent interrupt * @data: Pointer to interrupt specific data diff --git a/kernel/irq/generic-chip.c b/kernel/irq/generic-chip.c index 61024e8..8d77e4c 100644 --- a/kernel/irq/generic-chip.c +++ b/kernel/irq/generic-chip.c @@ -36,12 +36,13 @@ void irq_gc_mask_disable_reg(struct irq_data *d) { struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); struct irq_chip_type *ct = irq_data_get_chip_type(d); + unsigned long flags; u32 mask = d->mask; - irq_gc_lock(gc); + flags = irq_gc_lock(gc); irq_reg_writel(gc, mask, ct->regs.disable); *ct->mask_cache &= ~mask; - irq_gc_unlock(gc); + irq_gc_unlock(gc, flags); } /** @@ -55,12 +56,13 @@ void irq_gc_mask_set_bit(struct irq_data *d) { struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); struct irq_chip_type *ct = irq_data_get_chip_type(d); + unsigned long flags; u32 mask = d->mask; - irq_gc_lock(gc); + flags = irq_gc_lock(gc); *ct->mask_cache |= mask; irq_reg_writel(gc, *ct->mask_cache, ct->regs.mask); - irq_gc_unlock(gc); + irq_gc_unlock(gc, flags); } EXPORT_SYMBOL_GPL(irq_gc_mask_set_bit); @@ -75,12 +77,13 @@ void irq_gc_mask_clr_bit(struct irq_data *d) { struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); struct irq_chip_type *ct = irq_data_get_chip_type(d); + unsigned long flags; u32 mask = d->mask; - irq_gc_lock(gc); + flags = irq_gc_lock(gc); *ct->mask_cache &= ~mask; irq_reg_writel(gc, *ct->mask_cache, ct->regs.mask); - irq_gc_unlock(gc); + irq_gc_unlock(gc, flags); } EXPORT_SYMBOL_GPL(irq_gc_mask_clr_bit); @@ -95,12 +98,13 @@ void irq_gc_unmask_enable_reg(struct irq_data *d) { struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); struct irq_chip_type *ct = irq_data_get_chip_type(d); + unsigned long flags; u32 mask = d->mask; - irq_gc_lock(gc); + flags = irq_gc_lock(gc); irq_reg_writel(gc, mask, ct->regs.enable); *ct->mask_cache |= mask; - irq_gc_unlock(gc); + irq_gc_unlock(gc, flags); } /** @@ -111,11 +115,12 @@ void irq_gc_ack_set_bit(struct irq_data *d) { struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); struct irq_chip_type *ct = irq_data_get_chip_type(d); + unsigned long flags; u32 mask = d->mask; - irq_gc_lock(gc); + flags = irq_gc_lock(gc); irq_reg_writel(gc, mask, ct->regs.ack); - irq_gc_unlock(gc); + irq_gc_unlock(gc, flags); } EXPORT_SYMBOL_GPL(irq_gc_ack_set_bit); @@ -127,11 +132,12 @@ void irq_gc_ack_clr_bit(struct irq_data *d) { struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); struct irq_chip_type *ct = irq_data_get_chip_type(d); + unsigned long flags; u32 mask = ~d->mask; - irq_gc_lock(gc); + flags = irq_gc_lock(gc); irq_reg_writel(gc, mask, ct->regs.ack); - irq_gc_unlock(gc); + irq_gc_unlock(gc, flags); } /** @@ -142,12 +148,13 @@ void irq_gc_mask_disable_reg_and_ack(struct irq_data *d) { struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); struct irq_chip_type *ct = irq_data_get_chip_type(d); + unsigned long flags; u32 mask = d->mask; - irq_gc_lock(gc); + flags = irq_gc_lock(gc); irq_reg_writel(gc, mask, ct->regs.mask); irq_reg_writel(gc, mask, ct->regs.ack); - irq_gc_unlock(gc); + irq_gc_unlock(gc, flags); } /** @@ -158,11 +165,12 @@ void irq_gc_eoi(struct irq_data *d) { struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); struct irq_chip_type *ct = irq_data_get_chip_type(d); + unsigned long flags; u32 mask = d->mask; - irq_gc_lock(gc); + flags = irq_gc_lock(gc); irq_reg_writel(gc, mask, ct->regs.eoi); - irq_gc_unlock(gc); + irq_gc_unlock(gc, flags); } /** @@ -177,17 +185,18 @@ void irq_gc_eoi(struct irq_data *d) int irq_gc_set_wake(struct irq_data *d, unsigned int on) { struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); + unsigned long flags; u32 mask = d->mask; if (!(mask & gc->wake_enabled)) return -EINVAL; - irq_gc_lock(gc); + flags = irq_gc_lock(gc); if (on) gc->wake_active |= mask; else gc->wake_active &= ~mask; - irq_gc_unlock(gc); + irq_gc_unlock(gc, flags); return 0; } diff --git a/kernel/irq/internals.h b/kernel/irq/internals.h index df553b0..065468d 100644 --- a/kernel/irq/internals.h +++ b/kernel/irq/internals.h @@ -54,6 +54,9 @@ enum { IRQS_WAITING = 0x00000080, IRQS_PENDING = 0x00000200, IRQS_SUSPENDED = 0x00000800, +#ifdef CONFIG_IPIPE + IPIPE_IRQS_NEEDS_STARTUP= 0x80000000, +#endif }; #include "debug.h" diff --git a/kernel/irq/irqdesc.c b/kernel/irq/irqdesc.c index 99793b9..16cda25 100644 --- a/kernel/irq/irqdesc.c +++ b/kernel/irq/irqdesc.c @@ -92,6 +92,9 @@ static void desc_set_defaults(unsigned int irq, struct irq_desc *desc, int node, for_each_possible_cpu(cpu) *per_cpu_ptr(desc->kstat_irqs, cpu) = 0; desc_smp_init(desc, node); +#ifdef CONFIG_IPIPE + desc->istate |= IPIPE_IRQS_NEEDS_STARTUP; +#endif } int nr_irqs = NR_IRQS; @@ -287,11 +290,13 @@ int __init early_irq_init(void) return arch_early_irq_init(); } +#ifndef CONFIG_IPIPE struct irq_desc *irq_to_desc(unsigned int irq) { return (irq < NR_IRQS) ? irq_desc + irq : NULL; } EXPORT_SYMBOL(irq_to_desc); +#endif /* CONFIG_IPIPE */ static void free_desc(unsigned int irq) { diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c index e68932b..d5dc452 100644 --- a/kernel/irq/manage.c +++ b/kernel/irq/manage.c @@ -760,9 +760,14 @@ again: desc->threads_oneshot &= ~action->thread_mask; +#ifndef CONFIG_IPIPE if (!desc->threads_oneshot && !irqd_irq_disabled(&desc->irq_data) && irqd_irq_masked(&desc->irq_data)) unmask_threaded_irq(desc); +#else /* CONFIG_IPIPE */ + if (!desc->threads_oneshot && !irqd_irq_disabled(&desc->irq_data)) + desc->ipipe_end(desc->irq_data.irq, desc); +#endif /* CONFIG_IPIPE */ out_unlock: raw_spin_unlock_irq(&desc->lock); diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c index aaeae88..eecaf07 100644 --- a/kernel/locking/lockdep.c +++ b/kernel/locking/lockdep.c @@ -2592,7 +2592,7 @@ __visible void trace_hardirqs_on_caller(unsigned long ip) * already enabled, yet we find the hardware thinks they are in fact * enabled.. someone messed up their IRQ state tracing. */ - if (DEBUG_LOCKS_WARN_ON(!irqs_disabled())) + if (DEBUG_LOCKS_WARN_ON(!irqs_disabled() && !hard_irqs_disabled())) return; /* @@ -2636,7 +2636,7 @@ __visible void trace_hardirqs_off_caller(unsigned long ip) * So we're supposed to get called after you mask local IRQs, but for * some reason the hardware doesn't quite think you did a proper job. */ - if (DEBUG_LOCKS_WARN_ON(!irqs_disabled())) + if (DEBUG_LOCKS_WARN_ON(!irqs_disabled() && !hard_irqs_disabled())) return; if (curr->hardirqs_enabled) { @@ -2672,7 +2672,7 @@ void trace_softirqs_on(unsigned long ip) * We fancy IRQs being disabled here, see softirq.c, avoids * funny state and nesting things. */ - if (DEBUG_LOCKS_WARN_ON(!irqs_disabled())) + if (DEBUG_LOCKS_WARN_ON(!irqs_disabled() && !hard_irqs_disabled())) return; if (curr->softirqs_enabled) { @@ -2711,7 +2711,7 @@ void trace_softirqs_off(unsigned long ip) /* * We fancy IRQs being disabled here, see softirq.c */ - if (DEBUG_LOCKS_WARN_ON(!irqs_disabled())) + if (DEBUG_LOCKS_WARN_ON(!irqs_disabled() && !hard_irqs_disabled())) return; if (curr->softirqs_enabled) { diff --git a/kernel/locking/spinlock.c b/kernel/locking/spinlock.c index db3ccb1..da9b4d0 100644 --- a/kernel/locking/spinlock.c +++ b/kernel/locking/spinlock.c @@ -26,7 +26,9 @@ * even on CONFIG_PREEMPT, because lockdep assumes that interrupts are * not re-enabled during lock-acquire (which the preempt-spin-ops do): */ -#if !defined(CONFIG_GENERIC_LOCKBREAK) || defined(CONFIG_DEBUG_LOCK_ALLOC) +#if !defined(CONFIG_GENERIC_LOCKBREAK) || \ + defined(CONFIG_DEBUG_LOCK_ALLOC) || \ + defined(CONFIG_IPIPE) /* * The __lock_function inlines are taken from * include/linux/spinlock_api_smp.h diff --git a/kernel/module.c b/kernel/module.c index 3b9ff96..9bc0581 100644 --- a/kernel/module.c +++ b/kernel/module.c @@ -943,7 +943,7 @@ bool try_module_get(struct module *module) bool ret = true; if (module) { - preempt_disable(); + unsigned long flags = hard_preempt_disable(); /* Note: here, we can fail to get a reference */ if (likely(module_is_live(module) && atomic_inc_not_zero(&module->refcnt) != 0)) @@ -951,7 +951,7 @@ bool try_module_get(struct module *module) else ret = false; - preempt_enable(); + hard_preempt_enable(flags); } return ret; } @@ -962,11 +962,11 @@ void module_put(struct module *module) int ret; if (module) { - preempt_disable(); + unsigned long flags = hard_preempt_disable(); ret = atomic_dec_if_positive(&module->refcnt); WARN_ON(ret < 0); /* Failed to put refcount */ trace_module_put(module, _RET_IP_); - preempt_enable(); + hard_preempt_enable(flags); } } EXPORT_SYMBOL(module_put); diff --git a/kernel/panic.c b/kernel/panic.c index a4f7820..a13096b 100644 --- a/kernel/panic.c +++ b/kernel/panic.c @@ -24,6 +24,7 @@ #include #include #include +#include #define PANIC_TIMER_STEP 100 #define PANIC_BLINK_SPD 18 @@ -388,6 +389,8 @@ void oops_enter(void) { tracing_off(); /* can't trust the integrity of the kernel anymore: */ + ipipe_trace_panic_freeze(); + ipipe_disable_context_check(); debug_locks_off(); do_oops_enter_exit(); } diff --git a/kernel/power/hibernate.c b/kernel/power/hibernate.c index 2329daa..79cfe9b 100644 --- a/kernel/power/hibernate.c +++ b/kernel/power/hibernate.c @@ -284,6 +284,7 @@ static int create_image(int platform_mode) goto Enable_cpus; local_irq_disable(); + hard_cond_local_irq_disable(); error = syscore_suspend(); if (error) { @@ -437,6 +438,7 @@ static int resume_target_kernel(bool platform_mode) goto Enable_cpus; local_irq_disable(); + hard_cond_local_irq_disable(); error = syscore_suspend(); if (error) @@ -555,6 +557,7 @@ int hibernation_platform_enter(void) goto Platform_finish; local_irq_disable(); + hard_cond_local_irq_disable(); syscore_suspend(); if (pm_wakeup_pending()) { error = -EAGAIN; diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index 3c1aca0..ae6b3d5 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -62,6 +62,9 @@ int console_printk[4] = { CONSOLE_LOGLEVEL_DEFAULT, /* default_console_loglevel */ }; +/* Deferred messaged from sched code are marked by this special level */ +#define SCHED_MESSAGE_LOGLEVEL -2 + /* * Low level drivers may need that to know if they can schedule in * their unblank() callback or not. So let's export it. @@ -1257,7 +1260,7 @@ static int syslog_print_all(char __user *buf, int size, bool clear) int do_syslog(int type, char __user *buf, int len, bool from_file) { bool clear = false; - static int saved_console_loglevel = LOGLEVEL_DEFAULT; + static int saved_console_loglevel = -1; int error; error = check_syslog_permissions(type, from_file); @@ -1310,15 +1313,15 @@ int do_syslog(int type, char __user *buf, int len, bool from_file) break; /* Disable logging to console */ case SYSLOG_ACTION_CONSOLE_OFF: - if (saved_console_loglevel == LOGLEVEL_DEFAULT) + if (saved_console_loglevel == -1) saved_console_loglevel = console_loglevel; console_loglevel = minimum_console_loglevel; break; /* Enable logging to console */ case SYSLOG_ACTION_CONSOLE_ON: - if (saved_console_loglevel != LOGLEVEL_DEFAULT) { + if (saved_console_loglevel != -1) { console_loglevel = saved_console_loglevel; - saved_console_loglevel = LOGLEVEL_DEFAULT; + saved_console_loglevel = -1; } break; /* Set level of messages printed to console */ @@ -1330,7 +1333,7 @@ int do_syslog(int type, char __user *buf, int len, bool from_file) len = minimum_console_loglevel; console_loglevel = len; /* Implicitly re-enable logging to console */ - saved_console_loglevel = LOGLEVEL_DEFAULT; + saved_console_loglevel = -1; error = 0; break; /* Number of chars in the log buffer */ @@ -1621,10 +1624,10 @@ asmlinkage int vprintk_emit(int facility, int level, int printed_len = 0; bool in_sched = false; /* cpu currently holding logbuf_lock in this function */ - static unsigned int logbuf_cpu = UINT_MAX; + static volatile unsigned int logbuf_cpu = UINT_MAX; - if (level == LOGLEVEL_SCHED) { - level = LOGLEVEL_DEFAULT; + if (level == SCHED_MESSAGE_LOGLEVEL) { + level = -1; in_sched = true; } @@ -1689,9 +1692,8 @@ asmlinkage int vprintk_emit(int facility, int level, const char *end_of_header = printk_skip_level(text); switch (kern_level) { case '0' ... '7': - if (level == LOGLEVEL_DEFAULT) + if (level == -1) level = kern_level - '0'; - /* fallthrough */ case 'd': /* KERN_DEFAULT */ lflags |= LOG_PREFIX; } @@ -1705,7 +1707,7 @@ asmlinkage int vprintk_emit(int facility, int level, } } - if (level == LOGLEVEL_DEFAULT) + if (level == -1) level = default_message_loglevel; if (dict) @@ -1783,7 +1785,7 @@ EXPORT_SYMBOL(vprintk_emit); asmlinkage int vprintk(const char *fmt, va_list args) { - return vprintk_emit(0, LOGLEVEL_DEFAULT, NULL, 0, fmt, args); + return vprintk_emit(0, -1, NULL, 0, fmt, args); } EXPORT_SYMBOL(vprintk); @@ -1826,6 +1828,43 @@ EXPORT_SYMBOL_GPL(vprintk_default); */ DEFINE_PER_CPU(printk_func_t, printk_func) = vprintk_default; +#ifdef CONFIG_IPIPE + +extern int __ipipe_printk_bypass; + +static IPIPE_DEFINE_SPINLOCK(__ipipe_printk_lock); + +static int __ipipe_printk_fill; + +static char __ipipe_printk_buf[__LOG_BUF_LEN]; + +void __ipipe_flush_printk (unsigned virq, void *cookie) +{ + char *p = __ipipe_printk_buf; + int len, lmax, out = 0; + unsigned long flags; + + goto start; + + do { + raw_spin_unlock_irqrestore(&__ipipe_printk_lock, flags); + start: + lmax = __ipipe_printk_fill; + while (out < lmax) { + len = strlen(p) + 1; + printk("%s",p); + p += len; + out += len; + } + raw_spin_lock_irqsave(&__ipipe_printk_lock, flags); + } + while (__ipipe_printk_fill != lmax); + + __ipipe_printk_fill = 0; + + raw_spin_unlock_irqrestore(&__ipipe_printk_lock, flags); +} + /** * printk - print a kernel message * @fmt: format string @@ -1849,6 +1888,59 @@ DEFINE_PER_CPU(printk_func_t, printk_func) = vprintk_default; */ asmlinkage __visible int printk(const char *fmt, ...) { + int sprintk = 1, cs = -1; + int r, fbytes, oldcount; + unsigned long flags; + va_list args; + + va_start(args, fmt); + + flags = hard_local_irq_save(); + + if (__ipipe_printk_bypass || oops_in_progress) + cs = ipipe_disable_context_check(); + else if (__ipipe_current_domain == ipipe_root_domain) { + if (ipipe_head_domain != ipipe_root_domain && + (raw_irqs_disabled_flags(flags) || + test_bit(IPIPE_STALL_FLAG, &__ipipe_head_status))) + sprintk = 0; + } else + sprintk = 0; + + hard_local_irq_restore(flags); + + if (sprintk) { + r = vprintk(fmt, args); + if (cs != -1) + ipipe_restore_context_check(cs); + goto out; + } + + raw_spin_lock_irqsave(&__ipipe_printk_lock, flags); + + oldcount = __ipipe_printk_fill; + fbytes = __LOG_BUF_LEN - oldcount; + if (fbytes > 1) { + r = vscnprintf(__ipipe_printk_buf + __ipipe_printk_fill, + fbytes, fmt, args) + 1; + __ipipe_printk_fill += r; + } else + r = 0; + + raw_spin_unlock_irqrestore(&__ipipe_printk_lock, flags); + + if (oldcount == 0) + ipipe_raise_irq(__ipipe_printk_virq); +out: + va_end(args); + + return r; +} + +#else /* !CONFIG_IPIPE */ + +asmlinkage __visible int printk(const char *fmt, ...) +{ printk_func_t vprintk_func; va_list args; int r; @@ -1868,6 +1960,8 @@ asmlinkage __visible int printk(const char *fmt, ...) return r; } +#endif /* CONFIG_IPIPE */ + EXPORT_SYMBOL(printk); #else /* CONFIG_PRINTK */ @@ -1905,21 +1999,80 @@ DEFINE_PER_CPU(printk_func_t, printk_func); #ifdef CONFIG_EARLY_PRINTK struct console *early_console; +void early_vprintk(const char *fmt, va_list ap) +{ + if (early_console) { + char buf[512]; + int n = vscnprintf(buf, sizeof(buf), fmt, ap); + + early_console->write(early_console, buf, n); + } +} + asmlinkage __visible void early_printk(const char *fmt, ...) { va_list ap; - char buf[512]; - int n; - if (!early_console) + va_start(ap, fmt); + early_vprintk(fmt, ap); + va_end(ap); +} +#endif + +#ifdef CONFIG_RAW_PRINTK +static struct console *raw_console; +static IPIPE_DEFINE_RAW_SPINLOCK(raw_console_lock); + +void raw_vprintk(const char *fmt, va_list ap) +{ + unsigned long flags; + char buf[256]; + int n; + + if (raw_console == NULL || console_suspended) return; - va_start(ap, fmt); n = vscnprintf(buf, sizeof(buf), fmt, ap); + touch_nmi_watchdog(); + raw_spin_lock_irqsave(&raw_console_lock, flags); + if (raw_console) + raw_console->write_raw(raw_console, buf, n); + raw_spin_unlock_irqrestore(&raw_console_lock, flags); +} + +asmlinkage __visible void raw_printk(const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + raw_vprintk(fmt, ap); va_end(ap); +} + +static inline void register_raw_console(struct console *newcon) +{ + if ((newcon->flags & CON_RAW) != 0 && newcon->write_raw) + raw_console = newcon; +} + +static inline void unregister_raw_console(struct console *oldcon) +{ + unsigned long flags; - early_console->write(early_console, buf, n); + raw_spin_lock_irqsave(&raw_console_lock, flags); + if (oldcon == raw_console) + raw_console = NULL; + raw_spin_unlock_irqrestore(&raw_console_lock, flags); } + +#else + +static inline void register_raw_console(struct console *newcon) +{ } + +static inline void unregister_raw_console(struct console *oldcon) +{ } + #endif static int __add_preferred_console(char *name, int idx, char *options, @@ -2527,6 +2680,10 @@ void register_console(struct console *newcon) newcon->next = console_drivers->next; console_drivers->next = newcon; } + + /* The latest raw console to register is current. */ + register_raw_console(newcon); + if (newcon->flags & CON_PRINTBUFFER) { /* * console_unlock(); will print out the buffered messages @@ -2579,6 +2736,8 @@ int unregister_console(struct console *console) (console->flags & CON_BOOT) ? "boot" : "" , console->name, console->index); + unregister_raw_console(console); + res = _braille_unregister_console(console); if (res) return res; @@ -2672,7 +2831,7 @@ int printk_deferred(const char *fmt, ...) preempt_disable(); va_start(args, fmt); - r = vprintk_emit(0, LOGLEVEL_SCHED, NULL, 0, fmt, args); + r = vprintk_emit(0, SCHED_MESSAGE_LOGLEVEL, NULL, 0, fmt, args); va_end(args); __this_cpu_or(printk_pending, PRINTK_PENDING_OUTPUT); diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 4d870eb..fa7217c 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -1565,7 +1565,9 @@ void scheduler_ipi(void) * however a fair share of IPIs are still resched only so this would * somewhat pessimize the simple resched case. */ +#ifndef IPIPE_ARCH_HAVE_VIRQ_IPI irq_enter(); +#endif sched_ttwu_pending(); /* @@ -1575,7 +1577,9 @@ void scheduler_ipi(void) this_rq()->idle_balance = 1; raise_softirq_irqoff(SCHED_SOFTIRQ); } +#ifndef IPIPE_ARCH_HAVE_VIRQ_IPI irq_exit(); +#endif } static void ttwu_queue_remote(struct task_struct *p, int cpu) @@ -1666,7 +1670,8 @@ try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) */ smp_mb__before_spinlock(); raw_spin_lock_irqsave(&p->pi_lock, flags); - if (!(p->state & state)) + if (!(p->state & state) || + (p->state & (TASK_NOWAKEUP|TASK_HARDENING))) goto out; success = 1; /* we're going to change ->state */ @@ -2283,6 +2288,8 @@ asmlinkage __visible void schedule_tail(struct task_struct *prev) { struct rq *rq; + __ipipe_complete_domain_migration(); + /* finish_task_switch() drops rq->lock and enables preemtion */ preempt_disable(); rq = finish_task_switch(prev); @@ -2337,6 +2344,9 @@ context_switch(struct rq *rq, struct task_struct *prev, switch_to(prev, next, prev); barrier(); + if (unlikely(__ipipe_switch_tail())) + return NULL; + return finish_task_switch(prev); } @@ -2558,6 +2568,7 @@ notrace unsigned long get_parent_ip(unsigned long addr) void preempt_count_add(int val) { + ipipe_preempt_root_only(); #ifdef CONFIG_DEBUG_PREEMPT /* * Underflow? @@ -2586,6 +2597,7 @@ NOKPROBE_SYMBOL(preempt_count_add); void preempt_count_sub(int val) { + ipipe_preempt_root_only(); #ifdef CONFIG_DEBUG_PREEMPT /* * Underflow? @@ -2640,6 +2652,7 @@ static noinline void __schedule_bug(struct task_struct *prev) */ static inline void schedule_debug(struct task_struct *prev) { + ipipe_root_only(); #ifdef CONFIG_SCHED_STACK_END_CHECK BUG_ON(unlikely(task_stack_end_corrupted(prev))); #endif @@ -2737,7 +2750,7 @@ again: * accordingly in case an event triggered the need for rescheduling (such as * an interrupt waking up a task) while preemption was disabled in __schedule(). */ -static void __sched __schedule(void) +static int __sched __schedule(void) { struct task_struct *prev, *next; unsigned long *switch_count; @@ -2750,6 +2763,10 @@ static void __sched __schedule(void) rcu_note_context_switch(); prev = rq->curr; + if (unlikely(prev->state & TASK_HARDENING)) + /* Pop one disable level -- one still remains. */ + preempt_enable(); + schedule_debug(prev); if (sched_feat(HRTICK)) @@ -2803,13 +2820,19 @@ static void __sched __schedule(void) ++*switch_count; rq = context_switch(rq, prev, next); /* unlocks the rq */ + if (rq == NULL) + return 1; /* task hijacked by head domain */ cpu = cpu_of(rq); - } else + } else { + prev->state &= ~TASK_HARDENING; raw_spin_unlock_irq(&rq->lock); + } post_schedule(rq); sched_preempt_enable_no_resched(); + + return 0; } static inline void sched_submit_work(struct task_struct *tsk) @@ -2830,7 +2853,8 @@ asmlinkage __visible void __sched schedule(void) sched_submit_work(tsk); do { - __schedule(); + if (__schedule()) + return; } while (need_resched()); } EXPORT_SYMBOL(schedule); @@ -2870,7 +2894,8 @@ static void __sched notrace preempt_schedule_common(void) { do { __preempt_count_add(PREEMPT_ACTIVE); - __schedule(); + if (__schedule()) + return; __preempt_count_sub(PREEMPT_ACTIVE); /* @@ -2893,7 +2918,7 @@ asmlinkage __visible void __sched notrace preempt_schedule(void) * If there is a non-zero preempt_count or interrupts are disabled, * we do not want to preempt the current task. Just return.. */ - if (likely(!preemptible())) + if (likely(!preemptible() || !ipipe_root_p)) return; preempt_schedule_common(); @@ -3622,6 +3647,7 @@ change: prev_class = p->sched_class; __setscheduler(rq, p, attr, true); + __ipipe_report_setsched(p); if (running) p->sched_class->set_curr_task(rq); @@ -4805,10 +4831,13 @@ int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask) do_set_cpus_allowed(p, new_mask); /* Can the task run on the task's current CPU? If so, we're done */ - if (cpumask_test_cpu(task_cpu(p), new_mask)) + if (cpumask_test_cpu(task_cpu(p), new_mask)) { + __ipipe_report_setaffinity(p, task_cpu(p)); goto out; + } dest_cpu = cpumask_any_and(cpu_active_mask, new_mask); + __ipipe_report_setaffinity(p, dest_cpu); if (task_running(rq, p) || p->state == TASK_WAKING) { struct migration_arg arg = { p, dest_cpu }; /* Need help from migration thread: drop lock and wait. */ @@ -8394,3 +8423,42 @@ void dump_cpu_task(int cpu) pr_info("Task dump for CPU %d:\n", cpu); sched_show_task(cpu_curr(cpu)); } + +#ifdef CONFIG_IPIPE + +int __ipipe_migrate_head(void) +{ + struct task_struct *p = current; + + preempt_disable(); + + IPIPE_WARN_ONCE(__this_cpu_read(ipipe_percpu.task_hijacked) != NULL); + + __this_cpu_write(ipipe_percpu.task_hijacked, p); + set_current_state(TASK_INTERRUPTIBLE | TASK_HARDENING); + sched_submit_work(p); + if (likely(__schedule())) + return 0; + + if (signal_pending(p)) + return -ERESTARTSYS; + + BUG(); +} +EXPORT_SYMBOL_GPL(__ipipe_migrate_head); + +void __ipipe_reenter_root(void) +{ + struct rq *rq; + struct task_struct *p; + + p = __this_cpu_read(ipipe_percpu.rqlock_owner); + BUG_ON(p == NULL); + ipipe_clear_thread_flag(TIP_HEAD); + rq = finish_task_switch(p); + post_schedule(rq); + sched_preempt_enable_no_resched(); +} +EXPORT_SYMBOL_GPL(__ipipe_reenter_root); + +#endif /* CONFIG_IPIPE */ diff --git a/kernel/sched/wait.c b/kernel/sched/wait.c index 852143a..efc4a6f 100644 --- a/kernel/sched/wait.c +++ b/kernel/sched/wait.c @@ -67,6 +67,8 @@ static void __wake_up_common(wait_queue_head_t *q, unsigned int mode, { wait_queue_t *curr, *next; + ipipe_root_only(); + list_for_each_entry_safe(curr, next, &q->task_list, task_list) { unsigned flags = curr->flags; diff --git a/kernel/signal.c b/kernel/signal.c index 0206be7..f77f96d 100644 --- a/kernel/signal.c +++ b/kernel/signal.c @@ -690,6 +690,10 @@ int dequeue_signal(struct task_struct *tsk, sigset_t *mask, siginfo_t *info) void signal_wake_up_state(struct task_struct *t, unsigned int state) { set_tsk_thread_flag(t, TIF_SIGPENDING); + + /* TIF_SIGPENDING must be prior to reporting. */ + __ipipe_report_sigwake(t); + /* * TASK_WAKEKILL also means wake it up in the stopped/traced/killable * case. We don't check t->state here because there is a race with it @@ -913,8 +917,11 @@ static inline int wants_signal(int sig, struct task_struct *p) return 0; if (sig == SIGKILL) return 1; - if (task_is_stopped_or_traced(p)) + if (task_is_stopped_or_traced(p)) { + if (!signal_pending(p)) + __ipipe_report_sigwake(p); return 0; + } return task_curr(p) || !signal_pending(p); } diff --git a/kernel/time/clockevents.c b/kernel/time/clockevents.c index 637a094..edd76be 100644 --- a/kernel/time/clockevents.c +++ b/kernel/time/clockevents.c @@ -17,6 +17,7 @@ #include #include #include +#include #include "tick-internal.h" @@ -482,6 +483,8 @@ void clockevents_register_device(struct clock_event_device *dev) /* Initialize state to DETACHED */ dev->state = CLOCK_EVT_STATE_DETACHED; + ipipe_host_timer_register(dev); + if (!dev->cpumask) { WARN_ON(num_possible_cpus() > 1); dev->cpumask = cpumask_of(smp_processor_id()); diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c index 15facb1..87ebf01 100644 --- a/kernel/time/clocksource.c +++ b/kernel/time/clocksource.c @@ -30,6 +30,7 @@ #include /* for spin_unlock_irq() using preempt_count() m68k */ #include #include +#include #include "tick-internal.h" #include "timekeeping_internal.h" @@ -170,6 +171,9 @@ static void clocksource_watchdog(unsigned long data) cycle_t csnow, wdnow, cslast, wdlast, delta; int64_t wd_nsec, cs_nsec; int next_cpu, reset_pending; +#ifdef CONFIG_IPIPE + cycle_t wdref; +#endif spin_lock(&watchdog_lock); if (!watchdog_running) @@ -186,11 +190,24 @@ static void clocksource_watchdog(unsigned long data) continue; } +#ifdef CONFIG_IPIPE +retry: +#endif local_irq_disable(); +#ifdef CONFIG_IPIPE + wdref = watchdog->read(watchdog); +#endif csnow = cs->read(cs); wdnow = watchdog->read(watchdog); local_irq_enable(); +#ifdef CONFIG_IPIPE + wd_nsec = clocksource_cyc2ns((wdnow - wdref) & watchdog->mask, + watchdog->mult, watchdog->shift); + if (wd_nsec > WATCHDOG_THRESHOLD) + goto retry; +#endif + /* Clocksource initialized ? */ if (!(cs->flags & CLOCK_SOURCE_WATCHDOG) || atomic_read(&watchdog_reset_pending)) { @@ -630,6 +647,95 @@ static int __init clocksource_done_booting(void) } fs_initcall(clocksource_done_booting); +#ifdef CONFIG_IPIPE_WANT_CLOCKSOURCE +unsigned long long __ipipe_cs_freq; +EXPORT_SYMBOL_GPL(__ipipe_cs_freq); + +struct clocksource *__ipipe_cs; +EXPORT_SYMBOL_GPL(__ipipe_cs); + +cycle_t (*__ipipe_cs_read)(struct clocksource *cs); +cycle_t __ipipe_cs_last_tsc; +cycle_t __ipipe_cs_mask; +unsigned __ipipe_cs_lat = 0xffffffff; + +static void ipipe_check_clocksource(struct clocksource *cs) +{ + cycle_t (*cread)(struct clocksource *cs); + cycle_t lat, mask, saved; + unsigned long long freq; + unsigned long flags; + unsigned i; + + if (cs->ipipe_read) { + mask = CLOCKSOURCE_MASK(64); + cread = cs->ipipe_read; + } else { + mask = cs->mask; + cread = cs->read; + + if ((cs->flags & CLOCK_SOURCE_IS_CONTINUOUS) == 0) + return; + + /* + * We only support masks such that cs->mask + 1 is a power of 2, + * 64 bits masks or masks lesser than 32 bits + */ + if (mask != CLOCKSOURCE_MASK(64) + && ((mask & (mask + 1)) != 0 || mask > 0xffffffff)) + return; + } + + /* + * We prefer a clocksource with a better resolution than 1us + */ + if (cs->shift <= 34) { + freq = 1000000000ULL << cs->shift; + do_div(freq, cs->mult); + } else { + freq = 1000000ULL << cs->shift; + do_div(freq, cs->mult); + freq *= 1000; + } + if (freq < 1000000) + return; + + /* Measure the clocksource latency */ + flags = hard_local_irq_save(); + saved = __ipipe_cs_last_tsc; + lat = cread(cs); + for (i = 0; i < 10; i++) + cread(cs); + lat = cread(cs) - lat; + __ipipe_cs_last_tsc = saved; + hard_local_irq_restore(flags); + lat = (lat * cs->mult) >> cs->shift; + do_div(lat, i + 1); + + if (!strcmp(cs->name, override_name)) + goto skip_tests; + + if (lat > __ipipe_cs_lat) + return; + + if (__ipipe_cs && !strcmp(__ipipe_cs->name, override_name)) + return; + + skip_tests: + flags = hard_local_irq_save(); + if (__ipipe_cs_last_tsc == 0) { + __ipipe_cs_lat = lat; + __ipipe_cs_freq = freq; + __ipipe_cs = cs; + __ipipe_cs_read = cread; + __ipipe_cs_mask = mask; + } + hard_local_irq_restore(flags); +} +#else /* !CONFIG_IPIPE_WANT_CLOCKSOURCE */ +#define ipipe_check_clocksource(cs) do { }while (0) +#endif /* !CONFIG_IPIPE_WANT_CLOCKSOURCE */ + /* * Enqueue the clocksource sorted by rating */ @@ -643,6 +749,8 @@ static void clocksource_enqueue(struct clocksource *cs) if (tmp->rating >= cs->rating) entry = &tmp->list; list_add(&cs->list, entry); + + ipipe_check_clocksource(cs); } /** diff --git a/kernel/time/tick-common.c b/kernel/time/tick-common.c index 3ae6afa..604b1d4 100644 --- a/kernel/time/tick-common.c +++ b/kernel/time/tick-common.c @@ -88,7 +88,7 @@ static void tick_periodic(int cpu) update_wall_time(); } - update_process_times(user_mode(get_irq_regs())); + update_root_process_times(get_irq_regs()); profile_tick(CPU_PROFILING); } diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c index 9142591..75c5887 100644 --- a/kernel/time/tick-sched.c +++ b/kernel/time/tick-sched.c @@ -148,7 +148,7 @@ static void tick_sched_handle(struct tick_sched *ts, struct pt_regs *regs) ts->idle_jiffies++; } #endif - update_process_times(user_mode(regs)); + update_root_process_times(regs); profile_tick(CPU_PROFILING); } diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 65dbf8a..71604d8 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -480,7 +480,7 @@ static inline void update_vsyscall(struct timekeeper *tk) xt = timespec64_to_timespec(tk_xtime(tk)); wm = timespec64_to_timespec(tk->wall_to_monotonic); update_vsyscall_old(&xt, &wm, tk->tkr_mono.clock, tk->tkr_mono.mult, - tk->tkr_mono.cycle_last); + tk->tkr_mono.shift, tk->tkr_mono.cycle_last); } static inline void old_vsyscall_fixup(struct timekeeper *tk) diff --git a/kernel/time/timer.c b/kernel/time/timer.c index 2ece3aa..511a184 100644 --- a/kernel/time/timer.c +++ b/kernel/time/timer.c @@ -1402,6 +1402,24 @@ void update_process_times(int user_tick) run_posix_cpu_timers(p); } +#ifdef CONFIG_IPIPE + +void update_root_process_times(struct pt_regs *regs) +{ + int user_tick = user_mode(regs); + + if (__ipipe_root_tick_p(regs)) { + update_process_times(user_tick); + return; + } + + run_local_timers(); + rcu_check_callbacks(user_tick); + run_posix_cpu_timers(current); +} + +#endif + /* * This function runs timers and the timer-tq in bottom half context. */ diff --git a/kernel/trace/Kconfig b/kernel/trace/Kconfig index 3b9a48a..edd9470 100644 --- a/kernel/trace/Kconfig +++ b/kernel/trace/Kconfig @@ -447,6 +447,7 @@ config DYNAMIC_FTRACE bool "enable/disable function tracing dynamically" depends on FUNCTION_TRACER depends on HAVE_DYNAMIC_FTRACE + depends on !IPIPE default y help This option will modify all the calls to function tracing diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c index eb11011..d0957bd 100644 --- a/kernel/trace/ftrace.c +++ b/kernel/trace/ftrace.c @@ -32,6 +32,7 @@ #include #include #include +#include #include @@ -262,8 +263,17 @@ static ftrace_func_t ftrace_ops_get_list_func(struct ftrace_ops *ops) static void update_ftrace_function(void) { + struct ftrace_ops *ops; ftrace_func_t func; + for (ops = ftrace_ops_list; + ops != &ftrace_list_end; ops = ops->next) + if (ops->flags & FTRACE_OPS_FL_IPIPE_EXCLUSIVE) { + set_function_trace_op = ops; + func = ops->func; + goto set_pointers; + } + /* * Prepare the ftrace_ops that the arch callback will use. * If there's only one ftrace_ops registered, the ftrace_ops_list @@ -291,6 +301,7 @@ static void update_ftrace_function(void) update_function_graph_func(); + set_pointers: /* If there's no change, then do nothing more here */ if (ftrace_trace_function == func) return; @@ -2518,6 +2529,9 @@ void __weak arch_ftrace_update_code(int command) static void ftrace_run_update_code(int command) { +#ifdef CONFIG_IPIPE + unsigned long flags; +#endif /* CONFIG_IPIPE */ int ret; ret = ftrace_arch_code_modify_prepare(); @@ -2531,7 +2545,13 @@ static void ftrace_run_update_code(int command) * is safe. The stop_machine() is the safest, but also * produces the most overhead. */ +#ifdef CONFIG_IPIPE + flags = ipipe_critical_enter(NULL); + __ftrace_modify_code(&command); + ipipe_critical_exit(flags); +#else /* !CONFIG_IPIPE */ arch_ftrace_update_code(command); +#endif /* !CONFIG_IPIPE */ ret = ftrace_arch_code_modify_post_process(); FTRACE_WARN_ON(ret); @@ -4877,10 +4897,10 @@ static int ftrace_process_locs(struct module *mod, * reason to cause large interrupt latencies while we do it. */ if (!mod) - local_irq_save(flags); + flags = hard_local_irq_save(); ftrace_update_code(mod, start_pg); if (!mod) - local_irq_restore(flags); + hard_local_irq_restore(flags); ret = 0; out: mutex_unlock(&ftrace_lock); @@ -4979,9 +4999,11 @@ void __init ftrace_init(void) unsigned long count, flags; int ret; - local_irq_save(flags); + flags = hard_local_irq_save_notrace(); ret = ftrace_dyn_arch_init(); - local_irq_restore(flags); + hard_local_irq_restore_notrace(flags); + + /* ftrace_dyn_arch_init places the return code in addr */ if (ret) goto failed; @@ -5174,7 +5196,16 @@ __ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip, } } while_for_each_ftrace_op(op); out: - preempt_enable_notrace(); +#ifdef CONFIG_IPIPE + if (hard_irqs_disabled() || !__ipipe_root_p) + /* + * Nothing urgent to schedule here. At latest the timer tick + * will pick up whatever the tracing functions kicked off. + */ + preempt_enable_no_resched_notrace(); + else +#endif + preempt_enable_notrace(); trace_clear_recursion(bit); } diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 0315d43..cb14649 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -2679,7 +2679,8 @@ static DEFINE_PER_CPU(unsigned int, current_context); static __always_inline int trace_recursive_lock(void) { - unsigned int val = __this_cpu_read(current_context); + unsigned long flags; + unsigned int val; int bit; if (in_interrupt()) { @@ -2692,18 +2693,29 @@ static __always_inline int trace_recursive_lock(void) } else bit = 3; - if (unlikely(val & (1 << bit))) + flags = hard_local_irq_save(); + + val = __this_cpu_read(current_context); + if (unlikely(val & (1 << bit))) { + hard_local_irq_restore(flags); return 1; + } val |= (1 << bit); __this_cpu_write(current_context, val); + hard_local_irq_restore(flags); + return 0; } static __always_inline void trace_recursive_unlock(void) { + unsigned long flags; + + flags = hard_local_irq_save(); __this_cpu_and(current_context, __this_cpu_read(current_context) - 1); + hard_local_irq_restore(flags); } #else diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 0533049..8cf4609 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -2144,8 +2144,9 @@ int trace_vbprintk(unsigned long ip, const char *fmt, va_list args) /* Don't pollute graph traces with trace_vprintk internals */ pause_graph_tracing(); + flags = hard_local_irq_save(); + pc = preempt_count(); - preempt_disable_notrace(); tbuffer = get_trace_buf(); if (!tbuffer) { @@ -2158,7 +2159,6 @@ int trace_vbprintk(unsigned long ip, const char *fmt, va_list args) if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0) goto out; - local_save_flags(flags); size = sizeof(*entry) + sizeof(u32) * len; buffer = tr->trace_buffer.buffer; event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size, @@ -2176,7 +2176,7 @@ int trace_vbprintk(unsigned long ip, const char *fmt, va_list args) } out: - preempt_enable_notrace(); + hard_local_irq_restore(flags); unpause_graph_tracing(); return len; diff --git a/kernel/trace/trace_clock.c b/kernel/trace/trace_clock.c index 57b67b1..66334fa 100644 --- a/kernel/trace/trace_clock.c +++ b/kernel/trace/trace_clock.c @@ -93,7 +93,7 @@ u64 notrace trace_clock_global(void) int this_cpu; u64 now; - local_irq_save(flags); + flags = hard_local_irq_save_notrace(); this_cpu = raw_smp_processor_id(); now = sched_clock_cpu(this_cpu); @@ -119,7 +119,7 @@ u64 notrace trace_clock_global(void) arch_spin_unlock(&trace_clock_struct.lock); out: - local_irq_restore(flags); + hard_local_irq_restore_notrace(flags); return now; } diff --git a/kernel/trace/trace_functions.c b/kernel/trace/trace_functions.c index fcd41a1..2c44ecd 100644 --- a/kernel/trace/trace_functions.c +++ b/kernel/trace/trace_functions.c @@ -171,7 +171,7 @@ function_stack_trace_call(unsigned long ip, unsigned long parent_ip, * Need to use raw, since this must be called before the * recursive protection is performed. */ - local_irq_save(flags); + flags = hard_local_irq_save(); cpu = raw_smp_processor_id(); data = per_cpu_ptr(tr->trace_buffer.data, cpu); disabled = atomic_inc_return(&data->disabled); @@ -191,7 +191,7 @@ function_stack_trace_call(unsigned long ip, unsigned long parent_ip, } atomic_dec(&data->disabled); - local_irq_restore(flags); + hard_local_irq_restore(flags); } static struct tracer_opt func_opts[] = { diff --git a/kernel/trace/trace_functions_graph.c b/kernel/trace/trace_functions_graph.c index a51e796..eb76f41 100644 --- a/kernel/trace/trace_functions_graph.c +++ b/kernel/trace/trace_functions_graph.c @@ -335,7 +335,7 @@ int trace_graph_entry(struct ftrace_graph_ent *trace) if (ftrace_graph_notrace_addr(trace->func)) return 1; - local_irq_save(flags); + flags = hard_local_irq_save_notrace(); cpu = raw_smp_processor_id(); data = per_cpu_ptr(tr->trace_buffer.data, cpu); disabled = atomic_inc_return(&data->disabled); @@ -347,7 +347,7 @@ int trace_graph_entry(struct ftrace_graph_ent *trace) } atomic_dec(&data->disabled); - local_irq_restore(flags); + hard_local_irq_restore_notrace(flags); return ret; } @@ -420,7 +420,7 @@ void trace_graph_return(struct ftrace_graph_ret *trace) int cpu; int pc; - local_irq_save(flags); + flags = hard_local_irq_save_notrace(); cpu = raw_smp_processor_id(); data = per_cpu_ptr(tr->trace_buffer.data, cpu); disabled = atomic_inc_return(&data->disabled); @@ -429,7 +429,7 @@ void trace_graph_return(struct ftrace_graph_ret *trace) __trace_graph_return(tr, trace, flags, pc); } atomic_dec(&data->disabled); - local_irq_restore(flags); + hard_local_irq_restore_notrace(flags); } void set_graph_array(struct trace_array *tr) diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index ba2b0c8..eb1dec9 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -363,6 +363,7 @@ config MAGIC_SYSRQ keys are documented in . Don't say Y unless you really know what this hack does. + config MAGIC_SYSRQ_DEFAULT_ENABLE hex "Enable magic SysRq key functions by default" depends on MAGIC_SYSRQ @@ -372,6 +373,8 @@ config MAGIC_SYSRQ_DEFAULT_ENABLE This may be set to 1 or 0 to enable or disable them all, or to a bitmask as described in Documentation/sysrq.txt. +source "kernel/ipipe/Kconfig.debug" + config DEBUG_KERNEL bool "Kernel debugging" help @@ -644,7 +647,7 @@ config HAVE_DEBUG_STACKOVERFLOW config DEBUG_STACKOVERFLOW bool "Check for stack overflows" - depends on DEBUG_KERNEL && HAVE_DEBUG_STACKOVERFLOW + depends on DEBUG_KERNEL && HAVE_DEBUG_STACKOVERFLOW && !IPIPE_LEGACY ---help--- Say Y here if you want to check for overflows of kernel, IRQ and exception stacks (if your architecture uses them). This @@ -1193,7 +1196,7 @@ config DEBUG_CREDENTIALS menu "RCU Debugging" config PROVE_RCU - def_bool PROVE_LOCKING + def_bool PROVE_LOCKING && !IPIPE config PROVE_RCU_REPEATEDLY bool "RCU debugging: don't disable PROVE_RCU on first splat" diff --git a/lib/atomic64.c b/lib/atomic64.c index 1298c05e..6ef2528 100644 --- a/lib/atomic64.c +++ b/lib/atomic64.c @@ -29,15 +29,15 @@ * Ensure each lock is in a separate cacheline. */ static union { - raw_spinlock_t lock; + ipipe_spinlock_t lock; char pad[L1_CACHE_BYTES]; } atomic64_lock[NR_LOCKS] __cacheline_aligned_in_smp = { [0 ... (NR_LOCKS - 1)] = { - .lock = __RAW_SPIN_LOCK_UNLOCKED(atomic64_lock.lock), + .lock = IPIPE_SPIN_LOCK_UNLOCKED, }, }; -static inline raw_spinlock_t *lock_addr(const atomic64_t *v) +static inline ipipe_spinlock_t *lock_addr(const atomic64_t *v) { unsigned long addr = (unsigned long) v; @@ -49,7 +49,7 @@ static inline raw_spinlock_t *lock_addr(const atomic64_t *v) long long atomic64_read(const atomic64_t *v) { unsigned long flags; - raw_spinlock_t *lock = lock_addr(v); + ipipe_spinlock_t *lock = lock_addr(v); long long val; raw_spin_lock_irqsave(lock, flags); @@ -62,7 +62,7 @@ EXPORT_SYMBOL(atomic64_read); void atomic64_set(atomic64_t *v, long long i) { unsigned long flags; - raw_spinlock_t *lock = lock_addr(v); + ipipe_spinlock_t *lock = lock_addr(v); raw_spin_lock_irqsave(lock, flags); v->counter = i; @@ -74,7 +74,7 @@ EXPORT_SYMBOL(atomic64_set); void atomic64_##op(long long a, atomic64_t *v) \ { \ unsigned long flags; \ - raw_spinlock_t *lock = lock_addr(v); \ + ipipe_spinlock_t *lock = lock_addr(v); \ \ raw_spin_lock_irqsave(lock, flags); \ v->counter c_op a; \ @@ -86,7 +86,7 @@ EXPORT_SYMBOL(atomic64_##op); long long atomic64_##op##_return(long long a, atomic64_t *v) \ { \ unsigned long flags; \ - raw_spinlock_t *lock = lock_addr(v); \ + ipipe_spinlock_t *lock = lock_addr(v); \ long long val; \ \ raw_spin_lock_irqsave(lock, flags); \ @@ -110,7 +110,7 @@ ATOMIC64_OPS(sub, -=) long long atomic64_dec_if_positive(atomic64_t *v) { unsigned long flags; - raw_spinlock_t *lock = lock_addr(v); + ipipe_spinlock_t *lock = lock_addr(v); long long val; raw_spin_lock_irqsave(lock, flags); @@ -125,7 +125,7 @@ EXPORT_SYMBOL(atomic64_dec_if_positive); long long atomic64_cmpxchg(atomic64_t *v, long long o, long long n) { unsigned long flags; - raw_spinlock_t *lock = lock_addr(v); + ipipe_spinlock_t *lock = lock_addr(v); long long val; raw_spin_lock_irqsave(lock, flags); @@ -140,7 +140,7 @@ EXPORT_SYMBOL(atomic64_cmpxchg); long long atomic64_xchg(atomic64_t *v, long long new) { unsigned long flags; - raw_spinlock_t *lock = lock_addr(v); + ipipe_spinlock_t *lock = lock_addr(v); long long val; raw_spin_lock_irqsave(lock, flags); @@ -154,7 +154,7 @@ EXPORT_SYMBOL(atomic64_xchg); int atomic64_add_unless(atomic64_t *v, long long a, long long u) { unsigned long flags; - raw_spinlock_t *lock = lock_addr(v); + ipipe_spinlock_t *lock = lock_addr(v); int ret = 0; raw_spin_lock_irqsave(lock, flags); diff --git a/lib/bust_spinlocks.c b/lib/bust_spinlocks.c index f8e0e53..02175aa3 100644 --- a/lib/bust_spinlocks.c +++ b/lib/bust_spinlocks.c @@ -14,6 +14,7 @@ #include #include #include +#include void __attribute__((weak)) bust_spinlocks(int yes) @@ -25,6 +26,7 @@ void __attribute__((weak)) bust_spinlocks(int yes) unblank_screen(); #endif console_unblank(); + ipipe_trace_panic_dump(); if (--oops_in_progress == 0) wake_up_klogd(); } diff --git a/lib/ioremap.c b/lib/ioremap.c index 86c8911..373a30b 100644 --- a/lib/ioremap.c +++ b/lib/ioremap.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -140,7 +141,12 @@ int ioremap_page_range(unsigned long addr, break; } while (pgd++, addr = next, addr != end); - flush_cache_vmap(start, end); + /* APEI may invoke this for temporarily remapping pages in interrupt + * context - nothing we can and need to propagate globally. */ + if (!in_interrupt()) { + __ipipe_pin_mapping_globally(start, end); + flush_cache_vmap(start, end); + } return err; } diff --git a/lib/smp_processor_id.c b/lib/smp_processor_id.c index 1afec32..f7c1a2a 100644 --- a/lib/smp_processor_id.c +++ b/lib/smp_processor_id.c @@ -12,6 +12,12 @@ notrace static unsigned int check_preemption_disabled(const char *what1, { int this_cpu = raw_smp_processor_id(); + if (hard_irqs_disabled()) + goto out; + + if (!ipipe_root_p) + goto out; + if (likely(preempt_count())) goto out; diff --git a/mm/memory.c b/mm/memory.c index 2a9e098..46ec4cd 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -796,6 +796,34 @@ out: return pfn_to_page(pfn); } +static inline void cow_user_page(struct page *dst, struct page *src, unsigned long va, struct vm_area_struct *vma) +{ + debug_dma_assert_idle(src); + + /* + * If the source page was a PFN mapping, we don't have + * a "struct page" for it. We do a best-effort copy by + * just copying from the original user address. If that + * fails, we just zero-fill it. Live with it. + */ + if (unlikely(!src)) { + void *kaddr = kmap_atomic(dst); + void __user *uaddr = (void __user *)(va & PAGE_MASK); + + /* + * This really shouldn't fail, because the page is there + * in the page tables. But it might just be unreadable, + * in which case we just give up and fill the result with + * zeroes. + */ + if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE)) + clear_page(kaddr); + kunmap_atomic(kaddr); + flush_dcache_page(dst); + } else + copy_user_highpage(dst, src, va, vma); +} + /* * copy one vm_area from one task to the other. Assumes the page tables * already present in the new task to be cleared in the whole range @@ -804,8 +832,8 @@ out: static inline unsigned long copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, - pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *vma, - unsigned long addr, int *rss) + pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *vma, + unsigned long addr, int *rss, struct page *uncow_page) { unsigned long vm_flags = vma->vm_flags; pte_t pte = *src_pte; @@ -857,6 +885,21 @@ copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, * in the parent and the child */ if (is_cow_mapping(vm_flags)) { +#ifdef CONFIG_IPIPE + if (uncow_page) { + struct page *old_page = vm_normal_page(vma, addr, pte); + cow_user_page(uncow_page, old_page, addr, vma); + pte = mk_pte(uncow_page, vma->vm_page_prot); + + if (vm_flags & VM_SHARED) + pte = pte_mkclean(pte); + pte = pte_mkold(pte); + + page_add_new_anon_rmap(uncow_page, vma, addr); + rss[!!PageAnon(uncow_page)]++; + goto out_set_pte; + } +#endif /* CONFIG_IPIPE */ ptep_set_wrprotect(src_mm, addr, src_pte); pte = pte_wrprotect(pte); } @@ -894,13 +937,27 @@ static int copy_pte_range(struct mm_struct *dst_mm, struct mm_struct *src_mm, int progress = 0; int rss[NR_MM_COUNTERS]; swp_entry_t entry = (swp_entry_t){0}; - + struct page *uncow_page = NULL; +#ifdef CONFIG_IPIPE + int do_cow_break = 0; again: + if (do_cow_break) { + uncow_page = alloc_page_vma(GFP_HIGHUSER, vma, addr); + if (uncow_page == NULL) + return -ENOMEM; + do_cow_break = 0; + } +#else +again: +#endif init_rss_vec(rss); dst_pte = pte_alloc_map_lock(dst_mm, dst_pmd, addr, &dst_ptl); - if (!dst_pte) + if (!dst_pte) { + if (uncow_page) + page_cache_release(uncow_page); return -ENOMEM; + } src_pte = pte_offset_map(src_pmd, addr); src_ptl = pte_lockptr(src_mm, src_pmd); spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING); @@ -923,8 +980,25 @@ again: progress++; continue; } +#ifdef CONFIG_IPIPE + if (likely(uncow_page == NULL) && likely(pte_present(*src_pte))) { + if (is_cow_mapping(vma->vm_flags) && + test_bit(MMF_VM_PINNED, &src_mm->flags) && + ((vma->vm_flags|src_mm->def_flags) & VM_LOCKED)) { + arch_leave_lazy_mmu_mode(); + spin_unlock(src_ptl); + pte_unmap(src_pte); + add_mm_rss_vec(dst_mm, rss); + pte_unmap_unlock(dst_pte, dst_ptl); + cond_resched(); + do_cow_break = 1; + goto again; + } + } +#endif entry.val = copy_one_pte(dst_mm, src_mm, dst_pte, src_pte, - vma, addr, rss); + vma, addr, rss, uncow_page); + uncow_page = NULL; if (entry.val) break; progress += 8; @@ -1920,34 +1994,6 @@ static inline int pte_unmap_same(struct mm_struct *mm, pmd_t *pmd, return same; } -static inline void cow_user_page(struct page *dst, struct page *src, unsigned long va, struct vm_area_struct *vma) -{ - debug_dma_assert_idle(src); - - /* - * If the source page was a PFN mapping, we don't have - * a "struct page" for it. We do a best-effort copy by - * just copying from the original user address. If that - * fails, we just zero-fill it. Live with it. - */ - if (unlikely(!src)) { - void *kaddr = kmap_atomic(dst); - void __user *uaddr = (void __user *)(va & PAGE_MASK); - - /* - * This really shouldn't fail, because the page is there - * in the page tables. But it might just be unreadable, - * in which case we just give up and fill the result with - * zeroes. - */ - if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE)) - clear_page(kaddr); - kunmap_atomic(kaddr); - flush_dcache_page(dst); - } else - copy_user_highpage(dst, src, va, vma); -} - /* * Notify the address space that the page is about to become writable so that * it can prohibit this or wait for the page to get into an appropriate state. @@ -3841,6 +3887,41 @@ void copy_user_huge_page(struct page *dst, struct page *src, } #endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */ +#ifdef CONFIG_IPIPE + +int __ipipe_disable_ondemand_mappings(struct task_struct *tsk) +{ + struct vm_area_struct *vma; + struct mm_struct *mm; + int result = 0; + + mm = get_task_mm(tsk); + if (!mm) + return -EPERM; + + down_write(&mm->mmap_sem); + if (test_bit(MMF_VM_PINNED, &mm->flags)) + goto done_mm; + + for (vma = mm->mmap; vma; vma = vma->vm_next) { + if (is_cow_mapping(vma->vm_flags) && + (vma->vm_flags & VM_WRITE)) { + result = __ipipe_pin_vma(mm, vma); + if (result < 0) + goto done_mm; + } + } + set_bit(MMF_VM_PINNED, &mm->flags); + + done_mm: + up_write(&mm->mmap_sem); + mmput(mm); + return result; +} +EXPORT_SYMBOL_GPL(__ipipe_disable_ondemand_mappings); + +#endif /* CONFIG_IPIPE */ + #if USE_SPLIT_PTE_PTLOCKS && ALLOC_SPLIT_PTLOCKS static struct kmem_cache *page_ptl_cachep; diff --git a/mm/mlock.c b/mm/mlock.c index 3d3ee6ca..ed47c08 100644 --- a/mm/mlock.c +++ b/mm/mlock.c @@ -756,3 +756,28 @@ void user_shm_unlock(size_t size, struct user_struct *user) spin_unlock(&shmlock_user_lock); free_uid(user); } + +#ifdef CONFIG_IPIPE +int __ipipe_pin_vma(struct mm_struct *mm, struct vm_area_struct *vma) +{ + int ret, write, len; + + if (vma->vm_flags & (VM_IO | VM_PFNMAP)) + return 0; + + if (!((vma->vm_flags & VM_DONTEXPAND) || + is_vm_hugetlb_page(vma) || vma == get_gate_vma(mm))) { + ret = populate_vma_page_range(vma, vma->vm_start, vma->vm_end, + NULL); + return ret < 0 ? ret : 0; + } + + write = (vma->vm_flags & (VM_WRITE | VM_SHARED)) == VM_WRITE; + len = DIV_ROUND_UP(vma->vm_end, PAGE_SIZE) - vma->vm_start/PAGE_SIZE; + ret = get_user_pages(current, mm, vma->vm_start, + len, write, 0, NULL, NULL); + if (ret < 0) + return ret; + return ret == len ? 0 : -EFAULT; +} +#endif diff --git a/mm/mmap.c b/mm/mmap.c index b639fa2..101fcd74 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -49,6 +49,10 @@ #include "internal.h" +#ifndef MMAP_BRK +#define MMAP_BRK 0 +#endif + #ifndef arch_mmap_check #define arch_mmap_check(addr, len, flags) (0) #endif @@ -2717,7 +2721,7 @@ static unsigned long do_brk(unsigned long addr, unsigned long len) flags = VM_DATA_DEFAULT_FLAGS | VM_ACCOUNT | mm->def_flags; - error = get_unmapped_area(NULL, addr, len, 0, MAP_FIXED); + error = get_unmapped_area(NULL, addr, len, 0, MAP_FIXED | MAP_BRK); if (error & ~PAGE_MASK) return error; diff --git a/mm/mmu_context.c b/mm/mmu_context.c index f802c2d..b320432 100644 --- a/mm/mmu_context.c +++ b/mm/mmu_context.c @@ -21,15 +21,18 @@ void use_mm(struct mm_struct *mm) { struct mm_struct *active_mm; struct task_struct *tsk = current; + unsigned long flags; task_lock(tsk); active_mm = tsk->active_mm; + ipipe_mm_switch_protect(flags); if (active_mm != mm) { atomic_inc(&mm->mm_count); tsk->active_mm = mm; } tsk->mm = mm; - switch_mm(active_mm, mm, tsk); + __switch_mm(active_mm, mm, tsk); + ipipe_mm_switch_unprotect(flags); task_unlock(tsk); #ifdef finish_arch_post_lock_switch finish_arch_post_lock_switch(); diff --git a/mm/mprotect.c b/mm/mprotect.c index 8858483..6bf09bf 100644 --- a/mm/mprotect.c +++ b/mm/mprotect.c @@ -64,7 +64,7 @@ static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd, struct mm_struct *mm = vma->vm_mm; pte_t *pte, oldpte; spinlock_t *ptl; - unsigned long pages = 0; + unsigned long pages = 0, flags; pte = lock_pte_protection(vma, pmd, addr, prot_numa, &ptl); if (!pte) @@ -93,6 +93,7 @@ static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd, continue; } + flags = hard_local_irq_save(); ptent = ptep_modify_prot_start(mm, addr, pte); ptent = pte_modify(ptent, newprot); if (preserve_write) @@ -105,6 +106,7 @@ static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd, ptent = pte_mkwrite(ptent); } ptep_modify_prot_commit(mm, addr, pte, ptent); + hard_local_irq_restore(flags); pages++; } else if (IS_ENABLED(CONFIG_MIGRATION)) { swp_entry_t entry = pte_to_swp_entry(oldpte); @@ -248,6 +250,12 @@ unsigned long change_protection(struct vm_area_struct *vma, unsigned long start, pages = hugetlb_change_protection(vma, start, end, newprot); else pages = change_protection_range(vma, start, end, newprot, dirty_accountable, prot_numa); +#ifdef CONFIG_IPIPE + if (test_bit(MMF_VM_PINNED, &vma->vm_mm->flags) && + ((vma->vm_flags | vma->vm_mm->def_flags) & VM_LOCKED) && + (vma->vm_flags & (VM_READ | VM_WRITE | VM_EXEC))) + __ipipe_pin_vma(vma->vm_mm, vma); +#endif return pages; } diff --git a/mm/vmalloc.c b/mm/vmalloc.c index 2faaa29..ef00f26 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -198,6 +198,8 @@ static int vmap_page_range_noflush(unsigned long start, unsigned long end, return err; } while (pgd++, addr = next, addr != end); + __ipipe_pin_mapping_globally(start, end); + return nr; } diff --git a/sound/soc/intel/atom/sst/sst.c b/sound/soc/intel/atom/sst/sst.c index 96c2e42..2e6e3cf 100644 --- a/sound/soc/intel/atom/sst/sst.c +++ b/sound/soc/intel/atom/sst/sst.c @@ -369,7 +369,7 @@ static inline void sst_restore_shim64(struct intel_sst_drv *ctx, */ spin_lock_irqsave(&ctx->ipc_spin_lock, irq_flags); sst_shim_write64(shim, SST_IMRX, shim_regs->imrx), - sst_shim_write64(shim, SST_CSR, shim_regs->csr), + sst_shim_write64(shim, SST_CSR, shim_regs->csr); spin_unlock_irqrestore(&ctx->ipc_spin_lock, irq_flags); }