diff options
Diffstat (limited to 'arch')
76 files changed, 1529 insertions, 1319 deletions
diff --git a/arch/arm/Kconfig.debug b/arch/arm/Kconfig.debug index 970de7518341..8b0183a9a300 100644 --- a/arch/arm/Kconfig.debug +++ b/arch/arm/Kconfig.debug @@ -1610,59 +1610,6 @@ config DEBUG_SET_MODULE_RONX against certain classes of kernel exploits. If in doubt, say "N". -menuconfig CORESIGHT - bool "CoreSight Tracing Support" - select ARM_AMBA - help - This framework provides a kernel interface for the CoreSight debug - and trace drivers to register themselves with. It's intended to build - a topological view of the CoreSight components based on a DT - specification and configure the right serie of components when a - trace source gets enabled. - -if CORESIGHT -config CORESIGHT_LINKS_AND_SINKS - bool "CoreSight Link and Sink drivers" - help - This enables support for CoreSight link and sink drivers that are - responsible for transporting and collecting the trace data - respectively. Link and sinks are dynamically aggregated with a trace - entity at run time to form a complete trace path. - -config CORESIGHT_LINK_AND_SINK_TMC - bool "Coresight generic TMC driver" - depends on CORESIGHT_LINKS_AND_SINKS - help - This enables support for the Trace Memory Controller driver. Depending - on its configuration the device can act as a link (embedded trace router - - ETR) or sink (embedded trace FIFO). The driver complies with the - generic implementation of the component without special enhancement or - added features. - -config CORESIGHT_SINK_TPIU - bool "Coresight generic TPIU driver" - depends on CORESIGHT_LINKS_AND_SINKS - help - This enables support for the Trace Port Interface Unit driver, responsible - for bridging the gap between the on-chip coresight components and a trace - port collection engine, typically connected to an external host for use - case capturing more traces than the on-board coresight memory can handle. - -config CORESIGHT_SINK_ETBV10 - bool "Coresight ETBv1.0 driver" - depends on CORESIGHT_LINKS_AND_SINKS - help - This enables support for the Embedded Trace Buffer version 1.0 driver - that complies with the generic implementation of the component without - special enhancement or added features. +source "drivers/hwtracing/coresight/Kconfig" -config CORESIGHT_SOURCE_ETM3X - bool "CoreSight Embedded Trace Macrocell 3.x driver" - select CORESIGHT_LINKS_AND_SINKS - help - This driver provides support for processor ETM3.x and PTM1.x modules, - which allows tracing the instructions that a processor is executing - This is primarily useful for instruction level tracing. Depending - the ETM version data tracing may also be available. -endif endmenu diff --git a/arch/arm/boot/dts/hip04.dtsi b/arch/arm/boot/dts/hip04.dtsi index 238814596a87..44044f275115 100644 --- a/arch/arm/boot/dts/hip04.dtsi +++ b/arch/arm/boot/dts/hip04.dtsi @@ -275,7 +275,6 @@ compatible = "arm,coresight-etb10", "arm,primecell"; reg = <0 0xe3c42000 0 0x1000>; - coresight-default-sink; clocks = <&clk_375m>; clock-names = "apb_pclk"; port { diff --git a/arch/arm/boot/dts/omap3-beagle-xm.dts b/arch/arm/boot/dts/omap3-beagle-xm.dts index 25f7b0a22114..8cdca51b6984 100644 --- a/arch/arm/boot/dts/omap3-beagle-xm.dts +++ b/arch/arm/boot/dts/omap3-beagle-xm.dts @@ -150,7 +150,6 @@ compatible = "arm,coresight-etb10", "arm,primecell"; reg = <0x5401b000 0x1000>; - coresight-default-sink; clocks = <&emu_src_ck>; clock-names = "apb_pclk"; port { diff --git a/arch/arm/boot/dts/omap3-beagle.dts b/arch/arm/boot/dts/omap3-beagle.dts index c792391ef090..6d4c46be8c39 100644 --- a/arch/arm/boot/dts/omap3-beagle.dts +++ b/arch/arm/boot/dts/omap3-beagle.dts @@ -145,7 +145,6 @@ compatible = "arm,coresight-etb10", "arm,primecell"; reg = <0x5401b000 0x1000>; - coresight-default-sink; clocks = <&emu_src_ck>; clock-names = "apb_pclk"; port { diff --git a/arch/arm/boot/dts/omap3-n900.dts b/arch/arm/boot/dts/omap3-n900.dts index db80f9d376fa..2cab149b191c 100644 --- a/arch/arm/boot/dts/omap3-n900.dts +++ b/arch/arm/boot/dts/omap3-n900.dts @@ -609,6 +609,58 @@ pinctrl-0 = <&i2c3_pins>; clock-frequency = <400000>; + + lis302dl: lis3lv02d@1d { + compatible = "st,lis3lv02d"; + reg = <0x1d>; + + Vdd-supply = <&vaux1>; + Vdd_IO-supply = <&vio>; + + interrupt-parent = <&gpio6>; + interrupts = <21 20>; /* 181 and 180 */ + + /* click flags */ + st,click-single-x; + st,click-single-y; + st,click-single-z; + + /* Limits are 0.5g * value */ + st,click-threshold-x = <8>; + st,click-threshold-y = <8>; + st,click-threshold-z = <10>; + + /* Click must be longer than time limit */ + st,click-time-limit = <9>; + + /* Kind of debounce filter */ + st,click-latency = <50>; + + /* Interrupt line 2 for click detection */ + st,irq2-click; + + st,wakeup-x-hi; + st,wakeup-y-hi; + st,wakeup-threshold = <(800/18)>; /* millig-value / 18 to get HW values */ + + st,wakeup2-z-hi; + st,wakeup2-threshold = <(900/18)>; /* millig-value / 18 to get HW values */ + + st,hipass1-disable; + st,hipass2-disable; + + st,axis-x = <1>; /* LIS3_DEV_X */ + st,axis-y = <(-2)>; /* LIS3_INV_DEV_Y */ + st,axis-z = <(-3)>; /* LIS3_INV_DEV_Z */ + + st,min-limit-x = <(-32)>; + st,min-limit-y = <3>; + st,min-limit-z = <3>; + + st,max-limit-x = <(-3)>; + st,max-limit-y = <32>; + st,max-limit-z = <32>; + }; }; &mmc1 { diff --git a/arch/arm/boot/dts/vexpress-v2p-ca15_a7.dts b/arch/arm/boot/dts/vexpress-v2p-ca15_a7.dts index 33920df03640..7a2aeacd62c0 100644 --- a/arch/arm/boot/dts/vexpress-v2p-ca15_a7.dts +++ b/arch/arm/boot/dts/vexpress-v2p-ca15_a7.dts @@ -362,7 +362,6 @@ compatible = "arm,coresight-etb10", "arm,primecell"; reg = <0 0x20010000 0 0x1000>; - coresight-default-sink; clocks = <&oscclk6a>; clock-names = "apb_pclk"; port { diff --git a/arch/arm/mach-omap2/board-cm-t35.c b/arch/arm/mach-omap2/board-cm-t35.c index 91738a14ecbe..b5dfbc1b1fc6 100644 --- a/arch/arm/mach-omap2/board-cm-t35.c +++ b/arch/arm/mach-omap2/board-cm-t35.c @@ -492,51 +492,36 @@ static struct twl4030_platform_data cm_t35_twldata = { #include <media/omap3isp.h> #include "devices.h" -static struct i2c_board_info cm_t35_isp_i2c_boardinfo[] = { +static struct isp_platform_subdev cm_t35_isp_subdevs[] = { { - I2C_BOARD_INFO("mt9t001", 0x5d), - }, - { - I2C_BOARD_INFO("tvp5150", 0x5c), - }, -}; - -static struct isp_subdev_i2c_board_info cm_t35_isp_primary_subdevs[] = { - { - .board_info = &cm_t35_isp_i2c_boardinfo[0], - .i2c_adapter_id = 3, - }, - { NULL, 0, }, -}; - -static struct isp_subdev_i2c_board_info cm_t35_isp_secondary_subdevs[] = { - { - .board_info = &cm_t35_isp_i2c_boardinfo[1], + .board_info = &(struct i2c_board_info){ + I2C_BOARD_INFO("mt9t001", 0x5d) + }, .i2c_adapter_id = 3, - }, - { NULL, 0, }, -}; - -static struct isp_v4l2_subdevs_group cm_t35_isp_subdevs[] = { - { - .subdevs = cm_t35_isp_primary_subdevs, - .interface = ISP_INTERFACE_PARALLEL, - .bus = { - .parallel = { - .clk_pol = 1, + .bus = &(struct isp_bus_cfg){ + .interface = ISP_INTERFACE_PARALLEL, + .bus = { + .parallel = { + .clk_pol = 1, + }, }, }, }, { - .subdevs = cm_t35_isp_secondary_subdevs, - .interface = ISP_INTERFACE_PARALLEL, - .bus = { - .parallel = { - .clk_pol = 0, + .board_info = &(struct i2c_board_info){ + I2C_BOARD_INFO("tvp5150", 0x5c), + }, + .i2c_adapter_id = 3, + .bus = &(struct isp_bus_cfg){ + .interface = ISP_INTERFACE_PARALLEL, + .bus = { + .parallel = { + .clk_pol = 0, + }, }, }, }, - { NULL, 0, }, + { 0 }, }; static struct isp_platform_data cm_t35_isp_pdata = { diff --git a/arch/arm/mach-omap2/devices.c b/arch/arm/mach-omap2/devices.c index 1afb50d6d636..990338fbaa59 100644 --- a/arch/arm/mach-omap2/devices.c +++ b/arch/arm/mach-omap2/devices.c @@ -74,82 +74,12 @@ omap_postcore_initcall(omap3_l3_init); static struct resource omap3isp_resources[] = { { .start = OMAP3430_ISP_BASE, - .end = OMAP3430_ISP_END, + .end = OMAP3430_ISP_BASE + 0x12fc, .flags = IORESOURCE_MEM, }, { - .start = OMAP3430_ISP_CCP2_BASE, - .end = OMAP3430_ISP_CCP2_END, - .flags = IORESOURCE_MEM, - }, - { - .start = OMAP3430_ISP_CCDC_BASE, - .end = OMAP3430_ISP_CCDC_END, - .flags = IORESOURCE_MEM, - }, - { - .start = OMAP3430_ISP_HIST_BASE, - .end = OMAP3430_ISP_HIST_END, - .flags = IORESOURCE_MEM, - }, - { - .start = OMAP3430_ISP_H3A_BASE, - .end = OMAP3430_ISP_H3A_END, - .flags = IORESOURCE_MEM, - }, - { - .start = OMAP3430_ISP_PREV_BASE, - .end = OMAP3430_ISP_PREV_END, - .flags = IORESOURCE_MEM, - }, - { - .start = OMAP3430_ISP_RESZ_BASE, - .end = OMAP3430_ISP_RESZ_END, - .flags = IORESOURCE_MEM, - }, - { - .start = OMAP3430_ISP_SBL_BASE, - .end = OMAP3430_ISP_SBL_END, - .flags = IORESOURCE_MEM, - }, - { - .start = OMAP3430_ISP_CSI2A_REGS1_BASE, - .end = OMAP3430_ISP_CSI2A_REGS1_END, - .flags = IORESOURCE_MEM, - }, - { - .start = OMAP3430_ISP_CSIPHY2_BASE, - .end = OMAP3430_ISP_CSIPHY2_END, - .flags = IORESOURCE_MEM, - }, - { - .start = OMAP3630_ISP_CSI2A_REGS2_BASE, - .end = OMAP3630_ISP_CSI2A_REGS2_END, - .flags = IORESOURCE_MEM, - }, - { - .start = OMAP3630_ISP_CSI2C_REGS1_BASE, - .end = OMAP3630_ISP_CSI2C_REGS1_END, - .flags = IORESOURCE_MEM, - }, - { - .start = OMAP3630_ISP_CSIPHY1_BASE, - .end = OMAP3630_ISP_CSIPHY1_END, - .flags = IORESOURCE_MEM, - }, - { - .start = OMAP3630_ISP_CSI2C_REGS2_BASE, - .end = OMAP3630_ISP_CSI2C_REGS2_END, - .flags = IORESOURCE_MEM, - }, - { - .start = OMAP343X_CTRL_BASE + OMAP343X_CONTROL_CSIRXFE, - .end = OMAP343X_CTRL_BASE + OMAP343X_CONTROL_CSIRXFE + 3, - .flags = IORESOURCE_MEM, - }, - { - .start = OMAP343X_CTRL_BASE + OMAP3630_CONTROL_CAMERA_PHY_CTRL, - .end = OMAP343X_CTRL_BASE + OMAP3630_CONTROL_CAMERA_PHY_CTRL + 3, + .start = OMAP3430_ISP_BASE2, + .end = OMAP3430_ISP_BASE2 + 0x0600, .flags = IORESOURCE_MEM, }, { diff --git a/arch/arm/mach-omap2/omap34xx.h b/arch/arm/mach-omap2/omap34xx.h index c0d1b4b1653f..ed0024dda133 100644 --- a/arch/arm/mach-omap2/omap34xx.h +++ b/arch/arm/mach-omap2/omap34xx.h @@ -46,39 +46,9 @@ #define OMAP34XX_IC_BASE 0x48200000 -#define OMAP3430_ISP_BASE (L4_34XX_BASE + 0xBC000) -#define OMAP3430_ISP_CBUFF_BASE (OMAP3430_ISP_BASE + 0x0100) -#define OMAP3430_ISP_CCP2_BASE (OMAP3430_ISP_BASE + 0x0400) -#define OMAP3430_ISP_CCDC_BASE (OMAP3430_ISP_BASE + 0x0600) -#define OMAP3430_ISP_HIST_BASE (OMAP3430_ISP_BASE + 0x0A00) -#define OMAP3430_ISP_H3A_BASE (OMAP3430_ISP_BASE + 0x0C00) -#define OMAP3430_ISP_PREV_BASE (OMAP3430_ISP_BASE + 0x0E00) -#define OMAP3430_ISP_RESZ_BASE (OMAP3430_ISP_BASE + 0x1000) -#define OMAP3430_ISP_SBL_BASE (OMAP3430_ISP_BASE + 0x1200) -#define OMAP3430_ISP_MMU_BASE (OMAP3430_ISP_BASE + 0x1400) -#define OMAP3430_ISP_CSI2A_REGS1_BASE (OMAP3430_ISP_BASE + 0x1800) -#define OMAP3430_ISP_CSIPHY2_BASE (OMAP3430_ISP_BASE + 0x1970) -#define OMAP3630_ISP_CSI2A_REGS2_BASE (OMAP3430_ISP_BASE + 0x19C0) -#define OMAP3630_ISP_CSI2C_REGS1_BASE (OMAP3430_ISP_BASE + 0x1C00) -#define OMAP3630_ISP_CSIPHY1_BASE (OMAP3430_ISP_BASE + 0x1D70) -#define OMAP3630_ISP_CSI2C_REGS2_BASE (OMAP3430_ISP_BASE + 0x1DC0) - -#define OMAP3430_ISP_END (OMAP3430_ISP_BASE + 0x06F) -#define OMAP3430_ISP_CBUFF_END (OMAP3430_ISP_CBUFF_BASE + 0x077) -#define OMAP3430_ISP_CCP2_END (OMAP3430_ISP_CCP2_BASE + 0x1EF) -#define OMAP3430_ISP_CCDC_END (OMAP3430_ISP_CCDC_BASE + 0x0A7) -#define OMAP3430_ISP_HIST_END (OMAP3430_ISP_HIST_BASE + 0x047) -#define OMAP3430_ISP_H3A_END (OMAP3430_ISP_H3A_BASE + 0x05F) -#define OMAP3430_ISP_PREV_END (OMAP3430_ISP_PREV_BASE + 0x09F) -#define OMAP3430_ISP_RESZ_END (OMAP3430_ISP_RESZ_BASE + 0x0AB) -#define OMAP3430_ISP_SBL_END (OMAP3430_ISP_SBL_BASE + 0x0FB) -#define OMAP3430_ISP_MMU_END (OMAP3430_ISP_MMU_BASE + 0x06F) -#define OMAP3430_ISP_CSI2A_REGS1_END (OMAP3430_ISP_CSI2A_REGS1_BASE + 0x16F) -#define OMAP3430_ISP_CSIPHY2_END (OMAP3430_ISP_CSIPHY2_BASE + 0x00B) -#define OMAP3630_ISP_CSI2A_REGS2_END (OMAP3630_ISP_CSI2A_REGS2_BASE + 0x3F) -#define OMAP3630_ISP_CSI2C_REGS1_END (OMAP3630_ISP_CSI2C_REGS1_BASE + 0x16F) -#define OMAP3630_ISP_CSIPHY1_END (OMAP3630_ISP_CSIPHY1_BASE + 0x00B) -#define OMAP3630_ISP_CSI2C_REGS2_END (OMAP3630_ISP_CSI2C_REGS2_BASE + 0x3F) +#define OMAP3430_ISP_BASE (L4_34XX_BASE + 0xBC000) +#define OMAP3430_ISP_MMU_BASE (OMAP3430_ISP_BASE + 0x1400) +#define OMAP3430_ISP_BASE2 (OMAP3430_ISP_BASE + 0x1800) #define OMAP34XX_HSUSB_OTG_BASE (L4_34XX_BASE + 0xAB000) #define OMAP34XX_USBTLL_BASE (L4_34XX_BASE + 0x62000) diff --git a/arch/arm64/Kconfig.debug b/arch/arm64/Kconfig.debug index 4a8741073c90..d6285ef9b5f9 100644 --- a/arch/arm64/Kconfig.debug +++ b/arch/arm64/Kconfig.debug @@ -89,4 +89,6 @@ config DEBUG_ALIGN_RODATA If in doubt, say N +source "drivers/hwtracing/coresight/Kconfig" + endmenu diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c index ffe8e1b814e0..714411f62391 100644 --- a/arch/arm64/kernel/smp.c +++ b/arch/arm64/kernel/smp.c @@ -636,7 +636,7 @@ void smp_send_stop(void) cpumask_t mask; cpumask_copy(&mask, cpu_online_mask); - cpu_clear(smp_processor_id(), mask); + cpumask_clear_cpu(smp_processor_id(), &mask); smp_cross_call(&mask, IPI_CPU_STOP); } diff --git a/arch/blackfin/mach-bf561/smp.c b/arch/blackfin/mach-bf561/smp.c index 11789beca75a..8c0c80fd1a45 100644 --- a/arch/blackfin/mach-bf561/smp.c +++ b/arch/blackfin/mach-bf561/smp.c @@ -124,7 +124,7 @@ void platform_send_ipi(cpumask_t callmap, int irq) unsigned int cpu; int offset = (irq == IRQ_SUPPLE_0) ? 6 : 8; - for_each_cpu_mask(cpu, callmap) { + for_each_cpu(cpu, &callmap) { BUG_ON(cpu >= 2); SSYNC(); bfin_write_SICB_SYSCR(bfin_read_SICB_SYSCR() | (1 << (offset + cpu))); diff --git a/arch/ia64/include/asm/acpi.h b/arch/ia64/include/asm/acpi.h index a1d91ab4c5ef..aa0fdf125aba 100644 --- a/arch/ia64/include/asm/acpi.h +++ b/arch/ia64/include/asm/acpi.h @@ -117,7 +117,7 @@ static inline void arch_acpi_set_pdc_bits(u32 *buf) #ifdef CONFIG_ACPI_NUMA extern cpumask_t early_cpu_possible_map; #define for_each_possible_early_cpu(cpu) \ - for_each_cpu_mask((cpu), early_cpu_possible_map) + for_each_cpu((cpu), &early_cpu_possible_map) static inline void per_cpu_scan_finalize(int min_cpus, int reserve_cpus) { @@ -125,13 +125,13 @@ static inline void per_cpu_scan_finalize(int min_cpus, int reserve_cpus) int cpu; int next_nid = 0; - low_cpu = cpus_weight(early_cpu_possible_map); + low_cpu = cpumask_weight(&early_cpu_possible_map); high_cpu = max(low_cpu, min_cpus); high_cpu = min(high_cpu + reserve_cpus, NR_CPUS); for (cpu = low_cpu; cpu < high_cpu; cpu++) { - cpu_set(cpu, early_cpu_possible_map); + cpumask_set_cpu(cpu, &early_cpu_possible_map); if (node_cpuid[cpu].nid == NUMA_NO_NODE) { node_cpuid[cpu].nid = next_nid; next_nid++; diff --git a/arch/ia64/kernel/acpi.c b/arch/ia64/kernel/acpi.c index 2c4498919d3c..35bf22cc71b7 100644 --- a/arch/ia64/kernel/acpi.c +++ b/arch/ia64/kernel/acpi.c @@ -483,7 +483,7 @@ acpi_numa_processor_affinity_init(struct acpi_srat_cpu_affinity *pa) (pa->apic_id << 8) | (pa->local_sapic_eid); /* nid should be overridden as logical node id later */ node_cpuid[srat_num_cpus].nid = pxm; - cpu_set(srat_num_cpus, early_cpu_possible_map); + cpumask_set_cpu(srat_num_cpus, &early_cpu_possible_map); srat_num_cpus++; } diff --git a/arch/ia64/kernel/iosapic.c b/arch/ia64/kernel/iosapic.c index cd44a57c73be..bc9501e36e77 100644 --- a/arch/ia64/kernel/iosapic.c +++ b/arch/ia64/kernel/iosapic.c @@ -690,7 +690,7 @@ skip_numa_setup: do { if (++cpu >= nr_cpu_ids) cpu = 0; - } while (!cpu_online(cpu) || !cpu_isset(cpu, domain)); + } while (!cpu_online(cpu) || !cpumask_test_cpu(cpu, &domain)); return cpu_physical_id(cpu); #else /* CONFIG_SMP */ diff --git a/arch/ia64/kernel/irq_ia64.c b/arch/ia64/kernel/irq_ia64.c index 698d8fefde6c..eaa3199f98c8 100644 --- a/arch/ia64/kernel/irq_ia64.c +++ b/arch/ia64/kernel/irq_ia64.c @@ -109,13 +109,13 @@ static inline int find_unassigned_vector(cpumask_t domain) int pos, vector; cpumask_and(&mask, &domain, cpu_online_mask); - if (cpus_empty(mask)) + if (cpumask_empty(&mask)) return -EINVAL; for (pos = 0; pos < IA64_NUM_DEVICE_VECTORS; pos++) { vector = IA64_FIRST_DEVICE_VECTOR + pos; - cpus_and(mask, domain, vector_table[vector]); - if (!cpus_empty(mask)) + cpumask_and(&mask, &domain, &vector_table[vector]); + if (!cpumask_empty(&mask)) continue; return vector; } @@ -132,18 +132,18 @@ static int __bind_irq_vector(int irq, int vector, cpumask_t domain) BUG_ON((unsigned)vector >= IA64_NUM_VECTORS); cpumask_and(&mask, &domain, cpu_online_mask); - if (cpus_empty(mask)) + if (cpumask_empty(&mask)) return -EINVAL; - if ((cfg->vector == vector) && cpus_equal(cfg->domain, domain)) + if ((cfg->vector == vector) && cpumask_equal(&cfg->domain, &domain)) return 0; if (cfg->vector != IRQ_VECTOR_UNASSIGNED) return -EBUSY; - for_each_cpu_mask(cpu, mask) + for_each_cpu(cpu, &mask) per_cpu(vector_irq, cpu)[vector] = irq; cfg->vector = vector; cfg->domain = domain; irq_status[irq] = IRQ_USED; - cpus_or(vector_table[vector], vector_table[vector], domain); + cpumask_or(&vector_table[vector], &vector_table[vector], &domain); return 0; } @@ -161,7 +161,6 @@ int bind_irq_vector(int irq, int vector, cpumask_t domain) static void __clear_irq_vector(int irq) { int vector, cpu; - cpumask_t mask; cpumask_t domain; struct irq_cfg *cfg = &irq_cfg[irq]; @@ -169,13 +168,12 @@ static void __clear_irq_vector(int irq) BUG_ON(cfg->vector == IRQ_VECTOR_UNASSIGNED); vector = cfg->vector; domain = cfg->domain; - cpumask_and(&mask, &cfg->domain, cpu_online_mask); - for_each_cpu_mask(cpu, mask) + for_each_cpu_and(cpu, &cfg->domain, cpu_online_mask) per_cpu(vector_irq, cpu)[vector] = -1; cfg->vector = IRQ_VECTOR_UNASSIGNED; cfg->domain = CPU_MASK_NONE; irq_status[irq] = IRQ_UNUSED; - cpus_andnot(vector_table[vector], vector_table[vector], domain); + cpumask_andnot(&vector_table[vector], &vector_table[vector], &domain); } static void clear_irq_vector(int irq) @@ -244,7 +242,7 @@ void __setup_vector_irq(int cpu) per_cpu(vector_irq, cpu)[vector] = -1; /* Mark the inuse vectors */ for (irq = 0; irq < NR_IRQS; ++irq) { - if (!cpu_isset(cpu, irq_cfg[irq].domain)) + if (!cpumask_test_cpu(cpu, &irq_cfg[irq].domain)) continue; vector = irq_to_vector(irq); per_cpu(vector_irq, cpu)[vector] = irq; @@ -261,7 +259,7 @@ static enum vector_domain_type { static cpumask_t vector_allocation_domain(int cpu) { if (vector_domain_type == VECTOR_DOMAIN_PERCPU) - return cpumask_of_cpu(cpu); + return *cpumask_of(cpu); return CPU_MASK_ALL; } @@ -275,7 +273,7 @@ static int __irq_prepare_move(int irq, int cpu) return -EBUSY; if (cfg->vector == IRQ_VECTOR_UNASSIGNED || !cpu_online(cpu)) return -EINVAL; - if (cpu_isset(cpu, cfg->domain)) + if (cpumask_test_cpu(cpu, &cfg->domain)) return 0; domain = vector_allocation_domain(cpu); vector = find_unassigned_vector(domain); @@ -309,12 +307,12 @@ void irq_complete_move(unsigned irq) if (likely(!cfg->move_in_progress)) return; - if (unlikely(cpu_isset(smp_processor_id(), cfg->old_domain))) + if (unlikely(cpumask_test_cpu(smp_processor_id(), &cfg->old_domain))) return; cpumask_and(&cleanup_mask, &cfg->old_domain, cpu_online_mask); - cfg->move_cleanup_count = cpus_weight(cleanup_mask); - for_each_cpu_mask(i, cleanup_mask) + cfg->move_cleanup_count = cpumask_weight(&cleanup_mask); + for_each_cpu(i, &cleanup_mask) platform_send_ipi(i, IA64_IRQ_MOVE_VECTOR, IA64_IPI_DM_INT, 0); cfg->move_in_progress = 0; } @@ -340,12 +338,12 @@ static irqreturn_t smp_irq_move_cleanup_interrupt(int irq, void *dev_id) if (!cfg->move_cleanup_count) goto unlock; - if (!cpu_isset(me, cfg->old_domain)) + if (!cpumask_test_cpu(me, &cfg->old_domain)) goto unlock; spin_lock_irqsave(&vector_lock, flags); __this_cpu_write(vector_irq[vector], -1); - cpu_clear(me, vector_table[vector]); + cpumask_clear_cpu(me, &vector_table[vector]); spin_unlock_irqrestore(&vector_lock, flags); cfg->move_cleanup_count--; unlock: diff --git a/arch/ia64/kernel/mca.c b/arch/ia64/kernel/mca.c index 8bfd36af46f8..dd5801eb4c69 100644 --- a/arch/ia64/kernel/mca.c +++ b/arch/ia64/kernel/mca.c @@ -1293,7 +1293,7 @@ ia64_mca_handler(struct pt_regs *regs, struct switch_stack *sw, monarch_cpu = cpu; sos->monarch = 1; } else { - cpu_set(cpu, mca_cpu); + cpumask_set_cpu(cpu, &mca_cpu); sos->monarch = 0; } mprintk(KERN_INFO "Entered OS MCA handler. PSP=%lx cpu=%d " @@ -1316,7 +1316,7 @@ ia64_mca_handler(struct pt_regs *regs, struct switch_stack *sw, */ ia64_mca_wakeup_all(); } else { - while (cpu_isset(cpu, mca_cpu)) + while (cpumask_test_cpu(cpu, &mca_cpu)) cpu_relax(); /* spin until monarch wakes us */ } @@ -1355,9 +1355,9 @@ ia64_mca_handler(struct pt_regs *regs, struct switch_stack *sw, * and put this cpu in the rendez loop. */ for_each_online_cpu(i) { - if (cpu_isset(i, mca_cpu)) { + if (cpumask_test_cpu(i, &mca_cpu)) { monarch_cpu = i; - cpu_clear(i, mca_cpu); /* wake next cpu */ + cpumask_clear_cpu(i, &mca_cpu); /* wake next cpu */ while (monarch_cpu != -1) cpu_relax(); /* spin until last cpu leaves */ set_curr_task(cpu, previous_current); @@ -1822,7 +1822,7 @@ format_mca_init_stack(void *mca_data, unsigned long offset, ti->cpu = cpu; p->stack = ti; p->state = TASK_UNINTERRUPTIBLE; - cpu_set(cpu, p->cpus_allowed); + cpumask_set_cpu(cpu, &p->cpus_allowed); INIT_LIST_HEAD(&p->tasks); p->parent = p->real_parent = p->group_leader = p; INIT_LIST_HEAD(&p->children); diff --git a/arch/ia64/kernel/msi_ia64.c b/arch/ia64/kernel/msi_ia64.c index 8ae36ea177d3..9dd7464f8c17 100644 --- a/arch/ia64/kernel/msi_ia64.c +++ b/arch/ia64/kernel/msi_ia64.c @@ -47,15 +47,14 @@ int ia64_setup_msi_irq(struct pci_dev *pdev, struct msi_desc *desc) struct msi_msg msg; unsigned long dest_phys_id; int irq, vector; - cpumask_t mask; irq = create_irq(); if (irq < 0) return irq; irq_set_msi_desc(irq, desc); - cpumask_and(&mask, &(irq_to_domain(irq)), cpu_online_mask); - dest_phys_id = cpu_physical_id(first_cpu(mask)); + dest_phys_id = cpu_physical_id(cpumask_any_and(&(irq_to_domain(irq)), + cpu_online_mask)); vector = irq_to_vector(irq); msg.address_hi = 0; @@ -171,10 +170,9 @@ msi_compose_msg(struct pci_dev *pdev, unsigned int irq, struct msi_msg *msg) { struct irq_cfg *cfg = irq_cfg + irq; unsigned dest; - cpumask_t mask; - cpumask_and(&mask, &(irq_to_domain(irq)), cpu_online_mask); - dest = cpu_physical_id(first_cpu(mask)); + dest = cpu_physical_id(cpumask_first_and(&(irq_to_domain(irq)), + cpu_online_mask)); msg->address_hi = 0; msg->address_lo = diff --git a/arch/ia64/kernel/numa.c b/arch/ia64/kernel/numa.c index d288cde93606..92c376279c6d 100644 --- a/arch/ia64/kernel/numa.c +++ b/arch/ia64/kernel/numa.c @@ -39,7 +39,7 @@ void map_cpu_to_node(int cpu, int nid) } /* sanity check first */ oldnid = cpu_to_node_map[cpu]; - if (cpu_isset(cpu, node_to_cpu_mask[oldnid])) { + if (cpumask_test_cpu(cpu, &node_to_cpu_mask[oldnid])) { return; /* nothing to do */ } /* we don't have cpu-driven node hot add yet... @@ -47,16 +47,16 @@ void map_cpu_to_node(int cpu, int nid) if (!node_online(nid)) nid = first_online_node; cpu_to_node_map[cpu] = nid; - cpu_set(cpu, node_to_cpu_mask[nid]); + cpumask_set_cpu(cpu, &node_to_cpu_mask[nid]); return; } void unmap_cpu_from_node(int cpu, int nid) { - WARN_ON(!cpu_isset(cpu, node_to_cpu_mask[nid])); + WARN_ON(!cpumask_test_cpu(cpu, &node_to_cpu_mask[nid])); WARN_ON(cpu_to_node_map[cpu] != nid); cpu_to_node_map[cpu] = 0; - cpu_clear(cpu, node_to_cpu_mask[nid]); + cpumask_clear_cpu(cpu, &node_to_cpu_mask[nid]); } @@ -71,7 +71,7 @@ void __init build_cpu_to_node_map(void) int cpu, i, node; for(node=0; node < MAX_NUMNODES; node++) - cpus_clear(node_to_cpu_mask[node]); + cpumask_clear(&node_to_cpu_mask[node]); for_each_possible_early_cpu(cpu) { node = -1; diff --git a/arch/ia64/kernel/salinfo.c b/arch/ia64/kernel/salinfo.c index ee9719eebb1e..1eeffb7fbb16 100644 --- a/arch/ia64/kernel/salinfo.c +++ b/arch/ia64/kernel/salinfo.c @@ -256,7 +256,7 @@ salinfo_log_wakeup(int type, u8 *buffer, u64 size, int irqsafe) data_saved->buffer = buffer; } } - cpu_set(smp_processor_id(), data->cpu_event); + cpumask_set_cpu(smp_processor_id(), &data->cpu_event); if (irqsafe) { salinfo_work_to_do(data); spin_unlock_irqrestore(&data_saved_lock, flags); @@ -274,7 +274,7 @@ salinfo_timeout_check(struct salinfo_data *data) unsigned long flags; if (!data->open) return; - if (!cpus_empty(data->cpu_event)) { + if (!cpumask_empty(&data->cpu_event)) { spin_lock_irqsave(&data_saved_lock, flags); salinfo_work_to_do(data); spin_unlock_irqrestore(&data_saved_lock, flags); @@ -308,7 +308,7 @@ salinfo_event_read(struct file *file, char __user *buffer, size_t count, loff_t int i, n, cpu = -1; retry: - if (cpus_empty(data->cpu_event) && down_trylock(&data->mutex)) { + if (cpumask_empty(&data->cpu_event) && down_trylock(&data->mutex)) { if (file->f_flags & O_NONBLOCK) return -EAGAIN; if (down_interruptible(&data->mutex)) @@ -317,9 +317,9 @@ retry: n = data->cpu_check; for (i = 0; i < nr_cpu_ids; i++) { - if (cpu_isset(n, data->cpu_event)) { + if (cpumask_test_cpu(n, &data->cpu_event)) { if (!cpu_online(n)) { - cpu_clear(n, data->cpu_event); + cpumask_clear_cpu(n, &data->cpu_event); continue; } cpu = n; @@ -451,7 +451,7 @@ retry: call_on_cpu(cpu, salinfo_log_read_cpu, data); if (!data->log_size) { data->state = STATE_NO_DATA; - cpu_clear(cpu, data->cpu_event); + cpumask_clear_cpu(cpu, &data->cpu_event); } else { data->state = STATE_LOG_RECORD; } @@ -491,11 +491,11 @@ salinfo_log_clear(struct salinfo_data *data, int cpu) unsigned long flags; spin_lock_irqsave(&data_saved_lock, flags); data->state = STATE_NO_DATA; - if (!cpu_isset(cpu, data->cpu_event)) { + if (!cpumask_test_cpu(cpu, &data->cpu_event)) { spin_unlock_irqrestore(&data_saved_lock, flags); return 0; } - cpu_clear(cpu, data->cpu_event); + cpumask_clear_cpu(cpu, &data->cpu_event); if (data->saved_num) { shift1_data_saved(data, data->saved_num - 1); data->saved_num = 0; @@ -509,7 +509,7 @@ salinfo_log_clear(struct salinfo_data *data, int cpu) salinfo_log_new_read(cpu, data); if (data->state == STATE_LOG_RECORD) { spin_lock_irqsave(&data_saved_lock, flags); - cpu_set(cpu, data->cpu_event); + cpumask_set_cpu(cpu, &data->cpu_event); salinfo_work_to_do(data); spin_unlock_irqrestore(&data_saved_lock, flags); } @@ -581,7 +581,7 @@ salinfo_cpu_callback(struct notifier_block *nb, unsigned long action, void *hcpu for (i = 0, data = salinfo_data; i < ARRAY_SIZE(salinfo_data); ++i, ++data) { - cpu_set(cpu, data->cpu_event); + cpumask_set_cpu(cpu, &data->cpu_event); salinfo_work_to_do(data); } spin_unlock_irqrestore(&data_saved_lock, flags); @@ -601,7 +601,7 @@ salinfo_cpu_callback(struct notifier_block *nb, unsigned long action, void *hcpu shift1_data_saved(data, j); } } - cpu_clear(cpu, data->cpu_event); + cpumask_clear_cpu(cpu, &data->cpu_event); } spin_unlock_irqrestore(&data_saved_lock, flags); break; @@ -659,7 +659,7 @@ salinfo_init(void) /* we missed any events before now */ for_each_online_cpu(j) - cpu_set(j, data->cpu_event); + cpumask_set_cpu(j, &data->cpu_event); *sdir++ = dir; } diff --git a/arch/ia64/kernel/setup.c b/arch/ia64/kernel/setup.c index d86669bcdfb2..b9761389cb8d 100644 --- a/arch/ia64/kernel/setup.c +++ b/arch/ia64/kernel/setup.c @@ -562,8 +562,8 @@ setup_arch (char **cmdline_p) # ifdef CONFIG_ACPI_HOTPLUG_CPU prefill_possible_map(); # endif - per_cpu_scan_finalize((cpus_weight(early_cpu_possible_map) == 0 ? - 32 : cpus_weight(early_cpu_possible_map)), + per_cpu_scan_finalize((cpumask_weight(&early_cpu_possible_map) == 0 ? + 32 : cpumask_weight(&early_cpu_possible_map)), additional_cpus > 0 ? additional_cpus : 0); # endif #endif /* CONFIG_APCI_BOOT */ @@ -702,7 +702,8 @@ show_cpuinfo (struct seq_file *m, void *v) c->itc_freq / 1000000, c->itc_freq % 1000000, lpj*HZ/500000, (lpj*HZ/5000) % 100); #ifdef CONFIG_SMP - seq_printf(m, "siblings : %u\n", cpus_weight(cpu_core_map[cpunum])); + seq_printf(m, "siblings : %u\n", + cpumask_weight(&cpu_core_map[cpunum])); if (c->socket_id != -1) seq_printf(m, "physical id: %u\n", c->socket_id); if (c->threads_per_core > 1 || c->cores_per_socket > 1) @@ -933,8 +934,8 @@ cpu_init (void) * (must be done after per_cpu area is setup) */ if (smp_processor_id() == 0) { - cpu_set(0, per_cpu(cpu_sibling_map, 0)); - cpu_set(0, cpu_core_map[0]); + cpumask_set_cpu(0, &per_cpu(cpu_sibling_map, 0)); + cpumask_set_cpu(0, &cpu_core_map[0]); } else { /* * Set ar.k3 so that assembly code in MCA handler can compute diff --git a/arch/ia64/kernel/smp.c b/arch/ia64/kernel/smp.c index 9fcd4e63048f..7f706d4f84f7 100644 --- a/arch/ia64/kernel/smp.c +++ b/arch/ia64/kernel/smp.c @@ -262,11 +262,11 @@ smp_flush_tlb_cpumask(cpumask_t xcpumask) preempt_disable(); mycpu = smp_processor_id(); - for_each_cpu_mask(cpu, cpumask) + for_each_cpu(cpu, &cpumask) counts[cpu] = local_tlb_flush_counts[cpu].count & 0xffff; mb(); - for_each_cpu_mask(cpu, cpumask) { + for_each_cpu(cpu, &cpumask) { if (cpu == mycpu) flush_mycpu = 1; else @@ -276,7 +276,7 @@ smp_flush_tlb_cpumask(cpumask_t xcpumask) if (flush_mycpu) smp_local_flush_tlb(); - for_each_cpu_mask(cpu, cpumask) + for_each_cpu(cpu, &cpumask) while(counts[cpu] == (local_tlb_flush_counts[cpu].count & 0xffff)) udelay(FLUSH_DELAY); diff --git a/arch/ia64/kernel/smpboot.c b/arch/ia64/kernel/smpboot.c index 547a48d78bd7..15051e9c2c6f 100644 --- a/arch/ia64/kernel/smpboot.c +++ b/arch/ia64/kernel/smpboot.c @@ -434,7 +434,7 @@ smp_callin (void) /* * Allow the master to continue. */ - cpu_set(cpuid, cpu_callin_map); + cpumask_set_cpu(cpuid, &cpu_callin_map); Dprintk("Stack on CPU %d at about %p\n",cpuid, &cpuid); } @@ -475,13 +475,13 @@ do_boot_cpu (int sapicid, int cpu, struct task_struct *idle) */ Dprintk("Waiting on callin_map ..."); for (timeout = 0; timeout < 100000; timeout++) { - if (cpu_isset(cpu, cpu_callin_map)) + if (cpumask_test_cpu(cpu, &cpu_callin_map)) break; /* It has booted */ udelay(100); } Dprintk("\n"); - if (!cpu_isset(cpu, cpu_callin_map)) { + if (!cpumask_test_cpu(cpu, &cpu_callin_map)) { printk(KERN_ERR "Processor 0x%x/0x%x is stuck.\n", cpu, sapicid); ia64_cpu_to_sapicid[cpu] = -1; set_cpu_online(cpu, false); /* was set in smp_callin() */ @@ -541,7 +541,7 @@ smp_prepare_cpus (unsigned int max_cpus) smp_setup_percpu_timer(); - cpu_set(0, cpu_callin_map); + cpumask_set_cpu(0, &cpu_callin_map); local_cpu_data->loops_per_jiffy = loops_per_jiffy; ia64_cpu_to_sapicid[0] = boot_cpu_id; @@ -565,7 +565,7 @@ smp_prepare_cpus (unsigned int max_cpus) void smp_prepare_boot_cpu(void) { set_cpu_online(smp_processor_id(), true); - cpu_set(smp_processor_id(), cpu_callin_map); + cpumask_set_cpu(smp_processor_id(), &cpu_callin_map); set_numa_node(cpu_to_node_map[smp_processor_id()]); per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE; paravirt_post_smp_prepare_boot_cpu(); @@ -577,10 +577,10 @@ clear_cpu_sibling_map(int cpu) { int i; - for_each_cpu_mask(i, per_cpu(cpu_sibling_map, cpu)) - cpu_clear(cpu, per_cpu(cpu_sibling_map, i)); - for_each_cpu_mask(i, cpu_core_map[cpu]) - cpu_clear(cpu, cpu_core_map[i]); + for_each_cpu(i, &per_cpu(cpu_sibling_map, cpu)) + cpumask_clear_cpu(cpu, &per_cpu(cpu_sibling_map, i)); + for_each_cpu(i, &cpu_core_map[cpu]) + cpumask_clear_cpu(cpu, &cpu_core_map[i]); per_cpu(cpu_sibling_map, cpu) = cpu_core_map[cpu] = CPU_MASK_NONE; } @@ -592,12 +592,12 @@ remove_siblinginfo(int cpu) if (cpu_data(cpu)->threads_per_core == 1 && cpu_data(cpu)->cores_per_socket == 1) { - cpu_clear(cpu, cpu_core_map[cpu]); - cpu_clear(cpu, per_cpu(cpu_sibling_map, cpu)); + cpumask_clear_cpu(cpu, &cpu_core_map[cpu]); + cpumask_clear_cpu(cpu, &per_cpu(cpu_sibling_map, cpu)); return; } - last = (cpus_weight(cpu_core_map[cpu]) == 1 ? 1 : 0); + last = (cpumask_weight(&cpu_core_map[cpu]) == 1 ? 1 : 0); /* remove it from all sibling map's */ clear_cpu_sibling_map(cpu); @@ -673,7 +673,7 @@ int __cpu_disable(void) remove_siblinginfo(cpu); fixup_irqs(); local_flush_tlb_all(); - cpu_clear(cpu, cpu_callin_map); + cpumask_clear_cpu(cpu, &cpu_callin_map); return 0; } @@ -718,11 +718,13 @@ static inline void set_cpu_sibling_map(int cpu) for_each_online_cpu(i) { if ((cpu_data(cpu)->socket_id == cpu_data(i)->socket_id)) { - cpu_set(i, cpu_core_map[cpu]); - cpu_set(cpu, cpu_core_map[i]); + cpumask_set_cpu(i, &cpu_core_map[cpu]); + cpumask_set_cpu(cpu, &cpu_core_map[i]); if (cpu_data(cpu)->core_id == cpu_data(i)->core_id) { - cpu_set(i, per_cpu(cpu_sibling_map, cpu)); - cpu_set(cpu, per_cpu(cpu_sibling_map, i)); + cpumask_set_cpu(i, + &per_cpu(cpu_sibling_map, cpu)); + cpumask_set_cpu(cpu, + &per_cpu(cpu_sibling_map, i)); } } } @@ -742,7 +744,7 @@ __cpu_up(unsigned int cpu, struct task_struct *tidle) * Already booted cpu? not valid anymore since we dont * do idle loop tightspin anymore. */ - if (cpu_isset(cpu, cpu_callin_map)) + if (cpumask_test_cpu(cpu, &cpu_callin_map)) return -EINVAL; per_cpu(cpu_state, cpu) = CPU_UP_PREPARE; @@ -753,8 +755,8 @@ __cpu_up(unsigned int cpu, struct task_struct *tidle) if (cpu_data(cpu)->threads_per_core == 1 && cpu_data(cpu)->cores_per_socket == 1) { - cpu_set(cpu, per_cpu(cpu_sibling_map, cpu)); - cpu_set(cpu, cpu_core_map[cpu]); + cpumask_set_cpu(cpu, &per_cpu(cpu_sibling_map, cpu)); + cpumask_set_cpu(cpu, &cpu_core_map[cpu]); return 0; } diff --git a/arch/ia64/kernel/topology.c b/arch/ia64/kernel/topology.c index 965ab42fabb0..c01fe8991244 100644 --- a/arch/ia64/kernel/topology.c +++ b/arch/ia64/kernel/topology.c @@ -148,7 +148,7 @@ static void cache_shared_cpu_map_setup(unsigned int cpu, if (cpu_data(cpu)->threads_per_core <= 1 && cpu_data(cpu)->cores_per_socket <= 1) { - cpu_set(cpu, this_leaf->shared_cpu_map); + cpumask_set_cpu(cpu, &this_leaf->shared_cpu_map); return; } @@ -164,7 +164,7 @@ static void cache_shared_cpu_map_setup(unsigned int cpu, if (cpu_data(cpu)->socket_id == cpu_data(j)->socket_id && cpu_data(j)->core_id == csi.log1_cid && cpu_data(j)->thread_id == csi.log1_tid) - cpu_set(j, this_leaf->shared_cpu_map); + cpumask_set_cpu(j, &this_leaf->shared_cpu_map); i++; } while (i < num_shared && @@ -177,7 +177,7 @@ static void cache_shared_cpu_map_setup(unsigned int cpu, static void cache_shared_cpu_map_setup(unsigned int cpu, struct cache_info * this_leaf) { - cpu_set(cpu, this_leaf->shared_cpu_map); + cpumask_set_cpu(cpu, &this_leaf->shared_cpu_map); return; } #endif diff --git a/arch/m32r/include/asm/io.h b/arch/m32r/include/asm/io.h index 6e7787f3dac7..9cc00dbd59ce 100644 --- a/arch/m32r/include/asm/io.h +++ b/arch/m32r/include/asm/io.h @@ -67,6 +67,7 @@ static inline void __iomem *ioremap(unsigned long offset, unsigned long size) extern void iounmap(volatile void __iomem *addr); #define ioremap_nocache(off,size) ioremap(off,size) +#define ioremap_wc ioremap_nocache /* * IO bus memory addresses are also 1:1 with the physical address diff --git a/arch/m32r/kernel/smpboot.c b/arch/m32r/kernel/smpboot.c index bb21f4f63170..a468467542f4 100644 --- a/arch/m32r/kernel/smpboot.c +++ b/arch/m32r/kernel/smpboot.c @@ -376,7 +376,7 @@ void __init smp_cpus_done(unsigned int max_cpus) if (!cpumask_equal(&cpu_callin_map, cpu_online_mask)) BUG(); - for (cpu_id = 0 ; cpu_id < num_online_cpus() ; cpu_id++) + for_each_online_cpu(cpu_id) show_cpu_info(cpu_id); /* diff --git a/arch/m68k/coldfire/m527x.c b/arch/m68k/coldfire/m527x.c index 2ba470735bed..c0b3e28f91df 100644 --- a/arch/m68k/coldfire/m527x.c +++ b/arch/m68k/coldfire/m527x.c @@ -92,7 +92,6 @@ static void __init m527x_uarts_init(void) static void __init m527x_fec_init(void) { - u16 par; u8 v; /* Set multi-function pins to ethernet mode for fec0 */ @@ -100,6 +99,8 @@ static void __init m527x_fec_init(void) v = readb(MCFGPIO_PAR_FECI2C); writeb(v | 0xf0, MCFGPIO_PAR_FECI2C); #else + u16 par; + par = readw(MCFGPIO_PAR_FECI2C); writew(par | 0xf00, MCFGPIO_PAR_FECI2C); v = readb(MCFGPIO_PAR_FEC0HL); diff --git a/arch/m68k/include/asm/m527xsim.h b/arch/m68k/include/asm/m527xsim.h index 1bebbe78055a..2c648a043f24 100644 --- a/arch/m68k/include/asm/m527xsim.h +++ b/arch/m68k/include/asm/m527xsim.h @@ -103,8 +103,10 @@ */ #define MCFFEC_BASE0 (MCF_IPSBAR + 0x1000) #define MCFFEC_SIZE0 0x800 +#ifdef CONFIG_M5275 #define MCFFEC_BASE1 (MCF_IPSBAR + 0x1800) #define MCFFEC_SIZE1 0x800 +#endif /* * QSPI module. diff --git a/arch/m68k/include/asm/m68360_pram.h b/arch/m68k/include/asm/m68360_pram.h index e6088bbce93d..c0cbd96f09bc 100644 --- a/arch/m68k/include/asm/m68360_pram.h +++ b/arch/m68k/include/asm/m68360_pram.h @@ -170,7 +170,7 @@ struct uart_pram { unsigned short frmer; /* Rx framing error counter */ unsigned short nosec; /* Rx noise counter */ unsigned short brkec; /* Rx break character counter */ - unsigned short brkln; /* Reaceive break length */ + unsigned short brkln; /* Receive break length */ unsigned short uaddr1; /* address character 1 */ unsigned short uaddr2; /* address character 2 */ @@ -338,7 +338,7 @@ struct ethernet_pram { unsigned long c_pres; /* preset CRC */ unsigned long c_mask; /* constant mask for CRC */ unsigned long crcec; /* CRC error counter */ - unsigned long alec; /* alighnment error counter */ + unsigned long alec; /* alignment error counter */ unsigned long disfc; /* discard frame counter */ unsigned short pads; /* short frame PAD characters */ unsigned short ret_lim; /* retry limit threshold */ diff --git a/arch/mips/bcm63xx/irq.c b/arch/mips/bcm63xx/irq.c index b94bf44d8d8e..e3e808a6c542 100644 --- a/arch/mips/bcm63xx/irq.c +++ b/arch/mips/bcm63xx/irq.c @@ -58,9 +58,9 @@ static inline int enable_irq_for_cpu(int cpu, struct irq_data *d, #ifdef CONFIG_SMP if (m) - enable &= cpu_isset(cpu, *m); + enable &= cpumask_test_cpu(cpu, m); else if (irqd_affinity_was_set(d)) - enable &= cpu_isset(cpu, *d->affinity); + enable &= cpumask_test_cpu(cpu, d->affinity); #endif return enable; } diff --git a/arch/mips/cavium-octeon/smp.c b/arch/mips/cavium-octeon/smp.c index 8b1eeffa12ed..56f5d080ef9d 100644 --- a/arch/mips/cavium-octeon/smp.c +++ b/arch/mips/cavium-octeon/smp.c @@ -72,7 +72,7 @@ static inline void octeon_send_ipi_mask(const struct cpumask *mask, { unsigned int i; - for_each_cpu_mask(i, *mask) + for_each_cpu(i, mask) octeon_send_ipi_single(i, action); } @@ -239,7 +239,7 @@ static int octeon_cpu_disable(void) return -ENOTSUPP; set_cpu_online(cpu, false); - cpu_clear(cpu, cpu_callin_map); + cpumask_clear_cpu(cpu, &cpu_callin_map); octeon_fixup_irqs(); flush_cache_all(); diff --git a/arch/mips/include/asm/smp.h b/arch/mips/include/asm/smp.h index eacf865d21c2..bb02fac9b4fa 100644 --- a/arch/mips/include/asm/smp.h +++ b/arch/mips/include/asm/smp.h @@ -88,7 +88,7 @@ static inline void arch_send_call_function_single_ipi(int cpu) { extern struct plat_smp_ops *mp_ops; /* private */ - mp_ops->send_ipi_mask(&cpumask_of_cpu(cpu), SMP_CALL_FUNCTION); + mp_ops->send_ipi_mask(cpumask_of(cpu), SMP_CALL_FUNCTION); } static inline void arch_send_call_function_ipi_mask(const struct cpumask *mask) diff --git a/arch/mips/kernel/crash.c b/arch/mips/kernel/crash.c index d21264681e97..d434d5d5ae6e 100644 --- a/arch/mips/kernel/crash.c +++ b/arch/mips/kernel/crash.c @@ -25,9 +25,9 @@ static void crash_shutdown_secondary(void *ignore) return; local_irq_disable(); - if (!cpu_isset(cpu, cpus_in_crash)) + if (!cpumask_test_cpu(cpu, &cpus_in_crash)) crash_save_cpu(regs, cpu); - cpu_set(cpu, cpus_in_crash); + cpumask_set_cpu(cpu, &cpus_in_crash); while (!atomic_read(&kexec_ready_to_reboot)) cpu_relax(); @@ -50,7 +50,7 @@ static void crash_kexec_prepare_cpus(void) */ pr_emerg("Sending IPI to other cpus...\n"); msecs = 10000; - while ((cpus_weight(cpus_in_crash) < ncpus) && (--msecs > 0)) { + while ((cpumask_weight(&cpus_in_crash) < ncpus) && (--msecs > 0)) { cpu_relax(); mdelay(1); } @@ -66,5 +66,5 @@ void default_machine_crash_shutdown(struct pt_regs *regs) crashing_cpu = smp_processor_id(); crash_save_cpu(regs, crashing_cpu); crash_kexec_prepare_cpus(); - cpu_set(crashing_cpu, cpus_in_crash); + cpumask_set_cpu(crashing_cpu, &cpus_in_crash); } diff --git a/arch/mips/kernel/mips-mt-fpaff.c b/arch/mips/kernel/mips-mt-fpaff.c index 362bb3707e62..3e4491aa6d6b 100644 --- a/arch/mips/kernel/mips-mt-fpaff.c +++ b/arch/mips/kernel/mips-mt-fpaff.c @@ -114,8 +114,8 @@ asmlinkage long mipsmt_sys_sched_setaffinity(pid_t pid, unsigned int len, /* Compute new global allowed CPU set if necessary */ ti = task_thread_info(p); if (test_ti_thread_flag(ti, TIF_FPUBOUND) && - cpus_intersects(*new_mask, mt_fpu_cpumask)) { - cpus_and(*effective_mask, *new_mask, mt_fpu_cpumask); + cpumask_intersects(new_mask, &mt_fpu_cpumask)) { + cpumask_and(effective_mask, new_mask, &mt_fpu_cpumask); retval = set_cpus_allowed_ptr(p, effective_mask); } else { cpumask_copy(effective_mask, new_mask); diff --git a/arch/mips/kernel/process.c b/arch/mips/kernel/process.c index d295bd1e4996..f2975d4d1e44 100644 --- a/arch/mips/kernel/process.c +++ b/arch/mips/kernel/process.c @@ -49,7 +49,7 @@ void arch_cpu_idle_dead(void) { /* What the heck is this check doing ? */ - if (!cpu_isset(smp_processor_id(), cpu_callin_map)) + if (!cpumask_test_cpu(smp_processor_id(), &cpu_callin_map)) play_dead(); } #endif diff --git a/arch/mips/kernel/smp-bmips.c b/arch/mips/kernel/smp-bmips.c index b8bd9340c9c7..fd528d7ea278 100644 --- a/arch/mips/kernel/smp-bmips.c +++ b/arch/mips/kernel/smp-bmips.c @@ -362,7 +362,7 @@ static int bmips_cpu_disable(void) pr_info("SMP: CPU%d is offline\n", cpu); set_cpu_online(cpu, false); - cpu_clear(cpu, cpu_callin_map); + cpumask_clear_cpu(cpu, &cpu_callin_map); clear_c0_status(IE_IRQ5); local_flush_tlb_all(); diff --git a/arch/mips/kernel/smp-cmp.c b/arch/mips/kernel/smp-cmp.c index e36a859af666..d5e0f949dc48 100644 --- a/arch/mips/kernel/smp-cmp.c +++ b/arch/mips/kernel/smp-cmp.c @@ -66,7 +66,7 @@ static void cmp_smp_finish(void) #ifdef CONFIG_MIPS_MT_FPAFF /* If we have an FPU, enroll ourselves in the FPU-full mask */ if (cpu_has_fpu) - cpu_set(smp_processor_id(), mt_fpu_cpumask); + cpumask_set_cpu(smp_processor_id(), &mt_fpu_cpumask); #endif /* CONFIG_MIPS_MT_FPAFF */ local_irq_enable(); @@ -110,7 +110,7 @@ void __init cmp_smp_setup(void) #ifdef CONFIG_MIPS_MT_FPAFF /* If we have an FPU, enroll ourselves in the FPU-full mask */ if (cpu_has_fpu) - cpu_set(0, mt_fpu_cpumask); + cpumask_set_cpu(0, &mt_fpu_cpumask); #endif /* CONFIG_MIPS_MT_FPAFF */ for (i = 1; i < NR_CPUS; i++) { diff --git a/arch/mips/kernel/smp-cps.c b/arch/mips/kernel/smp-cps.c index d5589bedd0a4..7e011f95bb8e 100644 --- a/arch/mips/kernel/smp-cps.c +++ b/arch/mips/kernel/smp-cps.c @@ -290,7 +290,7 @@ static void cps_smp_finish(void) #ifdef CONFIG_MIPS_MT_FPAFF /* If we have an FPU, enroll ourselves in the FPU-full mask */ if (cpu_has_fpu) - cpu_set(smp_processor_id(), mt_fpu_cpumask); + cpumask_set_cpu(smp_processor_id(), &mt_fpu_cpumask); #endif /* CONFIG_MIPS_MT_FPAFF */ local_irq_enable(); @@ -313,7 +313,7 @@ static int cps_cpu_disable(void) atomic_sub(1 << cpu_vpe_id(¤t_cpu_data), &core_cfg->vpe_mask); smp_mb__after_atomic(); set_cpu_online(cpu, false); - cpu_clear(cpu, cpu_callin_map); + cpumask_clear_cpu(cpu, &cpu_callin_map); return 0; } diff --git a/arch/mips/kernel/smp-mt.c b/arch/mips/kernel/smp-mt.c index 17ea705f6c40..86311a164ef1 100644 --- a/arch/mips/kernel/smp-mt.c +++ b/arch/mips/kernel/smp-mt.c @@ -178,7 +178,7 @@ static void vsmp_smp_finish(void) #ifdef CONFIG_MIPS_MT_FPAFF /* If we have an FPU, enroll ourselves in the FPU-full mask */ if (cpu_has_fpu) - cpu_set(smp_processor_id(), mt_fpu_cpumask); + cpumask_set_cpu(smp_processor_id(), &mt_fpu_cpumask); #endif /* CONFIG_MIPS_MT_FPAFF */ local_irq_enable(); @@ -239,7 +239,7 @@ static void __init vsmp_smp_setup(void) #ifdef CONFIG_MIPS_MT_FPAFF /* If we have an FPU, enroll ourselves in the FPU-full mask */ if (cpu_has_fpu) - cpu_set(0, mt_fpu_cpumask); + cpumask_set_cpu(0, &mt_fpu_cpumask); #endif /* CONFIG_MIPS_MT_FPAFF */ if (!cpu_has_mipsmt) return; diff --git a/arch/mips/kernel/smp.c b/arch/mips/kernel/smp.c index 5b020bda3e05..193ace7955fb 100644 --- a/arch/mips/kernel/smp.c +++ b/arch/mips/kernel/smp.c @@ -75,30 +75,30 @@ static inline void set_cpu_sibling_map(int cpu) { int i; - cpu_set(cpu, cpu_sibling_setup_map); + cpumask_set_cpu(cpu, &cpu_sibling_setup_map); if (smp_num_siblings > 1) { - for_each_cpu_mask(i, cpu_sibling_setup_map) { + for_each_cpu(i, &cpu_sibling_setup_map) { if (cpu_data[cpu].package == cpu_data[i].package && cpu_data[cpu].core == cpu_data[i].core) { - cpu_set(i, cpu_sibling_map[cpu]); - cpu_set(cpu, cpu_sibling_map[i]); + cpumask_set_cpu(i, &cpu_sibling_map[cpu]); + cpumask_set_cpu(cpu, &cpu_sibling_map[i]); } } } else - cpu_set(cpu, cpu_sibling_map[cpu]); + cpumask_set_cpu(cpu, &cpu_sibling_map[cpu]); } static inline void set_cpu_core_map(int cpu) { int i; - cpu_set(cpu, cpu_core_setup_map); + cpumask_set_cpu(cpu, &cpu_core_setup_map); - for_each_cpu_mask(i, cpu_core_setup_map) { + for_each_cpu(i, &cpu_core_setup_map) { if (cpu_data[cpu].package == cpu_data[i].package) { - cpu_set(i, cpu_core_map[cpu]); - cpu_set(cpu, cpu_core_map[i]); + cpumask_set_cpu(i, &cpu_core_map[cpu]); + cpumask_set_cpu(cpu, &cpu_core_map[i]); } } } @@ -138,7 +138,7 @@ asmlinkage void start_secondary(void) cpu = smp_processor_id(); cpu_data[cpu].udelay_val = loops_per_jiffy; - cpu_set(cpu, cpu_coherent_mask); + cpumask_set_cpu(cpu, &cpu_coherent_mask); notify_cpu_starting(cpu); set_cpu_online(cpu, true); @@ -146,7 +146,7 @@ asmlinkage void start_secondary(void) set_cpu_sibling_map(cpu); set_cpu_core_map(cpu); - cpu_set(cpu, cpu_callin_map); + cpumask_set_cpu(cpu, &cpu_callin_map); synchronise_count_slave(cpu); @@ -208,7 +208,7 @@ void smp_prepare_boot_cpu(void) { set_cpu_possible(0, true); set_cpu_online(0, true); - cpu_set(0, cpu_callin_map); + cpumask_set_cpu(0, &cpu_callin_map); } int __cpu_up(unsigned int cpu, struct task_struct *tidle) @@ -218,7 +218,7 @@ int __cpu_up(unsigned int cpu, struct task_struct *tidle) /* * Trust is futile. We should really have timeouts ... */ - while (!cpu_isset(cpu, cpu_callin_map)) + while (!cpumask_test_cpu(cpu, &cpu_callin_map)) udelay(100); synchronise_count_master(cpu); diff --git a/arch/mips/kernel/traps.c b/arch/mips/kernel/traps.c index e334c641a81b..ba32e48d4697 100644 --- a/arch/mips/kernel/traps.c +++ b/arch/mips/kernel/traps.c @@ -1153,13 +1153,13 @@ static void mt_ase_fp_affinity(void) * restricted the allowed set to exclude any CPUs with FPUs, * we'll skip the procedure. */ - if (cpus_intersects(current->cpus_allowed, mt_fpu_cpumask)) { + if (cpumask_intersects(¤t->cpus_allowed, &mt_fpu_cpumask)) { cpumask_t tmask; current->thread.user_cpus_allowed = current->cpus_allowed; - cpus_and(tmask, current->cpus_allowed, - mt_fpu_cpumask); + cpumask_and(&tmask, ¤t->cpus_allowed, + &mt_fpu_cpumask); set_cpus_allowed_ptr(current, &tmask); set_thread_flag(TIF_FPUBOUND); } diff --git a/arch/mips/loongson/loongson-3/numa.c b/arch/mips/loongson/loongson-3/numa.c index 6cae0e75de27..12d14ed48778 100644 --- a/arch/mips/loongson/loongson-3/numa.c +++ b/arch/mips/loongson/loongson-3/numa.c @@ -233,7 +233,7 @@ static __init void prom_meminit(void) if (node_online(node)) { szmem(node); node_mem_init(node); - cpus_clear(__node_data[(node)]->cpumask); + cpumask_clear(&__node_data[(node)]->cpumask); } } for (cpu = 0; cpu < loongson_sysconf.nr_cpus; cpu++) { @@ -244,7 +244,7 @@ static __init void prom_meminit(void) if (loongson_sysconf.reserved_cpus_mask & (1<<cpu)) continue; - cpu_set(active_cpu, __node_data[(node)]->cpumask); + cpumask_set_cpu(active_cpu, &__node_data[(node)]->cpumask); pr_info("NUMA: set cpumask cpu %d on node %d\n", active_cpu, node); active_cpu++; diff --git a/arch/mips/loongson/loongson-3/smp.c b/arch/mips/loongson/loongson-3/smp.c index e2eb688b5434..e3c68b5da18d 100644 --- a/arch/mips/loongson/loongson-3/smp.c +++ b/arch/mips/loongson/loongson-3/smp.c @@ -408,7 +408,7 @@ static int loongson3_cpu_disable(void) return -EBUSY; set_cpu_online(cpu, false); - cpu_clear(cpu, cpu_callin_map); + cpumask_clear_cpu(cpu, &cpu_callin_map); local_irq_save(flags); fixup_irqs(); local_irq_restore(flags); diff --git a/arch/mips/mti-malta/malta-init.c b/arch/mips/mti-malta/malta-init.c index 6849f533154f..cec3e187c48f 100644 --- a/arch/mips/mti-malta/malta-init.c +++ b/arch/mips/mti-malta/malta-init.c @@ -14,7 +14,7 @@ #include <linux/init.h> #include <linux/string.h> #include <linux/kernel.h> -#include <linux/serial_8250.h> +#include <linux/serial_core.h> #include <asm/cacheflush.h> #include <asm/smp-ops.h> @@ -75,7 +75,7 @@ static void __init console_config(void) if ((strstr(fw_getcmdline(), "earlycon=")) == NULL) { sprintf(console_string, "uart8250,io,0x3f8,%d%c%c", baud, parity, bits); - setup_early_serial8250_console(console_string); + setup_earlycon(console_string); } if ((strstr(fw_getcmdline(), "console=")) == NULL) { diff --git a/arch/mips/paravirt/paravirt-smp.c b/arch/mips/paravirt/paravirt-smp.c index 0164b0c48352..42181c7105df 100644 --- a/arch/mips/paravirt/paravirt-smp.c +++ b/arch/mips/paravirt/paravirt-smp.c @@ -75,7 +75,7 @@ static void paravirt_send_ipi_mask(const struct cpumask *mask, unsigned int acti { unsigned int cpu; - for_each_cpu_mask(cpu, *mask) + for_each_cpu(cpu, mask) paravirt_send_ipi_single(cpu, action); } diff --git a/arch/mips/sgi-ip27/ip27-init.c b/arch/mips/sgi-ip27/ip27-init.c index ee736bd103f8..570098bfdf87 100644 --- a/arch/mips/sgi-ip27/ip27-init.c +++ b/arch/mips/sgi-ip27/ip27-init.c @@ -60,7 +60,7 @@ static void per_hub_init(cnodeid_t cnode) nasid_t nasid = COMPACT_TO_NASID_NODEID(cnode); int i; - cpu_set(smp_processor_id(), hub->h_cpus); + cpumask_set_cpu(smp_processor_id(), &hub->h_cpus); if (test_and_set_bit(cnode, hub_init_mask)) return; diff --git a/arch/mips/sgi-ip27/ip27-klnuma.c b/arch/mips/sgi-ip27/ip27-klnuma.c index ecbb62f339c5..bda90cf87e8c 100644 --- a/arch/mips/sgi-ip27/ip27-klnuma.c +++ b/arch/mips/sgi-ip27/ip27-klnuma.c @@ -29,8 +29,8 @@ static cpumask_t ktext_repmask; void __init setup_replication_mask(void) { /* Set only the master cnode's bit. The master cnode is always 0. */ - cpus_clear(ktext_repmask); - cpu_set(0, ktext_repmask); + cpumask_clear(&ktext_repmask); + cpumask_set_cpu(0, &ktext_repmask); #ifdef CONFIG_REPLICATE_KTEXT #ifndef CONFIG_MAPPED_KERNEL @@ -43,7 +43,7 @@ void __init setup_replication_mask(void) if (cnode == 0) continue; /* Advertise that we have a copy of the kernel */ - cpu_set(cnode, ktext_repmask); + cpumask_set_cpu(cnode, &ktext_repmask); } } #endif @@ -99,7 +99,7 @@ void __init replicate_kernel_text() client_nasid = COMPACT_TO_NASID_NODEID(cnode); /* Check if this node should get a copy of the kernel */ - if (cpu_isset(cnode, ktext_repmask)) { + if (cpumask_test_cpu(cnode, &ktext_repmask)) { server_nasid = client_nasid; copy_kernel(server_nasid); } @@ -124,7 +124,7 @@ unsigned long node_getfirstfree(cnodeid_t cnode) loadbase += 16777216; #endif offset = PAGE_ALIGN((unsigned long)(&_end)) - loadbase; - if ((cnode == 0) || (cpu_isset(cnode, ktext_repmask))) + if ((cnode == 0) || (cpumask_test_cpu(cnode, &ktext_repmask))) return TO_NODE(nasid, offset) >> PAGE_SHIFT; else return KDM_TO_PHYS(PAGE_ALIGN(SYMMON_STK_ADDR(nasid, 0))) >> PAGE_SHIFT; diff --git a/arch/mips/sgi-ip27/ip27-memory.c b/arch/mips/sgi-ip27/ip27-memory.c index 0b68469e063f..8d0eb2643248 100644 --- a/arch/mips/sgi-ip27/ip27-memory.c +++ b/arch/mips/sgi-ip27/ip27-memory.c @@ -404,7 +404,7 @@ static void __init node_mem_init(cnodeid_t node) NODE_DATA(node)->node_start_pfn = start_pfn; NODE_DATA(node)->node_spanned_pages = end_pfn - start_pfn; - cpus_clear(hub_data(node)->h_cpus); + cpumask_clear(&hub_data(node)->h_cpus); slot_freepfn += PFN_UP(sizeof(struct pglist_data) + sizeof(struct hub_data)); diff --git a/arch/parisc/include/asm/Kbuild b/arch/parisc/include/asm/Kbuild index 12b341d04f88..7a4bcc36303d 100644 --- a/arch/parisc/include/asm/Kbuild +++ b/arch/parisc/include/asm/Kbuild @@ -20,6 +20,7 @@ generic-y += param.h generic-y += percpu.h generic-y += poll.h generic-y += preempt.h +generic-y += scatterlist.h generic-y += seccomp.h generic-y += segment.h generic-y += topology.h diff --git a/arch/parisc/include/asm/pgalloc.h b/arch/parisc/include/asm/pgalloc.h index 1ba29369257c..3a08eae3318f 100644 --- a/arch/parisc/include/asm/pgalloc.h +++ b/arch/parisc/include/asm/pgalloc.h @@ -26,7 +26,7 @@ static inline pgd_t *pgd_alloc(struct mm_struct *mm) if (likely(pgd != NULL)) { memset(pgd, 0, PAGE_SIZE<<PGD_ALLOC_ORDER); -#if PT_NLEVELS == 3 +#if CONFIG_PGTABLE_LEVELS == 3 actual_pgd += PTRS_PER_PGD; /* Populate first pmd with allocated memory. We mark it * with PxD_FLAG_ATTACHED as a signal to the system that this @@ -45,7 +45,7 @@ static inline pgd_t *pgd_alloc(struct mm_struct *mm) static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) { -#if PT_NLEVELS == 3 +#if CONFIG_PGTABLE_LEVELS == 3 pgd -= PTRS_PER_PGD; #endif free_pages((unsigned long)pgd, PGD_ALLOC_ORDER); @@ -102,7 +102,7 @@ static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd, pte_t *pte) { -#if PT_NLEVELS == 3 +#if CONFIG_PGTABLE_LEVELS == 3 /* preserve the gateway marker if this is the beginning of * the permanent pmd */ if(pmd_flag(*pmd) & PxD_FLAG_ATTACHED) diff --git a/arch/parisc/include/asm/scatterlist.h b/arch/parisc/include/asm/scatterlist.h deleted file mode 100644 index 8bf1f0dd1f15..000000000000 --- a/arch/parisc/include/asm/scatterlist.h +++ /dev/null @@ -1,10 +0,0 @@ -#ifndef _ASM_PARISC_SCATTERLIST_H -#define _ASM_PARISC_SCATTERLIST_H - -#include <asm/page.h> -#include <asm/types.h> -#include <asm-generic/scatterlist.h> - -#define sg_virt_addr(sg) ((unsigned long)sg_virt(sg)) - -#endif /* _ASM_PARISC_SCATTERLIST_H */ diff --git a/arch/parisc/kernel/irq.c b/arch/parisc/kernel/irq.c index cfe056fe7f5c..f3191db6e2e9 100644 --- a/arch/parisc/kernel/irq.c +++ b/arch/parisc/kernel/irq.c @@ -525,8 +525,8 @@ void do_cpu_irq_mask(struct pt_regs *regs) desc = irq_to_desc(irq); cpumask_copy(&dest, desc->irq_data.affinity); if (irqd_is_per_cpu(&desc->irq_data) && - !cpu_isset(smp_processor_id(), dest)) { - int cpu = first_cpu(dest); + !cpumask_test_cpu(smp_processor_id(), &dest)) { + int cpu = cpumask_first(&dest); printk(KERN_DEBUG "redirecting irq %d from CPU %d to %d\n", irq, smp_processor_id(), cpu); diff --git a/arch/parisc/kernel/pci-dma.c b/arch/parisc/kernel/pci-dma.c index d87d1c476d85..ff834fd67478 100644 --- a/arch/parisc/kernel/pci-dma.c +++ b/arch/parisc/kernel/pci-dma.c @@ -482,7 +482,7 @@ static int pa11_dma_map_sg(struct device *dev, struct scatterlist *sglist, int n BUG_ON(direction == DMA_NONE); for (i = 0; i < nents; i++, sglist++ ) { - unsigned long vaddr = sg_virt_addr(sglist); + unsigned long vaddr = (unsigned long)sg_virt(sglist); sg_dma_address(sglist) = (dma_addr_t) virt_to_phys(vaddr); sg_dma_len(sglist) = sglist->length; flush_kernel_dcache_range(vaddr, sglist->length); @@ -502,7 +502,7 @@ static void pa11_dma_unmap_sg(struct device *dev, struct scatterlist *sglist, in /* once we do combining we'll need to use phys_to_virt(sg_dma_address(sglist)) */ for (i = 0; i < nents; i++, sglist++ ) - flush_kernel_dcache_range(sg_virt_addr(sglist), sglist->length); + flush_kernel_vmap_range(sg_virt(sglist), sglist->length); return; } @@ -527,7 +527,7 @@ static void pa11_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sgl /* once we do combining we'll need to use phys_to_virt(sg_dma_address(sglist)) */ for (i = 0; i < nents; i++, sglist++ ) - flush_kernel_dcache_range(sg_virt_addr(sglist), sglist->length); + flush_kernel_vmap_range(sg_virt(sglist), sglist->length); } static void pa11_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sglist, int nents, enum dma_data_direction direction) @@ -537,7 +537,7 @@ static void pa11_dma_sync_sg_for_device(struct device *dev, struct scatterlist * /* once we do combining we'll need to use phys_to_virt(sg_dma_address(sglist)) */ for (i = 0; i < nents; i++, sglist++ ) - flush_kernel_dcache_range(sg_virt_addr(sglist), sglist->length); + flush_kernel_vmap_range(sg_virt(sglist), sglist->length); } struct hppa_dma_ops pcxl_dma_ops = { diff --git a/arch/powerpc/include/asm/cputhreads.h b/arch/powerpc/include/asm/cputhreads.h index 4c8ad592ae33..5be6c4753667 100644 --- a/arch/powerpc/include/asm/cputhreads.h +++ b/arch/powerpc/include/asm/cputhreads.h @@ -25,7 +25,7 @@ extern cpumask_t threads_core_mask; #define threads_per_core 1 #define threads_per_subcore 1 #define threads_shift 0 -#define threads_core_mask (CPU_MASK_CPU0) +#define threads_core_mask (*get_cpu_mask(0)) #endif /* cpu_thread_mask_to_cores - Return a cpumask of one per cores diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index de2726a487b0..8e58c614c37d 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -115,7 +115,7 @@ config S390 select HAVE_ARCH_SECCOMP_FILTER select HAVE_ARCH_TRACEHOOK select HAVE_ARCH_TRANSPARENT_HUGEPAGE - select HAVE_BPF_JIT if PACK_STACK + select HAVE_BPF_JIT if PACK_STACK && HAVE_MARCH_Z9_109_FEATURES select HAVE_CMPXCHG_DOUBLE select HAVE_CMPXCHG_LOCAL select HAVE_DEBUG_KMEMLEAK diff --git a/arch/s390/include/asm/dma-mapping.h b/arch/s390/include/asm/dma-mapping.h index 709955ddaa4d..9d395961e713 100644 --- a/arch/s390/include/asm/dma-mapping.h +++ b/arch/s390/include/asm/dma-mapping.h @@ -42,7 +42,7 @@ static inline int dma_supported(struct device *dev, u64 mask) static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size) { if (!dev->dma_mask) - return 0; + return false; return addr + size - 1 <= *dev->dma_mask; } diff --git a/arch/s390/include/asm/pci.h b/arch/s390/include/asm/pci.h index ef803c202d42..a648338c434a 100644 --- a/arch/s390/include/asm/pci.h +++ b/arch/s390/include/asm/pci.h @@ -7,6 +7,7 @@ #define PCI_BAR_COUNT 6 #include <linux/pci.h> +#include <linux/mutex.h> #include <asm-generic/pci.h> #include <asm-generic/pci-dma-compat.h> #include <asm/pci_clp.h> @@ -44,10 +45,6 @@ struct zpci_fmb { u64 rpcit_ops; u64 dma_rbytes; u64 dma_wbytes; - /* software counters */ - atomic64_t allocated_pages; - atomic64_t mapped_pages; - atomic64_t unmapped_pages; } __packed __aligned(16); enum zpci_state { @@ -80,6 +77,7 @@ struct zpci_dev { u8 pft; /* pci function type */ u16 domain; + struct mutex lock; u8 pfip[CLP_PFIP_NR_SEGMENTS]; /* pci function internal path */ u32 uid; /* user defined id */ u8 util_str[CLP_UTIL_STR_LEN]; /* utility string */ @@ -111,6 +109,10 @@ struct zpci_dev { /* Function measurement block */ struct zpci_fmb *fmb; u16 fmb_update; /* update interval */ + /* software counters */ + atomic64_t allocated_pages; + atomic64_t mapped_pages; + atomic64_t unmapped_pages; enum pci_bus_speed max_bus_speed; diff --git a/arch/s390/net/bpf_jit.S b/arch/s390/net/bpf_jit.S index ba44c9f55346..a1c917d881ec 100644 --- a/arch/s390/net/bpf_jit.S +++ b/arch/s390/net/bpf_jit.S @@ -1,134 +1,115 @@ /* * BPF Jit compiler for s390, help functions. * - * Copyright IBM Corp. 2012 + * Copyright IBM Corp. 2012,2015 * * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> + * Michael Holzheu <holzheu@linux.vnet.ibm.com> */ + #include <linux/linkage.h> +#include "bpf_jit.h" /* * Calling convention: - * registers %r2, %r6-%r8, %r10-%r11, %r13, %r15 are call saved - * %r2: skb pointer - * %r3: offset parameter - * %r5: BPF A accumulator - * %r8: return address - * %r9: save register for skb pointer - * %r10: skb->data - * %r11: skb->len - skb->data_len (headlen) - * %r12: BPF X accumulator + * registers %r7-%r10, %r11,%r13, and %r15 are call saved + * + * Input (64 bit): + * %r3 (%b2) = offset into skb data + * %r6 (%b5) = return address + * %r7 (%b6) = skb pointer + * %r12 = skb data pointer + * + * Output: + * %r14= %b0 = return value (read skb value) + * + * Work registers: %r2,%r4,%r5,%r14 * * skb_copy_bits takes 4 parameters: * %r2 = skb pointer * %r3 = offset into skb data * %r4 = pointer to temp buffer * %r5 = length to copy + * Return value in %r2: 0 = ok + * + * bpf_internal_load_pointer_neg_helper takes 3 parameters: + * %r2 = skb pointer + * %r3 = offset into data + * %r4 = length to copy + * Return value in %r2: Pointer to data */ -#define SKBDATA %r8 - - /* A = *(u32 *) (skb->data+K+X) */ -ENTRY(sk_load_word_ind) - ar %r3,%r12 # offset += X - bmr %r8 # < 0 -> return with cc - - /* A = *(u32 *) (skb->data+K) */ -ENTRY(sk_load_word) - llgfr %r1,%r3 # extend offset - ahi %r3,4 # offset + 4 - clr %r11,%r3 # hlen <= offset + 4 ? - jl sk_load_word_slow - l %r5,0(%r1,%r10) # get word from skb - xr %r1,%r1 # set cc to zero - br %r8 -sk_load_word_slow: - lgr %r9,%r2 # save %r2 - lgr %r3,%r1 # offset - la %r4,160(%r15) # pointer to temp buffer - lghi %r5,4 # 4 bytes - brasl %r14,skb_copy_bits # get data from skb - l %r5,160(%r15) # load result from temp buffer - ltgr %r2,%r2 # set cc to (%r2 != 0) - lgr %r2,%r9 # restore %r2 - br %r8 +#define SKF_MAX_NEG_OFF -0x200000 /* SKF_LL_OFF from filter.h */ - /* A = *(u16 *) (skb->data+K+X) */ -ENTRY(sk_load_half_ind) - ar %r3,%r12 # offset += X - bmr %r8 # < 0 -> return with cc - - /* A = *(u16 *) (skb->data+K) */ -ENTRY(sk_load_half) - llgfr %r1,%r3 # extend offset - ahi %r3,2 # offset + 2 - clr %r11,%r3 # hlen <= offset + 2 ? - jl sk_load_half_slow - llgh %r5,0(%r1,%r10) # get half from skb - xr %r1,%r1 # set cc to zero - br %r8 - -sk_load_half_slow: - lgr %r9,%r2 # save %r2 - lgr %r3,%r1 # offset - la %r4,162(%r15) # pointer to temp buffer - lghi %r5,2 # 2 bytes - brasl %r14,skb_copy_bits # get data from skb - xc 160(2,%r15),160(%r15) - l %r5,160(%r15) # load result from temp buffer - ltgr %r2,%r2 # set cc to (%r2 != 0) - lgr %r2,%r9 # restore %r2 - br %r8 +/* + * Load SIZE bytes from SKB + */ +#define sk_load_common(NAME, SIZE, LOAD) \ +ENTRY(sk_load_##NAME); \ + ltgr %r3,%r3; /* Is offset negative? */ \ + jl sk_load_##NAME##_slow_neg; \ +ENTRY(sk_load_##NAME##_pos); \ + aghi %r3,SIZE; /* Offset + SIZE */ \ + clg %r3,STK_OFF_HLEN(%r15); /* Offset + SIZE > hlen? */ \ + jh sk_load_##NAME##_slow; \ + LOAD %r14,-SIZE(%r3,%r12); /* Get data from skb */ \ + b OFF_OK(%r6); /* Return */ \ + \ +sk_load_##NAME##_slow:; \ + lgr %r2,%r7; /* Arg1 = skb pointer */ \ + aghi %r3,-SIZE; /* Arg2 = offset */ \ + la %r4,STK_OFF_TMP(%r15); /* Arg3 = temp bufffer */ \ + lghi %r5,SIZE; /* Arg4 = size */ \ + brasl %r14,skb_copy_bits; /* Get data from skb */ \ + LOAD %r14,STK_OFF_TMP(%r15); /* Load from temp bufffer */ \ + ltgr %r2,%r2; /* Set cc to (%r2 != 0) */ \ + br %r6; /* Return */ - /* A = *(u8 *) (skb->data+K+X) */ -ENTRY(sk_load_byte_ind) - ar %r3,%r12 # offset += X - bmr %r8 # < 0 -> return with cc +sk_load_common(word, 4, llgf) /* r14 = *(u32 *) (skb->data+offset) */ +sk_load_common(half, 2, llgh) /* r14 = *(u16 *) (skb->data+offset) */ - /* A = *(u8 *) (skb->data+K) */ +/* + * Load 1 byte from SKB (optimized version) + */ + /* r14 = *(u8 *) (skb->data+offset) */ ENTRY(sk_load_byte) - llgfr %r1,%r3 # extend offset - clr %r11,%r3 # hlen < offset ? - jle sk_load_byte_slow - lhi %r5,0 - ic %r5,0(%r1,%r10) # get byte from skb - xr %r1,%r1 # set cc to zero - br %r8 + ltgr %r3,%r3 # Is offset negative? + jl sk_load_byte_slow_neg +ENTRY(sk_load_byte_pos) + clg %r3,STK_OFF_HLEN(%r15) # Offset >= hlen? + jnl sk_load_byte_slow + llgc %r14,0(%r3,%r12) # Get byte from skb + b OFF_OK(%r6) # Return OK sk_load_byte_slow: - lgr %r9,%r2 # save %r2 - lgr %r3,%r1 # offset - la %r4,163(%r15) # pointer to temp buffer - lghi %r5,1 # 1 byte - brasl %r14,skb_copy_bits # get data from skb - xc 160(3,%r15),160(%r15) - l %r5,160(%r15) # load result from temp buffer - ltgr %r2,%r2 # set cc to (%r2 != 0) - lgr %r2,%r9 # restore %r2 - br %r8 + lgr %r2,%r7 # Arg1 = skb pointer + # Arg2 = offset + la %r4,STK_OFF_TMP(%r15) # Arg3 = pointer to temp buffer + lghi %r5,1 # Arg4 = size (1 byte) + brasl %r14,skb_copy_bits # Get data from skb + llgc %r14,STK_OFF_TMP(%r15) # Load result from temp buffer + ltgr %r2,%r2 # Set cc to (%r2 != 0) + br %r6 # Return cc + +#define sk_negative_common(NAME, SIZE, LOAD) \ +sk_load_##NAME##_slow_neg:; \ + cgfi %r3,SKF_MAX_NEG_OFF; \ + jl bpf_error; \ + lgr %r2,%r7; /* Arg1 = skb pointer */ \ + /* Arg2 = offset */ \ + lghi %r4,SIZE; /* Arg3 = size */ \ + brasl %r14,bpf_internal_load_pointer_neg_helper; \ + ltgr %r2,%r2; \ + jz bpf_error; \ + LOAD %r14,0(%r2); /* Get data from pointer */ \ + xr %r3,%r3; /* Set cc to zero */ \ + br %r6; /* Return cc */ - /* X = (*(u8 *)(skb->data+K) & 0xf) << 2 */ -ENTRY(sk_load_byte_msh) - llgfr %r1,%r3 # extend offset - clr %r11,%r3 # hlen < offset ? - jle sk_load_byte_msh_slow - lhi %r12,0 - ic %r12,0(%r1,%r10) # get byte from skb - nill %r12,0x0f - sll %r12,2 - xr %r1,%r1 # set cc to zero - br %r8 +sk_negative_common(word, 4, llgf) +sk_negative_common(half, 2, llgh) +sk_negative_common(byte, 1, llgc) -sk_load_byte_msh_slow: - lgr %r9,%r2 # save %r2 - lgr %r3,%r1 # offset - la %r4,163(%r15) # pointer to temp buffer - lghi %r5,1 # 1 byte - brasl %r14,skb_copy_bits # get data from skb - xc 160(3,%r15),160(%r15) - l %r12,160(%r15) # load result from temp buffer - nill %r12,0x0f - sll %r12,2 - ltgr %r2,%r2 # set cc to (%r2 != 0) - lgr %r2,%r9 # restore %r2 - br %r8 +bpf_error: +# force a return 0 from jit handler + ltgr %r15,%r15 # Set condition code + br %r6 diff --git a/arch/s390/net/bpf_jit.h b/arch/s390/net/bpf_jit.h new file mode 100644 index 000000000000..ba8593a515ba --- /dev/null +++ b/arch/s390/net/bpf_jit.h @@ -0,0 +1,58 @@ +/* + * BPF Jit compiler defines + * + * Copyright IBM Corp. 2012,2015 + * + * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> + * Michael Holzheu <holzheu@linux.vnet.ibm.com> + */ + +#ifndef __ARCH_S390_NET_BPF_JIT_H +#define __ARCH_S390_NET_BPF_JIT_H + +#ifndef __ASSEMBLY__ + +#include <linux/filter.h> +#include <linux/types.h> + +extern u8 sk_load_word_pos[], sk_load_half_pos[], sk_load_byte_pos[]; +extern u8 sk_load_word[], sk_load_half[], sk_load_byte[]; + +#endif /* __ASSEMBLY__ */ + +/* + * Stackframe layout (packed stack): + * + * ^ high + * +---------------+ | + * | old backchain | | + * +---------------+ | + * | r15 - r6 | | + * BFP -> +===============+ | + * | | | + * | BPF stack | | + * | | | + * +---------------+ | + * | 8 byte hlen | | + * R15+168 -> +---------------+ | + * | 4 byte align | | + * +---------------+ | + * | 4 byte temp | | + * | for bpf_jit.S | | + * R15+160 -> +---------------+ | + * | new backchain | | + * R15+152 -> +---------------+ | + * | + 152 byte SA | | + * R15 -> +---------------+ + low + * + * We get 160 bytes stack space from calling function, but only use + * 11 * 8 byte (old backchain + r15 - r6) for storing registers. + */ +#define STK_OFF (MAX_BPF_STACK + 8 + 4 + 4 + (160 - 11 * 8)) +#define STK_OFF_TMP 160 /* Offset of tmp buffer on stack */ +#define STK_OFF_HLEN 168 /* Offset of SKB header length on stack */ + +/* Offset to skip condition code check */ +#define OFF_OK 4 + +#endif /* __ARCH_S390_NET_BPF_JIT_H */ diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c index bbd1981cc150..7690dc8e1ab5 100644 --- a/arch/s390/net/bpf_jit_comp.c +++ b/arch/s390/net/bpf_jit_comp.c @@ -1,817 +1,1209 @@ /* * BPF Jit compiler for s390. * - * Copyright IBM Corp. 2012 + * Minimum build requirements: + * + * - HAVE_MARCH_Z196_FEATURES: laal, laalg + * - HAVE_MARCH_Z10_FEATURES: msfi, cgrj, clgrj + * - HAVE_MARCH_Z9_109_FEATURES: alfi, llilf, clfi, oilf, nilf + * - PACK_STACK + * - 64BIT + * + * Copyright IBM Corp. 2012,2015 * * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> + * Michael Holzheu <holzheu@linux.vnet.ibm.com> */ + +#define KMSG_COMPONENT "bpf_jit" +#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt + #include <linux/netdevice.h> -#include <linux/if_vlan.h> #include <linux/filter.h> #include <linux/init.h> #include <asm/cacheflush.h> -#include <asm/facility.h> #include <asm/dis.h> +#include "bpf_jit.h" -/* - * Conventions: - * %r2 = skb pointer - * %r3 = offset parameter - * %r4 = scratch register / length parameter - * %r5 = BPF A accumulator - * %r8 = return address - * %r9 = save register for skb pointer - * %r10 = skb->data - * %r11 = skb->len - skb->data_len (headlen) - * %r12 = BPF X accumulator - * %r13 = literal pool pointer - * 0(%r15) - 63(%r15) scratch memory array with BPF_MEMWORDS - */ int bpf_jit_enable __read_mostly; +struct bpf_jit { + u32 seen; /* Flags to remember seen eBPF instructions */ + u32 seen_reg[16]; /* Array to remember which registers are used */ + u32 *addrs; /* Array with relative instruction addresses */ + u8 *prg_buf; /* Start of program */ + int size; /* Size of program and literal pool */ + int size_prg; /* Size of program */ + int prg; /* Current position in program */ + int lit_start; /* Start of literal pool */ + int lit; /* Current position in literal pool */ + int base_ip; /* Base address for literal pool */ + int ret0_ip; /* Address of return 0 */ + int exit_ip; /* Address of exit */ +}; + +#define BPF_SIZE_MAX 4096 /* Max size for program */ + +#define SEEN_SKB 1 /* skb access */ +#define SEEN_MEM 2 /* use mem[] for temporary storage */ +#define SEEN_RET0 4 /* ret0_ip points to a valid return 0 */ +#define SEEN_LITERAL 8 /* code uses literals */ +#define SEEN_FUNC 16 /* calls C functions */ +#define SEEN_STACK (SEEN_FUNC | SEEN_MEM | SEEN_SKB) + /* - * assembly code in arch/x86/net/bpf_jit.S + * s390 registers */ -extern u8 sk_load_word[], sk_load_half[], sk_load_byte[], sk_load_byte_msh[]; -extern u8 sk_load_word_ind[], sk_load_half_ind[], sk_load_byte_ind[]; +#define REG_W0 (__MAX_BPF_REG+0) /* Work register 1 (even) */ +#define REG_W1 (__MAX_BPF_REG+1) /* Work register 2 (odd) */ +#define REG_SKB_DATA (__MAX_BPF_REG+2) /* SKB data register */ +#define REG_L (__MAX_BPF_REG+3) /* Literal pool register */ +#define REG_15 (__MAX_BPF_REG+4) /* Register 15 */ +#define REG_0 REG_W0 /* Register 0 */ +#define REG_2 BPF_REG_1 /* Register 2 */ +#define REG_14 BPF_REG_0 /* Register 14 */ -struct bpf_jit { - unsigned int seen; - u8 *start; - u8 *prg; - u8 *mid; - u8 *lit; - u8 *end; - u8 *base_ip; - u8 *ret0_ip; - u8 *exit_ip; - unsigned int off_load_word; - unsigned int off_load_half; - unsigned int off_load_byte; - unsigned int off_load_bmsh; - unsigned int off_load_iword; - unsigned int off_load_ihalf; - unsigned int off_load_ibyte; +/* + * Mapping of BPF registers to s390 registers + */ +static const int reg2hex[] = { + /* Return code */ + [BPF_REG_0] = 14, + /* Function parameters */ + [BPF_REG_1] = 2, + [BPF_REG_2] = 3, + [BPF_REG_3] = 4, + [BPF_REG_4] = 5, + [BPF_REG_5] = 6, + /* Call saved registers */ + [BPF_REG_6] = 7, + [BPF_REG_7] = 8, + [BPF_REG_8] = 9, + [BPF_REG_9] = 10, + /* BPF stack pointer */ + [BPF_REG_FP] = 13, + /* SKB data pointer */ + [REG_SKB_DATA] = 12, + /* Work registers for s390x backend */ + [REG_W0] = 0, + [REG_W1] = 1, + [REG_L] = 11, + [REG_15] = 15, }; -#define BPF_SIZE_MAX 4096 /* Max size for program */ +static inline u32 reg(u32 dst_reg, u32 src_reg) +{ + return reg2hex[dst_reg] << 4 | reg2hex[src_reg]; +} + +static inline u32 reg_high(u32 reg) +{ + return reg2hex[reg] << 4; +} + +static inline void reg_set_seen(struct bpf_jit *jit, u32 b1) +{ + u32 r1 = reg2hex[b1]; + + if (!jit->seen_reg[r1] && r1 >= 6 && r1 <= 15) + jit->seen_reg[r1] = 1; +} + +#define REG_SET_SEEN(b1) \ +({ \ + reg_set_seen(jit, b1); \ +}) + +#define REG_SEEN(b1) jit->seen_reg[reg2hex[(b1)]] + +/* + * EMIT macros for code generation + */ + +#define _EMIT2(op) \ +({ \ + if (jit->prg_buf) \ + *(u16 *) (jit->prg_buf + jit->prg) = op; \ + jit->prg += 2; \ +}) -#define SEEN_DATAREF 1 /* might call external helpers */ -#define SEEN_XREG 2 /* ebx is used */ -#define SEEN_MEM 4 /* use mem[] for temporary storage */ -#define SEEN_RET0 8 /* pc_ret0 points to a valid return 0 */ -#define SEEN_LITERAL 16 /* code uses literals */ -#define SEEN_LOAD_WORD 32 /* code uses sk_load_word */ -#define SEEN_LOAD_HALF 64 /* code uses sk_load_half */ -#define SEEN_LOAD_BYTE 128 /* code uses sk_load_byte */ -#define SEEN_LOAD_BMSH 256 /* code uses sk_load_byte_msh */ -#define SEEN_LOAD_IWORD 512 /* code uses sk_load_word_ind */ -#define SEEN_LOAD_IHALF 1024 /* code uses sk_load_half_ind */ -#define SEEN_LOAD_IBYTE 2048 /* code uses sk_load_byte_ind */ - -#define EMIT2(op) \ -({ \ - if (jit->prg + 2 <= jit->mid) \ - *(u16 *) jit->prg = op; \ - jit->prg += 2; \ +#define EMIT2(op, b1, b2) \ +({ \ + _EMIT2(op | reg(b1, b2)); \ + REG_SET_SEEN(b1); \ + REG_SET_SEEN(b2); \ }) -#define EMIT4(op) \ -({ \ - if (jit->prg + 4 <= jit->mid) \ - *(u32 *) jit->prg = op; \ - jit->prg += 4; \ +#define _EMIT4(op) \ +({ \ + if (jit->prg_buf) \ + *(u32 *) (jit->prg_buf + jit->prg) = op; \ + jit->prg += 4; \ }) -#define EMIT4_DISP(op, disp) \ -({ \ - unsigned int __disp = (disp) & 0xfff; \ - EMIT4(op | __disp); \ +#define EMIT4(op, b1, b2) \ +({ \ + _EMIT4(op | reg(b1, b2)); \ + REG_SET_SEEN(b1); \ + REG_SET_SEEN(b2); \ }) -#define EMIT4_IMM(op, imm) \ -({ \ - unsigned int __imm = (imm) & 0xffff; \ - EMIT4(op | __imm); \ +#define EMIT4_RRF(op, b1, b2, b3) \ +({ \ + _EMIT4(op | reg_high(b3) << 8 | reg(b1, b2)); \ + REG_SET_SEEN(b1); \ + REG_SET_SEEN(b2); \ + REG_SET_SEEN(b3); \ }) -#define EMIT4_PCREL(op, pcrel) \ -({ \ - long __pcrel = ((pcrel) >> 1) & 0xffff; \ - EMIT4(op | __pcrel); \ +#define _EMIT4_DISP(op, disp) \ +({ \ + unsigned int __disp = (disp) & 0xfff; \ + _EMIT4(op | __disp); \ }) -#define EMIT6(op1, op2) \ -({ \ - if (jit->prg + 6 <= jit->mid) { \ - *(u32 *) jit->prg = op1; \ - *(u16 *) (jit->prg + 4) = op2; \ - } \ - jit->prg += 6; \ +#define EMIT4_DISP(op, b1, b2, disp) \ +({ \ + _EMIT4_DISP(op | reg_high(b1) << 16 | \ + reg_high(b2) << 8, disp); \ + REG_SET_SEEN(b1); \ + REG_SET_SEEN(b2); \ }) -#define EMIT6_DISP(op1, op2, disp) \ -({ \ - unsigned int __disp = (disp) & 0xfff; \ - EMIT6(op1 | __disp, op2); \ +#define EMIT4_IMM(op, b1, imm) \ +({ \ + unsigned int __imm = (imm) & 0xffff; \ + _EMIT4(op | reg_high(b1) << 16 | __imm); \ + REG_SET_SEEN(b1); \ }) -#define EMIT6_IMM(op, imm) \ -({ \ - unsigned int __imm = (imm); \ - EMIT6(op | (__imm >> 16), __imm & 0xffff); \ +#define EMIT4_PCREL(op, pcrel) \ +({ \ + long __pcrel = ((pcrel) >> 1) & 0xffff; \ + _EMIT4(op | __pcrel); \ }) -#define EMIT_CONST(val) \ -({ \ - unsigned int ret; \ - ret = (unsigned int) (jit->lit - jit->base_ip); \ - jit->seen |= SEEN_LITERAL; \ - if (jit->lit + 4 <= jit->end) \ - *(u32 *) jit->lit = val; \ - jit->lit += 4; \ - ret; \ +#define _EMIT6(op1, op2) \ +({ \ + if (jit->prg_buf) { \ + *(u32 *) (jit->prg_buf + jit->prg) = op1; \ + *(u16 *) (jit->prg_buf + jit->prg + 4) = op2; \ + } \ + jit->prg += 6; \ }) -#define EMIT_FN_CONST(bit, fn) \ -({ \ - unsigned int ret; \ - ret = (unsigned int) (jit->lit - jit->base_ip); \ - if (jit->seen & bit) { \ - jit->seen |= SEEN_LITERAL; \ - if (jit->lit + 8 <= jit->end) \ - *(void **) jit->lit = fn; \ - jit->lit += 8; \ - } \ - ret; \ +#define _EMIT6_DISP(op1, op2, disp) \ +({ \ + unsigned int __disp = (disp) & 0xfff; \ + _EMIT6(op1 | __disp, op2); \ }) -static void bpf_jit_fill_hole(void *area, unsigned int size) +#define EMIT6_DISP(op1, op2, b1, b2, b3, disp) \ +({ \ + _EMIT6_DISP(op1 | reg(b1, b2) << 16 | \ + reg_high(b3) << 8, op2, disp); \ + REG_SET_SEEN(b1); \ + REG_SET_SEEN(b2); \ + REG_SET_SEEN(b3); \ +}) + +#define _EMIT6_DISP_LH(op1, op2, disp) \ +({ \ + unsigned int __disp_h = ((u32)disp) & 0xff000; \ + unsigned int __disp_l = ((u32)disp) & 0x00fff; \ + _EMIT6(op1 | __disp_l, op2 | __disp_h >> 4); \ +}) + +#define EMIT6_DISP_LH(op1, op2, b1, b2, b3, disp) \ +({ \ + _EMIT6_DISP_LH(op1 | reg(b1, b2) << 16 | \ + reg_high(b3) << 8, op2, disp); \ + REG_SET_SEEN(b1); \ + REG_SET_SEEN(b2); \ + REG_SET_SEEN(b3); \ +}) + +#define EMIT6_PCREL(op1, op2, b1, b2, i, off, mask) \ +({ \ + /* Branch instruction needs 6 bytes */ \ + int rel = (addrs[i + off + 1] - (addrs[i + 1] - 6)) / 2;\ + _EMIT6(op1 | reg(b1, b2) << 16 | rel, op2 | mask); \ + REG_SET_SEEN(b1); \ + REG_SET_SEEN(b2); \ +}) + +#define _EMIT6_IMM(op, imm) \ +({ \ + unsigned int __imm = (imm); \ + _EMIT6(op | (__imm >> 16), __imm & 0xffff); \ +}) + +#define EMIT6_IMM(op, b1, imm) \ +({ \ + _EMIT6_IMM(op | reg_high(b1) << 16, imm); \ + REG_SET_SEEN(b1); \ +}) + +#define EMIT_CONST_U32(val) \ +({ \ + unsigned int ret; \ + ret = jit->lit - jit->base_ip; \ + jit->seen |= SEEN_LITERAL; \ + if (jit->prg_buf) \ + *(u32 *) (jit->prg_buf + jit->lit) = (u32) val; \ + jit->lit += 4; \ + ret; \ +}) + +#define EMIT_CONST_U64(val) \ +({ \ + unsigned int ret; \ + ret = jit->lit - jit->base_ip; \ + jit->seen |= SEEN_LITERAL; \ + if (jit->prg_buf) \ + *(u64 *) (jit->prg_buf + jit->lit) = (u64) val; \ + jit->lit += 8; \ + ret; \ +}) + +#define EMIT_ZERO(b1) \ +({ \ + /* llgfr %dst,%dst (zero extend to 64 bit) */ \ + EMIT4(0xb9160000, b1, b1); \ + REG_SET_SEEN(b1); \ +}) + +/* + * Fill whole space with illegal instructions + */ +static void jit_fill_hole(void *area, unsigned int size) { - /* Fill whole space with illegal instructions */ memset(area, 0, size); } -static void bpf_jit_prologue(struct bpf_jit *jit) +/* + * Save registers from "rs" (register start) to "re" (register end) on stack + */ +static void save_regs(struct bpf_jit *jit, u32 rs, u32 re) +{ + u32 off = 72 + (rs - 6) * 8; + + if (rs == re) + /* stg %rs,off(%r15) */ + _EMIT6(0xe300f000 | rs << 20 | off, 0x0024); + else + /* stmg %rs,%re,off(%r15) */ + _EMIT6_DISP(0xeb00f000 | rs << 20 | re << 16, 0x0024, off); +} + +/* + * Restore registers from "rs" (register start) to "re" (register end) on stack + */ +static void restore_regs(struct bpf_jit *jit, u32 rs, u32 re) { - /* Save registers and create stack frame if necessary */ - if (jit->seen & SEEN_DATAREF) { - /* stmg %r8,%r15,88(%r15) */ - EMIT6(0xeb8ff058, 0x0024); - /* lgr %r14,%r15 */ - EMIT4(0xb90400ef); - /* aghi %r15,<offset> */ - EMIT4_IMM(0xa7fb0000, (jit->seen & SEEN_MEM) ? -112 : -80); - /* stg %r14,152(%r15) */ - EMIT6(0xe3e0f098, 0x0024); - } else if ((jit->seen & SEEN_XREG) && (jit->seen & SEEN_LITERAL)) - /* stmg %r12,%r13,120(%r15) */ - EMIT6(0xebcdf078, 0x0024); - else if (jit->seen & SEEN_XREG) - /* stg %r12,120(%r15) */ - EMIT6(0xe3c0f078, 0x0024); - else if (jit->seen & SEEN_LITERAL) - /* stg %r13,128(%r15) */ - EMIT6(0xe3d0f080, 0x0024); + u32 off = 72 + (rs - 6) * 8; + + if (jit->seen & SEEN_STACK) + off += STK_OFF; + + if (rs == re) + /* lg %rs,off(%r15) */ + _EMIT6(0xe300f000 | rs << 20 | off, 0x0004); + else + /* lmg %rs,%re,off(%r15) */ + _EMIT6_DISP(0xeb00f000 | rs << 20 | re << 16, 0x0004, off); +} +/* + * Return first seen register (from start) + */ +static int get_start(struct bpf_jit *jit, int start) +{ + int i; + + for (i = start; i <= 15; i++) { + if (jit->seen_reg[i]) + return i; + } + return 0; +} + +/* + * Return last seen register (from start) (gap >= 2) + */ +static int get_end(struct bpf_jit *jit, int start) +{ + int i; + + for (i = start; i < 15; i++) { + if (!jit->seen_reg[i] && !jit->seen_reg[i + 1]) + return i - 1; + } + return jit->seen_reg[15] ? 15 : 14; +} + +#define REGS_SAVE 1 +#define REGS_RESTORE 0 +/* + * Save and restore clobbered registers (6-15) on stack. + * We save/restore registers in chunks with gap >= 2 registers. + */ +static void save_restore_regs(struct bpf_jit *jit, int op) +{ + + int re = 6, rs; + + do { + rs = get_start(jit, re); + if (!rs) + break; + re = get_end(jit, rs + 1); + if (op == REGS_SAVE) + save_regs(jit, rs, re); + else + restore_regs(jit, rs, re); + re++; + } while (re <= 15); +} + +/* + * Emit function prologue + * + * Save registers and create stack frame if necessary. + * See stack frame layout desription in "bpf_jit.h"! + */ +static void bpf_jit_prologue(struct bpf_jit *jit) +{ + /* Save registers */ + save_restore_regs(jit, REGS_SAVE); /* Setup literal pool */ if (jit->seen & SEEN_LITERAL) { /* basr %r13,0 */ - EMIT2(0x0dd0); + EMIT2(0x0d00, REG_L, REG_0); jit->base_ip = jit->prg; } - jit->off_load_word = EMIT_FN_CONST(SEEN_LOAD_WORD, sk_load_word); - jit->off_load_half = EMIT_FN_CONST(SEEN_LOAD_HALF, sk_load_half); - jit->off_load_byte = EMIT_FN_CONST(SEEN_LOAD_BYTE, sk_load_byte); - jit->off_load_bmsh = EMIT_FN_CONST(SEEN_LOAD_BMSH, sk_load_byte_msh); - jit->off_load_iword = EMIT_FN_CONST(SEEN_LOAD_IWORD, sk_load_word_ind); - jit->off_load_ihalf = EMIT_FN_CONST(SEEN_LOAD_IHALF, sk_load_half_ind); - jit->off_load_ibyte = EMIT_FN_CONST(SEEN_LOAD_IBYTE, sk_load_byte_ind); - - /* Filter needs to access skb data */ - if (jit->seen & SEEN_DATAREF) { - /* l %r11,<len>(%r2) */ - EMIT4_DISP(0x58b02000, offsetof(struct sk_buff, len)); - /* s %r11,<data_len>(%r2) */ - EMIT4_DISP(0x5bb02000, offsetof(struct sk_buff, data_len)); - /* lg %r10,<data>(%r2) */ - EMIT6_DISP(0xe3a02000, 0x0004, - offsetof(struct sk_buff, data)); + /* Setup stack and backchain */ + if (jit->seen & SEEN_STACK) { + /* lgr %bfp,%r15 (BPF frame pointer) */ + EMIT4(0xb9040000, BPF_REG_FP, REG_15); + /* aghi %r15,-STK_OFF */ + EMIT4_IMM(0xa70b0000, REG_15, -STK_OFF); + if (jit->seen & SEEN_FUNC) + /* stg %bfp,152(%r15) (backchain) */ + EMIT6_DISP_LH(0xe3000000, 0x0024, BPF_REG_FP, REG_0, + REG_15, 152); + } + /* + * For SKB access %b1 contains the SKB pointer. For "bpf_jit.S" + * we store the SKB header length on the stack and the SKB data + * pointer in REG_SKB_DATA. + */ + if (jit->seen & SEEN_SKB) { + /* Header length: llgf %w1,<len>(%b1) */ + EMIT6_DISP_LH(0xe3000000, 0x0016, REG_W1, REG_0, BPF_REG_1, + offsetof(struct sk_buff, len)); + /* s %w1,<data_len>(%b1) */ + EMIT4_DISP(0x5b000000, REG_W1, BPF_REG_1, + offsetof(struct sk_buff, data_len)); + /* stg %w1,ST_OFF_HLEN(%r0,%r15) */ + EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W1, REG_0, REG_15, + STK_OFF_HLEN); + /* lg %skb_data,data_off(%b1) */ + EMIT6_DISP_LH(0xe3000000, 0x0004, REG_SKB_DATA, REG_0, + BPF_REG_1, offsetof(struct sk_buff, data)); } + /* BPF compatibility: clear A (%b7) and X (%b8) registers */ + if (REG_SEEN(BPF_REG_7)) + /* lghi %b7,0 */ + EMIT4_IMM(0xa7090000, BPF_REG_7, 0); + if (REG_SEEN(BPF_REG_8)) + /* lghi %b8,0 */ + EMIT4_IMM(0xa7090000, BPF_REG_8, 0); } +/* + * Function epilogue + */ static void bpf_jit_epilogue(struct bpf_jit *jit) { /* Return 0 */ if (jit->seen & SEEN_RET0) { jit->ret0_ip = jit->prg; - /* lghi %r2,0 */ - EMIT4(0xa7290000); + /* lghi %b0,0 */ + EMIT4_IMM(0xa7090000, BPF_REG_0, 0); } jit->exit_ip = jit->prg; + /* Load exit code: lgr %r2,%b0 */ + EMIT4(0xb9040000, REG_2, BPF_REG_0); /* Restore registers */ - if (jit->seen & SEEN_DATAREF) - /* lmg %r8,%r15,<offset>(%r15) */ - EMIT6_DISP(0xeb8ff000, 0x0004, - (jit->seen & SEEN_MEM) ? 200 : 168); - else if ((jit->seen & SEEN_XREG) && (jit->seen & SEEN_LITERAL)) - /* lmg %r12,%r13,120(%r15) */ - EMIT6(0xebcdf078, 0x0004); - else if (jit->seen & SEEN_XREG) - /* lg %r12,120(%r15) */ - EMIT6(0xe3c0f078, 0x0004); - else if (jit->seen & SEEN_LITERAL) - /* lg %r13,128(%r15) */ - EMIT6(0xe3d0f080, 0x0004); + save_restore_regs(jit, REGS_RESTORE); /* br %r14 */ - EMIT2(0x07fe); + _EMIT2(0x07fe); } /* - * make sure we dont leak kernel information to user + * Compile one eBPF instruction into s390x code */ -static void bpf_jit_noleaks(struct bpf_jit *jit, struct sock_filter *filter) +static int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, int i) { - /* Clear temporary memory if (seen & SEEN_MEM) */ - if (jit->seen & SEEN_MEM) - /* xc 0(64,%r15),0(%r15) */ - EMIT6(0xd73ff000, 0xf000); - /* Clear X if (seen & SEEN_XREG) */ - if (jit->seen & SEEN_XREG) - /* lhi %r12,0 */ - EMIT4(0xa7c80000); - /* Clear A if the first register does not set it. */ - switch (filter[0].code) { - case BPF_LD | BPF_W | BPF_ABS: - case BPF_LD | BPF_H | BPF_ABS: - case BPF_LD | BPF_B | BPF_ABS: - case BPF_LD | BPF_W | BPF_LEN: - case BPF_LD | BPF_W | BPF_IND: - case BPF_LD | BPF_H | BPF_IND: - case BPF_LD | BPF_B | BPF_IND: - case BPF_LD | BPF_IMM: - case BPF_LD | BPF_MEM: - case BPF_MISC | BPF_TXA: - case BPF_RET | BPF_K: - /* first instruction sets A register */ - break; - default: /* A = 0 */ - /* lhi %r5,0 */ - EMIT4(0xa7580000); - } -} + struct bpf_insn *insn = &fp->insnsi[i]; + int jmp_off, last, insn_count = 1; + unsigned int func_addr, mask; + u32 dst_reg = insn->dst_reg; + u32 src_reg = insn->src_reg; + u32 *addrs = jit->addrs; + s32 imm = insn->imm; + s16 off = insn->off; -static int bpf_jit_insn(struct bpf_jit *jit, struct sock_filter *filter, - unsigned int *addrs, int i, int last) -{ - unsigned int K; - int offset; - unsigned int mask; - u16 code; - - K = filter->k; - code = bpf_anc_helper(filter); - - switch (code) { - case BPF_ALU | BPF_ADD | BPF_X: /* A += X */ - jit->seen |= SEEN_XREG; - /* ar %r5,%r12 */ - EMIT2(0x1a5c); - break; - case BPF_ALU | BPF_ADD | BPF_K: /* A += K */ - if (!K) + switch (insn->code) { + /* + * BPF_MOV + */ + case BPF_ALU | BPF_MOV | BPF_X: /* dst = (u32) src */ + /* llgfr %dst,%src */ + EMIT4(0xb9160000, dst_reg, src_reg); + break; + case BPF_ALU64 | BPF_MOV | BPF_X: /* dst = src */ + /* lgr %dst,%src */ + EMIT4(0xb9040000, dst_reg, src_reg); + break; + case BPF_ALU | BPF_MOV | BPF_K: /* dst = (u32) imm */ + /* llilf %dst,imm */ + EMIT6_IMM(0xc00f0000, dst_reg, imm); + break; + case BPF_ALU64 | BPF_MOV | BPF_K: /* dst = imm */ + /* lgfi %dst,imm */ + EMIT6_IMM(0xc0010000, dst_reg, imm); + break; + /* + * BPF_LD 64 + */ + case BPF_LD | BPF_IMM | BPF_DW: /* dst = (u64) imm */ + { + /* 16 byte instruction that uses two 'struct bpf_insn' */ + u64 imm64; + + imm64 = (u64)(u32) insn[0].imm | ((u64)(u32) insn[1].imm) << 32; + /* lg %dst,<d(imm)>(%l) */ + EMIT6_DISP_LH(0xe3000000, 0x0004, dst_reg, REG_0, REG_L, + EMIT_CONST_U64(imm64)); + insn_count = 2; + break; + } + /* + * BPF_ADD + */ + case BPF_ALU | BPF_ADD | BPF_X: /* dst = (u32) dst + (u32) src */ + /* ar %dst,%src */ + EMIT2(0x1a00, dst_reg, src_reg); + EMIT_ZERO(dst_reg); + break; + case BPF_ALU64 | BPF_ADD | BPF_X: /* dst = dst + src */ + /* agr %dst,%src */ + EMIT4(0xb9080000, dst_reg, src_reg); + break; + case BPF_ALU | BPF_ADD | BPF_K: /* dst = (u32) dst + (u32) imm */ + if (!imm) break; - if (K <= 16383) - /* ahi %r5,<K> */ - EMIT4_IMM(0xa75a0000, K); - else if (test_facility(21)) - /* alfi %r5,<K> */ - EMIT6_IMM(0xc25b0000, K); - else - /* a %r5,<d(K)>(%r13) */ - EMIT4_DISP(0x5a50d000, EMIT_CONST(K)); + /* alfi %dst,imm */ + EMIT6_IMM(0xc20b0000, dst_reg, imm); + EMIT_ZERO(dst_reg); + break; + case BPF_ALU64 | BPF_ADD | BPF_K: /* dst = dst + imm */ + if (!imm) + break; + /* agfi %dst,imm */ + EMIT6_IMM(0xc2080000, dst_reg, imm); + break; + /* + * BPF_SUB + */ + case BPF_ALU | BPF_SUB | BPF_X: /* dst = (u32) dst - (u32) src */ + /* sr %dst,%src */ + EMIT2(0x1b00, dst_reg, src_reg); + EMIT_ZERO(dst_reg); break; - case BPF_ALU | BPF_SUB | BPF_X: /* A -= X */ - jit->seen |= SEEN_XREG; - /* sr %r5,%r12 */ - EMIT2(0x1b5c); + case BPF_ALU64 | BPF_SUB | BPF_X: /* dst = dst - src */ + /* sgr %dst,%src */ + EMIT4(0xb9090000, dst_reg, src_reg); break; - case BPF_ALU | BPF_SUB | BPF_K: /* A -= K */ - if (!K) + case BPF_ALU | BPF_SUB | BPF_K: /* dst = (u32) dst - (u32) imm */ + if (!imm) break; - if (K <= 16384) - /* ahi %r5,-K */ - EMIT4_IMM(0xa75a0000, -K); - else if (test_facility(21)) - /* alfi %r5,-K */ - EMIT6_IMM(0xc25b0000, -K); - else - /* s %r5,<d(K)>(%r13) */ - EMIT4_DISP(0x5b50d000, EMIT_CONST(K)); - break; - case BPF_ALU | BPF_MUL | BPF_X: /* A *= X */ - jit->seen |= SEEN_XREG; - /* msr %r5,%r12 */ - EMIT4(0xb252005c); - break; - case BPF_ALU | BPF_MUL | BPF_K: /* A *= K */ - if (K <= 16383) - /* mhi %r5,K */ - EMIT4_IMM(0xa75c0000, K); - else if (test_facility(34)) - /* msfi %r5,<K> */ - EMIT6_IMM(0xc2510000, K); - else - /* ms %r5,<d(K)>(%r13) */ - EMIT4_DISP(0x7150d000, EMIT_CONST(K)); + /* alfi %dst,-imm */ + EMIT6_IMM(0xc20b0000, dst_reg, -imm); + EMIT_ZERO(dst_reg); break; - case BPF_ALU | BPF_DIV | BPF_X: /* A /= X */ - jit->seen |= SEEN_XREG | SEEN_RET0; - /* ltr %r12,%r12 */ - EMIT2(0x12cc); - /* jz <ret0> */ - EMIT4_PCREL(0xa7840000, (jit->ret0_ip - jit->prg)); - /* lhi %r4,0 */ - EMIT4(0xa7480000); - /* dlr %r4,%r12 */ - EMIT4(0xb997004c); - break; - case BPF_ALU | BPF_DIV | BPF_K: /* A /= K */ - if (K == 1) + case BPF_ALU64 | BPF_SUB | BPF_K: /* dst = dst - imm */ + if (!imm) + break; + /* agfi %dst,-imm */ + EMIT6_IMM(0xc2080000, dst_reg, -imm); + break; + /* + * BPF_MUL + */ + case BPF_ALU | BPF_MUL | BPF_X: /* dst = (u32) dst * (u32) src */ + /* msr %dst,%src */ + EMIT4(0xb2520000, dst_reg, src_reg); + EMIT_ZERO(dst_reg); + break; + case BPF_ALU64 | BPF_MUL | BPF_X: /* dst = dst * src */ + /* msgr %dst,%src */ + EMIT4(0xb90c0000, dst_reg, src_reg); + break; + case BPF_ALU | BPF_MUL | BPF_K: /* dst = (u32) dst * (u32) imm */ + if (imm == 1) + break; + /* msfi %r5,imm */ + EMIT6_IMM(0xc2010000, dst_reg, imm); + EMIT_ZERO(dst_reg); + break; + case BPF_ALU64 | BPF_MUL | BPF_K: /* dst = dst * imm */ + if (imm == 1) break; - /* lhi %r4,0 */ - EMIT4(0xa7480000); - /* dl %r4,<d(K)>(%r13) */ - EMIT6_DISP(0xe340d000, 0x0097, EMIT_CONST(K)); - break; - case BPF_ALU | BPF_MOD | BPF_X: /* A %= X */ - jit->seen |= SEEN_XREG | SEEN_RET0; - /* ltr %r12,%r12 */ - EMIT2(0x12cc); + /* msgfi %dst,imm */ + EMIT6_IMM(0xc2000000, dst_reg, imm); + break; + /* + * BPF_DIV / BPF_MOD + */ + case BPF_ALU | BPF_DIV | BPF_X: /* dst = (u32) dst / (u32) src */ + case BPF_ALU | BPF_MOD | BPF_X: /* dst = (u32) dst % (u32) src */ + { + int rc_reg = BPF_OP(insn->code) == BPF_DIV ? REG_W1 : REG_W0; + + jit->seen |= SEEN_RET0; + /* ltr %src,%src (if src == 0 goto fail) */ + EMIT2(0x1200, src_reg, src_reg); + /* jz <ret0> */ + EMIT4_PCREL(0xa7840000, jit->ret0_ip - jit->prg); + /* lhi %w0,0 */ + EMIT4_IMM(0xa7080000, REG_W0, 0); + /* lr %w1,%dst */ + EMIT2(0x1800, REG_W1, dst_reg); + /* dlr %w0,%src */ + EMIT4(0xb9970000, REG_W0, src_reg); + /* llgfr %dst,%rc */ + EMIT4(0xb9160000, dst_reg, rc_reg); + break; + } + case BPF_ALU64 | BPF_DIV | BPF_X: /* dst = dst / (u32) src */ + case BPF_ALU64 | BPF_MOD | BPF_X: /* dst = dst % (u32) src */ + { + int rc_reg = BPF_OP(insn->code) == BPF_DIV ? REG_W1 : REG_W0; + + jit->seen |= SEEN_RET0; + /* ltgr %src,%src (if src == 0 goto fail) */ + EMIT4(0xb9020000, src_reg, src_reg); /* jz <ret0> */ - EMIT4_PCREL(0xa7840000, (jit->ret0_ip - jit->prg)); - /* lhi %r4,0 */ - EMIT4(0xa7480000); - /* dlr %r4,%r12 */ - EMIT4(0xb997004c); - /* lr %r5,%r4 */ - EMIT2(0x1854); - break; - case BPF_ALU | BPF_MOD | BPF_K: /* A %= K */ - if (K == 1) { - /* lhi %r5,0 */ - EMIT4(0xa7580000); + EMIT4_PCREL(0xa7840000, jit->ret0_ip - jit->prg); + /* lghi %w0,0 */ + EMIT4_IMM(0xa7090000, REG_W0, 0); + /* lgr %w1,%dst */ + EMIT4(0xb9040000, REG_W1, dst_reg); + /* llgfr %dst,%src (u32 cast) */ + EMIT4(0xb9160000, dst_reg, src_reg); + /* dlgr %w0,%dst */ + EMIT4(0xb9870000, REG_W0, dst_reg); + /* lgr %dst,%rc */ + EMIT4(0xb9040000, dst_reg, rc_reg); + break; + } + case BPF_ALU | BPF_DIV | BPF_K: /* dst = (u32) dst / (u32) imm */ + case BPF_ALU | BPF_MOD | BPF_K: /* dst = (u32) dst % (u32) imm */ + { + int rc_reg = BPF_OP(insn->code) == BPF_DIV ? REG_W1 : REG_W0; + + if (imm == 1) { + if (BPF_OP(insn->code) == BPF_MOD) + /* lhgi %dst,0 */ + EMIT4_IMM(0xa7090000, dst_reg, 0); break; } - /* lhi %r4,0 */ - EMIT4(0xa7480000); - /* dl %r4,<d(K)>(%r13) */ - EMIT6_DISP(0xe340d000, 0x0097, EMIT_CONST(K)); - /* lr %r5,%r4 */ - EMIT2(0x1854); - break; - case BPF_ALU | BPF_AND | BPF_X: /* A &= X */ - jit->seen |= SEEN_XREG; - /* nr %r5,%r12 */ - EMIT2(0x145c); - break; - case BPF_ALU | BPF_AND | BPF_K: /* A &= K */ - if (test_facility(21)) - /* nilf %r5,<K> */ - EMIT6_IMM(0xc05b0000, K); - else - /* n %r5,<d(K)>(%r13) */ - EMIT4_DISP(0x5450d000, EMIT_CONST(K)); - break; - case BPF_ALU | BPF_OR | BPF_X: /* A |= X */ - jit->seen |= SEEN_XREG; - /* or %r5,%r12 */ - EMIT2(0x165c); - break; - case BPF_ALU | BPF_OR | BPF_K: /* A |= K */ - if (test_facility(21)) - /* oilf %r5,<K> */ - EMIT6_IMM(0xc05d0000, K); - else - /* o %r5,<d(K)>(%r13) */ - EMIT4_DISP(0x5650d000, EMIT_CONST(K)); + /* lhi %w0,0 */ + EMIT4_IMM(0xa7080000, REG_W0, 0); + /* lr %w1,%dst */ + EMIT2(0x1800, REG_W1, dst_reg); + /* dl %w0,<d(imm)>(%l) */ + EMIT6_DISP_LH(0xe3000000, 0x0097, REG_W0, REG_0, REG_L, + EMIT_CONST_U32(imm)); + /* llgfr %dst,%rc */ + EMIT4(0xb9160000, dst_reg, rc_reg); + break; + } + case BPF_ALU64 | BPF_DIV | BPF_K: /* dst = dst / (u32) imm */ + case BPF_ALU64 | BPF_MOD | BPF_K: /* dst = dst % (u32) imm */ + { + int rc_reg = BPF_OP(insn->code) == BPF_DIV ? REG_W1 : REG_W0; + + if (imm == 1) { + if (BPF_OP(insn->code) == BPF_MOD) + /* lhgi %dst,0 */ + EMIT4_IMM(0xa7090000, dst_reg, 0); + break; + } + /* lghi %w0,0 */ + EMIT4_IMM(0xa7090000, REG_W0, 0); + /* lgr %w1,%dst */ + EMIT4(0xb9040000, REG_W1, dst_reg); + /* dlg %w0,<d(imm)>(%l) */ + EMIT6_DISP_LH(0xe3000000, 0x0087, REG_W0, REG_0, REG_L, + EMIT_CONST_U64((u32) imm)); + /* lgr %dst,%rc */ + EMIT4(0xb9040000, dst_reg, rc_reg); + break; + } + /* + * BPF_AND + */ + case BPF_ALU | BPF_AND | BPF_X: /* dst = (u32) dst & (u32) src */ + /* nr %dst,%src */ + EMIT2(0x1400, dst_reg, src_reg); + EMIT_ZERO(dst_reg); + break; + case BPF_ALU64 | BPF_AND | BPF_X: /* dst = dst & src */ + /* ngr %dst,%src */ + EMIT4(0xb9800000, dst_reg, src_reg); + break; + case BPF_ALU | BPF_AND | BPF_K: /* dst = (u32) dst & (u32) imm */ + /* nilf %dst,imm */ + EMIT6_IMM(0xc00b0000, dst_reg, imm); + EMIT_ZERO(dst_reg); break; - case BPF_ANC | SKF_AD_ALU_XOR_X: /* A ^= X; */ - case BPF_ALU | BPF_XOR | BPF_X: - jit->seen |= SEEN_XREG; - /* xr %r5,%r12 */ - EMIT2(0x175c); + case BPF_ALU64 | BPF_AND | BPF_K: /* dst = dst & imm */ + /* ng %dst,<d(imm)>(%l) */ + EMIT6_DISP_LH(0xe3000000, 0x0080, dst_reg, REG_0, REG_L, + EMIT_CONST_U64(imm)); break; - case BPF_ALU | BPF_XOR | BPF_K: /* A ^= K */ - if (!K) + /* + * BPF_OR + */ + case BPF_ALU | BPF_OR | BPF_X: /* dst = (u32) dst | (u32) src */ + /* or %dst,%src */ + EMIT2(0x1600, dst_reg, src_reg); + EMIT_ZERO(dst_reg); + break; + case BPF_ALU64 | BPF_OR | BPF_X: /* dst = dst | src */ + /* ogr %dst,%src */ + EMIT4(0xb9810000, dst_reg, src_reg); + break; + case BPF_ALU | BPF_OR | BPF_K: /* dst = (u32) dst | (u32) imm */ + /* oilf %dst,imm */ + EMIT6_IMM(0xc00d0000, dst_reg, imm); + EMIT_ZERO(dst_reg); + break; + case BPF_ALU64 | BPF_OR | BPF_K: /* dst = dst | imm */ + /* og %dst,<d(imm)>(%l) */ + EMIT6_DISP_LH(0xe3000000, 0x0081, dst_reg, REG_0, REG_L, + EMIT_CONST_U64(imm)); + break; + /* + * BPF_XOR + */ + case BPF_ALU | BPF_XOR | BPF_X: /* dst = (u32) dst ^ (u32) src */ + /* xr %dst,%src */ + EMIT2(0x1700, dst_reg, src_reg); + EMIT_ZERO(dst_reg); + break; + case BPF_ALU64 | BPF_XOR | BPF_X: /* dst = dst ^ src */ + /* xgr %dst,%src */ + EMIT4(0xb9820000, dst_reg, src_reg); + break; + case BPF_ALU | BPF_XOR | BPF_K: /* dst = (u32) dst ^ (u32) imm */ + if (!imm) break; - /* x %r5,<d(K)>(%r13) */ - EMIT4_DISP(0x5750d000, EMIT_CONST(K)); + /* xilf %dst,imm */ + EMIT6_IMM(0xc0070000, dst_reg, imm); + EMIT_ZERO(dst_reg); + break; + case BPF_ALU64 | BPF_XOR | BPF_K: /* dst = dst ^ imm */ + /* xg %dst,<d(imm)>(%l) */ + EMIT6_DISP_LH(0xe3000000, 0x0082, dst_reg, REG_0, REG_L, + EMIT_CONST_U64(imm)); + break; + /* + * BPF_LSH + */ + case BPF_ALU | BPF_LSH | BPF_X: /* dst = (u32) dst << (u32) src */ + /* sll %dst,0(%src) */ + EMIT4_DISP(0x89000000, dst_reg, src_reg, 0); + EMIT_ZERO(dst_reg); break; - case BPF_ALU | BPF_LSH | BPF_X: /* A <<= X; */ - jit->seen |= SEEN_XREG; - /* sll %r5,0(%r12) */ - EMIT4(0x8950c000); + case BPF_ALU64 | BPF_LSH | BPF_X: /* dst = dst << src */ + /* sllg %dst,%dst,0(%src) */ + EMIT6_DISP_LH(0xeb000000, 0x000d, dst_reg, dst_reg, src_reg, 0); break; - case BPF_ALU | BPF_LSH | BPF_K: /* A <<= K */ - if (K == 0) + case BPF_ALU | BPF_LSH | BPF_K: /* dst = (u32) dst << (u32) imm */ + if (imm == 0) break; - /* sll %r5,K */ - EMIT4_DISP(0x89500000, K); + /* sll %dst,imm(%r0) */ + EMIT4_DISP(0x89000000, dst_reg, REG_0, imm); + EMIT_ZERO(dst_reg); break; - case BPF_ALU | BPF_RSH | BPF_X: /* A >>= X; */ - jit->seen |= SEEN_XREG; - /* srl %r5,0(%r12) */ - EMIT4(0x8850c000); + case BPF_ALU64 | BPF_LSH | BPF_K: /* dst = dst << imm */ + if (imm == 0) + break; + /* sllg %dst,%dst,imm(%r0) */ + EMIT6_DISP_LH(0xeb000000, 0x000d, dst_reg, dst_reg, REG_0, imm); + break; + /* + * BPF_RSH + */ + case BPF_ALU | BPF_RSH | BPF_X: /* dst = (u32) dst >> (u32) src */ + /* srl %dst,0(%src) */ + EMIT4_DISP(0x88000000, dst_reg, src_reg, 0); + EMIT_ZERO(dst_reg); break; - case BPF_ALU | BPF_RSH | BPF_K: /* A >>= K; */ - if (K == 0) + case BPF_ALU64 | BPF_RSH | BPF_X: /* dst = dst >> src */ + /* srlg %dst,%dst,0(%src) */ + EMIT6_DISP_LH(0xeb000000, 0x000c, dst_reg, dst_reg, src_reg, 0); + break; + case BPF_ALU | BPF_RSH | BPF_K: /* dst = (u32) dst >> (u32) imm */ + if (imm == 0) break; - /* srl %r5,K */ - EMIT4_DISP(0x88500000, K); - break; - case BPF_ALU | BPF_NEG: /* A = -A */ - /* lcr %r5,%r5 */ - EMIT2(0x1355); - break; - case BPF_JMP | BPF_JA: /* ip += K */ - offset = addrs[i + K] + jit->start - jit->prg; - EMIT4_PCREL(0xa7f40000, offset); - break; - case BPF_JMP | BPF_JGT | BPF_K: /* ip += (A > K) ? jt : jf */ - mask = 0x200000; /* jh */ - goto kbranch; - case BPF_JMP | BPF_JGE | BPF_K: /* ip += (A >= K) ? jt : jf */ - mask = 0xa00000; /* jhe */ - goto kbranch; - case BPF_JMP | BPF_JEQ | BPF_K: /* ip += (A == K) ? jt : jf */ - mask = 0x800000; /* je */ -kbranch: /* Emit compare if the branch targets are different */ - if (filter->jt != filter->jf) { - if (test_facility(21)) - /* clfi %r5,<K> */ - EMIT6_IMM(0xc25f0000, K); - else - /* cl %r5,<d(K)>(%r13) */ - EMIT4_DISP(0x5550d000, EMIT_CONST(K)); - } -branch: if (filter->jt == filter->jf) { - if (filter->jt == 0) - break; - /* j <jt> */ - offset = addrs[i + filter->jt] + jit->start - jit->prg; - EMIT4_PCREL(0xa7f40000, offset); + /* srl %dst,imm(%r0) */ + EMIT4_DISP(0x88000000, dst_reg, REG_0, imm); + EMIT_ZERO(dst_reg); + break; + case BPF_ALU64 | BPF_RSH | BPF_K: /* dst = dst >> imm */ + if (imm == 0) break; - } - if (filter->jt != 0) { - /* brc <mask>,<jt> */ - offset = addrs[i + filter->jt] + jit->start - jit->prg; - EMIT4_PCREL(0xa7040000 | mask, offset); - } - if (filter->jf != 0) { - /* brc <mask^15>,<jf> */ - offset = addrs[i + filter->jf] + jit->start - jit->prg; - EMIT4_PCREL(0xa7040000 | (mask ^ 0xf00000), offset); - } + /* srlg %dst,%dst,imm(%r0) */ + EMIT6_DISP_LH(0xeb000000, 0x000c, dst_reg, dst_reg, REG_0, imm); break; - case BPF_JMP | BPF_JSET | BPF_K: /* ip += (A & K) ? jt : jf */ - mask = 0x700000; /* jnz */ - /* Emit test if the branch targets are different */ - if (filter->jt != filter->jf) { - if (K > 65535) { - /* lr %r4,%r5 */ - EMIT2(0x1845); - /* n %r4,<d(K)>(%r13) */ - EMIT4_DISP(0x5440d000, EMIT_CONST(K)); - } else - /* tmll %r5,K */ - EMIT4_IMM(0xa7510000, K); - } - goto branch; - case BPF_JMP | BPF_JGT | BPF_X: /* ip += (A > X) ? jt : jf */ - mask = 0x200000; /* jh */ - goto xbranch; - case BPF_JMP | BPF_JGE | BPF_X: /* ip += (A >= X) ? jt : jf */ - mask = 0xa00000; /* jhe */ - goto xbranch; - case BPF_JMP | BPF_JEQ | BPF_X: /* ip += (A == X) ? jt : jf */ - mask = 0x800000; /* je */ -xbranch: /* Emit compare if the branch targets are different */ - if (filter->jt != filter->jf) { - jit->seen |= SEEN_XREG; - /* clr %r5,%r12 */ - EMIT2(0x155c); - } - goto branch; - case BPF_JMP | BPF_JSET | BPF_X: /* ip += (A & X) ? jt : jf */ - mask = 0x700000; /* jnz */ - /* Emit test if the branch targets are different */ - if (filter->jt != filter->jf) { - jit->seen |= SEEN_XREG; - /* lr %r4,%r5 */ - EMIT2(0x1845); - /* nr %r4,%r12 */ - EMIT2(0x144c); + /* + * BPF_ARSH + */ + case BPF_ALU64 | BPF_ARSH | BPF_X: /* ((s64) dst) >>= src */ + /* srag %dst,%dst,0(%src) */ + EMIT6_DISP_LH(0xeb000000, 0x000a, dst_reg, dst_reg, src_reg, 0); + break; + case BPF_ALU64 | BPF_ARSH | BPF_K: /* ((s64) dst) >>= imm */ + if (imm == 0) + break; + /* srag %dst,%dst,imm(%r0) */ + EMIT6_DISP_LH(0xeb000000, 0x000a, dst_reg, dst_reg, REG_0, imm); + break; + /* + * BPF_NEG + */ + case BPF_ALU | BPF_NEG: /* dst = (u32) -dst */ + /* lcr %dst,%dst */ + EMIT2(0x1300, dst_reg, dst_reg); + EMIT_ZERO(dst_reg); + break; + case BPF_ALU64 | BPF_NEG: /* dst = -dst */ + /* lcgr %dst,%dst */ + EMIT4(0xb9130000, dst_reg, dst_reg); + break; + /* + * BPF_FROM_BE/LE + */ + case BPF_ALU | BPF_END | BPF_FROM_BE: + /* s390 is big endian, therefore only clear high order bytes */ + switch (imm) { + case 16: /* dst = (u16) cpu_to_be16(dst) */ + /* llghr %dst,%dst */ + EMIT4(0xb9850000, dst_reg, dst_reg); + break; + case 32: /* dst = (u32) cpu_to_be32(dst) */ + /* llgfr %dst,%dst */ + EMIT4(0xb9160000, dst_reg, dst_reg); + break; + case 64: /* dst = (u64) cpu_to_be64(dst) */ + break; } - goto branch; - case BPF_LD | BPF_W | BPF_ABS: /* A = *(u32 *) (skb->data+K) */ - jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_WORD; - offset = jit->off_load_word; - goto load_abs; - case BPF_LD | BPF_H | BPF_ABS: /* A = *(u16 *) (skb->data+K) */ - jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_HALF; - offset = jit->off_load_half; - goto load_abs; - case BPF_LD | BPF_B | BPF_ABS: /* A = *(u8 *) (skb->data+K) */ - jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_BYTE; - offset = jit->off_load_byte; -load_abs: if ((int) K < 0) - goto out; -call_fn: /* lg %r1,<d(function)>(%r13) */ - EMIT6_DISP(0xe310d000, 0x0004, offset); - /* l %r3,<d(K)>(%r13) */ - EMIT4_DISP(0x5830d000, EMIT_CONST(K)); - /* basr %r8,%r1 */ - EMIT2(0x0d81); - /* jnz <ret0> */ - EMIT4_PCREL(0xa7740000, (jit->ret0_ip - jit->prg)); break; - case BPF_LD | BPF_W | BPF_IND: /* A = *(u32 *) (skb->data+K+X) */ - jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_IWORD; - offset = jit->off_load_iword; - goto call_fn; - case BPF_LD | BPF_H | BPF_IND: /* A = *(u16 *) (skb->data+K+X) */ - jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_IHALF; - offset = jit->off_load_ihalf; - goto call_fn; - case BPF_LD | BPF_B | BPF_IND: /* A = *(u8 *) (skb->data+K+X) */ - jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_IBYTE; - offset = jit->off_load_ibyte; - goto call_fn; - case BPF_LDX | BPF_B | BPF_MSH: - /* X = (*(u8 *)(skb->data+K) & 0xf) << 2 */ - jit->seen |= SEEN_RET0; - if ((int) K < 0) { - /* j <ret0> */ - EMIT4_PCREL(0xa7f40000, (jit->ret0_ip - jit->prg)); + case BPF_ALU | BPF_END | BPF_FROM_LE: + switch (imm) { + case 16: /* dst = (u16) cpu_to_le16(dst) */ + /* lrvr %dst,%dst */ + EMIT4(0xb91f0000, dst_reg, dst_reg); + /* srl %dst,16(%r0) */ + EMIT4_DISP(0x88000000, dst_reg, REG_0, 16); + /* llghr %dst,%dst */ + EMIT4(0xb9850000, dst_reg, dst_reg); + break; + case 32: /* dst = (u32) cpu_to_le32(dst) */ + /* lrvr %dst,%dst */ + EMIT4(0xb91f0000, dst_reg, dst_reg); + /* llgfr %dst,%dst */ + EMIT4(0xb9160000, dst_reg, dst_reg); + break; + case 64: /* dst = (u64) cpu_to_le64(dst) */ + /* lrvgr %dst,%dst */ + EMIT4(0xb90f0000, dst_reg, dst_reg); break; } - jit->seen |= SEEN_DATAREF | SEEN_LOAD_BMSH; - offset = jit->off_load_bmsh; - goto call_fn; - case BPF_LD | BPF_W | BPF_LEN: /* A = skb->len; */ - BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, len) != 4); - /* l %r5,<d(len)>(%r2) */ - EMIT4_DISP(0x58502000, offsetof(struct sk_buff, len)); - break; - case BPF_LDX | BPF_W | BPF_LEN: /* X = skb->len; */ - jit->seen |= SEEN_XREG; - /* l %r12,<d(len)>(%r2) */ - EMIT4_DISP(0x58c02000, offsetof(struct sk_buff, len)); - break; - case BPF_LD | BPF_IMM: /* A = K */ - if (K <= 16383) - /* lhi %r5,K */ - EMIT4_IMM(0xa7580000, K); - else if (test_facility(21)) - /* llilf %r5,<K> */ - EMIT6_IMM(0xc05f0000, K); - else - /* l %r5,<d(K)>(%r13) */ - EMIT4_DISP(0x5850d000, EMIT_CONST(K)); - break; - case BPF_LDX | BPF_IMM: /* X = K */ - jit->seen |= SEEN_XREG; - if (K <= 16383) - /* lhi %r12,<K> */ - EMIT4_IMM(0xa7c80000, K); - else if (test_facility(21)) - /* llilf %r12,<K> */ - EMIT6_IMM(0xc0cf0000, K); - else - /* l %r12,<d(K)>(%r13) */ - EMIT4_DISP(0x58c0d000, EMIT_CONST(K)); break; - case BPF_LD | BPF_MEM: /* A = mem[K] */ + /* + * BPF_ST(X) + */ + case BPF_STX | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = src_reg */ + /* stcy %src,off(%dst) */ + EMIT6_DISP_LH(0xe3000000, 0x0072, src_reg, dst_reg, REG_0, off); + jit->seen |= SEEN_MEM; + break; + case BPF_STX | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = src */ + /* sthy %src,off(%dst) */ + EMIT6_DISP_LH(0xe3000000, 0x0070, src_reg, dst_reg, REG_0, off); + jit->seen |= SEEN_MEM; + break; + case BPF_STX | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = src */ + /* sty %src,off(%dst) */ + EMIT6_DISP_LH(0xe3000000, 0x0050, src_reg, dst_reg, REG_0, off); + jit->seen |= SEEN_MEM; + break; + case BPF_STX | BPF_MEM | BPF_DW: /* (u64 *)(dst + off) = src */ + /* stg %src,off(%dst) */ + EMIT6_DISP_LH(0xe3000000, 0x0024, src_reg, dst_reg, REG_0, off); + jit->seen |= SEEN_MEM; + break; + case BPF_ST | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = imm */ + /* lhi %w0,imm */ + EMIT4_IMM(0xa7080000, REG_W0, (u8) imm); + /* stcy %w0,off(dst) */ + EMIT6_DISP_LH(0xe3000000, 0x0072, REG_W0, dst_reg, REG_0, off); + jit->seen |= SEEN_MEM; + break; + case BPF_ST | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = imm */ + /* lhi %w0,imm */ + EMIT4_IMM(0xa7080000, REG_W0, (u16) imm); + /* sthy %w0,off(dst) */ + EMIT6_DISP_LH(0xe3000000, 0x0070, REG_W0, dst_reg, REG_0, off); jit->seen |= SEEN_MEM; - /* l %r5,<K>(%r15) */ - EMIT4_DISP(0x5850f000, - (jit->seen & SEEN_DATAREF) ? 160 + K*4 : K*4); - break; - case BPF_LDX | BPF_MEM: /* X = mem[K] */ - jit->seen |= SEEN_XREG | SEEN_MEM; - /* l %r12,<K>(%r15) */ - EMIT4_DISP(0x58c0f000, - (jit->seen & SEEN_DATAREF) ? 160 + K*4 : K*4); - break; - case BPF_MISC | BPF_TAX: /* X = A */ - jit->seen |= SEEN_XREG; - /* lr %r12,%r5 */ - EMIT2(0x18c5); - break; - case BPF_MISC | BPF_TXA: /* A = X */ - jit->seen |= SEEN_XREG; - /* lr %r5,%r12 */ - EMIT2(0x185c); - break; - case BPF_RET | BPF_K: - if (K == 0) { - jit->seen |= SEEN_RET0; - if (last) - break; - /* j <ret0> */ - EMIT4_PCREL(0xa7f40000, jit->ret0_ip - jit->prg); - } else { - if (K <= 16383) - /* lghi %r2,K */ - EMIT4_IMM(0xa7290000, K); - else - /* llgf %r2,<K>(%r13) */ - EMIT6_DISP(0xe320d000, 0x0016, EMIT_CONST(K)); - /* j <exit> */ - if (last && !(jit->seen & SEEN_RET0)) - break; - EMIT4_PCREL(0xa7f40000, jit->exit_ip - jit->prg); - } break; - case BPF_RET | BPF_A: - /* llgfr %r2,%r5 */ - EMIT4(0xb9160025); + case BPF_ST | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = imm */ + /* llilf %w0,imm */ + EMIT6_IMM(0xc00f0000, REG_W0, (u32) imm); + /* sty %w0,off(%dst) */ + EMIT6_DISP_LH(0xe3000000, 0x0050, REG_W0, dst_reg, REG_0, off); + jit->seen |= SEEN_MEM; + break; + case BPF_ST | BPF_MEM | BPF_DW: /* *(u64 *)(dst + off) = imm */ + /* lgfi %w0,imm */ + EMIT6_IMM(0xc0010000, REG_W0, imm); + /* stg %w0,off(%dst) */ + EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W0, dst_reg, REG_0, off); + jit->seen |= SEEN_MEM; + break; + /* + * BPF_STX XADD (atomic_add) + */ + case BPF_STX | BPF_XADD | BPF_W: /* *(u32 *)(dst + off) += src */ + /* laal %w0,%src,off(%dst) */ + EMIT6_DISP_LH(0xeb000000, 0x00fa, REG_W0, src_reg, + dst_reg, off); + jit->seen |= SEEN_MEM; + break; + case BPF_STX | BPF_XADD | BPF_DW: /* *(u64 *)(dst + off) += src */ + /* laalg %w0,%src,off(%dst) */ + EMIT6_DISP_LH(0xeb000000, 0x00ea, REG_W0, src_reg, + dst_reg, off); + jit->seen |= SEEN_MEM; + break; + /* + * BPF_LDX + */ + case BPF_LDX | BPF_MEM | BPF_B: /* dst = *(u8 *)(ul) (src + off) */ + /* llgc %dst,0(off,%src) */ + EMIT6_DISP_LH(0xe3000000, 0x0090, dst_reg, src_reg, REG_0, off); + jit->seen |= SEEN_MEM; + break; + case BPF_LDX | BPF_MEM | BPF_H: /* dst = *(u16 *)(ul) (src + off) */ + /* llgh %dst,0(off,%src) */ + EMIT6_DISP_LH(0xe3000000, 0x0091, dst_reg, src_reg, REG_0, off); + jit->seen |= SEEN_MEM; + break; + case BPF_LDX | BPF_MEM | BPF_W: /* dst = *(u32 *)(ul) (src + off) */ + /* llgf %dst,off(%src) */ + jit->seen |= SEEN_MEM; + EMIT6_DISP_LH(0xe3000000, 0x0016, dst_reg, src_reg, REG_0, off); + break; + case BPF_LDX | BPF_MEM | BPF_DW: /* dst = *(u64 *)(ul) (src + off) */ + /* lg %dst,0(off,%src) */ + jit->seen |= SEEN_MEM; + EMIT6_DISP_LH(0xe3000000, 0x0004, dst_reg, src_reg, REG_0, off); + break; + /* + * BPF_JMP / CALL + */ + case BPF_JMP | BPF_CALL: + { + /* + * b0 = (__bpf_call_base + imm)(b1, b2, b3, b4, b5) + */ + const u64 func = (u64)__bpf_call_base + imm; + + REG_SET_SEEN(BPF_REG_5); + jit->seen |= SEEN_FUNC; + /* lg %w1,<d(imm)>(%l) */ + EMIT6_DISP(0xe3000000, 0x0004, REG_W1, REG_0, REG_L, + EMIT_CONST_U64(func)); + /* basr %r14,%w1 */ + EMIT2(0x0d00, REG_14, REG_W1); + /* lgr %b0,%r2: load return value into %b0 */ + EMIT4(0xb9040000, BPF_REG_0, REG_2); + break; + } + case BPF_JMP | BPF_EXIT: /* return b0 */ + last = (i == fp->len - 1) ? 1 : 0; + if (last && !(jit->seen & SEEN_RET0)) + break; /* j <exit> */ EMIT4_PCREL(0xa7f40000, jit->exit_ip - jit->prg); break; - case BPF_ST: /* mem[K] = A */ - jit->seen |= SEEN_MEM; - /* st %r5,<K>(%r15) */ - EMIT4_DISP(0x5050f000, - (jit->seen & SEEN_DATAREF) ? 160 + K*4 : K*4); - break; - case BPF_STX: /* mem[K] = X : mov %ebx,off8(%rbp) */ - jit->seen |= SEEN_XREG | SEEN_MEM; - /* st %r12,<K>(%r15) */ - EMIT4_DISP(0x50c0f000, - (jit->seen & SEEN_DATAREF) ? 160 + K*4 : K*4); - break; - case BPF_ANC | SKF_AD_PROTOCOL: /* A = ntohs(skb->protocol); */ - BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, protocol) != 2); - /* lhi %r5,0 */ - EMIT4(0xa7580000); - /* icm %r5,3,<d(protocol)>(%r2) */ - EMIT4_DISP(0xbf532000, offsetof(struct sk_buff, protocol)); - break; - case BPF_ANC | SKF_AD_IFINDEX: /* if (!skb->dev) return 0; - * A = skb->dev->ifindex */ - BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, ifindex) != 4); - jit->seen |= SEEN_RET0; - /* lg %r1,<d(dev)>(%r2) */ - EMIT6_DISP(0xe3102000, 0x0004, offsetof(struct sk_buff, dev)); - /* ltgr %r1,%r1 */ - EMIT4(0xb9020011); - /* jz <ret0> */ - EMIT4_PCREL(0xa7840000, jit->ret0_ip - jit->prg); - /* l %r5,<d(ifindex)>(%r1) */ - EMIT4_DISP(0x58501000, offsetof(struct net_device, ifindex)); - break; - case BPF_ANC | SKF_AD_MARK: /* A = skb->mark */ - BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, mark) != 4); - /* l %r5,<d(mark)>(%r2) */ - EMIT4_DISP(0x58502000, offsetof(struct sk_buff, mark)); - break; - case BPF_ANC | SKF_AD_QUEUE: /* A = skb->queue_mapping */ - BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, queue_mapping) != 2); - /* lhi %r5,0 */ - EMIT4(0xa7580000); - /* icm %r5,3,<d(queue_mapping)>(%r2) */ - EMIT4_DISP(0xbf532000, offsetof(struct sk_buff, queue_mapping)); - break; - case BPF_ANC | SKF_AD_HATYPE: /* if (!skb->dev) return 0; - * A = skb->dev->type */ - BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, type) != 2); - jit->seen |= SEEN_RET0; - /* lg %r1,<d(dev)>(%r2) */ - EMIT6_DISP(0xe3102000, 0x0004, offsetof(struct sk_buff, dev)); - /* ltgr %r1,%r1 */ - EMIT4(0xb9020011); - /* jz <ret0> */ - EMIT4_PCREL(0xa7840000, jit->ret0_ip - jit->prg); - /* lhi %r5,0 */ - EMIT4(0xa7580000); - /* icm %r5,3,<d(type)>(%r1) */ - EMIT4_DISP(0xbf531000, offsetof(struct net_device, type)); - break; - case BPF_ANC | SKF_AD_RXHASH: /* A = skb->hash */ - BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, hash) != 4); - /* l %r5,<d(hash)>(%r2) */ - EMIT4_DISP(0x58502000, offsetof(struct sk_buff, hash)); - break; - case BPF_ANC | SKF_AD_VLAN_TAG: - case BPF_ANC | SKF_AD_VLAN_TAG_PRESENT: - BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, vlan_tci) != 2); - BUILD_BUG_ON(VLAN_TAG_PRESENT != 0x1000); - /* lhi %r5,0 */ - EMIT4(0xa7580000); - /* icm %r5,3,<d(vlan_tci)>(%r2) */ - EMIT4_DISP(0xbf532000, offsetof(struct sk_buff, vlan_tci)); - if (code == (BPF_ANC | SKF_AD_VLAN_TAG)) { - /* nill %r5,0xefff */ - EMIT4_IMM(0xa5570000, ~VLAN_TAG_PRESENT); - } else { - /* nill %r5,0x1000 */ - EMIT4_IMM(0xa5570000, VLAN_TAG_PRESENT); - /* srl %r5,12 */ - EMIT4_DISP(0x88500000, 12); - } + /* + * Branch relative (number of skipped instructions) to offset on + * condition. + * + * Condition code to mask mapping: + * + * CC | Description | Mask + * ------------------------------ + * 0 | Operands equal | 8 + * 1 | First operand low | 4 + * 2 | First operand high | 2 + * 3 | Unused | 1 + * + * For s390x relative branches: ip = ip + off_bytes + * For BPF relative branches: insn = insn + off_insns + 1 + * + * For example for s390x with offset 0 we jump to the branch + * instruction itself (loop) and for BPF with offset 0 we + * branch to the instruction behind the branch. + */ + case BPF_JMP | BPF_JA: /* if (true) */ + mask = 0xf000; /* j */ + goto branch_oc; + case BPF_JMP | BPF_JSGT | BPF_K: /* ((s64) dst > (s64) imm) */ + mask = 0x2000; /* jh */ + goto branch_ks; + case BPF_JMP | BPF_JSGE | BPF_K: /* ((s64) dst >= (s64) imm) */ + mask = 0xa000; /* jhe */ + goto branch_ks; + case BPF_JMP | BPF_JGT | BPF_K: /* (dst_reg > imm) */ + mask = 0x2000; /* jh */ + goto branch_ku; + case BPF_JMP | BPF_JGE | BPF_K: /* (dst_reg >= imm) */ + mask = 0xa000; /* jhe */ + goto branch_ku; + case BPF_JMP | BPF_JNE | BPF_K: /* (dst_reg != imm) */ + mask = 0x7000; /* jne */ + goto branch_ku; + case BPF_JMP | BPF_JEQ | BPF_K: /* (dst_reg == imm) */ + mask = 0x8000; /* je */ + goto branch_ku; + case BPF_JMP | BPF_JSET | BPF_K: /* (dst_reg & imm) */ + mask = 0x7000; /* jnz */ + /* lgfi %w1,imm (load sign extend imm) */ + EMIT6_IMM(0xc0010000, REG_W1, imm); + /* ngr %w1,%dst */ + EMIT4(0xb9800000, REG_W1, dst_reg); + goto branch_oc; + + case BPF_JMP | BPF_JSGT | BPF_X: /* ((s64) dst > (s64) src) */ + mask = 0x2000; /* jh */ + goto branch_xs; + case BPF_JMP | BPF_JSGE | BPF_X: /* ((s64) dst >= (s64) src) */ + mask = 0xa000; /* jhe */ + goto branch_xs; + case BPF_JMP | BPF_JGT | BPF_X: /* (dst > src) */ + mask = 0x2000; /* jh */ + goto branch_xu; + case BPF_JMP | BPF_JGE | BPF_X: /* (dst >= src) */ + mask = 0xa000; /* jhe */ + goto branch_xu; + case BPF_JMP | BPF_JNE | BPF_X: /* (dst != src) */ + mask = 0x7000; /* jne */ + goto branch_xu; + case BPF_JMP | BPF_JEQ | BPF_X: /* (dst == src) */ + mask = 0x8000; /* je */ + goto branch_xu; + case BPF_JMP | BPF_JSET | BPF_X: /* (dst & src) */ + mask = 0x7000; /* jnz */ + /* ngrk %w1,%dst,%src */ + EMIT4_RRF(0xb9e40000, REG_W1, dst_reg, src_reg); + goto branch_oc; +branch_ks: + /* lgfi %w1,imm (load sign extend imm) */ + EMIT6_IMM(0xc0010000, REG_W1, imm); + /* cgrj %dst,%w1,mask,off */ + EMIT6_PCREL(0xec000000, 0x0064, dst_reg, REG_W1, i, off, mask); + break; +branch_ku: + /* lgfi %w1,imm (load sign extend imm) */ + EMIT6_IMM(0xc0010000, REG_W1, imm); + /* clgrj %dst,%w1,mask,off */ + EMIT6_PCREL(0xec000000, 0x0065, dst_reg, REG_W1, i, off, mask); + break; +branch_xs: + /* cgrj %dst,%src,mask,off */ + EMIT6_PCREL(0xec000000, 0x0064, dst_reg, src_reg, i, off, mask); + break; +branch_xu: + /* clgrj %dst,%src,mask,off */ + EMIT6_PCREL(0xec000000, 0x0065, dst_reg, src_reg, i, off, mask); + break; +branch_oc: + /* brc mask,jmp_off (branch instruction needs 4 bytes) */ + jmp_off = addrs[i + off + 1] - (addrs[i + 1] - 4); + EMIT4_PCREL(0xa7040000 | mask << 8, jmp_off); break; - case BPF_ANC | SKF_AD_PKTTYPE: - /* lhi %r5,0 */ - EMIT4(0xa7580000); - /* ic %r5,<d(pkt_type_offset)>(%r2) */ - EMIT4_DISP(0x43502000, PKT_TYPE_OFFSET()); - /* srl %r5,5 */ - EMIT4_DISP(0x88500000, 5); - break; - case BPF_ANC | SKF_AD_CPU: /* A = smp_processor_id() */ -#ifdef CONFIG_SMP - /* l %r5,<d(cpu_nr)> */ - EMIT4_DISP(0x58500000, offsetof(struct _lowcore, cpu_nr)); -#else - /* lhi %r5,0 */ - EMIT4(0xa7580000); -#endif + /* + * BPF_LD + */ + case BPF_LD | BPF_ABS | BPF_B: /* b0 = *(u8 *) (skb->data+imm) */ + case BPF_LD | BPF_IND | BPF_B: /* b0 = *(u8 *) (skb->data+imm+src) */ + if ((BPF_MODE(insn->code) == BPF_ABS) && (imm >= 0)) + func_addr = __pa(sk_load_byte_pos); + else + func_addr = __pa(sk_load_byte); + goto call_fn; + case BPF_LD | BPF_ABS | BPF_H: /* b0 = *(u16 *) (skb->data+imm) */ + case BPF_LD | BPF_IND | BPF_H: /* b0 = *(u16 *) (skb->data+imm+src) */ + if ((BPF_MODE(insn->code) == BPF_ABS) && (imm >= 0)) + func_addr = __pa(sk_load_half_pos); + else + func_addr = __pa(sk_load_half); + goto call_fn; + case BPF_LD | BPF_ABS | BPF_W: /* b0 = *(u32 *) (skb->data+imm) */ + case BPF_LD | BPF_IND | BPF_W: /* b0 = *(u32 *) (skb->data+imm+src) */ + if ((BPF_MODE(insn->code) == BPF_ABS) && (imm >= 0)) + func_addr = __pa(sk_load_word_pos); + else + func_addr = __pa(sk_load_word); + goto call_fn; +call_fn: + jit->seen |= SEEN_SKB | SEEN_RET0 | SEEN_FUNC; + REG_SET_SEEN(REG_14); /* Return address of possible func call */ + + /* + * Implicit input: + * BPF_REG_6 (R7) : skb pointer + * REG_SKB_DATA (R12): skb data pointer + * + * Calculated input: + * BPF_REG_2 (R3) : offset of byte(s) to fetch in skb + * BPF_REG_5 (R6) : return address + * + * Output: + * BPF_REG_0 (R14): data read from skb + * + * Scratch registers (BPF_REG_1-5) + */ + + /* Call function: llilf %w1,func_addr */ + EMIT6_IMM(0xc00f0000, REG_W1, func_addr); + + /* Offset: lgfi %b2,imm */ + EMIT6_IMM(0xc0010000, BPF_REG_2, imm); + if (BPF_MODE(insn->code) == BPF_IND) + /* agfr %b2,%src (%src is s32 here) */ + EMIT4(0xb9180000, BPF_REG_2, src_reg); + + /* basr %b5,%w1 (%b5 is call saved) */ + EMIT2(0x0d00, BPF_REG_5, REG_W1); + + /* + * Note: For fast access we jump directly after the + * jnz instruction from bpf_jit.S + */ + /* jnz <ret0> */ + EMIT4_PCREL(0xa7740000, jit->ret0_ip - jit->prg); break; default: /* too complex, give up */ - goto out; + pr_err("Unknown opcode %02x\n", insn->code); + return -1; + } + return insn_count; +} + +/* + * Compile eBPF program into s390x code + */ +static int bpf_jit_prog(struct bpf_jit *jit, struct bpf_prog *fp) +{ + int i, insn_count; + + jit->lit = jit->lit_start; + jit->prg = 0; + + bpf_jit_prologue(jit); + for (i = 0; i < fp->len; i += insn_count) { + insn_count = bpf_jit_insn(jit, fp, i); + if (insn_count < 0) + return -1; + jit->addrs[i + 1] = jit->prg; /* Next instruction address */ } - addrs[i] = jit->prg - jit->start; + bpf_jit_epilogue(jit); + + jit->lit_start = jit->prg; + jit->size = jit->lit; + jit->size_prg = jit->prg; return 0; -out: - return -1; } +/* + * Classic BPF function stub. BPF programs will be converted into + * eBPF and then bpf_int_jit_compile() will be called. + */ void bpf_jit_compile(struct bpf_prog *fp) { - struct bpf_binary_header *header = NULL; - unsigned long size, prg_len, lit_len; - struct bpf_jit jit, cjit; - unsigned int *addrs; - int pass, i; +} + +/* + * Compile eBPF program "fp" + */ +void bpf_int_jit_compile(struct bpf_prog *fp) +{ + struct bpf_binary_header *header; + struct bpf_jit jit; + int pass; if (!bpf_jit_enable) return; - addrs = kcalloc(fp->len, sizeof(*addrs), GFP_KERNEL); - if (addrs == NULL) + memset(&jit, 0, sizeof(jit)); + jit.addrs = kcalloc(fp->len + 1, sizeof(*jit.addrs), GFP_KERNEL); + if (jit.addrs == NULL) return; - memset(&jit, 0, sizeof(cjit)); - memset(&cjit, 0, sizeof(cjit)); - - for (pass = 0; pass < 10; pass++) { - jit.prg = jit.start; - jit.lit = jit.mid; - - bpf_jit_prologue(&jit); - bpf_jit_noleaks(&jit, fp->insns); - for (i = 0; i < fp->len; i++) { - if (bpf_jit_insn(&jit, fp->insns + i, addrs, i, - i == fp->len - 1)) - goto out; - } - bpf_jit_epilogue(&jit); - if (jit.start) { - WARN_ON(jit.prg > cjit.prg || jit.lit > cjit.lit); - if (memcmp(&jit, &cjit, sizeof(jit)) == 0) - break; - } else if (jit.prg == cjit.prg && jit.lit == cjit.lit) { - prg_len = jit.prg - jit.start; - lit_len = jit.lit - jit.mid; - size = prg_len + lit_len; - if (size >= BPF_SIZE_MAX) - goto out; - header = bpf_jit_binary_alloc(size, &jit.start, - 2, bpf_jit_fill_hole); - if (!header) - goto out; - jit.prg = jit.mid = jit.start + prg_len; - jit.lit = jit.end = jit.start + prg_len + lit_len; - jit.base_ip += (unsigned long) jit.start; - jit.exit_ip += (unsigned long) jit.start; - jit.ret0_ip += (unsigned long) jit.start; - } - cjit = jit; + /* + * Three initial passes: + * - 1/2: Determine clobbered registers + * - 3: Calculate program size and addrs arrray + */ + for (pass = 1; pass <= 3; pass++) { + if (bpf_jit_prog(&jit, fp)) + goto free_addrs; } + /* + * Final pass: Allocate and generate program + */ + if (jit.size >= BPF_SIZE_MAX) + goto free_addrs; + header = bpf_jit_binary_alloc(jit.size, &jit.prg_buf, 2, jit_fill_hole); + if (!header) + goto free_addrs; + if (bpf_jit_prog(&jit, fp)) + goto free_addrs; if (bpf_jit_enable > 1) { - bpf_jit_dump(fp->len, jit.end - jit.start, pass, jit.start); - if (jit.start) - print_fn_code(jit.start, jit.mid - jit.start); + bpf_jit_dump(fp->len, jit.size, pass, jit.prg_buf); + if (jit.prg_buf) + print_fn_code(jit.prg_buf, jit.size_prg); } - if (jit.start) { + if (jit.prg_buf) { set_memory_ro((unsigned long)header, header->pages); - fp->bpf_func = (void *) jit.start; + fp->bpf_func = (void *) jit.prg_buf; fp->jited = true; } -out: - kfree(addrs); +free_addrs: + kfree(jit.addrs); } +/* + * Free eBPF program + */ void bpf_jit_free(struct bpf_prog *fp) { unsigned long addr = (unsigned long)fp->bpf_func & PAGE_MASK; diff --git a/arch/s390/pci/pci.c b/arch/s390/pci/pci.c index 98336200c7b2..598f023cf8a6 100644 --- a/arch/s390/pci/pci.c +++ b/arch/s390/pci/pci.c @@ -190,6 +190,11 @@ int zpci_fmb_enable_device(struct zpci_dev *zdev) return -ENOMEM; WARN_ON((u64) zdev->fmb & 0xf); + /* reset software counters */ + atomic64_set(&zdev->allocated_pages, 0); + atomic64_set(&zdev->mapped_pages, 0); + atomic64_set(&zdev->unmapped_pages, 0); + args.fmb_addr = virt_to_phys(zdev->fmb); return mod_pci(zdev, ZPCI_MOD_FC_SET_MEASURE, 0, &args); } @@ -822,6 +827,7 @@ int zpci_create_device(struct zpci_dev *zdev) if (rc) goto out; + mutex_init(&zdev->lock); if (zdev->state == ZPCI_FN_STATE_CONFIGURED) { rc = zpci_enable_device(zdev); if (rc) diff --git a/arch/s390/pci/pci_debug.c b/arch/s390/pci/pci_debug.c index c22d4402ae45..4129b0a5fd78 100644 --- a/arch/s390/pci/pci_debug.c +++ b/arch/s390/pci/pci_debug.c @@ -31,12 +31,25 @@ static char *pci_perf_names[] = { "Refresh operations", "DMA read bytes", "DMA write bytes", - /* software counters */ +}; + +static char *pci_sw_names[] = { "Allocated pages", "Mapped pages", "Unmapped pages", }; +static void pci_sw_counter_show(struct seq_file *m) +{ + struct zpci_dev *zdev = m->private; + atomic64_t *counter = &zdev->allocated_pages; + int i; + + for (i = 0; i < ARRAY_SIZE(pci_sw_names); i++, counter++) + seq_printf(m, "%26s:\t%llu\n", pci_sw_names[i], + atomic64_read(counter)); +} + static int pci_perf_show(struct seq_file *m, void *v) { struct zpci_dev *zdev = m->private; @@ -45,7 +58,10 @@ static int pci_perf_show(struct seq_file *m, void *v) if (!zdev) return 0; + + mutex_lock(&zdev->lock); if (!zdev->fmb) { + mutex_unlock(&zdev->lock); seq_puts(m, "FMB statistics disabled\n"); return 0; } @@ -65,12 +81,9 @@ static int pci_perf_show(struct seq_file *m, void *v) for (i = 4; i < 6; i++) seq_printf(m, "%26s:\t%llu\n", pci_perf_names[i], *(stat + i)); - /* software counters */ - for (i = 6; i < ARRAY_SIZE(pci_perf_names); i++) - seq_printf(m, "%26s:\t%llu\n", - pci_perf_names[i], - atomic64_read((atomic64_t *) (stat + i))); + pci_sw_counter_show(m); + mutex_unlock(&zdev->lock); return 0; } @@ -88,19 +101,17 @@ static ssize_t pci_perf_seq_write(struct file *file, const char __user *ubuf, if (rc) return rc; + mutex_lock(&zdev->lock); switch (val) { case 0: rc = zpci_fmb_disable_device(zdev); - if (rc) - return rc; break; case 1: rc = zpci_fmb_enable_device(zdev); - if (rc) - return rc; break; } - return count; + mutex_unlock(&zdev->lock); + return rc ? rc : count; } static int pci_perf_seq_open(struct inode *inode, struct file *filp) diff --git a/arch/s390/pci/pci_dma.c b/arch/s390/pci/pci_dma.c index 4cbb29a4d615..6fd8d5836138 100644 --- a/arch/s390/pci/pci_dma.c +++ b/arch/s390/pci/pci_dma.c @@ -300,7 +300,7 @@ static dma_addr_t s390_dma_map_pages(struct device *dev, struct page *page, flags |= ZPCI_TABLE_PROTECTED; if (!dma_update_trans(zdev, pa, dma_addr, size, flags)) { - atomic64_add(nr_pages, &zdev->fmb->mapped_pages); + atomic64_add(nr_pages, &zdev->mapped_pages); return dma_addr + (offset & ~PAGE_MASK); } @@ -328,7 +328,7 @@ static void s390_dma_unmap_pages(struct device *dev, dma_addr_t dma_addr, zpci_err_hex(&dma_addr, sizeof(dma_addr)); } - atomic64_add(npages, &zdev->fmb->unmapped_pages); + atomic64_add(npages, &zdev->unmapped_pages); iommu_page_index = (dma_addr - zdev->start_dma) >> PAGE_SHIFT; dma_free_iommu(zdev, iommu_page_index, npages); } @@ -357,7 +357,7 @@ static void *s390_dma_alloc(struct device *dev, size_t size, return NULL; } - atomic64_add(size / PAGE_SIZE, &zdev->fmb->allocated_pages); + atomic64_add(size / PAGE_SIZE, &zdev->allocated_pages); if (dma_handle) *dma_handle = map; return (void *) pa; @@ -370,7 +370,7 @@ static void s390_dma_free(struct device *dev, size_t size, struct zpci_dev *zdev = get_zdev(to_pci_dev(dev)); size = PAGE_ALIGN(size); - atomic64_sub(size / PAGE_SIZE, &zdev->fmb->allocated_pages); + atomic64_sub(size / PAGE_SIZE, &zdev->allocated_pages); s390_dma_unmap_pages(dev, dma_handle, size, DMA_BIDIRECTIONAL, NULL); free_pages((unsigned long) pa, get_order(size)); } diff --git a/arch/sh/include/asm/mmu_context.h b/arch/sh/include/asm/mmu_context.h index b9d9489a5012..9f417feaf6e8 100644 --- a/arch/sh/include/asm/mmu_context.h +++ b/arch/sh/include/asm/mmu_context.h @@ -99,7 +99,7 @@ static inline int init_new_context(struct task_struct *tsk, { int i; - for (i = 0; i < num_online_cpus(); i++) + for_each_online_cpu(i) cpu_context(i, mm) = NO_CONTEXT; return 0; diff --git a/arch/sh/kernel/smp.c b/arch/sh/kernel/smp.c index fc5acfc93c92..de6be008fc01 100644 --- a/arch/sh/kernel/smp.c +++ b/arch/sh/kernel/smp.c @@ -363,7 +363,7 @@ void flush_tlb_mm(struct mm_struct *mm) smp_call_function(flush_tlb_mm_ipi, (void *)mm, 1); } else { int i; - for (i = 0; i < num_online_cpus(); i++) + for_each_online_cpu(i) if (smp_processor_id() != i) cpu_context(i, mm) = 0; } @@ -400,7 +400,7 @@ void flush_tlb_range(struct vm_area_struct *vma, smp_call_function(flush_tlb_range_ipi, (void *)&fd, 1); } else { int i; - for (i = 0; i < num_online_cpus(); i++) + for_each_online_cpu(i) if (smp_processor_id() != i) cpu_context(i, mm) = 0; } @@ -443,7 +443,7 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long page) smp_call_function(flush_tlb_page_ipi, (void *)&fd, 1); } else { int i; - for (i = 0; i < num_online_cpus(); i++) + for_each_online_cpu(i) if (smp_processor_id() != i) cpu_context(i, vma->vm_mm) = 0; } diff --git a/arch/sparc/kernel/time_32.c b/arch/sparc/kernel/time_32.c index 18147a5523d9..8caf45ee81d9 100644 --- a/arch/sparc/kernel/time_32.c +++ b/arch/sparc/kernel/time_32.c @@ -194,7 +194,7 @@ static __init int setup_timer_cs(void) static void percpu_ce_setup(enum clock_event_mode mode, struct clock_event_device *evt) { - int cpu = __first_cpu(evt->cpumask); + int cpu = cpumask_first(evt->cpumask); switch (mode) { case CLOCK_EVT_MODE_PERIODIC: @@ -214,7 +214,7 @@ static void percpu_ce_setup(enum clock_event_mode mode, static int percpu_ce_set_next_event(unsigned long delta, struct clock_event_device *evt) { - int cpu = __first_cpu(evt->cpumask); + int cpu = cpumask_first(evt->cpumask); unsigned int next = (unsigned int)delta; sparc_config.load_profile_irq(cpu, next); diff --git a/arch/tile/kernel/setup.c b/arch/tile/kernel/setup.c index 7833b2ccdfbc..6873f006f7d0 100644 --- a/arch/tile/kernel/setup.c +++ b/arch/tile/kernel/setup.c @@ -774,7 +774,7 @@ static void __init zone_sizes_init(void) * though, there'll be no lowmem, so we just alloc_bootmem * the memmap. There will be no percpu memory either. */ - if (i != 0 && cpu_isset(i, isolnodes)) { + if (i != 0 && cpumask_test_cpu(i, &isolnodes)) { node_memmap_pfn[i] = alloc_bootmem_pfn(0, memmap_size, 0); BUG_ON(node_percpu[i] != 0); diff --git a/arch/x86/Kconfig.debug b/arch/x86/Kconfig.debug index 20028da8ae18..72484a645f05 100644 --- a/arch/x86/Kconfig.debug +++ b/arch/x86/Kconfig.debug @@ -43,10 +43,6 @@ config EARLY_PRINTK with klogd/syslogd or the X server. You should normally N here, unless you want to debug such a crash. -config EARLY_PRINTK_INTEL_MID - bool "Early printk for Intel MID platform support" - depends on EARLY_PRINTK && X86_INTEL_MID - config EARLY_PRINTK_DBGP bool "Early printk via EHCI debug port" depends on EARLY_PRINTK && PCI diff --git a/arch/x86/include/asm/intel-mid.h b/arch/x86/include/asm/intel-mid.h index 705d35708a50..7c5af123bdbd 100644 --- a/arch/x86/include/asm/intel-mid.h +++ b/arch/x86/include/asm/intel-mid.h @@ -136,9 +136,6 @@ extern enum intel_mid_timer_options intel_mid_timer_options; #define SFI_MTMR_MAX_NUM 8 #define SFI_MRTC_MAX 8 -extern struct console early_hsu_console; -extern void hsu_early_console_init(const char *); - extern void intel_scu_devices_create(void); extern void intel_scu_devices_destroy(void); diff --git a/arch/x86/include/asm/serial.h b/arch/x86/include/asm/serial.h index 460b84f64556..8378b8c9109c 100644 --- a/arch/x86/include/asm/serial.h +++ b/arch/x86/include/asm/serial.h @@ -12,11 +12,11 @@ /* Standard COM flags (except for COM4, because of the 8514 problem) */ #ifdef CONFIG_SERIAL_DETECT_IRQ -# define STD_COMX_FLAGS (ASYNC_BOOT_AUTOCONF | ASYNC_SKIP_TEST | ASYNC_AUTO_IRQ) -# define STD_COM4_FLAGS (ASYNC_BOOT_AUTOCONF | 0 | ASYNC_AUTO_IRQ) +# define STD_COMX_FLAGS (UPF_BOOT_AUTOCONF | UPF_SKIP_TEST | UPF_AUTO_IRQ) +# define STD_COM4_FLAGS (UPF_BOOT_AUTOCONF | 0 | UPF_AUTO_IRQ) #else -# define STD_COMX_FLAGS (ASYNC_BOOT_AUTOCONF | ASYNC_SKIP_TEST | 0 ) -# define STD_COM4_FLAGS (ASYNC_BOOT_AUTOCONF | 0 | 0 ) +# define STD_COMX_FLAGS (UPF_BOOT_AUTOCONF | UPF_SKIP_TEST | 0 ) +# define STD_COM4_FLAGS (UPF_BOOT_AUTOCONF | 0 | 0 ) #endif #define SERIAL_PORT_DFNS \ diff --git a/arch/x86/include/uapi/asm/hyperv.h b/arch/x86/include/uapi/asm/hyperv.h index 90c458e66e13..ce6068dbcfbc 100644 --- a/arch/x86/include/uapi/asm/hyperv.h +++ b/arch/x86/include/uapi/asm/hyperv.h @@ -225,6 +225,8 @@ #define HV_STATUS_INVALID_HYPERCALL_CODE 2 #define HV_STATUS_INVALID_HYPERCALL_INPUT 3 #define HV_STATUS_INVALID_ALIGNMENT 4 +#define HV_STATUS_INSUFFICIENT_MEMORY 11 +#define HV_STATUS_INVALID_CONNECTION_ID 18 #define HV_STATUS_INSUFFICIENT_BUFFERS 19 typedef struct _HV_REFERENCE_TSC_PAGE { diff --git a/arch/x86/include/uapi/asm/msr-index.h b/arch/x86/include/uapi/asm/msr-index.h index 1a4eae695ca8..c469490db4a8 100644 --- a/arch/x86/include/uapi/asm/msr-index.h +++ b/arch/x86/include/uapi/asm/msr-index.h @@ -61,6 +61,9 @@ #define MSR_OFFCORE_RSP_1 0x000001a7 #define MSR_NHM_TURBO_RATIO_LIMIT 0x000001ad #define MSR_IVT_TURBO_RATIO_LIMIT 0x000001ae +#define MSR_TURBO_RATIO_LIMIT 0x000001ad +#define MSR_TURBO_RATIO_LIMIT1 0x000001ae +#define MSR_TURBO_RATIO_LIMIT2 0x000001af #define MSR_LBR_SELECT 0x000001c8 #define MSR_LBR_TOS 0x000001c9 @@ -165,6 +168,11 @@ #define MSR_PP1_ENERGY_STATUS 0x00000641 #define MSR_PP1_POLICY 0x00000642 +#define MSR_PKG_WEIGHTED_CORE_C0_RES 0x00000658 +#define MSR_PKG_ANY_CORE_C0_RES 0x00000659 +#define MSR_PKG_ANY_GFXE_C0_RES 0x0000065A +#define MSR_PKG_BOTH_CORE_GFXE_C0_RES 0x0000065B + #define MSR_CORE_C1_RES 0x00000660 #define MSR_CC6_DEMOTION_POLICY_CONFIG 0x00000668 diff --git a/arch/x86/kernel/apic/x2apic_cluster.c b/arch/x86/kernel/apic/x2apic_cluster.c index d9d0bd2faaf4..ab3219b3fbda 100644 --- a/arch/x86/kernel/apic/x2apic_cluster.c +++ b/arch/x86/kernel/apic/x2apic_cluster.c @@ -171,8 +171,8 @@ update_clusterinfo(struct notifier_block *nfb, unsigned long action, void *hcpu) for_each_online_cpu(cpu) { if (x2apic_cluster(this_cpu) != x2apic_cluster(cpu)) continue; - __cpu_clear(this_cpu, per_cpu(cpus_in_cluster, cpu)); - __cpu_clear(cpu, per_cpu(cpus_in_cluster, this_cpu)); + cpumask_clear_cpu(this_cpu, per_cpu(cpus_in_cluster, cpu)); + cpumask_clear_cpu(cpu, per_cpu(cpus_in_cluster, this_cpu)); } free_cpumask_var(per_cpu(cpus_in_cluster, this_cpu)); free_cpumask_var(per_cpu(ipi_mask, this_cpu)); diff --git a/arch/x86/kernel/early_printk.c b/arch/x86/kernel/early_printk.c index 49ff55ef9b26..89427d8d4fc5 100644 --- a/arch/x86/kernel/early_printk.c +++ b/arch/x86/kernel/early_printk.c @@ -375,12 +375,6 @@ static int __init setup_early_printk(char *buf) if (!strncmp(buf, "xen", 3)) early_console_register(&xenboot_console, keep); #endif -#ifdef CONFIG_EARLY_PRINTK_INTEL_MID - if (!strncmp(buf, "hsu", 3)) { - hsu_early_console_init(buf + 3); - early_console_register(&early_hsu_console, keep); - } -#endif #ifdef CONFIG_EARLY_PRINTK_EFI if (!strncmp(buf, "efi", 3)) early_console_register(&early_efi_console, keep); diff --git a/arch/x86/platform/intel-mid/Makefile b/arch/x86/platform/intel-mid/Makefile index 0a8ee703b9fa..0ce1b1913673 100644 --- a/arch/x86/platform/intel-mid/Makefile +++ b/arch/x86/platform/intel-mid/Makefile @@ -1,5 +1,4 @@ obj-$(CONFIG_X86_INTEL_MID) += intel-mid.o intel_mid_vrtc.o mfld.o mrfl.o -obj-$(CONFIG_EARLY_PRINTK_INTEL_MID) += early_printk_intel_mid.o # SFI specific code ifdef CONFIG_X86_INTEL_MID diff --git a/arch/x86/platform/intel-mid/early_printk_intel_mid.c b/arch/x86/platform/intel-mid/early_printk_intel_mid.c deleted file mode 100644 index 4e720829ab90..000000000000 --- a/arch/x86/platform/intel-mid/early_printk_intel_mid.c +++ /dev/null @@ -1,112 +0,0 @@ -/* - * early_printk_intel_mid.c - early consoles for Intel MID platforms - * - * Copyright (c) 2008-2010, Intel Corporation - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; version 2 - * of the License. - */ - -/* - * This file implements early console named hsu. - * hsu is based on a High Speed UART device which only exists in the Medfield - * platform - */ - -#include <linux/serial_reg.h> -#include <linux/serial_mfd.h> -#include <linux/console.h> -#include <linux/kernel.h> -#include <linux/delay.h> -#include <linux/io.h> - -#include <asm/fixmap.h> -#include <asm/pgtable.h> -#include <asm/intel-mid.h> - -/* - * Following is the early console based on Medfield HSU (High - * Speed UART) device. - */ -#define HSU_PORT_BASE 0xffa28080 - -static void __iomem *phsu; - -void hsu_early_console_init(const char *s) -{ - unsigned long paddr, port = 0; - u8 lcr; - - /* - * Select the early HSU console port if specified by user in the - * kernel command line. - */ - if (*s && !kstrtoul(s, 10, &port)) - port = clamp_val(port, 0, 2); - - paddr = HSU_PORT_BASE + port * 0x80; - phsu = (void __iomem *)set_fixmap_offset_nocache(FIX_EARLYCON_MEM_BASE, paddr); - - /* Disable FIFO */ - writeb(0x0, phsu + UART_FCR); - - /* Set to default 115200 bps, 8n1 */ - lcr = readb(phsu + UART_LCR); - writeb((0x80 | lcr), phsu + UART_LCR); - writeb(0x18, phsu + UART_DLL); - writeb(lcr, phsu + UART_LCR); - writel(0x3600, phsu + UART_MUL*4); - - writeb(0x8, phsu + UART_MCR); - writeb(0x7, phsu + UART_FCR); - writeb(0x3, phsu + UART_LCR); - - /* Clear IRQ status */ - readb(phsu + UART_LSR); - readb(phsu + UART_RX); - readb(phsu + UART_IIR); - readb(phsu + UART_MSR); - - /* Enable FIFO */ - writeb(0x7, phsu + UART_FCR); -} - -#define BOTH_EMPTY (UART_LSR_TEMT | UART_LSR_THRE) - -static void early_hsu_putc(char ch) -{ - unsigned int timeout = 10000; /* 10ms */ - u8 status; - - while (--timeout) { - status = readb(phsu + UART_LSR); - if (status & BOTH_EMPTY) - break; - udelay(1); - } - - /* Only write the char when there was no timeout */ - if (timeout) - writeb(ch, phsu + UART_TX); -} - -static void early_hsu_write(struct console *con, const char *str, unsigned n) -{ - int i; - - for (i = 0; i < n && *str; i++) { - if (*str == '\n') - early_hsu_putc('\r'); - early_hsu_putc(*str); - str++; - } -} - -struct console early_hsu_console = { - .name = "earlyhsu", - .write = early_hsu_write, - .flags = CON_PRINTBUFFER, - .index = -1, -}; |