From d6f39be098dacce6ca8b2362cf82fd0dcd0b8a6d Mon Sep 17 00:00:00 2001 From: Waldemar Brodkorb Date: Mon, 12 Oct 2015 22:34:38 +0200 Subject: cleanup .orig files from patch --- target/linux/patches/4.1.10/realtime.patch | 79343 ++++++--------------------- 1 file changed, 18015 insertions(+), 61328 deletions(-) (limited to 'target') diff --git a/target/linux/patches/4.1.10/realtime.patch b/target/linux/patches/4.1.10/realtime.patch index af7a748cb..fef472a8b 100644 --- a/target/linux/patches/4.1.10/realtime.patch +++ b/target/linux/patches/4.1.10/realtime.patch @@ -1,6 +1,6 @@ diff -Nur linux-4.1.10.orig/arch/alpha/mm/fault.c linux-4.1.10/arch/alpha/mm/fault.c --- linux-4.1.10.orig/arch/alpha/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/alpha/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/alpha/mm/fault.c 2015-10-12 22:33:32.144685475 +0200 @@ -23,8 +23,7 @@ #include #include @@ -22,7 +22,7 @@ diff -Nur linux-4.1.10.orig/arch/alpha/mm/fault.c linux-4.1.10/arch/alpha/mm/fau #ifdef CONFIG_ALPHA_LARGE_VMALLOC diff -Nur linux-4.1.10.orig/arch/arc/include/asm/futex.h linux-4.1.10/arch/arc/include/asm/futex.h --- linux-4.1.10.orig/arch/arc/include/asm/futex.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/arc/include/asm/futex.h 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/arc/include/asm/futex.h 2015-10-12 22:33:32.144685475 +0200 @@ -53,7 +53,7 @@ if (!access_ok(VERIFY_WRITE, uaddr, sizeof(int))) return -EFAULT; @@ -70,7 +70,7 @@ diff -Nur linux-4.1.10.orig/arch/arc/include/asm/futex.h linux-4.1.10/arch/arc/i return val; diff -Nur linux-4.1.10.orig/arch/arc/mm/fault.c linux-4.1.10/arch/arc/mm/fault.c --- linux-4.1.10.orig/arch/arc/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/arc/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/arc/mm/fault.c 2015-10-12 22:33:32.144685475 +0200 @@ -86,7 +86,7 @@ * If we're in an interrupt or have no user * context, we must not take the fault.. @@ -82,7 +82,7 @@ diff -Nur linux-4.1.10.orig/arch/arc/mm/fault.c linux-4.1.10/arch/arc/mm/fault.c if (user_mode(regs)) diff -Nur linux-4.1.10.orig/arch/arm/include/asm/cmpxchg.h linux-4.1.10/arch/arm/include/asm/cmpxchg.h --- linux-4.1.10.orig/arch/arm/include/asm/cmpxchg.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/arm/include/asm/cmpxchg.h 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/arm/include/asm/cmpxchg.h 2015-10-12 22:33:32.148685211 +0200 @@ -129,6 +129,8 @@ #else /* min ARCH >= ARMv6 */ @@ -94,7 +94,7 @@ diff -Nur linux-4.1.10.orig/arch/arm/include/asm/cmpxchg.h linux-4.1.10/arch/arm /* diff -Nur linux-4.1.10.orig/arch/arm/include/asm/futex.h linux-4.1.10/arch/arm/include/asm/futex.h --- linux-4.1.10.orig/arch/arm/include/asm/futex.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/arm/include/asm/futex.h 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/arm/include/asm/futex.h 2015-10-12 22:33:32.148685211 +0200 @@ -93,6 +93,7 @@ if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32))) return -EFAULT; @@ -138,7 +138,7 @@ diff -Nur linux-4.1.10.orig/arch/arm/include/asm/futex.h linux-4.1.10/arch/arm/i switch (cmp) { diff -Nur linux-4.1.10.orig/arch/arm/include/asm/switch_to.h linux-4.1.10/arch/arm/include/asm/switch_to.h --- linux-4.1.10.orig/arch/arm/include/asm/switch_to.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/arm/include/asm/switch_to.h 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/arm/include/asm/switch_to.h 2015-10-12 22:33:32.148685211 +0200 @@ -3,6 +3,13 @@ #include @@ -163,7 +163,7 @@ diff -Nur linux-4.1.10.orig/arch/arm/include/asm/switch_to.h linux-4.1.10/arch/a diff -Nur linux-4.1.10.orig/arch/arm/include/asm/thread_info.h linux-4.1.10/arch/arm/include/asm/thread_info.h --- linux-4.1.10.orig/arch/arm/include/asm/thread_info.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/arm/include/asm/thread_info.h 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/arm/include/asm/thread_info.h 2015-10-12 22:33:32.148685211 +0200 @@ -50,6 +50,7 @@ struct thread_info { unsigned long flags; /* low level flags */ @@ -190,7 +190,7 @@ diff -Nur linux-4.1.10.orig/arch/arm/include/asm/thread_info.h linux-4.1.10/arch #define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT) diff -Nur linux-4.1.10.orig/arch/arm/Kconfig linux-4.1.10/arch/arm/Kconfig --- linux-4.1.10.orig/arch/arm/Kconfig 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/arm/Kconfig 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/arm/Kconfig 2015-10-12 22:33:32.148685211 +0200 @@ -31,7 +31,7 @@ select HARDIRQS_SW_RESEND select HAVE_ARCH_AUDITSYSCALL if (AEABI && !OABI_COMPAT) @@ -210,7 +210,7 @@ diff -Nur linux-4.1.10.orig/arch/arm/Kconfig linux-4.1.10/arch/arm/Kconfig select HAVE_SYSCALL_TRACEPOINTS diff -Nur linux-4.1.10.orig/arch/arm/kernel/asm-offsets.c linux-4.1.10/arch/arm/kernel/asm-offsets.c --- linux-4.1.10.orig/arch/arm/kernel/asm-offsets.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/arm/kernel/asm-offsets.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/arm/kernel/asm-offsets.c 2015-10-12 22:33:32.148685211 +0200 @@ -65,6 +65,7 @@ BLANK(); DEFINE(TI_FLAGS, offsetof(struct thread_info, flags)); @@ -221,7 +221,7 @@ diff -Nur linux-4.1.10.orig/arch/arm/kernel/asm-offsets.c linux-4.1.10/arch/arm/ DEFINE(TI_CPU, offsetof(struct thread_info, cpu)); diff -Nur linux-4.1.10.orig/arch/arm/kernel/entry-armv.S linux-4.1.10/arch/arm/kernel/entry-armv.S --- linux-4.1.10.orig/arch/arm/kernel/entry-armv.S 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/arm/kernel/entry-armv.S 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/arm/kernel/entry-armv.S 2015-10-12 22:33:32.152684946 +0200 @@ -208,11 +208,18 @@ #ifdef CONFIG_PREEMPT get_thread_info tsk @@ -254,7 +254,7 @@ diff -Nur linux-4.1.10.orig/arch/arm/kernel/entry-armv.S linux-4.1.10/arch/arm/k #endif diff -Nur linux-4.1.10.orig/arch/arm/kernel/process.c linux-4.1.10/arch/arm/kernel/process.c --- linux-4.1.10.orig/arch/arm/kernel/process.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/arm/kernel/process.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/arm/kernel/process.c 2015-10-12 22:33:32.152684946 +0200 @@ -290,6 +290,30 @@ } @@ -288,7 +288,7 @@ diff -Nur linux-4.1.10.orig/arch/arm/kernel/process.c linux-4.1.10/arch/arm/kern * The vectors page is always readable from user space for the diff -Nur linux-4.1.10.orig/arch/arm/kernel/signal.c linux-4.1.10/arch/arm/kernel/signal.c --- linux-4.1.10.orig/arch/arm/kernel/signal.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/arm/kernel/signal.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/arm/kernel/signal.c 2015-10-12 22:33:32.152684946 +0200 @@ -563,7 +563,8 @@ do_work_pending(struct pt_regs *regs, unsigned int thread_flags, int syscall) { @@ -301,7 +301,7 @@ diff -Nur linux-4.1.10.orig/arch/arm/kernel/signal.c linux-4.1.10/arch/arm/kerne if (unlikely(!user_mode(regs))) diff -Nur linux-4.1.10.orig/arch/arm/kernel/smp.c linux-4.1.10/arch/arm/kernel/smp.c --- linux-4.1.10.orig/arch/arm/kernel/smp.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/arm/kernel/smp.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/arm/kernel/smp.c 2015-10-12 22:33:32.152684946 +0200 @@ -213,8 +213,6 @@ flush_cache_louis(); local_flush_tlb_all(); @@ -323,7 +323,7 @@ diff -Nur linux-4.1.10.orig/arch/arm/kernel/smp.c linux-4.1.10/arch/arm/kernel/s /* diff -Nur linux-4.1.10.orig/arch/arm/kernel/unwind.c linux-4.1.10/arch/arm/kernel/unwind.c --- linux-4.1.10.orig/arch/arm/kernel/unwind.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/arm/kernel/unwind.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/arm/kernel/unwind.c 2015-10-12 22:33:32.152684946 +0200 @@ -93,7 +93,7 @@ static const struct unwind_idx *__origin_unwind_idx; extern const struct unwind_idx __stop_unwind_idx[]; @@ -377,7 +377,7 @@ diff -Nur linux-4.1.10.orig/arch/arm/kernel/unwind.c linux-4.1.10/arch/arm/kerne } diff -Nur linux-4.1.10.orig/arch/arm/kvm/arm.c linux-4.1.10/arch/arm/kvm/arm.c --- linux-4.1.10.orig/arch/arm/kvm/arm.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/arm/kvm/arm.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/arm/kvm/arm.c 2015-10-12 22:33:32.156684682 +0200 @@ -474,9 +474,9 @@ static void vcpu_pause(struct kvm_vcpu *vcpu) @@ -392,7 +392,7 @@ diff -Nur linux-4.1.10.orig/arch/arm/kvm/arm.c linux-4.1.10/arch/arm/kvm/arm.c static int kvm_vcpu_initialized(struct kvm_vcpu *vcpu) diff -Nur linux-4.1.10.orig/arch/arm/kvm/psci.c linux-4.1.10/arch/arm/kvm/psci.c --- linux-4.1.10.orig/arch/arm/kvm/psci.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/arm/kvm/psci.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/arm/kvm/psci.c 2015-10-12 22:33:32.156684682 +0200 @@ -68,7 +68,7 @@ { struct kvm *kvm = source_vcpu->kvm; @@ -413,7 +413,7 @@ diff -Nur linux-4.1.10.orig/arch/arm/kvm/psci.c linux-4.1.10/arch/arm/kvm/psci.c } diff -Nur linux-4.1.10.orig/arch/arm/mach-exynos/platsmp.c linux-4.1.10/arch/arm/mach-exynos/platsmp.c --- linux-4.1.10.orig/arch/arm/mach-exynos/platsmp.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/arm/mach-exynos/platsmp.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/arm/mach-exynos/platsmp.c 2015-10-12 22:33:32.156684682 +0200 @@ -231,7 +231,7 @@ return (void __iomem *)(S5P_VA_SCU); } @@ -463,7 +463,7 @@ diff -Nur linux-4.1.10.orig/arch/arm/mach-exynos/platsmp.c linux-4.1.10/arch/arm } diff -Nur linux-4.1.10.orig/arch/arm/mach-hisi/platmcpm.c linux-4.1.10/arch/arm/mach-hisi/platmcpm.c --- linux-4.1.10.orig/arch/arm/mach-hisi/platmcpm.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/arm/mach-hisi/platmcpm.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/arm/mach-hisi/platmcpm.c 2015-10-12 22:33:32.156684682 +0200 @@ -57,7 +57,7 @@ static void __iomem *sysctrl, *fabric; @@ -568,7 +568,7 @@ diff -Nur linux-4.1.10.orig/arch/arm/mach-hisi/platmcpm.c linux-4.1.10/arch/arm/ static void __naked hip04_mcpm_power_up_setup(unsigned int affinity_level) diff -Nur linux-4.1.10.orig/arch/arm/mach-omap2/omap-smp.c linux-4.1.10/arch/arm/mach-omap2/omap-smp.c --- linux-4.1.10.orig/arch/arm/mach-omap2/omap-smp.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/arm/mach-omap2/omap-smp.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/arm/mach-omap2/omap-smp.c 2015-10-12 22:33:32.156684682 +0200 @@ -43,7 +43,7 @@ /* SCU base address */ static void __iomem *scu_base; @@ -609,7 +609,7 @@ diff -Nur linux-4.1.10.orig/arch/arm/mach-omap2/omap-smp.c linux-4.1.10/arch/arm } diff -Nur linux-4.1.10.orig/arch/arm/mach-prima2/platsmp.c linux-4.1.10/arch/arm/mach-prima2/platsmp.c --- linux-4.1.10.orig/arch/arm/mach-prima2/platsmp.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/arm/mach-prima2/platsmp.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/arm/mach-prima2/platsmp.c 2015-10-12 22:33:32.156684682 +0200 @@ -22,7 +22,7 @@ static void __iomem *clk_base; @@ -650,7 +650,7 @@ diff -Nur linux-4.1.10.orig/arch/arm/mach-prima2/platsmp.c linux-4.1.10/arch/arm } diff -Nur linux-4.1.10.orig/arch/arm/mach-qcom/platsmp.c linux-4.1.10/arch/arm/mach-qcom/platsmp.c --- linux-4.1.10.orig/arch/arm/mach-qcom/platsmp.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/arm/mach-qcom/platsmp.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/arm/mach-qcom/platsmp.c 2015-10-12 22:33:32.160684418 +0200 @@ -46,7 +46,7 @@ extern void secondary_startup_arm(void); @@ -691,7 +691,7 @@ diff -Nur linux-4.1.10.orig/arch/arm/mach-qcom/platsmp.c linux-4.1.10/arch/arm/m } diff -Nur linux-4.1.10.orig/arch/arm/mach-spear/platsmp.c linux-4.1.10/arch/arm/mach-spear/platsmp.c --- linux-4.1.10.orig/arch/arm/mach-spear/platsmp.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/arm/mach-spear/platsmp.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/arm/mach-spear/platsmp.c 2015-10-12 22:33:32.160684418 +0200 @@ -32,7 +32,7 @@ sync_cache_w(&pen_release); } @@ -732,7 +732,7 @@ diff -Nur linux-4.1.10.orig/arch/arm/mach-spear/platsmp.c linux-4.1.10/arch/arm/ } diff -Nur linux-4.1.10.orig/arch/arm/mach-sti/platsmp.c linux-4.1.10/arch/arm/mach-sti/platsmp.c --- linux-4.1.10.orig/arch/arm/mach-sti/platsmp.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/arm/mach-sti/platsmp.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/arm/mach-sti/platsmp.c 2015-10-12 22:33:32.160684418 +0200 @@ -34,7 +34,7 @@ sync_cache_w(&pen_release); } @@ -773,7 +773,7 @@ diff -Nur linux-4.1.10.orig/arch/arm/mach-sti/platsmp.c linux-4.1.10/arch/arm/ma } diff -Nur linux-4.1.10.orig/arch/arm/mach-ux500/platsmp.c linux-4.1.10/arch/arm/mach-ux500/platsmp.c --- linux-4.1.10.orig/arch/arm/mach-ux500/platsmp.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/arm/mach-ux500/platsmp.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/arm/mach-ux500/platsmp.c 2015-10-12 22:33:32.160684418 +0200 @@ -51,7 +51,7 @@ return NULL; } @@ -814,7 +814,7 @@ diff -Nur linux-4.1.10.orig/arch/arm/mach-ux500/platsmp.c linux-4.1.10/arch/arm/ } diff -Nur linux-4.1.10.orig/arch/arm/mm/fault.c linux-4.1.10/arch/arm/mm/fault.c --- linux-4.1.10.orig/arch/arm/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/arm/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/arm/mm/fault.c 2015-10-12 22:33:32.160684418 +0200 @@ -276,7 +276,7 @@ * If we're in an interrupt or have no user * context, we must not take the fault.. @@ -846,7 +846,7 @@ diff -Nur linux-4.1.10.orig/arch/arm/mm/fault.c linux-4.1.10/arch/arm/mm/fault.c } diff -Nur linux-4.1.10.orig/arch/arm/mm/highmem.c linux-4.1.10/arch/arm/mm/highmem.c --- linux-4.1.10.orig/arch/arm/mm/highmem.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/arm/mm/highmem.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/arm/mm/highmem.c 2015-10-12 22:33:32.160684418 +0200 @@ -54,11 +54,13 @@ void *kmap_atomic(struct page *page) @@ -951,7 +951,7 @@ diff -Nur linux-4.1.10.orig/arch/arm/mm/highmem.c linux-4.1.10/arch/arm/mm/highm +#endif diff -Nur linux-4.1.10.orig/arch/arm/plat-versatile/platsmp.c linux-4.1.10/arch/arm/plat-versatile/platsmp.c --- linux-4.1.10.orig/arch/arm/plat-versatile/platsmp.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/arm/plat-versatile/platsmp.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/arm/plat-versatile/platsmp.c 2015-10-12 22:33:32.160684418 +0200 @@ -30,7 +30,7 @@ sync_cache_w(&pen_release); } @@ -992,7 +992,7 @@ diff -Nur linux-4.1.10.orig/arch/arm/plat-versatile/platsmp.c linux-4.1.10/arch/ } diff -Nur linux-4.1.10.orig/arch/arm64/include/asm/futex.h linux-4.1.10/arch/arm64/include/asm/futex.h --- linux-4.1.10.orig/arch/arm64/include/asm/futex.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/arm64/include/asm/futex.h 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/arm64/include/asm/futex.h 2015-10-12 22:33:32.160684418 +0200 @@ -58,7 +58,7 @@ if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32))) return -EFAULT; @@ -1013,7 +1013,7 @@ diff -Nur linux-4.1.10.orig/arch/arm64/include/asm/futex.h linux-4.1.10/arch/arm switch (cmp) { diff -Nur linux-4.1.10.orig/arch/arm64/include/asm/thread_info.h linux-4.1.10/arch/arm64/include/asm/thread_info.h --- linux-4.1.10.orig/arch/arm64/include/asm/thread_info.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/arm64/include/asm/thread_info.h 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/arm64/include/asm/thread_info.h 2015-10-12 22:33:32.164684154 +0200 @@ -47,6 +47,7 @@ mm_segment_t addr_limit; /* address limit */ struct task_struct *task; /* main task structure */ @@ -1040,7 +1040,7 @@ diff -Nur linux-4.1.10.orig/arch/arm64/include/asm/thread_info.h linux-4.1.10/ar #define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT) diff -Nur linux-4.1.10.orig/arch/arm64/Kconfig linux-4.1.10/arch/arm64/Kconfig --- linux-4.1.10.orig/arch/arm64/Kconfig 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/arm64/Kconfig 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/arm64/Kconfig 2015-10-12 22:33:32.164684154 +0200 @@ -69,8 +69,10 @@ select HAVE_PERF_REGS select HAVE_PERF_USER_STACK_DUMP @@ -1054,7 +1054,7 @@ diff -Nur linux-4.1.10.orig/arch/arm64/Kconfig linux-4.1.10/arch/arm64/Kconfig select OF diff -Nur linux-4.1.10.orig/arch/arm64/kernel/asm-offsets.c linux-4.1.10/arch/arm64/kernel/asm-offsets.c --- linux-4.1.10.orig/arch/arm64/kernel/asm-offsets.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/arm64/kernel/asm-offsets.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/arm64/kernel/asm-offsets.c 2015-10-12 22:33:32.164684154 +0200 @@ -35,6 +35,7 @@ BLANK(); DEFINE(TI_FLAGS, offsetof(struct thread_info, flags)); @@ -1065,7 +1065,7 @@ diff -Nur linux-4.1.10.orig/arch/arm64/kernel/asm-offsets.c linux-4.1.10/arch/ar DEFINE(TI_CPU, offsetof(struct thread_info, cpu)); diff -Nur linux-4.1.10.orig/arch/arm64/kernel/entry.S linux-4.1.10/arch/arm64/kernel/entry.S --- linux-4.1.10.orig/arch/arm64/kernel/entry.S 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/arm64/kernel/entry.S 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/arm64/kernel/entry.S 2015-10-12 22:33:32.164684154 +0200 @@ -367,11 +367,16 @@ #ifdef CONFIG_PREEMPT get_thread_info tsk @@ -1104,7 +1104,7 @@ diff -Nur linux-4.1.10.orig/arch/arm64/kernel/entry.S linux-4.1.10/arch/arm64/ke mov x0, sp // 'regs' diff -Nur linux-4.1.10.orig/arch/arm64/kernel/perf_event.c linux-4.1.10/arch/arm64/kernel/perf_event.c --- linux-4.1.10.orig/arch/arm64/kernel/perf_event.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/arm64/kernel/perf_event.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/arm64/kernel/perf_event.c 2015-10-12 22:33:32.164684154 +0200 @@ -488,7 +488,7 @@ } @@ -1116,7 +1116,7 @@ diff -Nur linux-4.1.10.orig/arch/arm64/kernel/perf_event.c linux-4.1.10/arch/arm pr_err("unable to request IRQ%d for ARM PMU counters\n", diff -Nur linux-4.1.10.orig/arch/arm64/mm/fault.c linux-4.1.10/arch/arm64/mm/fault.c --- linux-4.1.10.orig/arch/arm64/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/arm64/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/arm64/mm/fault.c 2015-10-12 22:33:32.168683889 +0200 @@ -211,7 +211,7 @@ * If we're in an interrupt or have no user context, we must not take * the fault. @@ -1128,7 +1128,7 @@ diff -Nur linux-4.1.10.orig/arch/arm64/mm/fault.c linux-4.1.10/arch/arm64/mm/fau if (user_mode(regs)) diff -Nur linux-4.1.10.orig/arch/avr32/include/asm/uaccess.h linux-4.1.10/arch/avr32/include/asm/uaccess.h --- linux-4.1.10.orig/arch/avr32/include/asm/uaccess.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/avr32/include/asm/uaccess.h 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/avr32/include/asm/uaccess.h 2015-10-12 22:33:32.180683097 +0200 @@ -97,7 +97,8 @@ * @x: Value to copy to user space. * @ptr: Destination address, in user space. @@ -1171,7 +1171,7 @@ diff -Nur linux-4.1.10.orig/arch/avr32/include/asm/uaccess.h linux-4.1.10/arch/a * space. It supports simple types like char and int, but not larger diff -Nur linux-4.1.10.orig/arch/avr32/mm/fault.c linux-4.1.10/arch/avr32/mm/fault.c --- linux-4.1.10.orig/arch/avr32/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/avr32/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/avr32/mm/fault.c 2015-10-12 22:33:32.180683097 +0200 @@ -14,11 +14,11 @@ #include #include @@ -1196,7 +1196,7 @@ diff -Nur linux-4.1.10.orig/arch/avr32/mm/fault.c linux-4.1.10/arch/avr32/mm/fau local_irq_enable(); diff -Nur linux-4.1.10.orig/arch/cris/mm/fault.c linux-4.1.10/arch/cris/mm/fault.c --- linux-4.1.10.orig/arch/cris/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/cris/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/cris/mm/fault.c 2015-10-12 22:33:32.180683097 +0200 @@ -8,7 +8,7 @@ #include #include @@ -1222,7 +1222,7 @@ diff -Nur linux-4.1.10.orig/arch/cris/mm/fault.c linux-4.1.10/arch/cris/mm/fault if (user_mode(regs)) diff -Nur linux-4.1.10.orig/arch/frv/mm/fault.c linux-4.1.10/arch/frv/mm/fault.c --- linux-4.1.10.orig/arch/frv/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/frv/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/frv/mm/fault.c 2015-10-12 22:33:32.180683097 +0200 @@ -19,9 +19,9 @@ #include #include @@ -1245,7 +1245,7 @@ diff -Nur linux-4.1.10.orig/arch/frv/mm/fault.c linux-4.1.10/arch/frv/mm/fault.c if (user_mode(__frame)) diff -Nur linux-4.1.10.orig/arch/frv/mm/highmem.c linux-4.1.10/arch/frv/mm/highmem.c --- linux-4.1.10.orig/arch/frv/mm/highmem.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/frv/mm/highmem.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/frv/mm/highmem.c 2015-10-12 22:33:32.180683097 +0200 @@ -42,6 +42,7 @@ unsigned long paddr; int type; @@ -1263,7 +1263,7 @@ diff -Nur linux-4.1.10.orig/arch/frv/mm/highmem.c linux-4.1.10/arch/frv/mm/highm EXPORT_SYMBOL(__kunmap_atomic); diff -Nur linux-4.1.10.orig/arch/hexagon/include/asm/uaccess.h linux-4.1.10/arch/hexagon/include/asm/uaccess.h --- linux-4.1.10.orig/arch/hexagon/include/asm/uaccess.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/hexagon/include/asm/uaccess.h 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/hexagon/include/asm/uaccess.h 2015-10-12 22:33:32.180683097 +0200 @@ -36,7 +36,8 @@ * @addr: User space pointer to start of block to check * @size: Size of block to check @@ -1276,7 +1276,7 @@ diff -Nur linux-4.1.10.orig/arch/hexagon/include/asm/uaccess.h linux-4.1.10/arch * diff -Nur linux-4.1.10.orig/arch/ia64/mm/fault.c linux-4.1.10/arch/ia64/mm/fault.c --- linux-4.1.10.orig/arch/ia64/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/ia64/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/ia64/mm/fault.c 2015-10-12 22:33:32.184682832 +0200 @@ -11,10 +11,10 @@ #include #include @@ -1300,7 +1300,7 @@ diff -Nur linux-4.1.10.orig/arch/ia64/mm/fault.c linux-4.1.10/arch/ia64/mm/fault #ifdef CONFIG_VIRTUAL_MEM_MAP diff -Nur linux-4.1.10.orig/arch/Kconfig linux-4.1.10/arch/Kconfig --- linux-4.1.10.orig/arch/Kconfig 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/Kconfig 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/Kconfig 2015-10-12 22:33:32.184682832 +0200 @@ -6,6 +6,7 @@ tristate "OProfile system profiling" depends on PROFILING @@ -1311,7 +1311,7 @@ diff -Nur linux-4.1.10.orig/arch/Kconfig linux-4.1.10/arch/Kconfig help diff -Nur linux-4.1.10.orig/arch/m32r/include/asm/uaccess.h linux-4.1.10/arch/m32r/include/asm/uaccess.h --- linux-4.1.10.orig/arch/m32r/include/asm/uaccess.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/m32r/include/asm/uaccess.h 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/m32r/include/asm/uaccess.h 2015-10-12 22:33:32.184682832 +0200 @@ -91,7 +91,8 @@ * @addr: User space pointer to start of block to check * @size: Size of block to check @@ -1414,7 +1414,7 @@ diff -Nur linux-4.1.10.orig/arch/m32r/include/asm/uaccess.h linux-4.1.10/arch/m3 * diff -Nur linux-4.1.10.orig/arch/m32r/mm/fault.c linux-4.1.10/arch/m32r/mm/fault.c --- linux-4.1.10.orig/arch/m32r/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/m32r/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/m32r/mm/fault.c 2015-10-12 22:33:32.184682832 +0200 @@ -24,9 +24,9 @@ #include /* For unblank_screen() */ #include @@ -1442,7 +1442,7 @@ diff -Nur linux-4.1.10.orig/arch/m32r/mm/fault.c linux-4.1.10/arch/m32r/mm/fault if (error_code & ACE_USERMODE) diff -Nur linux-4.1.10.orig/arch/m68k/mm/fault.c linux-4.1.10/arch/m68k/mm/fault.c --- linux-4.1.10.orig/arch/m68k/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/m68k/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/m68k/mm/fault.c 2015-10-12 22:33:32.184682832 +0200 @@ -10,10 +10,10 @@ #include #include @@ -1466,7 +1466,7 @@ diff -Nur linux-4.1.10.orig/arch/m68k/mm/fault.c linux-4.1.10/arch/m68k/mm/fault if (user_mode(regs)) diff -Nur linux-4.1.10.orig/arch/metag/mm/fault.c linux-4.1.10/arch/metag/mm/fault.c --- linux-4.1.10.orig/arch/metag/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/metag/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/metag/mm/fault.c 2015-10-12 22:33:32.184682832 +0200 @@ -105,7 +105,7 @@ mm = tsk->mm; @@ -1478,7 +1478,7 @@ diff -Nur linux-4.1.10.orig/arch/metag/mm/fault.c linux-4.1.10/arch/metag/mm/fau if (user_mode(regs)) diff -Nur linux-4.1.10.orig/arch/metag/mm/highmem.c linux-4.1.10/arch/metag/mm/highmem.c --- linux-4.1.10.orig/arch/metag/mm/highmem.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/metag/mm/highmem.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/metag/mm/highmem.c 2015-10-12 22:33:32.184682832 +0200 @@ -43,7 +43,7 @@ unsigned long vaddr; int type; @@ -1506,7 +1506,7 @@ diff -Nur linux-4.1.10.orig/arch/metag/mm/highmem.c linux-4.1.10/arch/metag/mm/h type = kmap_atomic_idx_push(); diff -Nur linux-4.1.10.orig/arch/microblaze/include/asm/uaccess.h linux-4.1.10/arch/microblaze/include/asm/uaccess.h --- linux-4.1.10.orig/arch/microblaze/include/asm/uaccess.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/microblaze/include/asm/uaccess.h 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/microblaze/include/asm/uaccess.h 2015-10-12 22:33:32.188682568 +0200 @@ -178,7 +178,8 @@ * @x: Variable to store result. * @ptr: Source address, in user space. @@ -1529,7 +1529,7 @@ diff -Nur linux-4.1.10.orig/arch/microblaze/include/asm/uaccess.h linux-4.1.10/a * space. It supports simple types like char and int, but not larger diff -Nur linux-4.1.10.orig/arch/microblaze/mm/fault.c linux-4.1.10/arch/microblaze/mm/fault.c --- linux-4.1.10.orig/arch/microblaze/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/microblaze/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/microblaze/mm/fault.c 2015-10-12 22:33:32.188682568 +0200 @@ -107,14 +107,14 @@ if ((error_code & 0x13) == 0x13 || (error_code & 0x11) == 0x11) is_write = 0; @@ -1551,7 +1551,7 @@ diff -Nur linux-4.1.10.orig/arch/microblaze/mm/fault.c linux-4.1.10/arch/microbl die("Weird page fault", regs, SIGSEGV); diff -Nur linux-4.1.10.orig/arch/microblaze/mm/highmem.c linux-4.1.10/arch/microblaze/mm/highmem.c --- linux-4.1.10.orig/arch/microblaze/mm/highmem.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/microblaze/mm/highmem.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/microblaze/mm/highmem.c 2015-10-12 22:33:32.188682568 +0200 @@ -37,7 +37,7 @@ unsigned long vaddr; int idx, type; @@ -1578,7 +1578,7 @@ diff -Nur linux-4.1.10.orig/arch/microblaze/mm/highmem.c linux-4.1.10/arch/micro EXPORT_SYMBOL(__kunmap_atomic); diff -Nur linux-4.1.10.orig/arch/mips/include/asm/uaccess.h linux-4.1.10/arch/mips/include/asm/uaccess.h --- linux-4.1.10.orig/arch/mips/include/asm/uaccess.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/mips/include/asm/uaccess.h 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/mips/include/asm/uaccess.h 2015-10-12 22:33:32.188682568 +0200 @@ -103,7 +103,8 @@ * @addr: User space pointer to start of block to check * @size: Size of block to check @@ -1731,7 +1731,7 @@ diff -Nur linux-4.1.10.orig/arch/mips/include/asm/uaccess.h linux-4.1.10/arch/mi * diff -Nur linux-4.1.10.orig/arch/mips/Kconfig linux-4.1.10/arch/mips/Kconfig --- linux-4.1.10.orig/arch/mips/Kconfig 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/mips/Kconfig 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/mips/Kconfig 2015-10-12 22:33:32.192682304 +0200 @@ -2366,7 +2366,7 @@ # config HIGHMEM @@ -1743,7 +1743,7 @@ diff -Nur linux-4.1.10.orig/arch/mips/Kconfig linux-4.1.10/arch/mips/Kconfig bool diff -Nur linux-4.1.10.orig/arch/mips/kernel/signal-common.h linux-4.1.10/arch/mips/kernel/signal-common.h --- linux-4.1.10.orig/arch/mips/kernel/signal-common.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/mips/kernel/signal-common.h 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/mips/kernel/signal-common.h 2015-10-12 22:33:32.192682304 +0200 @@ -28,12 +28,7 @@ extern int fpcsr_pending(unsigned int __user *fpcsr); @@ -1761,7 +1761,7 @@ diff -Nur linux-4.1.10.orig/arch/mips/kernel/signal-common.h linux-4.1.10/arch/m #endif /* __SIGNAL_COMMON_H */ diff -Nur linux-4.1.10.orig/arch/mips/mm/fault.c linux-4.1.10/arch/mips/mm/fault.c --- linux-4.1.10.orig/arch/mips/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/mips/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/mips/mm/fault.c 2015-10-12 22:33:32.192682304 +0200 @@ -21,10 +21,10 @@ #include #include @@ -1785,7 +1785,7 @@ diff -Nur linux-4.1.10.orig/arch/mips/mm/fault.c linux-4.1.10/arch/mips/mm/fault if (user_mode(regs)) diff -Nur linux-4.1.10.orig/arch/mips/mm/highmem.c linux-4.1.10/arch/mips/mm/highmem.c --- linux-4.1.10.orig/arch/mips/mm/highmem.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/mips/mm/highmem.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/mips/mm/highmem.c 2015-10-12 22:33:32.192682304 +0200 @@ -47,7 +47,7 @@ unsigned long vaddr; int idx, type; @@ -1821,7 +1821,7 @@ diff -Nur linux-4.1.10.orig/arch/mips/mm/highmem.c linux-4.1.10/arch/mips/mm/hig type = kmap_atomic_idx_push(); diff -Nur linux-4.1.10.orig/arch/mips/mm/init.c linux-4.1.10/arch/mips/mm/init.c --- linux-4.1.10.orig/arch/mips/mm/init.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/mips/mm/init.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/mips/mm/init.c 2015-10-12 22:33:32.192682304 +0200 @@ -90,6 +90,7 @@ BUG_ON(Page_dcache_dirty(page)); @@ -1840,7 +1840,7 @@ diff -Nur linux-4.1.10.orig/arch/mips/mm/init.c linux-4.1.10/arch/mips/mm/init.c void copy_user_highpage(struct page *to, struct page *from, diff -Nur linux-4.1.10.orig/arch/mn10300/include/asm/highmem.h linux-4.1.10/arch/mn10300/include/asm/highmem.h --- linux-4.1.10.orig/arch/mn10300/include/asm/highmem.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/mn10300/include/asm/highmem.h 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/mn10300/include/asm/highmem.h 2015-10-12 22:33:32.192682304 +0200 @@ -75,6 +75,7 @@ unsigned long vaddr; int idx, type; @@ -1867,7 +1867,7 @@ diff -Nur linux-4.1.10.orig/arch/mn10300/include/asm/highmem.h linux-4.1.10/arch diff -Nur linux-4.1.10.orig/arch/mn10300/mm/fault.c linux-4.1.10/arch/mn10300/mm/fault.c --- linux-4.1.10.orig/arch/mn10300/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/mn10300/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/mn10300/mm/fault.c 2015-10-12 22:33:32.192682304 +0200 @@ -23,8 +23,8 @@ #include #include @@ -1889,7 +1889,7 @@ diff -Nur linux-4.1.10.orig/arch/mn10300/mm/fault.c linux-4.1.10/arch/mn10300/mm if ((fault_code & MMUFCR_xFC_ACCESS) == MMUFCR_xFC_ACCESS_USR) diff -Nur linux-4.1.10.orig/arch/nios2/mm/fault.c linux-4.1.10/arch/nios2/mm/fault.c --- linux-4.1.10.orig/arch/nios2/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/nios2/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/nios2/mm/fault.c 2015-10-12 22:33:32.192682304 +0200 @@ -77,7 +77,7 @@ * If we're in an interrupt or have no user * context, we must not take the fault.. @@ -1901,7 +1901,7 @@ diff -Nur linux-4.1.10.orig/arch/nios2/mm/fault.c linux-4.1.10/arch/nios2/mm/fau if (user_mode(regs)) diff -Nur linux-4.1.10.orig/arch/parisc/include/asm/cacheflush.h linux-4.1.10/arch/parisc/include/asm/cacheflush.h --- linux-4.1.10.orig/arch/parisc/include/asm/cacheflush.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/parisc/include/asm/cacheflush.h 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/parisc/include/asm/cacheflush.h 2015-10-12 22:33:32.192682304 +0200 @@ -142,6 +142,7 @@ static inline void *kmap_atomic(struct page *page) @@ -1920,7 +1920,7 @@ diff -Nur linux-4.1.10.orig/arch/parisc/include/asm/cacheflush.h linux-4.1.10/ar #define kmap_atomic_prot(page, prot) kmap_atomic(page) diff -Nur linux-4.1.10.orig/arch/parisc/kernel/traps.c linux-4.1.10/arch/parisc/kernel/traps.c --- linux-4.1.10.orig/arch/parisc/kernel/traps.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/parisc/kernel/traps.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/parisc/kernel/traps.c 2015-10-12 22:33:32.196682039 +0200 @@ -26,9 +26,9 @@ #include #include @@ -1943,7 +1943,7 @@ diff -Nur linux-4.1.10.orig/arch/parisc/kernel/traps.c linux-4.1.10/arch/parisc/ parisc_terminate("Kernel Fault", regs, code, fault_address); diff -Nur linux-4.1.10.orig/arch/parisc/mm/fault.c linux-4.1.10/arch/parisc/mm/fault.c --- linux-4.1.10.orig/arch/parisc/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/parisc/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/parisc/mm/fault.c 2015-10-12 22:33:32.196682039 +0200 @@ -15,8 +15,8 @@ #include #include @@ -1965,7 +1965,7 @@ diff -Nur linux-4.1.10.orig/arch/parisc/mm/fault.c linux-4.1.10/arch/parisc/mm/f tsk = current; diff -Nur linux-4.1.10.orig/arch/powerpc/include/asm/kvm_host.h linux-4.1.10/arch/powerpc/include/asm/kvm_host.h --- linux-4.1.10.orig/arch/powerpc/include/asm/kvm_host.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/powerpc/include/asm/kvm_host.h 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/powerpc/include/asm/kvm_host.h 2015-10-12 22:33:32.196682039 +0200 @@ -280,7 +280,7 @@ u8 in_guest; struct list_head runnable_threads; @@ -1986,7 +1986,7 @@ diff -Nur linux-4.1.10.orig/arch/powerpc/include/asm/kvm_host.h linux-4.1.10/arc int trap; diff -Nur linux-4.1.10.orig/arch/powerpc/include/asm/thread_info.h linux-4.1.10/arch/powerpc/include/asm/thread_info.h --- linux-4.1.10.orig/arch/powerpc/include/asm/thread_info.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/powerpc/include/asm/thread_info.h 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/powerpc/include/asm/thread_info.h 2015-10-12 22:33:32.196682039 +0200 @@ -42,6 +42,8 @@ int cpu; /* cpu we're on */ int preempt_count; /* 0 => preemptable, @@ -2035,7 +2035,7 @@ diff -Nur linux-4.1.10.orig/arch/powerpc/include/asm/thread_info.h linux-4.1.10/ /* Don't move TLF_NAPPING without adjusting the code in entry_32.S */ diff -Nur linux-4.1.10.orig/arch/powerpc/Kconfig linux-4.1.10/arch/powerpc/Kconfig --- linux-4.1.10.orig/arch/powerpc/Kconfig 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/powerpc/Kconfig 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/powerpc/Kconfig 2015-10-12 22:33:32.196682039 +0200 @@ -60,10 +60,11 @@ config RWSEM_GENERIC_SPINLOCK @@ -2068,7 +2068,7 @@ diff -Nur linux-4.1.10.orig/arch/powerpc/Kconfig linux-4.1.10/arch/powerpc/Kconf source kernel/Kconfig.preempt diff -Nur linux-4.1.10.orig/arch/powerpc/kernel/asm-offsets.c linux-4.1.10/arch/powerpc/kernel/asm-offsets.c --- linux-4.1.10.orig/arch/powerpc/kernel/asm-offsets.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/powerpc/kernel/asm-offsets.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/powerpc/kernel/asm-offsets.c 2015-10-12 22:33:32.196682039 +0200 @@ -160,6 +160,7 @@ DEFINE(TI_FLAGS, offsetof(struct thread_info, flags)); DEFINE(TI_LOCAL_FLAGS, offsetof(struct thread_info, local_flags)); @@ -2079,7 +2079,7 @@ diff -Nur linux-4.1.10.orig/arch/powerpc/kernel/asm-offsets.c linux-4.1.10/arch/ diff -Nur linux-4.1.10.orig/arch/powerpc/kernel/entry_32.S linux-4.1.10/arch/powerpc/kernel/entry_32.S --- linux-4.1.10.orig/arch/powerpc/kernel/entry_32.S 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/powerpc/kernel/entry_32.S 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/powerpc/kernel/entry_32.S 2015-10-12 22:33:32.196682039 +0200 @@ -813,7 +813,14 @@ cmpwi 0,r0,0 /* if non-zero, just restore regs and return */ bne restore @@ -2130,7 +2130,7 @@ diff -Nur linux-4.1.10.orig/arch/powerpc/kernel/entry_32.S linux-4.1.10/arch/pow beq restore_user diff -Nur linux-4.1.10.orig/arch/powerpc/kernel/entry_64.S linux-4.1.10/arch/powerpc/kernel/entry_64.S --- linux-4.1.10.orig/arch/powerpc/kernel/entry_64.S 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/powerpc/kernel/entry_64.S 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/powerpc/kernel/entry_64.S 2015-10-12 22:33:32.196682039 +0200 @@ -636,7 +636,7 @@ #else beq restore @@ -2171,7 +2171,7 @@ diff -Nur linux-4.1.10.orig/arch/powerpc/kernel/entry_64.S linux-4.1.10/arch/pow /* diff -Nur linux-4.1.10.orig/arch/powerpc/kernel/irq.c linux-4.1.10/arch/powerpc/kernel/irq.c --- linux-4.1.10.orig/arch/powerpc/kernel/irq.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/powerpc/kernel/irq.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/powerpc/kernel/irq.c 2015-10-12 22:33:32.196682039 +0200 @@ -614,6 +614,7 @@ } } @@ -2190,7 +2190,7 @@ diff -Nur linux-4.1.10.orig/arch/powerpc/kernel/irq.c linux-4.1.10/arch/powerpc/ { diff -Nur linux-4.1.10.orig/arch/powerpc/kernel/misc_32.S linux-4.1.10/arch/powerpc/kernel/misc_32.S --- linux-4.1.10.orig/arch/powerpc/kernel/misc_32.S 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/powerpc/kernel/misc_32.S 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/powerpc/kernel/misc_32.S 2015-10-12 22:33:32.196682039 +0200 @@ -40,6 +40,7 @@ * We store the saved ksp_limit in the unused part * of the STACK_FRAME_OVERHEAD @@ -2209,7 +2209,7 @@ diff -Nur linux-4.1.10.orig/arch/powerpc/kernel/misc_32.S linux-4.1.10/arch/powe * void call_do_irq(struct pt_regs *regs, struct thread_info *irqtp); diff -Nur linux-4.1.10.orig/arch/powerpc/kernel/misc_64.S linux-4.1.10/arch/powerpc/kernel/misc_64.S --- linux-4.1.10.orig/arch/powerpc/kernel/misc_64.S 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/powerpc/kernel/misc_64.S 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/powerpc/kernel/misc_64.S 2015-10-12 22:33:32.208681247 +0200 @@ -29,6 +29,7 @@ .text @@ -2228,7 +2228,7 @@ diff -Nur linux-4.1.10.orig/arch/powerpc/kernel/misc_64.S linux-4.1.10/arch/powe mflr r0 diff -Nur linux-4.1.10.orig/arch/powerpc/kvm/book3s_hv.c linux-4.1.10/arch/powerpc/kvm/book3s_hv.c --- linux-4.1.10.orig/arch/powerpc/kvm/book3s_hv.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/powerpc/kvm/book3s_hv.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/powerpc/kvm/book3s_hv.c 2015-10-12 22:33:32.208681247 +0200 @@ -115,11 +115,11 @@ static void kvmppc_fast_vcpu_kick_hv(struct kvm_vcpu *vcpu) { @@ -2306,7 +2306,7 @@ diff -Nur linux-4.1.10.orig/arch/powerpc/kvm/book3s_hv.c linux-4.1.10/arch/power } diff -Nur linux-4.1.10.orig/arch/powerpc/kvm/Kconfig linux-4.1.10/arch/powerpc/kvm/Kconfig --- linux-4.1.10.orig/arch/powerpc/kvm/Kconfig 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/powerpc/kvm/Kconfig 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/powerpc/kvm/Kconfig 2015-10-12 22:33:32.208681247 +0200 @@ -172,6 +172,7 @@ config KVM_MPIC bool "KVM in-kernel MPIC emulation" @@ -2317,7 +2317,7 @@ diff -Nur linux-4.1.10.orig/arch/powerpc/kvm/Kconfig linux-4.1.10/arch/powerpc/k select HAVE_KVM_IRQ_ROUTING diff -Nur linux-4.1.10.orig/arch/powerpc/mm/fault.c linux-4.1.10/arch/powerpc/mm/fault.c --- linux-4.1.10.orig/arch/powerpc/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/powerpc/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/powerpc/mm/fault.c 2015-10-12 22:33:32.212680982 +0200 @@ -33,13 +33,13 @@ #include #include @@ -2355,7 +2355,7 @@ diff -Nur linux-4.1.10.orig/arch/powerpc/mm/fault.c linux-4.1.10/arch/powerpc/mm die("Weird page fault", regs, SIGSEGV); diff -Nur linux-4.1.10.orig/arch/powerpc/mm/highmem.c linux-4.1.10/arch/powerpc/mm/highmem.c --- linux-4.1.10.orig/arch/powerpc/mm/highmem.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/powerpc/mm/highmem.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/powerpc/mm/highmem.c 2015-10-12 22:33:32.212680982 +0200 @@ -34,7 +34,7 @@ unsigned long vaddr; int idx, type; @@ -2382,7 +2382,7 @@ diff -Nur linux-4.1.10.orig/arch/powerpc/mm/highmem.c linux-4.1.10/arch/powerpc/ EXPORT_SYMBOL(__kunmap_atomic); diff -Nur linux-4.1.10.orig/arch/powerpc/platforms/ps3/device-init.c linux-4.1.10/arch/powerpc/platforms/ps3/device-init.c --- linux-4.1.10.orig/arch/powerpc/platforms/ps3/device-init.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/powerpc/platforms/ps3/device-init.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/powerpc/platforms/ps3/device-init.c 2015-10-12 22:33:32.212680982 +0200 @@ -752,7 +752,7 @@ } pr_debug("%s:%u: notification %s issued\n", __func__, __LINE__, op); @@ -2394,7 +2394,7 @@ diff -Nur linux-4.1.10.orig/arch/powerpc/platforms/ps3/device-init.c linux-4.1.1 res = -EINTR; diff -Nur linux-4.1.10.orig/arch/s390/include/asm/kvm_host.h linux-4.1.10/arch/s390/include/asm/kvm_host.h --- linux-4.1.10.orig/arch/s390/include/asm/kvm_host.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/s390/include/asm/kvm_host.h 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/s390/include/asm/kvm_host.h 2015-10-12 22:33:32.212680982 +0200 @@ -419,7 +419,7 @@ struct kvm_s390_local_interrupt { spinlock_t lock; @@ -2406,7 +2406,7 @@ diff -Nur linux-4.1.10.orig/arch/s390/include/asm/kvm_host.h linux-4.1.10/arch/s struct kvm_s390_irq_payload irq; diff -Nur linux-4.1.10.orig/arch/s390/include/asm/uaccess.h linux-4.1.10/arch/s390/include/asm/uaccess.h --- linux-4.1.10.orig/arch/s390/include/asm/uaccess.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/s390/include/asm/uaccess.h 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/s390/include/asm/uaccess.h 2015-10-12 22:33:32.212680982 +0200 @@ -98,7 +98,8 @@ * @from: Source address, in user space. * @n: Number of bytes to copy. @@ -2459,7 +2459,7 @@ diff -Nur linux-4.1.10.orig/arch/s390/include/asm/uaccess.h linux-4.1.10/arch/s3 * diff -Nur linux-4.1.10.orig/arch/s390/kvm/interrupt.c linux-4.1.10/arch/s390/kvm/interrupt.c --- linux-4.1.10.orig/arch/s390/kvm/interrupt.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/s390/kvm/interrupt.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/s390/kvm/interrupt.c 2015-10-12 22:33:32.212680982 +0200 @@ -875,13 +875,13 @@ void kvm_s390_vcpu_wakeup(struct kvm_vcpu *vcpu) @@ -2496,7 +2496,7 @@ diff -Nur linux-4.1.10.orig/arch/s390/kvm/interrupt.c linux-4.1.10/arch/s390/kvm } diff -Nur linux-4.1.10.orig/arch/s390/mm/fault.c linux-4.1.10/arch/s390/mm/fault.c --- linux-4.1.10.orig/arch/s390/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/s390/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/s390/mm/fault.c 2015-10-12 22:33:32.212680982 +0200 @@ -399,7 +399,7 @@ * user context. */ @@ -2508,7 +2508,7 @@ diff -Nur linux-4.1.10.orig/arch/s390/mm/fault.c linux-4.1.10/arch/s390/mm/fault address = trans_exc_code & __FAIL_ADDR_MASK; diff -Nur linux-4.1.10.orig/arch/score/include/asm/uaccess.h linux-4.1.10/arch/score/include/asm/uaccess.h --- linux-4.1.10.orig/arch/score/include/asm/uaccess.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/score/include/asm/uaccess.h 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/score/include/asm/uaccess.h 2015-10-12 22:33:32.212680982 +0200 @@ -36,7 +36,8 @@ * @addr: User space pointer to start of block to check * @size: Size of block to check @@ -2561,7 +2561,7 @@ diff -Nur linux-4.1.10.orig/arch/score/include/asm/uaccess.h linux-4.1.10/arch/s * space. It supports simple types like char and int, but not larger diff -Nur linux-4.1.10.orig/arch/score/mm/fault.c linux-4.1.10/arch/score/mm/fault.c --- linux-4.1.10.orig/arch/score/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/score/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/score/mm/fault.c 2015-10-12 22:33:32.212680982 +0200 @@ -34,6 +34,7 @@ #include #include @@ -2581,7 +2581,7 @@ diff -Nur linux-4.1.10.orig/arch/score/mm/fault.c linux-4.1.10/arch/score/mm/fau if (user_mode(regs)) diff -Nur linux-4.1.10.orig/arch/sh/kernel/irq.c linux-4.1.10/arch/sh/kernel/irq.c --- linux-4.1.10.orig/arch/sh/kernel/irq.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/sh/kernel/irq.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/sh/kernel/irq.c 2015-10-12 22:33:32.212680982 +0200 @@ -147,6 +147,7 @@ hardirq_ctx[cpu] = NULL; } @@ -2600,7 +2600,7 @@ diff -Nur linux-4.1.10.orig/arch/sh/kernel/irq.c linux-4.1.10/arch/sh/kernel/irq { diff -Nur linux-4.1.10.orig/arch/sh/mm/fault.c linux-4.1.10/arch/sh/mm/fault.c --- linux-4.1.10.orig/arch/sh/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/sh/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/sh/mm/fault.c 2015-10-12 22:33:32.212680982 +0200 @@ -17,6 +17,7 @@ #include #include @@ -2623,7 +2623,7 @@ diff -Nur linux-4.1.10.orig/arch/sh/mm/fault.c linux-4.1.10/arch/sh/mm/fault.c } diff -Nur linux-4.1.10.orig/arch/sparc/Kconfig linux-4.1.10/arch/sparc/Kconfig --- linux-4.1.10.orig/arch/sparc/Kconfig 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/sparc/Kconfig 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/sparc/Kconfig 2015-10-12 22:33:32.212680982 +0200 @@ -189,12 +189,10 @@ source kernel/Kconfig.hz @@ -2641,7 +2641,7 @@ diff -Nur linux-4.1.10.orig/arch/sparc/Kconfig linux-4.1.10/arch/sparc/Kconfig bool diff -Nur linux-4.1.10.orig/arch/sparc/kernel/irq_64.c linux-4.1.10/arch/sparc/kernel/irq_64.c --- linux-4.1.10.orig/arch/sparc/kernel/irq_64.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/sparc/kernel/irq_64.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/sparc/kernel/irq_64.c 2015-10-12 22:33:32.212680982 +0200 @@ -849,6 +849,7 @@ set_irq_regs(old_regs); } @@ -2660,7 +2660,7 @@ diff -Nur linux-4.1.10.orig/arch/sparc/kernel/irq_64.c linux-4.1.10/arch/sparc/k void fixup_irqs(void) diff -Nur linux-4.1.10.orig/arch/sparc/mm/fault_32.c linux-4.1.10/arch/sparc/mm/fault_32.c --- linux-4.1.10.orig/arch/sparc/mm/fault_32.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/sparc/mm/fault_32.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/sparc/mm/fault_32.c 2015-10-12 22:33:32.212680982 +0200 @@ -21,6 +21,7 @@ #include #include @@ -2688,7 +2688,7 @@ diff -Nur linux-4.1.10.orig/arch/sparc/mm/fault_32.c linux-4.1.10/arch/sparc/mm/ perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); diff -Nur linux-4.1.10.orig/arch/sparc/mm/fault_64.c linux-4.1.10/arch/sparc/mm/fault_64.c --- linux-4.1.10.orig/arch/sparc/mm/fault_64.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/sparc/mm/fault_64.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/sparc/mm/fault_64.c 2015-10-12 22:33:32.212680982 +0200 @@ -22,12 +22,12 @@ #include #include @@ -2714,7 +2714,7 @@ diff -Nur linux-4.1.10.orig/arch/sparc/mm/fault_64.c linux-4.1.10/arch/sparc/mm/ perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); diff -Nur linux-4.1.10.orig/arch/sparc/mm/highmem.c linux-4.1.10/arch/sparc/mm/highmem.c --- linux-4.1.10.orig/arch/sparc/mm/highmem.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/sparc/mm/highmem.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/sparc/mm/highmem.c 2015-10-12 22:33:32.216680718 +0200 @@ -53,7 +53,7 @@ unsigned long vaddr; long idx, type; @@ -2741,7 +2741,7 @@ diff -Nur linux-4.1.10.orig/arch/sparc/mm/highmem.c linux-4.1.10/arch/sparc/mm/h EXPORT_SYMBOL(__kunmap_atomic); diff -Nur linux-4.1.10.orig/arch/sparc/mm/init_64.c linux-4.1.10/arch/sparc/mm/init_64.c --- linux-4.1.10.orig/arch/sparc/mm/init_64.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/sparc/mm/init_64.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/sparc/mm/init_64.c 2015-10-12 22:33:32.216680718 +0200 @@ -2738,7 +2738,7 @@ struct mm_struct *mm = current->mm; struct tsb_config *tp; @@ -2753,7 +2753,7 @@ diff -Nur linux-4.1.10.orig/arch/sparc/mm/init_64.c linux-4.1.10/arch/sparc/mm/i entry = search_exception_tables(regs->tpc); diff -Nur linux-4.1.10.orig/arch/tile/include/asm/uaccess.h linux-4.1.10/arch/tile/include/asm/uaccess.h --- linux-4.1.10.orig/arch/tile/include/asm/uaccess.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/tile/include/asm/uaccess.h 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/tile/include/asm/uaccess.h 2015-10-12 22:33:32.216680718 +0200 @@ -78,7 +78,8 @@ * @addr: User space pointer to start of block to check * @size: Size of block to check @@ -2816,7 +2816,7 @@ diff -Nur linux-4.1.10.orig/arch/tile/include/asm/uaccess.h linux-4.1.10/arch/ti * the specified blocks with access_ok() before calling this function. diff -Nur linux-4.1.10.orig/arch/tile/mm/fault.c linux-4.1.10/arch/tile/mm/fault.c --- linux-4.1.10.orig/arch/tile/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/tile/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/tile/mm/fault.c 2015-10-12 22:33:32.216680718 +0200 @@ -354,9 +354,9 @@ /* @@ -2831,7 +2831,7 @@ diff -Nur linux-4.1.10.orig/arch/tile/mm/fault.c linux-4.1.10/arch/tile/mm/fault } diff -Nur linux-4.1.10.orig/arch/tile/mm/highmem.c linux-4.1.10/arch/tile/mm/highmem.c --- linux-4.1.10.orig/arch/tile/mm/highmem.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/tile/mm/highmem.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/tile/mm/highmem.c 2015-10-12 22:33:32.216680718 +0200 @@ -201,7 +201,7 @@ int idx, type; pte_t *pte; @@ -2851,7 +2851,7 @@ diff -Nur linux-4.1.10.orig/arch/tile/mm/highmem.c linux-4.1.10/arch/tile/mm/hig diff -Nur linux-4.1.10.orig/arch/um/kernel/trap.c linux-4.1.10/arch/um/kernel/trap.c --- linux-4.1.10.orig/arch/um/kernel/trap.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/um/kernel/trap.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/um/kernel/trap.c 2015-10-12 22:33:32.216680718 +0200 @@ -35,10 +35,10 @@ *code_out = SEGV_MAPERR; @@ -2867,7 +2867,7 @@ diff -Nur linux-4.1.10.orig/arch/um/kernel/trap.c linux-4.1.10/arch/um/kernel/tr if (is_user) diff -Nur linux-4.1.10.orig/arch/unicore32/mm/fault.c linux-4.1.10/arch/unicore32/mm/fault.c --- linux-4.1.10.orig/arch/unicore32/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/unicore32/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/unicore32/mm/fault.c 2015-10-12 22:33:32.216680718 +0200 @@ -218,7 +218,7 @@ * If we're in an interrupt or have no user * context, we must not take the fault.. @@ -2879,7 +2879,7 @@ diff -Nur linux-4.1.10.orig/arch/unicore32/mm/fault.c linux-4.1.10/arch/unicore3 if (user_mode(regs)) diff -Nur linux-4.1.10.orig/arch/x86/crypto/aesni-intel_glue.c linux-4.1.10/arch/x86/crypto/aesni-intel_glue.c --- linux-4.1.10.orig/arch/x86/crypto/aesni-intel_glue.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/x86/crypto/aesni-intel_glue.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/x86/crypto/aesni-intel_glue.c 2015-10-12 22:33:32.216680718 +0200 @@ -382,14 +382,14 @@ err = blkcipher_walk_virt(desc, &walk); desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; @@ -2974,7 +2974,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/crypto/aesni-intel_glue.c linux-4.1.10/arch } diff -Nur linux-4.1.10.orig/arch/x86/crypto/cast5_avx_glue.c linux-4.1.10/arch/x86/crypto/cast5_avx_glue.c --- linux-4.1.10.orig/arch/x86/crypto/cast5_avx_glue.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/x86/crypto/cast5_avx_glue.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/x86/crypto/cast5_avx_glue.c 2015-10-12 22:33:32.216680718 +0200 @@ -60,7 +60,7 @@ static int ecb_crypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk, bool enc) @@ -3056,7 +3056,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/crypto/cast5_avx_glue.c linux-4.1.10/arch/x err = blkcipher_walk_done(desc, &walk, 0); diff -Nur linux-4.1.10.orig/arch/x86/crypto/glue_helper.c linux-4.1.10/arch/x86/crypto/glue_helper.c --- linux-4.1.10.orig/arch/x86/crypto/glue_helper.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/x86/crypto/glue_helper.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/x86/crypto/glue_helper.c 2015-10-12 22:33:32.216680718 +0200 @@ -39,7 +39,7 @@ void *ctx = crypto_blkcipher_ctx(desc->tfm); const unsigned int bsize = 128 / 8; @@ -3174,7 +3174,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/crypto/glue_helper.c linux-4.1.10/arch/x86/ EXPORT_SYMBOL_GPL(glue_xts_crypt_128bit); diff -Nur linux-4.1.10.orig/arch/x86/include/asm/preempt.h linux-4.1.10/arch/x86/include/asm/preempt.h --- linux-4.1.10.orig/arch/x86/include/asm/preempt.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/x86/include/asm/preempt.h 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/x86/include/asm/preempt.h 2015-10-12 22:33:32.216680718 +0200 @@ -82,17 +82,33 @@ * a decrement which hits zero means we have no preempt_count and should * reschedule. @@ -3212,7 +3212,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/include/asm/preempt.h linux-4.1.10/arch/x86 #ifdef CONFIG_PREEMPT diff -Nur linux-4.1.10.orig/arch/x86/include/asm/signal.h linux-4.1.10/arch/x86/include/asm/signal.h --- linux-4.1.10.orig/arch/x86/include/asm/signal.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/x86/include/asm/signal.h 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/x86/include/asm/signal.h 2015-10-12 22:33:32.216680718 +0200 @@ -23,6 +23,19 @@ unsigned long sig[_NSIG_WORDS]; } sigset_t; @@ -3235,7 +3235,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/include/asm/signal.h linux-4.1.10/arch/x86/ #endif diff -Nur linux-4.1.10.orig/arch/x86/include/asm/stackprotector.h linux-4.1.10/arch/x86/include/asm/stackprotector.h --- linux-4.1.10.orig/arch/x86/include/asm/stackprotector.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/x86/include/asm/stackprotector.h 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/x86/include/asm/stackprotector.h 2015-10-12 22:33:32.216680718 +0200 @@ -57,7 +57,7 @@ */ static __always_inline void boot_init_stack_canary(void) @@ -3264,7 +3264,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/include/asm/stackprotector.h linux-4.1.10/a diff -Nur linux-4.1.10.orig/arch/x86/include/asm/thread_info.h linux-4.1.10/arch/x86/include/asm/thread_info.h --- linux-4.1.10.orig/arch/x86/include/asm/thread_info.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/x86/include/asm/thread_info.h 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/x86/include/asm/thread_info.h 2015-10-12 22:33:32.220680454 +0200 @@ -55,6 +55,8 @@ __u32 status; /* thread synchronous flags */ __u32 cpu; /* current CPU */ @@ -3301,7 +3301,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/include/asm/thread_info.h linux-4.1.10/arch /* diff -Nur linux-4.1.10.orig/arch/x86/include/asm/uaccess_32.h linux-4.1.10/arch/x86/include/asm/uaccess_32.h --- linux-4.1.10.orig/arch/x86/include/asm/uaccess_32.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/x86/include/asm/uaccess_32.h 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/x86/include/asm/uaccess_32.h 2015-10-12 22:33:32.220680454 +0200 @@ -70,7 +70,8 @@ * @from: Source address, in kernel space. * @n: Number of bytes to copy. @@ -3324,7 +3324,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/include/asm/uaccess_32.h linux-4.1.10/arch/ * the specified block with access_ok() before calling this function. diff -Nur linux-4.1.10.orig/arch/x86/include/asm/uaccess.h linux-4.1.10/arch/x86/include/asm/uaccess.h --- linux-4.1.10.orig/arch/x86/include/asm/uaccess.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/x86/include/asm/uaccess.h 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/x86/include/asm/uaccess.h 2015-10-12 22:33:32.220680454 +0200 @@ -74,7 +74,8 @@ * @addr: User space pointer to start of block to check * @size: Size of block to check @@ -3377,7 +3377,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/include/asm/uaccess.h linux-4.1.10/arch/x86 * space. It supports simple types like char and int, but not larger diff -Nur linux-4.1.10.orig/arch/x86/include/asm/uv/uv_bau.h linux-4.1.10/arch/x86/include/asm/uv/uv_bau.h --- linux-4.1.10.orig/arch/x86/include/asm/uv/uv_bau.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/x86/include/asm/uv/uv_bau.h 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/x86/include/asm/uv/uv_bau.h 2015-10-12 22:33:32.220680454 +0200 @@ -615,9 +615,9 @@ cycles_t send_message; cycles_t period_end; @@ -3413,7 +3413,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/include/asm/uv/uv_bau.h linux-4.1.10/arch/x diff -Nur linux-4.1.10.orig/arch/x86/include/asm/uv/uv_hub.h linux-4.1.10/arch/x86/include/asm/uv/uv_hub.h --- linux-4.1.10.orig/arch/x86/include/asm/uv/uv_hub.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/x86/include/asm/uv/uv_hub.h 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/x86/include/asm/uv/uv_hub.h 2015-10-12 22:33:32.220680454 +0200 @@ -492,7 +492,7 @@ unsigned short nr_online_cpus; unsigned short pnode; @@ -3425,7 +3425,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/include/asm/uv/uv_hub.h linux-4.1.10/arch/x extern struct uv_blade_info *uv_blade_info; diff -Nur linux-4.1.10.orig/arch/x86/Kconfig linux-4.1.10/arch/x86/Kconfig --- linux-4.1.10.orig/arch/x86/Kconfig 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/x86/Kconfig 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/x86/Kconfig 2015-10-12 22:33:32.220680454 +0200 @@ -22,6 +22,7 @@ ### Arch settings config X86 @@ -3458,7 +3458,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/Kconfig linux-4.1.10/arch/x86/Kconfig If unsure, say N. diff -Nur linux-4.1.10.orig/arch/x86/kernel/apic/io_apic.c linux-4.1.10/arch/x86/kernel/apic/io_apic.c --- linux-4.1.10.orig/arch/x86/kernel/apic/io_apic.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/x86/kernel/apic/io_apic.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/x86/kernel/apic/io_apic.c 2015-10-12 22:33:32.220680454 +0200 @@ -1891,7 +1891,8 @@ static inline bool ioapic_irqd_mask(struct irq_data *data, struct irq_cfg *cfg) { @@ -3471,7 +3471,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/kernel/apic/io_apic.c linux-4.1.10/arch/x86 } diff -Nur linux-4.1.10.orig/arch/x86/kernel/apic/x2apic_uv_x.c linux-4.1.10/arch/x86/kernel/apic/x2apic_uv_x.c --- linux-4.1.10.orig/arch/x86/kernel/apic/x2apic_uv_x.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/x86/kernel/apic/x2apic_uv_x.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/x86/kernel/apic/x2apic_uv_x.c 2015-10-12 22:33:32.220680454 +0200 @@ -949,7 +949,7 @@ uv_blade_info[blade].pnode = pnode; uv_blade_info[blade].nr_possible_cpus = 0; @@ -3483,7 +3483,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/kernel/apic/x2apic_uv_x.c linux-4.1.10/arch blade++; diff -Nur linux-4.1.10.orig/arch/x86/kernel/asm-offsets.c linux-4.1.10/arch/x86/kernel/asm-offsets.c --- linux-4.1.10.orig/arch/x86/kernel/asm-offsets.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/x86/kernel/asm-offsets.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/x86/kernel/asm-offsets.c 2015-10-12 22:33:32.220680454 +0200 @@ -32,6 +32,7 @@ OFFSET(TI_flags, thread_info, flags); OFFSET(TI_status, thread_info, status); @@ -3500,7 +3500,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/kernel/asm-offsets.c linux-4.1.10/arch/x86/ } diff -Nur linux-4.1.10.orig/arch/x86/kernel/cpu/mcheck/mce.c linux-4.1.10/arch/x86/kernel/cpu/mcheck/mce.c --- linux-4.1.10.orig/arch/x86/kernel/cpu/mcheck/mce.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/x86/kernel/cpu/mcheck/mce.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/x86/kernel/cpu/mcheck/mce.c 2015-10-12 22:33:32.220680454 +0200 @@ -41,6 +41,8 @@ #include #include @@ -3744,7 +3744,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/kernel/cpu/mcheck/mce.c linux-4.1.10/arch/x goto err_out; diff -Nur linux-4.1.10.orig/arch/x86/kernel/dumpstack_32.c linux-4.1.10/arch/x86/kernel/dumpstack_32.c --- linux-4.1.10.orig/arch/x86/kernel/dumpstack_32.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/x86/kernel/dumpstack_32.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/x86/kernel/dumpstack_32.c 2015-10-12 22:33:32.224680189 +0200 @@ -42,7 +42,7 @@ unsigned long *stack, unsigned long bp, const struct stacktrace_ops *ops, void *data) @@ -3765,7 +3765,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/kernel/dumpstack_32.c linux-4.1.10/arch/x86 diff -Nur linux-4.1.10.orig/arch/x86/kernel/dumpstack_64.c linux-4.1.10/arch/x86/kernel/dumpstack_64.c --- linux-4.1.10.orig/arch/x86/kernel/dumpstack_64.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/x86/kernel/dumpstack_64.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/x86/kernel/dumpstack_64.c 2015-10-12 22:33:32.224680189 +0200 @@ -152,7 +152,7 @@ unsigned long *stack, unsigned long bp, const struct stacktrace_ops *ops, void *data) @@ -3804,7 +3804,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/kernel/dumpstack_64.c linux-4.1.10/arch/x86 show_trace_log_lvl(task, regs, sp, bp, log_lvl); diff -Nur linux-4.1.10.orig/arch/x86/kernel/entry_32.S linux-4.1.10/arch/x86/kernel/entry_32.S --- linux-4.1.10.orig/arch/x86/kernel/entry_32.S 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/x86/kernel/entry_32.S 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/x86/kernel/entry_32.S 2015-10-12 22:33:32.224680189 +0200 @@ -359,8 +359,24 @@ ENTRY(resume_kernel) DISABLE_INTERRUPTS(CLBR_ANY) @@ -3850,7 +3850,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/kernel/entry_32.S linux-4.1.10/arch/x86/ker work_notifysig: # deal with pending signals and diff -Nur linux-4.1.10.orig/arch/x86/kernel/entry_64.S linux-4.1.10/arch/x86/kernel/entry_64.S --- linux-4.1.10.orig/arch/x86/kernel/entry_64.S 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/x86/kernel/entry_64.S 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/x86/kernel/entry_64.S 2015-10-12 22:33:32.224680189 +0200 @@ -370,8 +370,8 @@ /* First do a reschedule test. */ /* edx: work, edi: workmask */ @@ -3915,7 +3915,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/kernel/entry_64.S linux-4.1.10/arch/x86/ker idtentry xen_hypervisor_callback xen_do_hypervisor_callback has_error_code=0 diff -Nur linux-4.1.10.orig/arch/x86/kernel/irq_32.c linux-4.1.10/arch/x86/kernel/irq_32.c --- linux-4.1.10.orig/arch/x86/kernel/irq_32.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/x86/kernel/irq_32.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/x86/kernel/irq_32.c 2015-10-12 22:33:32.224680189 +0200 @@ -135,6 +135,7 @@ cpu, per_cpu(hardirq_stack, cpu), per_cpu(softirq_stack, cpu)); } @@ -3934,7 +3934,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/kernel/irq_32.c linux-4.1.10/arch/x86/kerne { diff -Nur linux-4.1.10.orig/arch/x86/kernel/process_32.c linux-4.1.10/arch/x86/kernel/process_32.c --- linux-4.1.10.orig/arch/x86/kernel/process_32.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/x86/kernel/process_32.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/x86/kernel/process_32.c 2015-10-12 22:33:32.224680189 +0200 @@ -35,6 +35,7 @@ #include #include @@ -3990,7 +3990,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/kernel/process_32.c linux-4.1.10/arch/x86/k * This must be done before restoring TLS segments so diff -Nur linux-4.1.10.orig/arch/x86/kernel/signal.c linux-4.1.10/arch/x86/kernel/signal.c --- linux-4.1.10.orig/arch/x86/kernel/signal.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/x86/kernel/signal.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/x86/kernel/signal.c 2015-10-12 22:33:32.224680189 +0200 @@ -723,6 +723,14 @@ { user_exit(); @@ -4008,7 +4008,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/kernel/signal.c linux-4.1.10/arch/x86/kerne diff -Nur linux-4.1.10.orig/arch/x86/kernel/traps.c linux-4.1.10/arch/x86/kernel/traps.c --- linux-4.1.10.orig/arch/x86/kernel/traps.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/x86/kernel/traps.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/x86/kernel/traps.c 2015-10-12 22:33:32.224680189 +0200 @@ -88,9 +88,21 @@ local_irq_enable(); } @@ -4085,7 +4085,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/kernel/traps.c linux-4.1.10/arch/x86/kernel exit: diff -Nur linux-4.1.10.orig/arch/x86/kvm/lapic.c linux-4.1.10/arch/x86/kvm/lapic.c --- linux-4.1.10.orig/arch/x86/kvm/lapic.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/x86/kvm/lapic.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/x86/kvm/lapic.c 2015-10-12 22:33:32.224680189 +0200 @@ -1104,7 +1104,7 @@ static void apic_timer_expired(struct kvm_lapic *apic) { @@ -4188,7 +4188,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/kvm/lapic.c linux-4.1.10/arch/x86/kvm/lapic /* diff -Nur linux-4.1.10.orig/arch/x86/kvm/x86.c linux-4.1.10/arch/x86/kvm/x86.c --- linux-4.1.10.orig/arch/x86/kvm/x86.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/x86/kvm/x86.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/x86/kvm/x86.c 2015-10-12 22:33:32.228679925 +0200 @@ -5813,6 +5813,13 @@ goto out; } @@ -4205,7 +4205,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/kvm/x86.c linux-4.1.10/arch/x86/kvm/x86.c goto out_free_percpu; diff -Nur linux-4.1.10.orig/arch/x86/lib/usercopy_32.c linux-4.1.10/arch/x86/lib/usercopy_32.c --- linux-4.1.10.orig/arch/x86/lib/usercopy_32.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/x86/lib/usercopy_32.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/x86/lib/usercopy_32.c 2015-10-12 22:33:32.228679925 +0200 @@ -647,7 +647,8 @@ * @from: Source address, in kernel space. * @n: Number of bytes to copy. @@ -4228,7 +4228,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/lib/usercopy_32.c linux-4.1.10/arch/x86/lib * diff -Nur linux-4.1.10.orig/arch/x86/mm/fault.c linux-4.1.10/arch/x86/mm/fault.c --- linux-4.1.10.orig/arch/x86/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/x86/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/x86/mm/fault.c 2015-10-12 22:33:32.228679925 +0200 @@ -13,6 +13,7 @@ #include /* hstate_index_to_shift */ #include /* prefetchw */ @@ -4251,7 +4251,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/mm/fault.c linux-4.1.10/arch/x86/mm/fault.c } diff -Nur linux-4.1.10.orig/arch/x86/mm/highmem_32.c linux-4.1.10/arch/x86/mm/highmem_32.c --- linux-4.1.10.orig/arch/x86/mm/highmem_32.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/x86/mm/highmem_32.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/x86/mm/highmem_32.c 2015-10-12 22:33:32.228679925 +0200 @@ -32,10 +32,11 @@ */ void *kmap_atomic_prot(struct page *page, pgprot_t prot) @@ -4297,7 +4297,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/mm/highmem_32.c linux-4.1.10/arch/x86/mm/hi diff -Nur linux-4.1.10.orig/arch/x86/mm/iomap_32.c linux-4.1.10/arch/x86/mm/iomap_32.c --- linux-4.1.10.orig/arch/x86/mm/iomap_32.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/x86/mm/iomap_32.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/x86/mm/iomap_32.c 2015-10-12 22:33:32.228679925 +0200 @@ -56,15 +56,22 @@ void *kmap_atomic_prot_pfn(unsigned long pfn, pgprot_t prot) @@ -4339,7 +4339,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/mm/iomap_32.c linux-4.1.10/arch/x86/mm/ioma EXPORT_SYMBOL_GPL(iounmap_atomic); diff -Nur linux-4.1.10.orig/arch/x86/platform/uv/tlb_uv.c linux-4.1.10/arch/x86/platform/uv/tlb_uv.c --- linux-4.1.10.orig/arch/x86/platform/uv/tlb_uv.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/x86/platform/uv/tlb_uv.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/x86/platform/uv/tlb_uv.c 2015-10-12 22:33:32.228679925 +0200 @@ -714,9 +714,9 @@ quiesce_local_uvhub(hmaster); @@ -4428,7 +4428,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/platform/uv/tlb_uv.c linux-4.1.10/arch/x86/ diff -Nur linux-4.1.10.orig/arch/x86/platform/uv/uv_time.c linux-4.1.10/arch/x86/platform/uv/uv_time.c --- linux-4.1.10.orig/arch/x86/platform/uv/uv_time.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/x86/platform/uv/uv_time.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/x86/platform/uv/uv_time.c 2015-10-12 22:33:32.228679925 +0200 @@ -58,7 +58,7 @@ /* There is one of these allocated per node */ @@ -4511,7 +4511,7 @@ diff -Nur linux-4.1.10.orig/arch/x86/platform/uv/uv_time.c linux-4.1.10/arch/x86 /* diff -Nur linux-4.1.10.orig/arch/xtensa/mm/fault.c linux-4.1.10/arch/xtensa/mm/fault.c --- linux-4.1.10.orig/arch/xtensa/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/xtensa/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/xtensa/mm/fault.c 2015-10-12 22:33:32.228679925 +0200 @@ -15,10 +15,10 @@ #include #include @@ -4535,7 +4535,7 @@ diff -Nur linux-4.1.10.orig/arch/xtensa/mm/fault.c linux-4.1.10/arch/xtensa/mm/f } diff -Nur linux-4.1.10.orig/arch/xtensa/mm/highmem.c linux-4.1.10/arch/xtensa/mm/highmem.c --- linux-4.1.10.orig/arch/xtensa/mm/highmem.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/arch/xtensa/mm/highmem.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/arch/xtensa/mm/highmem.c 2015-10-12 22:33:32.228679925 +0200 @@ -42,6 +42,7 @@ enum fixed_addresses idx; unsigned long vaddr; @@ -4554,7 +4554,7 @@ diff -Nur linux-4.1.10.orig/arch/xtensa/mm/highmem.c linux-4.1.10/arch/xtensa/mm diff -Nur linux-4.1.10.orig/block/blk-core.c linux-4.1.10/block/blk-core.c --- linux-4.1.10.orig/block/blk-core.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/block/blk-core.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/block/blk-core.c 2015-10-12 22:33:32.228679925 +0200 @@ -100,6 +100,9 @@ INIT_LIST_HEAD(&rq->queuelist); @@ -4632,7 +4632,7 @@ diff -Nur linux-4.1.10.orig/block/blk-core.c linux-4.1.10/block/blk-core.c void blk_finish_plug(struct blk_plug *plug) diff -Nur linux-4.1.10.orig/block/blk-ioc.c linux-4.1.10/block/blk-ioc.c --- linux-4.1.10.orig/block/blk-ioc.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/block/blk-ioc.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/block/blk-ioc.c 2015-10-12 22:33:32.228679925 +0200 @@ -7,6 +7,7 @@ #include #include @@ -4661,7 +4661,7 @@ diff -Nur linux-4.1.10.orig/block/blk-ioc.c linux-4.1.10/block/blk-ioc.c } diff -Nur linux-4.1.10.orig/block/blk-iopoll.c linux-4.1.10/block/blk-iopoll.c --- linux-4.1.10.orig/block/blk-iopoll.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/block/blk-iopoll.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/block/blk-iopoll.c 2015-10-12 22:33:32.228679925 +0200 @@ -35,6 +35,7 @@ list_add_tail(&iop->list, this_cpu_ptr(&blk_cpu_iopoll)); __raise_softirq_irqoff(BLOCK_IOPOLL_SOFTIRQ); @@ -4688,7 +4688,7 @@ diff -Nur linux-4.1.10.orig/block/blk-iopoll.c linux-4.1.10/block/blk-iopoll.c return NOTIFY_OK; diff -Nur linux-4.1.10.orig/block/blk-mq.c linux-4.1.10/block/blk-mq.c --- linux-4.1.10.orig/block/blk-mq.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/block/blk-mq.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/block/blk-mq.c 2015-10-12 22:33:32.228679925 +0200 @@ -88,7 +88,7 @@ if (!(gfp & __GFP_WAIT)) return -EBUSY; @@ -4826,7 +4826,7 @@ diff -Nur linux-4.1.10.orig/block/blk-mq.c linux-4.1.10/block/blk-mq.c /* diff -Nur linux-4.1.10.orig/block/blk-mq-cpu.c linux-4.1.10/block/blk-mq-cpu.c --- linux-4.1.10.orig/block/blk-mq-cpu.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/block/blk-mq-cpu.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/block/blk-mq-cpu.c 2015-10-12 22:33:32.228679925 +0200 @@ -16,7 +16,7 @@ #include "blk-mq.h" @@ -4880,7 +4880,7 @@ diff -Nur linux-4.1.10.orig/block/blk-mq-cpu.c linux-4.1.10/block/blk-mq-cpu.c void blk_mq_init_cpu_notifier(struct blk_mq_cpu_notifier *notifier, diff -Nur linux-4.1.10.orig/block/blk-mq.h linux-4.1.10/block/blk-mq.h --- linux-4.1.10.orig/block/blk-mq.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/block/blk-mq.h 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/block/blk-mq.h 2015-10-12 22:33:32.228679925 +0200 @@ -76,7 +76,10 @@ static inline struct blk_mq_ctx *__blk_mq_get_ctx(struct request_queue *q, unsigned int cpu) @@ -4910,7 +4910,7 @@ diff -Nur linux-4.1.10.orig/block/blk-mq.h linux-4.1.10/block/blk-mq.h struct blk_mq_alloc_data { diff -Nur linux-4.1.10.orig/block/blk-softirq.c linux-4.1.10/block/blk-softirq.c --- linux-4.1.10.orig/block/blk-softirq.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/block/blk-softirq.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/block/blk-softirq.c 2015-10-12 22:33:32.228679925 +0200 @@ -51,6 +51,7 @@ raise_softirq_irqoff(BLOCK_SOFTIRQ); @@ -4937,7 +4937,7 @@ diff -Nur linux-4.1.10.orig/block/blk-softirq.c linux-4.1.10/block/blk-softirq.c /** diff -Nur linux-4.1.10.orig/block/bounce.c linux-4.1.10/block/bounce.c --- linux-4.1.10.orig/block/bounce.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/block/bounce.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/block/bounce.c 2015-10-12 22:33:32.228679925 +0200 @@ -54,11 +54,11 @@ unsigned long flags; unsigned char *vto; @@ -4954,7 +4954,7 @@ diff -Nur linux-4.1.10.orig/block/bounce.c linux-4.1.10/block/bounce.c #else /* CONFIG_HIGHMEM */ diff -Nur linux-4.1.10.orig/crypto/algapi.c linux-4.1.10/crypto/algapi.c --- linux-4.1.10.orig/crypto/algapi.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/crypto/algapi.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/crypto/algapi.c 2015-10-12 22:33:32.232679661 +0200 @@ -695,13 +695,13 @@ int crypto_register_notifier(struct notifier_block *nb) @@ -4973,7 +4973,7 @@ diff -Nur linux-4.1.10.orig/crypto/algapi.c linux-4.1.10/crypto/algapi.c diff -Nur linux-4.1.10.orig/crypto/api.c linux-4.1.10/crypto/api.c --- linux-4.1.10.orig/crypto/api.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/crypto/api.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/crypto/api.c 2015-10-12 22:33:32.232679661 +0200 @@ -31,7 +31,7 @@ DECLARE_RWSEM(crypto_alg_sem); EXPORT_SYMBOL_GPL(crypto_alg_sem); @@ -4998,7 +4998,7 @@ diff -Nur linux-4.1.10.orig/crypto/api.c linux-4.1.10/crypto/api.c return ok; diff -Nur linux-4.1.10.orig/crypto/internal.h linux-4.1.10/crypto/internal.h --- linux-4.1.10.orig/crypto/internal.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/crypto/internal.h 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/crypto/internal.h 2015-10-12 22:33:32.232679661 +0200 @@ -48,7 +48,7 @@ extern struct list_head crypto_alg_list; @@ -5019,7 +5019,7 @@ diff -Nur linux-4.1.10.orig/crypto/internal.h linux-4.1.10/crypto/internal.h #endif /* _CRYPTO_INTERNAL_H */ diff -Nur linux-4.1.10.orig/Documentation/hwlat_detector.txt linux-4.1.10/Documentation/hwlat_detector.txt --- linux-4.1.10.orig/Documentation/hwlat_detector.txt 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.10/Documentation/hwlat_detector.txt 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/Documentation/hwlat_detector.txt 2015-10-12 22:33:32.232679661 +0200 @@ -0,0 +1,64 @@ +Introduction: +------------- @@ -5087,7 +5087,7 @@ diff -Nur linux-4.1.10.orig/Documentation/hwlat_detector.txt linux-4.1.10/Docume +consumed by reading from the "sample" (pipe) debugfs file interface. diff -Nur linux-4.1.10.orig/Documentation/sysrq.txt linux-4.1.10/Documentation/sysrq.txt --- linux-4.1.10.orig/Documentation/sysrq.txt 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/Documentation/sysrq.txt 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/Documentation/sysrq.txt 2015-10-12 22:33:32.232679661 +0200 @@ -59,10 +59,17 @@ On other - If you know of the key combos for other architectures, please let me know so I can add them to this section. @@ -5110,7 +5110,7 @@ diff -Nur linux-4.1.10.orig/Documentation/sysrq.txt linux-4.1.10/Documentation/s 'b' - Will immediately reboot the system without syncing or unmounting diff -Nur linux-4.1.10.orig/Documentation/trace/histograms.txt linux-4.1.10/Documentation/trace/histograms.txt --- linux-4.1.10.orig/Documentation/trace/histograms.txt 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.10/Documentation/trace/histograms.txt 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/Documentation/trace/histograms.txt 2015-10-12 22:33:32.232679661 +0200 @@ -0,0 +1,186 @@ + Using the Linux Kernel Latency Histograms + @@ -5300,7 +5300,7 @@ diff -Nur linux-4.1.10.orig/Documentation/trace/histograms.txt linux-4.1.10/Docu +These data are also reset when the wakeup histogram is reset. diff -Nur linux-4.1.10.orig/drivers/acpi/acpica/acglobal.h linux-4.1.10/drivers/acpi/acpica/acglobal.h --- linux-4.1.10.orig/drivers/acpi/acpica/acglobal.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/acpi/acpica/acglobal.h 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/drivers/acpi/acpica/acglobal.h 2015-10-12 22:33:32.232679661 +0200 @@ -112,7 +112,7 @@ * interrupt level */ @@ -5312,7 +5312,7 @@ diff -Nur linux-4.1.10.orig/drivers/acpi/acpica/acglobal.h linux-4.1.10/drivers/ /* Mutex for _OSI support */ diff -Nur linux-4.1.10.orig/drivers/acpi/acpica/hwregs.c linux-4.1.10/drivers/acpi/acpica/hwregs.c --- linux-4.1.10.orig/drivers/acpi/acpica/hwregs.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/acpi/acpica/hwregs.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/drivers/acpi/acpica/hwregs.c 2015-10-12 22:33:32.232679661 +0200 @@ -269,14 +269,14 @@ ACPI_BITMASK_ALL_FIXED_STATUS, ACPI_FORMAT_UINT64(acpi_gbl_xpm1a_status.address))); @@ -5332,7 +5332,7 @@ diff -Nur linux-4.1.10.orig/drivers/acpi/acpica/hwregs.c linux-4.1.10/drivers/ac goto exit; diff -Nur linux-4.1.10.orig/drivers/acpi/acpica/hwxface.c linux-4.1.10/drivers/acpi/acpica/hwxface.c --- linux-4.1.10.orig/drivers/acpi/acpica/hwxface.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/acpi/acpica/hwxface.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/drivers/acpi/acpica/hwxface.c 2015-10-12 22:33:32.232679661 +0200 @@ -374,7 +374,7 @@ return_ACPI_STATUS(AE_BAD_PARAMETER); } @@ -5353,7 +5353,7 @@ diff -Nur linux-4.1.10.orig/drivers/acpi/acpica/hwxface.c linux-4.1.10/drivers/a diff -Nur linux-4.1.10.orig/drivers/acpi/acpica/utmutex.c linux-4.1.10/drivers/acpi/acpica/utmutex.c --- linux-4.1.10.orig/drivers/acpi/acpica/utmutex.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/acpi/acpica/utmutex.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/drivers/acpi/acpica/utmutex.c 2015-10-12 22:33:32.232679661 +0200 @@ -88,7 +88,7 @@ return_ACPI_STATUS (status); } @@ -5374,7 +5374,7 @@ diff -Nur linux-4.1.10.orig/drivers/acpi/acpica/utmutex.c linux-4.1.10/drivers/a /* Delete the reader/writer lock */ diff -Nur linux-4.1.10.orig/drivers/ata/libata-sff.c linux-4.1.10/drivers/ata/libata-sff.c --- linux-4.1.10.orig/drivers/ata/libata-sff.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/ata/libata-sff.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/drivers/ata/libata-sff.c 2015-10-12 22:33:32.232679661 +0200 @@ -678,9 +678,9 @@ unsigned long flags; unsigned int consumed; @@ -5425,7 +5425,7 @@ diff -Nur linux-4.1.10.orig/drivers/ata/libata-sff.c linux-4.1.10/drivers/ata/li consumed = ap->ops->sff_data_xfer(dev, buf + offset, diff -Nur linux-4.1.10.orig/drivers/char/random.c linux-4.1.10/drivers/char/random.c --- linux-4.1.10.orig/drivers/char/random.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/char/random.c 2015-10-07 18:00:07.000000000 +0200 ++++ linux-4.1.10/drivers/char/random.c 2015-10-12 22:33:32.232679661 +0200 @@ -776,8 +776,6 @@ } sample; long delta, delta2, delta3; @@ -5479,7 +5479,7 @@ diff -Nur linux-4.1.10.orig/drivers/char/random.c linux-4.1.10/drivers/char/rand add_interrupt_bench(cycles); diff -Nur linux-4.1.10.orig/drivers/clocksource/tcb_clksrc.c linux-4.1.10/drivers/clocksource/tcb_clksrc.c --- linux-4.1.10.orig/drivers/clocksource/tcb_clksrc.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/clocksource/tcb_clksrc.c 2015-10-07 18:00:08.000000000 +0200 ++++ linux-4.1.10/drivers/clocksource/tcb_clksrc.c 2015-10-12 22:33:32.232679661 +0200 @@ -23,8 +23,7 @@ * this 32 bit free-running counter. the second channel is not used. * @@ -5596,7 +5596,7 @@ diff -Nur linux-4.1.10.orig/drivers/clocksource/tcb_clksrc.c linux-4.1.10/driver diff -Nur linux-4.1.10.orig/drivers/clocksource/timer-atmel-pit.c linux-4.1.10/drivers/clocksource/timer-atmel-pit.c --- linux-4.1.10.orig/drivers/clocksource/timer-atmel-pit.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/clocksource/timer-atmel-pit.c 2015-10-07 18:00:08.000000000 +0200 ++++ linux-4.1.10/drivers/clocksource/timer-atmel-pit.c 2015-10-12 22:33:32.232679661 +0200 @@ -90,6 +90,7 @@ return elapsed; } @@ -5624,7 +5624,7 @@ diff -Nur linux-4.1.10.orig/drivers/clocksource/timer-atmel-pit.c linux-4.1.10/d break; diff -Nur linux-4.1.10.orig/drivers/clocksource/timer-atmel-st.c linux-4.1.10/drivers/clocksource/timer-atmel-st.c --- linux-4.1.10.orig/drivers/clocksource/timer-atmel-st.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/clocksource/timer-atmel-st.c 2015-10-07 18:00:08.000000000 +0200 ++++ linux-4.1.10/drivers/clocksource/timer-atmel-st.c 2015-10-12 22:33:32.232679661 +0200 @@ -131,6 +131,7 @@ break; case CLOCK_EVT_MODE_SHUTDOWN: @@ -5635,7 +5635,7 @@ diff -Nur linux-4.1.10.orig/drivers/clocksource/timer-atmel-st.c linux-4.1.10/dr break; diff -Nur linux-4.1.10.orig/drivers/cpufreq/cpufreq.c linux-4.1.10/drivers/cpufreq/cpufreq.c --- linux-4.1.10.orig/drivers/cpufreq/cpufreq.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/cpufreq/cpufreq.c 2015-10-07 18:00:08.000000000 +0200 ++++ linux-4.1.10/drivers/cpufreq/cpufreq.c 2015-10-12 22:33:32.232679661 +0200 @@ -64,12 +64,6 @@ return cpufreq_driver->target_index || cpufreq_driver->target; } @@ -5774,7 +5774,7 @@ diff -Nur linux-4.1.10.orig/drivers/cpufreq/cpufreq.c linux-4.1.10/drivers/cpufr } diff -Nur linux-4.1.10.orig/drivers/cpufreq/Kconfig.x86 linux-4.1.10/drivers/cpufreq/Kconfig.x86 --- linux-4.1.10.orig/drivers/cpufreq/Kconfig.x86 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/cpufreq/Kconfig.x86 2015-10-07 18:00:08.000000000 +0200 ++++ linux-4.1.10/drivers/cpufreq/Kconfig.x86 2015-10-12 22:33:32.232679661 +0200 @@ -123,7 +123,7 @@ config X86_POWERNOW_K8 @@ -5786,7 +5786,7 @@ diff -Nur linux-4.1.10.orig/drivers/cpufreq/Kconfig.x86 linux-4.1.10/drivers/cpu Support for K10 and newer processors is now in acpi-cpufreq. diff -Nur linux-4.1.10.orig/drivers/gpio/gpio-omap.c linux-4.1.10/drivers/gpio/gpio-omap.c --- linux-4.1.10.orig/drivers/gpio/gpio-omap.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/gpio/gpio-omap.c 2015-10-07 18:00:08.000000000 +0200 ++++ linux-4.1.10/drivers/gpio/gpio-omap.c 2015-10-12 22:33:32.236679397 +0200 @@ -57,7 +57,7 @@ u32 saved_datain; u32 level_mask; @@ -6069,7 +6069,7 @@ diff -Nur linux-4.1.10.orig/drivers/gpio/gpio-omap.c linux-4.1.10/drivers/gpio/g } diff -Nur linux-4.1.10.orig/drivers/gpu/drm/i915/i915_gem_execbuffer.c linux-4.1.10/drivers/gpu/drm/i915/i915_gem_execbuffer.c --- linux-4.1.10.orig/drivers/gpu/drm/i915/i915_gem_execbuffer.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/gpu/drm/i915/i915_gem_execbuffer.c 2015-10-07 18:00:08.000000000 +0200 ++++ linux-4.1.10/drivers/gpu/drm/i915/i915_gem_execbuffer.c 2015-10-12 22:33:32.236679397 +0200 @@ -32,6 +32,7 @@ #include "i915_trace.h" #include "intel_drv.h" @@ -6099,7 +6099,7 @@ diff -Nur linux-4.1.10.orig/drivers/gpu/drm/i915/i915_gem_execbuffer.c linux-4.1 i915_gem_execbuffer_retire_commands(dev, file, ring, batch_obj); diff -Nur linux-4.1.10.orig/drivers/gpu/drm/i915/i915_gem_shrinker.c linux-4.1.10/drivers/gpu/drm/i915/i915_gem_shrinker.c --- linux-4.1.10.orig/drivers/gpu/drm/i915/i915_gem_shrinker.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/gpu/drm/i915/i915_gem_shrinker.c 2015-10-07 18:00:08.000000000 +0200 ++++ linux-4.1.10/drivers/gpu/drm/i915/i915_gem_shrinker.c 2015-10-12 22:33:32.236679397 +0200 @@ -39,7 +39,7 @@ if (!mutex_is_locked(mutex)) return false; @@ -6111,7 +6111,7 @@ diff -Nur linux-4.1.10.orig/drivers/gpu/drm/i915/i915_gem_shrinker.c linux-4.1.1 /* Since UP may be pre-empted, we cannot assume that we own the lock */ diff -Nur linux-4.1.10.orig/drivers/gpu/drm/i915/intel_display.c linux-4.1.10/drivers/gpu/drm/i915/intel_display.c --- linux-4.1.10.orig/drivers/gpu/drm/i915/intel_display.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/gpu/drm/i915/intel_display.c 2015-10-07 18:00:08.000000000 +0200 ++++ linux-4.1.10/drivers/gpu/drm/i915/intel_display.c 2015-10-12 22:33:32.236679397 +0200 @@ -10086,7 +10086,7 @@ struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); @@ -6123,7 +6123,7 @@ diff -Nur linux-4.1.10.orig/drivers/gpu/drm/i915/intel_display.c linux-4.1.10/dr return; diff -Nur linux-4.1.10.orig/drivers/i2c/busses/i2c-omap.c linux-4.1.10/drivers/i2c/busses/i2c-omap.c --- linux-4.1.10.orig/drivers/i2c/busses/i2c-omap.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/i2c/busses/i2c-omap.c 2015-10-07 18:00:08.000000000 +0200 ++++ linux-4.1.10/drivers/i2c/busses/i2c-omap.c 2015-10-12 22:33:32.244678868 +0200 @@ -996,15 +996,12 @@ u16 mask; u16 stat; @@ -6143,7 +6143,7 @@ diff -Nur linux-4.1.10.orig/drivers/i2c/busses/i2c-omap.c linux-4.1.10/drivers/i diff -Nur linux-4.1.10.orig/drivers/ide/alim15x3.c linux-4.1.10/drivers/ide/alim15x3.c --- linux-4.1.10.orig/drivers/ide/alim15x3.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/ide/alim15x3.c 2015-10-07 18:00:08.000000000 +0200 ++++ linux-4.1.10/drivers/ide/alim15x3.c 2015-10-12 22:33:32.244678868 +0200 @@ -234,7 +234,7 @@ isa_dev = pci_get_device(PCI_VENDOR_ID_AL, PCI_DEVICE_ID_AL_M1533, NULL); @@ -6164,7 +6164,7 @@ diff -Nur linux-4.1.10.orig/drivers/ide/alim15x3.c linux-4.1.10/drivers/ide/alim diff -Nur linux-4.1.10.orig/drivers/ide/hpt366.c linux-4.1.10/drivers/ide/hpt366.c --- linux-4.1.10.orig/drivers/ide/hpt366.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/ide/hpt366.c 2015-10-07 18:00:08.000000000 +0200 ++++ linux-4.1.10/drivers/ide/hpt366.c 2015-10-12 22:33:32.244678868 +0200 @@ -1241,7 +1241,7 @@ dma_old = inb(base + 2); @@ -6185,7 +6185,7 @@ diff -Nur linux-4.1.10.orig/drivers/ide/hpt366.c linux-4.1.10/drivers/ide/hpt366 hwif->name, base, base + 7); diff -Nur linux-4.1.10.orig/drivers/ide/ide-io.c linux-4.1.10/drivers/ide/ide-io.c --- linux-4.1.10.orig/drivers/ide/ide-io.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/ide/ide-io.c 2015-10-07 18:00:08.000000000 +0200 ++++ linux-4.1.10/drivers/ide/ide-io.c 2015-10-12 22:33:32.244678868 +0200 @@ -659,7 +659,7 @@ /* disable_irq_nosync ?? */ disable_irq(hwif->irq); @@ -6197,7 +6197,7 @@ diff -Nur linux-4.1.10.orig/drivers/ide/ide-io.c linux-4.1.10/drivers/ide/ide-io } else if (drive_is_ready(drive)) { diff -Nur linux-4.1.10.orig/drivers/ide/ide-iops.c linux-4.1.10/drivers/ide/ide-iops.c --- linux-4.1.10.orig/drivers/ide/ide-iops.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/ide/ide-iops.c 2015-10-07 18:00:08.000000000 +0200 ++++ linux-4.1.10/drivers/ide/ide-iops.c 2015-10-12 22:33:32.244678868 +0200 @@ -129,12 +129,12 @@ if ((stat & ATA_BUSY) == 0) break; @@ -6215,7 +6215,7 @@ diff -Nur linux-4.1.10.orig/drivers/ide/ide-iops.c linux-4.1.10/drivers/ide/ide- * Allow status to settle, then read it again. diff -Nur linux-4.1.10.orig/drivers/ide/ide-io-std.c linux-4.1.10/drivers/ide/ide-io-std.c --- linux-4.1.10.orig/drivers/ide/ide-io-std.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/ide/ide-io-std.c 2015-10-07 18:00:08.000000000 +0200 ++++ linux-4.1.10/drivers/ide/ide-io-std.c 2015-10-12 22:33:32.244678868 +0200 @@ -175,7 +175,7 @@ unsigned long uninitialized_var(flags); @@ -6254,7 +6254,7 @@ diff -Nur linux-4.1.10.orig/drivers/ide/ide-io-std.c linux-4.1.10/drivers/ide/id return; diff -Nur linux-4.1.10.orig/drivers/ide/ide-probe.c linux-4.1.10/drivers/ide/ide-probe.c --- linux-4.1.10.orig/drivers/ide/ide-probe.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/ide/ide-probe.c 2015-10-07 18:00:08.000000000 +0200 ++++ linux-4.1.10/drivers/ide/ide-probe.c 2015-10-12 22:33:32.244678868 +0200 @@ -196,10 +196,10 @@ int bswap = 1; @@ -6270,7 +6270,7 @@ diff -Nur linux-4.1.10.orig/drivers/ide/ide-probe.c linux-4.1.10/drivers/ide/ide #ifdef DEBUG diff -Nur linux-4.1.10.orig/drivers/ide/ide-taskfile.c linux-4.1.10/drivers/ide/ide-taskfile.c --- linux-4.1.10.orig/drivers/ide/ide-taskfile.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/ide/ide-taskfile.c 2015-10-07 18:00:08.000000000 +0200 ++++ linux-4.1.10/drivers/ide/ide-taskfile.c 2015-10-12 22:33:32.244678868 +0200 @@ -250,7 +250,7 @@ page_is_high = PageHighMem(page); @@ -6300,7 +6300,7 @@ diff -Nur linux-4.1.10.orig/drivers/ide/ide-taskfile.c linux-4.1.10/drivers/ide/ diff -Nur linux-4.1.10.orig/drivers/infiniband/ulp/ipoib/ipoib_multicast.c linux-4.1.10/drivers/infiniband/ulp/ipoib/ipoib_multicast.c --- linux-4.1.10.orig/drivers/infiniband/ulp/ipoib/ipoib_multicast.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/infiniband/ulp/ipoib/ipoib_multicast.c 2015-10-07 18:00:08.000000000 +0200 ++++ linux-4.1.10/drivers/infiniband/ulp/ipoib/ipoib_multicast.c 2015-10-12 22:33:32.244678868 +0200 @@ -821,7 +821,7 @@ ipoib_dbg_mcast(priv, "restarting multicast task\n"); @@ -6321,7 +6321,7 @@ diff -Nur linux-4.1.10.orig/drivers/infiniband/ulp/ipoib/ipoib_multicast.c linux * make sure the in-flight joins have finished before we attempt diff -Nur linux-4.1.10.orig/drivers/input/gameport/gameport.c linux-4.1.10/drivers/input/gameport/gameport.c --- linux-4.1.10.orig/drivers/input/gameport/gameport.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/input/gameport/gameport.c 2015-10-07 18:00:08.000000000 +0200 ++++ linux-4.1.10/drivers/input/gameport/gameport.c 2015-10-12 22:33:32.244678868 +0200 @@ -124,12 +124,12 @@ tx = 1 << 30; @@ -6353,7 +6353,7 @@ diff -Nur linux-4.1.10.orig/drivers/input/gameport/gameport.c linux-4.1.10/drive } diff -Nur linux-4.1.10.orig/drivers/leds/trigger/Kconfig linux-4.1.10/drivers/leds/trigger/Kconfig --- linux-4.1.10.orig/drivers/leds/trigger/Kconfig 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/leds/trigger/Kconfig 2015-10-07 18:00:08.000000000 +0200 ++++ linux-4.1.10/drivers/leds/trigger/Kconfig 2015-10-12 22:33:32.244678868 +0200 @@ -61,7 +61,7 @@ config LEDS_TRIGGER_CPU @@ -6365,7 +6365,7 @@ diff -Nur linux-4.1.10.orig/drivers/leds/trigger/Kconfig linux-4.1.10/drivers/le the active CPUs across an array of LEDs so you can see which diff -Nur linux-4.1.10.orig/drivers/md/bcache/Kconfig linux-4.1.10/drivers/md/bcache/Kconfig --- linux-4.1.10.orig/drivers/md/bcache/Kconfig 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/md/bcache/Kconfig 2015-10-07 18:00:08.000000000 +0200 ++++ linux-4.1.10/drivers/md/bcache/Kconfig 2015-10-12 22:33:32.244678868 +0200 @@ -1,6 +1,7 @@ config BCACHE @@ -6376,7 +6376,7 @@ diff -Nur linux-4.1.10.orig/drivers/md/bcache/Kconfig linux-4.1.10/drivers/md/bc a btree for indexing and the layout is optimized for SSDs. diff -Nur linux-4.1.10.orig/drivers/md/dm.c linux-4.1.10/drivers/md/dm.c --- linux-4.1.10.orig/drivers/md/dm.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/md/dm.c 2015-10-07 18:00:08.000000000 +0200 ++++ linux-4.1.10/drivers/md/dm.c 2015-10-12 22:33:32.244678868 +0200 @@ -2132,7 +2132,7 @@ /* Establish tio->ti before queuing work (map_tio_request) */ tio->ti = ti; @@ -6388,7 +6388,7 @@ diff -Nur linux-4.1.10.orig/drivers/md/dm.c linux-4.1.10/drivers/md/dm.c goto out; diff -Nur linux-4.1.10.orig/drivers/md/raid5.c linux-4.1.10/drivers/md/raid5.c --- linux-4.1.10.orig/drivers/md/raid5.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/md/raid5.c 2015-10-07 18:00:08.000000000 +0200 ++++ linux-4.1.10/drivers/md/raid5.c 2015-10-12 22:33:32.248678604 +0200 @@ -1918,8 +1918,9 @@ struct raid5_percpu *percpu; unsigned long cpu; @@ -6418,63643 +6418,20330 @@ diff -Nur linux-4.1.10.orig/drivers/md/raid5.c linux-4.1.10/drivers/md/raid5.c } put_online_cpus(); -diff -Nur linux-4.1.10.orig/drivers/md/raid5.c.orig linux-4.1.10/drivers/md/raid5.c.orig ---- linux-4.1.10.orig/drivers/md/raid5.c.orig 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.10/drivers/md/raid5.c.orig 2015-10-03 13:49:38.000000000 +0200 -@@ -0,0 +1,7853 @@ +diff -Nur linux-4.1.10.orig/drivers/md/raid5.h linux-4.1.10/drivers/md/raid5.h +--- linux-4.1.10.orig/drivers/md/raid5.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/md/raid5.h 2015-10-12 22:33:32.252678339 +0200 +@@ -495,6 +495,7 @@ + int recovery_disabled; + /* per cpu variables */ + struct raid5_percpu { ++ spinlock_t lock; /* Protection for -RT */ + struct page *spare_page; /* Used when checking P/Q in raid6 */ + struct flex_array *scribble; /* space for constructing buffer + * lists and performing address +diff -Nur linux-4.1.10.orig/drivers/misc/hwlat_detector.c linux-4.1.10/drivers/misc/hwlat_detector.c +--- linux-4.1.10.orig/drivers/misc/hwlat_detector.c 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/drivers/misc/hwlat_detector.c 2015-10-12 22:33:32.252678339 +0200 +@@ -0,0 +1,1240 @@ +/* -+ * raid5.c : Multiple Devices driver for Linux -+ * Copyright (C) 1996, 1997 Ingo Molnar, Miguel de Icaza, Gadi Oxman -+ * Copyright (C) 1999, 2000 Ingo Molnar -+ * Copyright (C) 2002, 2003 H. Peter Anvin ++ * hwlat_detector.c - A simple Hardware Latency detector. + * -+ * RAID-4/5/6 management functions. -+ * Thanks to Penguin Computing for making the RAID-6 development possible -+ * by donating a test server! ++ * Use this module to detect large system latencies induced by the behavior of ++ * certain underlying system hardware or firmware, independent of Linux itself. ++ * The code was developed originally to detect the presence of SMIs on Intel ++ * and AMD systems, although there is no dependency upon x86 herein. + * -+ * This program is free software; you can redistribute it and/or modify -+ * it under the terms of the GNU General Public License as published by -+ * the Free Software Foundation; either version 2, or (at your option) -+ * any later version. ++ * The classical example usage of this module is in detecting the presence of ++ * SMIs or System Management Interrupts on Intel and AMD systems. An SMI is a ++ * somewhat special form of hardware interrupt spawned from earlier CPU debug ++ * modes in which the (BIOS/EFI/etc.) firmware arranges for the South Bridge ++ * LPC (or other device) to generate a special interrupt under certain ++ * circumstances, for example, upon expiration of a special SMI timer device, ++ * due to certain external thermal readings, on certain I/O address accesses, ++ * and other situations. An SMI hits a special CPU pin, triggers a special ++ * SMI mode (complete with special memory map), and the OS is unaware. + * -+ * You should have received a copy of the GNU General Public License -+ * (for example /usr/src/linux/COPYING); if not, write to the Free -+ * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. -+ */ -+ -+/* -+ * BITMAP UNPLUGGING: ++ * Although certain hardware-inducing latencies are necessary (for example, ++ * a modern system often requires an SMI handler for correct thermal control ++ * and remote management) they can wreak havoc upon any OS-level performance ++ * guarantees toward low-latency, especially when the OS is not even made ++ * aware of the presence of these interrupts. For this reason, we need a ++ * somewhat brute force mechanism to detect these interrupts. In this case, ++ * we do it by hogging all of the CPU(s) for configurable timer intervals, ++ * sampling the built-in CPU timer, looking for discontiguous readings. ++ * ++ * WARNING: This implementation necessarily introduces latencies. Therefore, ++ * you should NEVER use this module in a production environment ++ * requiring any kind of low-latency performance guarantee(s). ++ * ++ * Copyright (C) 2008-2009 Jon Masters, Red Hat, Inc. + * -+ * The sequencing for updating the bitmap reliably is a little -+ * subtle (and I got it wrong the first time) so it deserves some -+ * explanation. ++ * Includes useful feedback from Clark Williams + * -+ * We group bitmap updates into batches. Each batch has a number. -+ * We may write out several batches at once, but that isn't very important. -+ * conf->seq_write is the number of the last batch successfully written. -+ * conf->seq_flush is the number of the last batch that was closed to -+ * new additions. -+ * When we discover that we will need to write to any block in a stripe -+ * (in add_stripe_bio) we update the in-memory bitmap and record in sh->bm_seq -+ * the number of the batch it will be in. This is seq_flush+1. -+ * When we are ready to do a write, if that batch hasn't been written yet, -+ * we plug the array and queue the stripe for later. -+ * When an unplug happens, we increment bm_flush, thus closing the current -+ * batch. -+ * When we notice that bm_flush > bm_write, we write out all pending updates -+ * to the bitmap, and advance bm_write to where bm_flush was. -+ * This may occasionally write a bit out twice, but is sure never to -+ * miss any bits. ++ * This file is licensed under the terms of the GNU General Public ++ * License version 2. This program is licensed "as is" without any ++ * warranty of any kind, whether express or implied. + */ + -+#include -+#include -+#include -+#include +#include -+#include ++#include ++#include ++#include ++#include ++#include ++#include +#include -+#include ++#include ++#include ++#include +#include -+#include -+#include -+#include -+#include -+ -+#include "md.h" -+#include "raid5.h" -+#include "raid0.h" -+#include "bitmap.h" -+ -+#define cpu_to_group(cpu) cpu_to_node(cpu) -+#define ANY_GROUP NUMA_NO_NODE -+ -+static bool devices_handle_discard_safely = false; -+module_param(devices_handle_discard_safely, bool, 0644); -+MODULE_PARM_DESC(devices_handle_discard_safely, -+ "Set to Y if all devices in each array reliably return zeroes on reads from discarded regions"); -+static struct workqueue_struct *raid5_wq; -+/* -+ * Stripe cache -+ */ ++#include + -+#define NR_STRIPES 256 -+#define STRIPE_SIZE PAGE_SIZE -+#define STRIPE_SHIFT (PAGE_SHIFT - 9) -+#define STRIPE_SECTORS (STRIPE_SIZE>>9) -+#define IO_THRESHOLD 1 -+#define BYPASS_THRESHOLD 1 -+#define NR_HASH (PAGE_SIZE / sizeof(struct hlist_head)) -+#define HASH_MASK (NR_HASH - 1) -+#define MAX_STRIPE_BATCH 8 -+ -+static inline struct hlist_head *stripe_hash(struct r5conf *conf, sector_t sect) -+{ -+ int hash = (sect >> STRIPE_SHIFT) & HASH_MASK; -+ return &conf->stripe_hashtbl[hash]; -+} ++#define BUF_SIZE_DEFAULT 262144UL /* 8K*(sizeof(entry)) */ ++#define BUF_FLAGS (RB_FL_OVERWRITE) /* no block on full */ ++#define U64STR_SIZE 22 /* 20 digits max */ + -+static inline int stripe_hash_locks_hash(sector_t sect) -+{ -+ return (sect >> STRIPE_SHIFT) & STRIPE_HASH_LOCKS_MASK; -+} ++#define VERSION "1.0.0" ++#define BANNER "hwlat_detector: " ++#define DRVNAME "hwlat_detector" ++#define DEFAULT_SAMPLE_WINDOW 1000000 /* 1s */ ++#define DEFAULT_SAMPLE_WIDTH 500000 /* 0.5s */ ++#define DEFAULT_LAT_THRESHOLD 10 /* 10us */ + -+static inline void lock_device_hash_lock(struct r5conf *conf, int hash) -+{ -+ spin_lock_irq(conf->hash_locks + hash); -+ spin_lock(&conf->device_lock); -+} ++/* Module metadata */ + -+static inline void unlock_device_hash_lock(struct r5conf *conf, int hash) -+{ -+ spin_unlock(&conf->device_lock); -+ spin_unlock_irq(conf->hash_locks + hash); -+} ++MODULE_LICENSE("GPL"); ++MODULE_AUTHOR("Jon Masters "); ++MODULE_DESCRIPTION("A simple hardware latency detector"); ++MODULE_VERSION(VERSION); + -+static inline void lock_all_device_hash_locks_irq(struct r5conf *conf) -+{ -+ int i; -+ local_irq_disable(); -+ spin_lock(conf->hash_locks); -+ for (i = 1; i < NR_STRIPE_HASH_LOCKS; i++) -+ spin_lock_nest_lock(conf->hash_locks + i, conf->hash_locks); -+ spin_lock(&conf->device_lock); -+} ++/* Module parameters */ + -+static inline void unlock_all_device_hash_locks_irq(struct r5conf *conf) -+{ -+ int i; -+ spin_unlock(&conf->device_lock); -+ for (i = NR_STRIPE_HASH_LOCKS; i; i--) -+ spin_unlock(conf->hash_locks + i - 1); -+ local_irq_enable(); -+} ++static int debug; ++static int enabled; ++static int threshold; + -+/* bio's attached to a stripe+device for I/O are linked together in bi_sector -+ * order without overlap. There may be several bio's per stripe+device, and -+ * a bio could span several devices. -+ * When walking this list for a particular stripe+device, we must never proceed -+ * beyond a bio that extends past this device, as the next bio might no longer -+ * be valid. -+ * This function is used to determine the 'next' bio in the list, given the sector -+ * of the current stripe+device -+ */ -+static inline struct bio *r5_next_bio(struct bio *bio, sector_t sector) -+{ -+ int sectors = bio_sectors(bio); -+ if (bio->bi_iter.bi_sector + sectors < sector + STRIPE_SECTORS) -+ return bio->bi_next; -+ else -+ return NULL; -+} ++module_param(debug, int, 0); /* enable debug */ ++module_param(enabled, int, 0); /* enable detector */ ++module_param(threshold, int, 0); /* latency threshold */ + -+/* -+ * We maintain a biased count of active stripes in the bottom 16 bits of -+ * bi_phys_segments, and a count of processed stripes in the upper 16 bits -+ */ -+static inline int raid5_bi_processed_stripes(struct bio *bio) -+{ -+ atomic_t *segments = (atomic_t *)&bio->bi_phys_segments; -+ return (atomic_read(segments) >> 16) & 0xffff; -+} ++/* Buffering and sampling */ + -+static inline int raid5_dec_bi_active_stripes(struct bio *bio) -+{ -+ atomic_t *segments = (atomic_t *)&bio->bi_phys_segments; -+ return atomic_sub_return(1, segments) & 0xffff; -+} ++static struct ring_buffer *ring_buffer; /* sample buffer */ ++static DEFINE_MUTEX(ring_buffer_mutex); /* lock changes */ ++static unsigned long buf_size = BUF_SIZE_DEFAULT; ++static struct task_struct *kthread; /* sampling thread */ + -+static inline void raid5_inc_bi_active_stripes(struct bio *bio) -+{ -+ atomic_t *segments = (atomic_t *)&bio->bi_phys_segments; -+ atomic_inc(segments); -+} ++/* DebugFS filesystem entries */ + -+static inline void raid5_set_bi_processed_stripes(struct bio *bio, -+ unsigned int cnt) -+{ -+ atomic_t *segments = (atomic_t *)&bio->bi_phys_segments; -+ int old, new; ++static struct dentry *debug_dir; /* debugfs directory */ ++static struct dentry *debug_max; /* maximum TSC delta */ ++static struct dentry *debug_count; /* total detect count */ ++static struct dentry *debug_sample_width; /* sample width us */ ++static struct dentry *debug_sample_window; /* sample window us */ ++static struct dentry *debug_sample; /* raw samples us */ ++static struct dentry *debug_threshold; /* threshold us */ ++static struct dentry *debug_enable; /* enable/disable */ + -+ do { -+ old = atomic_read(segments); -+ new = (old & 0xffff) | (cnt << 16); -+ } while (atomic_cmpxchg(segments, old, new) != old); -+} ++/* Individual samples and global state */ + -+static inline void raid5_set_bi_stripes(struct bio *bio, unsigned int cnt) -+{ -+ atomic_t *segments = (atomic_t *)&bio->bi_phys_segments; -+ atomic_set(segments, cnt); -+} ++struct sample; /* latency sample */ ++struct data; /* Global state */ + -+/* Find first data disk in a raid6 stripe */ -+static inline int raid6_d0(struct stripe_head *sh) -+{ -+ if (sh->ddf_layout) -+ /* ddf always start from first device */ -+ return 0; -+ /* md starts just after Q block */ -+ if (sh->qd_idx == sh->disks - 1) -+ return 0; -+ else -+ return sh->qd_idx + 1; -+} -+static inline int raid6_next_disk(int disk, int raid_disks) -+{ -+ disk++; -+ return (disk < raid_disks) ? disk : 0; -+} ++/* Sampling functions */ ++static int __buffer_add_sample(struct sample *sample); ++static struct sample *buffer_get_sample(struct sample *sample); + -+/* When walking through the disks in a raid5, starting at raid6_d0, -+ * We need to map each disk to a 'slot', where the data disks are slot -+ * 0 .. raid_disks-3, the parity disk is raid_disks-2 and the Q disk -+ * is raid_disks-1. This help does that mapping. -+ */ -+static int raid6_idx_to_slot(int idx, struct stripe_head *sh, -+ int *count, int syndrome_disks) -+{ -+ int slot = *count; -+ -+ if (sh->ddf_layout) -+ (*count)++; -+ if (idx == sh->pd_idx) -+ return syndrome_disks; -+ if (idx == sh->qd_idx) -+ return syndrome_disks + 1; -+ if (!sh->ddf_layout) -+ (*count)++; -+ return slot; -+} ++/* Threading and state */ ++static int kthread_fn(void *unused); ++static int start_kthread(void); ++static int stop_kthread(void); ++static void __reset_stats(void); ++static int init_stats(void); + -+static void return_io(struct bio *return_bi) -+{ -+ struct bio *bi = return_bi; -+ while (bi) { -+ -+ return_bi = bi->bi_next; -+ bi->bi_next = NULL; -+ bi->bi_iter.bi_size = 0; -+ trace_block_bio_complete(bdev_get_queue(bi->bi_bdev), -+ bi, 0); -+ bio_endio(bi, 0); -+ bi = return_bi; -+ } -+} ++/* Debugfs interface */ ++static ssize_t simple_data_read(struct file *filp, char __user *ubuf, ++ size_t cnt, loff_t *ppos, const u64 *entry); ++static ssize_t simple_data_write(struct file *filp, const char __user *ubuf, ++ size_t cnt, loff_t *ppos, u64 *entry); ++static int debug_sample_fopen(struct inode *inode, struct file *filp); ++static ssize_t debug_sample_fread(struct file *filp, char __user *ubuf, ++ size_t cnt, loff_t *ppos); ++static int debug_sample_release(struct inode *inode, struct file *filp); ++static int debug_enable_fopen(struct inode *inode, struct file *filp); ++static ssize_t debug_enable_fread(struct file *filp, char __user *ubuf, ++ size_t cnt, loff_t *ppos); ++static ssize_t debug_enable_fwrite(struct file *file, ++ const char __user *user_buffer, ++ size_t user_size, loff_t *offset); + -+static void print_raid5_conf (struct r5conf *conf); ++/* Initialization functions */ ++static int init_debugfs(void); ++static void free_debugfs(void); ++static int detector_init(void); ++static void detector_exit(void); + -+static int stripe_operations_active(struct stripe_head *sh) -+{ -+ return sh->check_state || sh->reconstruct_state || -+ test_bit(STRIPE_BIOFILL_RUN, &sh->state) || -+ test_bit(STRIPE_COMPUTE_RUN, &sh->state); -+} ++/* Individual latency samples are stored here when detected and packed into ++ * the ring_buffer circular buffer, where they are overwritten when ++ * more than buf_size/sizeof(sample) samples are received. */ ++struct sample { ++ u64 seqnum; /* unique sequence */ ++ u64 duration; /* ktime delta */ ++ u64 outer_duration; /* ktime delta (outer loop) */ ++ struct timespec timestamp; /* wall time */ ++ unsigned long lost; ++}; + -+static void raid5_wakeup_stripe_thread(struct stripe_head *sh) -+{ -+ struct r5conf *conf = sh->raid_conf; -+ struct r5worker_group *group; -+ int thread_cnt; -+ int i, cpu = sh->cpu; -+ -+ if (!cpu_online(cpu)) { -+ cpu = cpumask_any(cpu_online_mask); -+ sh->cpu = cpu; -+ } ++/* keep the global state somewhere. */ ++static struct data { + -+ if (list_empty(&sh->lru)) { -+ struct r5worker_group *group; -+ group = conf->worker_groups + cpu_to_group(cpu); -+ list_add_tail(&sh->lru, &group->handle_list); -+ group->stripes_cnt++; -+ sh->group = group; -+ } ++ struct mutex lock; /* protect changes */ + -+ if (conf->worker_cnt_per_group == 0) { -+ md_wakeup_thread(conf->mddev->thread); -+ return; -+ } ++ u64 count; /* total since reset */ ++ u64 max_sample; /* max hardware latency */ ++ u64 threshold; /* sample threshold level */ + -+ group = conf->worker_groups + cpu_to_group(sh->cpu); ++ u64 sample_window; /* total sampling window (on+off) */ ++ u64 sample_width; /* active sampling portion of window */ + -+ group->workers[0].working = true; -+ /* at least one worker should run to avoid race */ -+ queue_work_on(sh->cpu, raid5_wq, &group->workers[0].work); ++ atomic_t sample_open; /* whether the sample file is open */ + -+ thread_cnt = group->stripes_cnt / MAX_STRIPE_BATCH - 1; -+ /* wakeup more workers */ -+ for (i = 1; i < conf->worker_cnt_per_group && thread_cnt > 0; i++) { -+ if (group->workers[i].working == false) { -+ group->workers[i].working = true; -+ queue_work_on(sh->cpu, raid5_wq, -+ &group->workers[i].work); -+ thread_cnt--; -+ } -+ } -+} ++ wait_queue_head_t wq; /* waitqeue for new sample values */ + -+static void do_release_stripe(struct r5conf *conf, struct stripe_head *sh, -+ struct list_head *temp_inactive_list) -+{ -+ BUG_ON(!list_empty(&sh->lru)); -+ BUG_ON(atomic_read(&conf->active_stripes)==0); -+ if (test_bit(STRIPE_HANDLE, &sh->state)) { -+ if (test_bit(STRIPE_DELAYED, &sh->state) && -+ !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) -+ list_add_tail(&sh->lru, &conf->delayed_list); -+ else if (test_bit(STRIPE_BIT_DELAY, &sh->state) && -+ sh->bm_seq - conf->seq_write > 0) -+ list_add_tail(&sh->lru, &conf->bitmap_list); -+ else { -+ clear_bit(STRIPE_DELAYED, &sh->state); -+ clear_bit(STRIPE_BIT_DELAY, &sh->state); -+ if (conf->worker_cnt_per_group == 0) { -+ list_add_tail(&sh->lru, &conf->handle_list); -+ } else { -+ raid5_wakeup_stripe_thread(sh); -+ return; -+ } -+ } -+ md_wakeup_thread(conf->mddev->thread); -+ } else { -+ BUG_ON(stripe_operations_active(sh)); -+ if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) -+ if (atomic_dec_return(&conf->preread_active_stripes) -+ < IO_THRESHOLD) -+ md_wakeup_thread(conf->mddev->thread); -+ atomic_dec(&conf->active_stripes); -+ if (!test_bit(STRIPE_EXPANDING, &sh->state)) -+ list_add_tail(&sh->lru, temp_inactive_list); -+ } -+} ++} data; + -+static void __release_stripe(struct r5conf *conf, struct stripe_head *sh, -+ struct list_head *temp_inactive_list) ++/** ++ * __buffer_add_sample - add a new latency sample recording to the ring buffer ++ * @sample: The new latency sample value ++ * ++ * This receives a new latency sample and records it in a global ring buffer. ++ * No additional locking is used in this case. ++ */ ++static int __buffer_add_sample(struct sample *sample) +{ -+ if (atomic_dec_and_test(&sh->count)) -+ do_release_stripe(conf, sh, temp_inactive_list); ++ return ring_buffer_write(ring_buffer, ++ sizeof(struct sample), sample); +} + -+/* -+ * @hash could be NR_STRIPE_HASH_LOCKS, then we have a list of inactive_list ++/** ++ * buffer_get_sample - remove a hardware latency sample from the ring buffer ++ * @sample: Pre-allocated storage for the sample + * -+ * Be careful: Only one task can add/delete stripes from temp_inactive_list at -+ * given time. Adding stripes only takes device lock, while deleting stripes -+ * only takes hash lock. ++ * This retrieves a hardware latency sample from the global circular buffer + */ -+static void release_inactive_stripe_list(struct r5conf *conf, -+ struct list_head *temp_inactive_list, -+ int hash) ++static struct sample *buffer_get_sample(struct sample *sample) +{ -+ int size; -+ bool do_wakeup = false; -+ unsigned long flags; ++ struct ring_buffer_event *e = NULL; ++ struct sample *s = NULL; ++ unsigned int cpu = 0; + -+ if (hash == NR_STRIPE_HASH_LOCKS) { -+ size = NR_STRIPE_HASH_LOCKS; -+ hash = NR_STRIPE_HASH_LOCKS - 1; -+ } else -+ size = 1; -+ while (size) { -+ struct list_head *list = &temp_inactive_list[size - 1]; ++ if (!sample) ++ return NULL; + -+ /* -+ * We don't hold any lock here yet, get_active_stripe() might -+ * remove stripes from the list -+ */ -+ if (!list_empty_careful(list)) { -+ spin_lock_irqsave(conf->hash_locks + hash, flags); -+ if (list_empty(conf->inactive_list + hash) && -+ !list_empty(list)) -+ atomic_dec(&conf->empty_inactive_list_nr); -+ list_splice_tail_init(list, conf->inactive_list + hash); -+ do_wakeup = true; -+ spin_unlock_irqrestore(conf->hash_locks + hash, flags); -+ } -+ size--; -+ hash--; ++ mutex_lock(&ring_buffer_mutex); ++ for_each_online_cpu(cpu) { ++ e = ring_buffer_consume(ring_buffer, cpu, NULL, &sample->lost); ++ if (e) ++ break; + } + -+ if (do_wakeup) { -+ wake_up(&conf->wait_for_stripe); -+ if (conf->retry_read_aligned) -+ md_wakeup_thread(conf->mddev->thread); -+ } ++ if (e) { ++ s = ring_buffer_event_data(e); ++ memcpy(sample, s, sizeof(struct sample)); ++ } else ++ sample = NULL; ++ mutex_unlock(&ring_buffer_mutex); ++ ++ return sample; +} + -+/* should hold conf->device_lock already */ -+static int release_stripe_list(struct r5conf *conf, -+ struct list_head *temp_inactive_list) ++#ifndef CONFIG_TRACING ++#define time_type ktime_t ++#define time_get() ktime_get() ++#define time_to_us(x) ktime_to_us(x) ++#define time_sub(a, b) ktime_sub(a, b) ++#define init_time(a, b) (a).tv64 = b ++#define time_u64(a) ((a).tv64) ++#else ++#define time_type u64 ++#define time_get() trace_clock_local() ++#define time_to_us(x) div_u64(x, 1000) ++#define time_sub(a, b) ((a) - (b)) ++#define init_time(a, b) (a = b) ++#define time_u64(a) a ++#endif ++/** ++ * get_sample - sample the CPU TSC and look for likely hardware latencies ++ * ++ * Used to repeatedly capture the CPU TSC (or similar), looking for potential ++ * hardware-induced latency. Called with interrupts disabled and with ++ * data.lock held. ++ */ ++static int get_sample(void) +{ -+ struct stripe_head *sh; -+ int count = 0; -+ struct llist_node *head; -+ -+ head = llist_del_all(&conf->released_stripes); -+ head = llist_reverse_order(head); -+ while (head) { -+ int hash; -+ -+ sh = llist_entry(head, struct stripe_head, release_list); -+ head = llist_next(head); -+ /* sh could be readded after STRIPE_ON_RELEASE_LIST is cleard */ -+ smp_mb(); -+ clear_bit(STRIPE_ON_RELEASE_LIST, &sh->state); -+ /* -+ * Don't worry the bit is set here, because if the bit is set -+ * again, the count is always > 1. This is true for -+ * STRIPE_ON_UNPLUG_LIST bit too. -+ */ -+ hash = sh->hash_lock_index; -+ __release_stripe(conf, sh, &temp_inactive_list[hash]); -+ count++; -+ } ++ time_type start, t1, t2, last_t2; ++ s64 diff, total = 0; ++ u64 sample = 0; ++ u64 outer_sample = 0; ++ int ret = -1; + -+ return count; -+} ++ init_time(last_t2, 0); ++ start = time_get(); /* start timestamp */ + -+static void release_stripe(struct stripe_head *sh) -+{ -+ struct r5conf *conf = sh->raid_conf; -+ unsigned long flags; -+ struct list_head list; -+ int hash; -+ bool wakeup; ++ do { + -+ /* Avoid release_list until the last reference. -+ */ -+ if (atomic_add_unless(&sh->count, -1, 1)) -+ return; ++ t1 = time_get(); /* we'll look for a discontinuity */ ++ t2 = time_get(); + -+ if (unlikely(!conf->mddev->thread) || -+ test_and_set_bit(STRIPE_ON_RELEASE_LIST, &sh->state)) -+ goto slow_path; -+ wakeup = llist_add(&sh->release_list, &conf->released_stripes); -+ if (wakeup) -+ md_wakeup_thread(conf->mddev->thread); -+ return; -+slow_path: -+ local_irq_save(flags); -+ /* we are ok here if STRIPE_ON_RELEASE_LIST is set or not */ -+ if (atomic_dec_and_lock(&sh->count, &conf->device_lock)) { -+ INIT_LIST_HEAD(&list); -+ hash = sh->hash_lock_index; -+ do_release_stripe(conf, sh, &list); -+ spin_unlock(&conf->device_lock); -+ release_inactive_stripe_list(conf, &list, hash); -+ } -+ local_irq_restore(flags); -+} ++ if (time_u64(last_t2)) { ++ /* Check the delta from outer loop (t2 to next t1) */ ++ diff = time_to_us(time_sub(t1, last_t2)); ++ /* This shouldn't happen */ ++ if (diff < 0) { ++ pr_err(BANNER "time running backwards\n"); ++ goto out; ++ } ++ if (diff > outer_sample) ++ outer_sample = diff; ++ } ++ last_t2 = t2; + -+static inline void remove_hash(struct stripe_head *sh) -+{ -+ pr_debug("remove_hash(), stripe %llu\n", -+ (unsigned long long)sh->sector); ++ total = time_to_us(time_sub(t2, start)); /* sample width */ + -+ hlist_del_init(&sh->hash); -+} ++ /* This checks the inner loop (t1 to t2) */ ++ diff = time_to_us(time_sub(t2, t1)); /* current diff */ + -+static inline void insert_hash(struct r5conf *conf, struct stripe_head *sh) -+{ -+ struct hlist_head *hp = stripe_hash(conf, sh->sector); ++ /* This shouldn't happen */ ++ if (diff < 0) { ++ pr_err(BANNER "time running backwards\n"); ++ goto out; ++ } + -+ pr_debug("insert_hash(), stripe %llu\n", -+ (unsigned long long)sh->sector); ++ if (diff > sample) ++ sample = diff; /* only want highest value */ + -+ hlist_add_head(&sh->hash, hp); -+} ++ } while (total <= data.sample_width); + -+/* find an idle stripe, make sure it is unhashed, and return it. */ -+static struct stripe_head *get_free_stripe(struct r5conf *conf, int hash) -+{ -+ struct stripe_head *sh = NULL; -+ struct list_head *first; ++ ret = 0; + -+ if (list_empty(conf->inactive_list + hash)) -+ goto out; -+ first = (conf->inactive_list + hash)->next; -+ sh = list_entry(first, struct stripe_head, lru); -+ list_del_init(first); -+ remove_hash(sh); -+ atomic_inc(&conf->active_stripes); -+ BUG_ON(hash != sh->hash_lock_index); -+ if (list_empty(conf->inactive_list + hash)) -+ atomic_inc(&conf->empty_inactive_list_nr); -+out: -+ return sh; -+} ++ /* If we exceed the threshold value, we have found a hardware latency */ ++ if (sample > data.threshold || outer_sample > data.threshold) { ++ struct sample s; + -+static void shrink_buffers(struct stripe_head *sh) -+{ -+ struct page *p; -+ int i; -+ int num = sh->raid_conf->pool_size; ++ ret = 1; + -+ for (i = 0; i < num ; i++) { -+ WARN_ON(sh->dev[i].page != sh->dev[i].orig_page); -+ p = sh->dev[i].page; -+ if (!p) -+ continue; -+ sh->dev[i].page = NULL; -+ put_page(p); ++ data.count++; ++ s.seqnum = data.count; ++ s.duration = sample; ++ s.outer_duration = outer_sample; ++ s.timestamp = CURRENT_TIME; ++ __buffer_add_sample(&s); ++ ++ /* Keep a running maximum ever recorded hardware latency */ ++ if (sample > data.max_sample) ++ data.max_sample = sample; + } ++ ++out: ++ return ret; +} + -+static int grow_buffers(struct stripe_head *sh, gfp_t gfp) ++/* ++ * kthread_fn - The CPU time sampling/hardware latency detection kernel thread ++ * @unused: A required part of the kthread API. ++ * ++ * Used to periodically sample the CPU TSC via a call to get_sample. We ++ * disable interrupts, which does (intentionally) introduce latency since we ++ * need to ensure nothing else might be running (and thus pre-empting). ++ * Obviously this should never be used in production environments. ++ * ++ * Currently this runs on which ever CPU it was scheduled on, but most ++ * real-worald hardware latency situations occur across several CPUs, ++ * but we might later generalize this if we find there are any actualy ++ * systems with alternate SMI delivery or other hardware latencies. ++ */ ++static int kthread_fn(void *unused) +{ -+ int i; -+ int num = sh->raid_conf->pool_size; ++ int ret; ++ u64 interval; + -+ for (i = 0; i < num; i++) { -+ struct page *page; ++ while (!kthread_should_stop()) { + -+ if (!(page = alloc_page(gfp))) { -+ return 1; -+ } -+ sh->dev[i].page = page; -+ sh->dev[i].orig_page = page; -+ } -+ return 0; -+} ++ mutex_lock(&data.lock); + -+static void raid5_build_block(struct stripe_head *sh, int i, int previous); -+static void stripe_set_idx(sector_t stripe, struct r5conf *conf, int previous, -+ struct stripe_head *sh); ++ local_irq_disable(); ++ ret = get_sample(); ++ local_irq_enable(); + -+static void init_stripe(struct stripe_head *sh, sector_t sector, int previous) -+{ -+ struct r5conf *conf = sh->raid_conf; -+ int i, seq; ++ if (ret > 0) ++ wake_up(&data.wq); /* wake up reader(s) */ + -+ BUG_ON(atomic_read(&sh->count) != 0); -+ BUG_ON(test_bit(STRIPE_HANDLE, &sh->state)); -+ BUG_ON(stripe_operations_active(sh)); -+ BUG_ON(sh->batch_head); ++ interval = data.sample_window - data.sample_width; ++ do_div(interval, USEC_PER_MSEC); /* modifies interval value */ + -+ pr_debug("init_stripe called, stripe %llu\n", -+ (unsigned long long)sector); -+retry: -+ seq = read_seqcount_begin(&conf->gen_lock); -+ sh->generation = conf->generation - previous; -+ sh->disks = previous ? conf->previous_raid_disks : conf->raid_disks; -+ sh->sector = sector; -+ stripe_set_idx(sector, conf, previous, sh); -+ sh->state = 0; -+ -+ for (i = sh->disks; i--; ) { -+ struct r5dev *dev = &sh->dev[i]; -+ -+ if (dev->toread || dev->read || dev->towrite || dev->written || -+ test_bit(R5_LOCKED, &dev->flags)) { -+ printk(KERN_ERR "sector=%llx i=%d %p %p %p %p %d\n", -+ (unsigned long long)sh->sector, i, dev->toread, -+ dev->read, dev->towrite, dev->written, -+ test_bit(R5_LOCKED, &dev->flags)); -+ WARN_ON(1); -+ } -+ dev->flags = 0; -+ raid5_build_block(sh, i, previous); -+ } -+ if (read_seqcount_retry(&conf->gen_lock, seq)) -+ goto retry; -+ sh->overwrite_disks = 0; -+ insert_hash(conf, sh); -+ sh->cpu = smp_processor_id(); -+ set_bit(STRIPE_BATCH_READY, &sh->state); -+} ++ mutex_unlock(&data.lock); + -+static struct stripe_head *__find_stripe(struct r5conf *conf, sector_t sector, -+ short generation) -+{ -+ struct stripe_head *sh; ++ if (msleep_interruptible(interval)) ++ break; ++ } + -+ pr_debug("__find_stripe, sector %llu\n", (unsigned long long)sector); -+ hlist_for_each_entry(sh, stripe_hash(conf, sector), hash) -+ if (sh->sector == sector && sh->generation == generation) -+ return sh; -+ pr_debug("__stripe %llu not in cache\n", (unsigned long long)sector); -+ return NULL; ++ return 0; +} + -+/* -+ * Need to check if array has failed when deciding whether to: -+ * - start an array -+ * - remove non-faulty devices -+ * - add a spare -+ * - allow a reshape -+ * This determination is simple when no reshape is happening. -+ * However if there is a reshape, we need to carefully check -+ * both the before and after sections. -+ * This is because some failed devices may only affect one -+ * of the two sections, and some non-in_sync devices may -+ * be insync in the section most affected by failed devices. ++/** ++ * start_kthread - Kick off the hardware latency sampling/detector kthread ++ * ++ * This starts a kernel thread that will sit and sample the CPU timestamp ++ * counter (TSC or similar) and look for potential hardware latencies. + */ -+static int calc_degraded(struct r5conf *conf) ++static int start_kthread(void) +{ -+ int degraded, degraded2; -+ int i; -+ -+ rcu_read_lock(); -+ degraded = 0; -+ for (i = 0; i < conf->previous_raid_disks; i++) { -+ struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev); -+ if (rdev && test_bit(Faulty, &rdev->flags)) -+ rdev = rcu_dereference(conf->disks[i].replacement); -+ if (!rdev || test_bit(Faulty, &rdev->flags)) -+ degraded++; -+ else if (test_bit(In_sync, &rdev->flags)) -+ ; -+ else -+ /* not in-sync or faulty. -+ * If the reshape increases the number of devices, -+ * this is being recovered by the reshape, so -+ * this 'previous' section is not in_sync. -+ * If the number of devices is being reduced however, -+ * the device can only be part of the array if -+ * we are reverting a reshape, so this section will -+ * be in-sync. -+ */ -+ if (conf->raid_disks >= conf->previous_raid_disks) -+ degraded++; -+ } -+ rcu_read_unlock(); -+ if (conf->raid_disks == conf->previous_raid_disks) -+ return degraded; -+ rcu_read_lock(); -+ degraded2 = 0; -+ for (i = 0; i < conf->raid_disks; i++) { -+ struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev); -+ if (rdev && test_bit(Faulty, &rdev->flags)) -+ rdev = rcu_dereference(conf->disks[i].replacement); -+ if (!rdev || test_bit(Faulty, &rdev->flags)) -+ degraded2++; -+ else if (test_bit(In_sync, &rdev->flags)) -+ ; -+ else -+ /* not in-sync or faulty. -+ * If reshape increases the number of devices, this -+ * section has already been recovered, else it -+ * almost certainly hasn't. -+ */ -+ if (conf->raid_disks <= conf->previous_raid_disks) -+ degraded2++; ++ kthread = kthread_run(kthread_fn, NULL, ++ DRVNAME); ++ if (IS_ERR(kthread)) { ++ pr_err(BANNER "could not start sampling thread\n"); ++ enabled = 0; ++ return -ENOMEM; + } -+ rcu_read_unlock(); -+ if (degraded2 > degraded) -+ return degraded2; -+ return degraded; -+} -+ -+static int has_failed(struct r5conf *conf) -+{ -+ int degraded; -+ -+ if (conf->mddev->reshape_position == MaxSector) -+ return conf->mddev->degraded > conf->max_degraded; + -+ degraded = calc_degraded(conf); -+ if (degraded > conf->max_degraded) -+ return 1; + return 0; +} + -+static struct stripe_head * -+get_active_stripe(struct r5conf *conf, sector_t sector, -+ int previous, int noblock, int noquiesce) ++/** ++ * stop_kthread - Inform the hardware latency samping/detector kthread to stop ++ * ++ * This kicks the running hardware latency sampling/detector kernel thread and ++ * tells it to stop sampling now. Use this on unload and at system shutdown. ++ */ ++static int stop_kthread(void) +{ -+ struct stripe_head *sh; -+ int hash = stripe_hash_locks_hash(sector); -+ -+ pr_debug("get_stripe, sector %llu\n", (unsigned long long)sector); -+ -+ spin_lock_irq(conf->hash_locks + hash); -+ -+ do { -+ wait_event_lock_irq(conf->wait_for_stripe, -+ conf->quiesce == 0 || noquiesce, -+ *(conf->hash_locks + hash)); -+ sh = __find_stripe(conf, sector, conf->generation - previous); -+ if (!sh) { -+ if (!test_bit(R5_INACTIVE_BLOCKED, &conf->cache_state)) { -+ sh = get_free_stripe(conf, hash); -+ if (!sh && llist_empty(&conf->released_stripes) && -+ !test_bit(R5_DID_ALLOC, &conf->cache_state)) -+ set_bit(R5_ALLOC_MORE, -+ &conf->cache_state); -+ } -+ if (noblock && sh == NULL) -+ break; -+ if (!sh) { -+ set_bit(R5_INACTIVE_BLOCKED, -+ &conf->cache_state); -+ wait_event_lock_irq( -+ conf->wait_for_stripe, -+ !list_empty(conf->inactive_list + hash) && -+ (atomic_read(&conf->active_stripes) -+ < (conf->max_nr_stripes * 3 / 4) -+ || !test_bit(R5_INACTIVE_BLOCKED, -+ &conf->cache_state)), -+ *(conf->hash_locks + hash)); -+ clear_bit(R5_INACTIVE_BLOCKED, -+ &conf->cache_state); -+ } else { -+ init_stripe(sh, sector, previous); -+ atomic_inc(&sh->count); -+ } -+ } else if (!atomic_inc_not_zero(&sh->count)) { -+ spin_lock(&conf->device_lock); -+ if (!atomic_read(&sh->count)) { -+ if (!test_bit(STRIPE_HANDLE, &sh->state)) -+ atomic_inc(&conf->active_stripes); -+ BUG_ON(list_empty(&sh->lru) && -+ !test_bit(STRIPE_EXPANDING, &sh->state)); -+ list_del_init(&sh->lru); -+ if (sh->group) { -+ sh->group->stripes_cnt--; -+ sh->group = NULL; -+ } -+ } -+ atomic_inc(&sh->count); -+ spin_unlock(&conf->device_lock); -+ } -+ } while (sh == NULL); -+ -+ spin_unlock_irq(conf->hash_locks + hash); -+ return sh; -+} ++ int ret; + -+static bool is_full_stripe_write(struct stripe_head *sh) -+{ -+ BUG_ON(sh->overwrite_disks > (sh->disks - sh->raid_conf->max_degraded)); -+ return sh->overwrite_disks == (sh->disks - sh->raid_conf->max_degraded); -+} ++ ret = kthread_stop(kthread); + -+static void lock_two_stripes(struct stripe_head *sh1, struct stripe_head *sh2) -+{ -+ local_irq_disable(); -+ if (sh1 > sh2) { -+ spin_lock(&sh2->stripe_lock); -+ spin_lock_nested(&sh1->stripe_lock, 1); -+ } else { -+ spin_lock(&sh1->stripe_lock); -+ spin_lock_nested(&sh2->stripe_lock, 1); -+ } ++ return ret; +} + -+static void unlock_two_stripes(struct stripe_head *sh1, struct stripe_head *sh2) ++/** ++ * __reset_stats - Reset statistics for the hardware latency detector ++ * ++ * We use data to store various statistics and global state. We call this ++ * function in order to reset those when "enable" is toggled on or off, and ++ * also at initialization. Should be called with data.lock held. ++ */ ++static void __reset_stats(void) +{ -+ spin_unlock(&sh1->stripe_lock); -+ spin_unlock(&sh2->stripe_lock); -+ local_irq_enable(); ++ data.count = 0; ++ data.max_sample = 0; ++ ring_buffer_reset(ring_buffer); /* flush out old sample entries */ +} + -+/* Only freshly new full stripe normal write stripe can be added to a batch list */ -+static bool stripe_can_batch(struct stripe_head *sh) ++/** ++ * init_stats - Setup global state statistics for the hardware latency detector ++ * ++ * We use data to store various statistics and global state. We also use ++ * a global ring buffer (ring_buffer) to keep raw samples of detected hardware ++ * induced system latencies. This function initializes these structures and ++ * allocates the global ring buffer also. ++ */ ++static int init_stats(void) +{ -+ return test_bit(STRIPE_BATCH_READY, &sh->state) && -+ !test_bit(STRIPE_BITMAP_PENDING, &sh->state) && -+ is_full_stripe_write(sh); -+} ++ int ret = -ENOMEM; + -+/* we only do back search */ -+static void stripe_add_to_batch_list(struct r5conf *conf, struct stripe_head *sh) -+{ -+ struct stripe_head *head; -+ sector_t head_sector, tmp_sec; -+ int hash; -+ int dd_idx; ++ mutex_init(&data.lock); ++ init_waitqueue_head(&data.wq); ++ atomic_set(&data.sample_open, 0); + -+ if (!stripe_can_batch(sh)) -+ return; -+ /* Don't cross chunks, so stripe pd_idx/qd_idx is the same */ -+ tmp_sec = sh->sector; -+ if (!sector_div(tmp_sec, conf->chunk_sectors)) -+ return; -+ head_sector = sh->sector - STRIPE_SECTORS; -+ -+ hash = stripe_hash_locks_hash(head_sector); -+ spin_lock_irq(conf->hash_locks + hash); -+ head = __find_stripe(conf, head_sector, conf->generation); -+ if (head && !atomic_inc_not_zero(&head->count)) { -+ spin_lock(&conf->device_lock); -+ if (!atomic_read(&head->count)) { -+ if (!test_bit(STRIPE_HANDLE, &head->state)) -+ atomic_inc(&conf->active_stripes); -+ BUG_ON(list_empty(&head->lru) && -+ !test_bit(STRIPE_EXPANDING, &head->state)); -+ list_del_init(&head->lru); -+ if (head->group) { -+ head->group->stripes_cnt--; -+ head->group = NULL; -+ } -+ } -+ atomic_inc(&head->count); -+ spin_unlock(&conf->device_lock); -+ } -+ spin_unlock_irq(conf->hash_locks + hash); ++ ring_buffer = ring_buffer_alloc(buf_size, BUF_FLAGS); + -+ if (!head) -+ return; -+ if (!stripe_can_batch(head)) ++ if (WARN(!ring_buffer, KERN_ERR BANNER ++ "failed to allocate ring buffer!\n")) + goto out; + -+ lock_two_stripes(head, sh); -+ /* clear_batch_ready clear the flag */ -+ if (!stripe_can_batch(head) || !stripe_can_batch(sh)) -+ goto unlock_out; -+ -+ if (sh->batch_head) -+ goto unlock_out; -+ -+ dd_idx = 0; -+ while (dd_idx == sh->pd_idx || dd_idx == sh->qd_idx) -+ dd_idx++; -+ if (head->dev[dd_idx].towrite->bi_rw != sh->dev[dd_idx].towrite->bi_rw) -+ goto unlock_out; -+ -+ if (head->batch_head) { -+ spin_lock(&head->batch_head->batch_lock); -+ /* This batch list is already running */ -+ if (!stripe_can_batch(head)) { -+ spin_unlock(&head->batch_head->batch_lock); -+ goto unlock_out; -+ } -+ -+ /* -+ * at this point, head's BATCH_READY could be cleared, but we -+ * can still add the stripe to batch list -+ */ -+ list_add(&sh->batch_list, &head->batch_list); -+ spin_unlock(&head->batch_head->batch_lock); -+ -+ sh->batch_head = head->batch_head; -+ } else { -+ head->batch_head = head; -+ sh->batch_head = head->batch_head; -+ spin_lock(&head->batch_lock); -+ list_add_tail(&sh->batch_list, &head->batch_list); -+ spin_unlock(&head->batch_lock); -+ } ++ __reset_stats(); ++ data.threshold = threshold ?: DEFAULT_LAT_THRESHOLD; /* threshold us */ ++ data.sample_window = DEFAULT_SAMPLE_WINDOW; /* window us */ ++ data.sample_width = DEFAULT_SAMPLE_WIDTH; /* width us */ + -+ if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) -+ if (atomic_dec_return(&conf->preread_active_stripes) -+ < IO_THRESHOLD) -+ md_wakeup_thread(conf->mddev->thread); -+ -+ if (test_and_clear_bit(STRIPE_BIT_DELAY, &sh->state)) { -+ int seq = sh->bm_seq; -+ if (test_bit(STRIPE_BIT_DELAY, &sh->batch_head->state) && -+ sh->batch_head->bm_seq > seq) -+ seq = sh->batch_head->bm_seq; -+ set_bit(STRIPE_BIT_DELAY, &sh->batch_head->state); -+ sh->batch_head->bm_seq = seq; -+ } ++ ret = 0; + -+ atomic_inc(&sh->count); -+unlock_out: -+ unlock_two_stripes(head, sh); +out: -+ release_stripe(head); ++ return ret; ++ +} + -+/* Determine if 'data_offset' or 'new_data_offset' should be used -+ * in this stripe_head. -+ */ -+static int use_new_offset(struct r5conf *conf, struct stripe_head *sh) -+{ -+ sector_t progress = conf->reshape_progress; -+ /* Need a memory barrier to make sure we see the value -+ * of conf->generation, or ->data_offset that was set before -+ * reshape_progress was updated. -+ */ -+ smp_rmb(); -+ if (progress == MaxSector) -+ return 0; -+ if (sh->generation == conf->generation - 1) -+ return 0; -+ /* We are in a reshape, and this is a new-generation stripe, -+ * so use new_data_offset. -+ */ -+ return 1; -+} -+ -+static void -+raid5_end_read_request(struct bio *bi, int error); -+static void -+raid5_end_write_request(struct bio *bi, int error); -+ -+static void ops_run_io(struct stripe_head *sh, struct stripe_head_state *s) -+{ -+ struct r5conf *conf = sh->raid_conf; -+ int i, disks = sh->disks; -+ struct stripe_head *head_sh = sh; -+ -+ might_sleep(); -+ -+ for (i = disks; i--; ) { -+ int rw; -+ int replace_only = 0; -+ struct bio *bi, *rbi; -+ struct md_rdev *rdev, *rrdev = NULL; -+ -+ sh = head_sh; -+ if (test_and_clear_bit(R5_Wantwrite, &sh->dev[i].flags)) { -+ if (test_and_clear_bit(R5_WantFUA, &sh->dev[i].flags)) -+ rw = WRITE_FUA; -+ else -+ rw = WRITE; -+ if (test_bit(R5_Discard, &sh->dev[i].flags)) -+ rw |= REQ_DISCARD; -+ } else if (test_and_clear_bit(R5_Wantread, &sh->dev[i].flags)) -+ rw = READ; -+ else if (test_and_clear_bit(R5_WantReplace, -+ &sh->dev[i].flags)) { -+ rw = WRITE; -+ replace_only = 1; -+ } else -+ continue; -+ if (test_and_clear_bit(R5_SyncIO, &sh->dev[i].flags)) -+ rw |= REQ_SYNC; -+ -+again: -+ bi = &sh->dev[i].req; -+ rbi = &sh->dev[i].rreq; /* For writing to replacement */ -+ -+ rcu_read_lock(); -+ rrdev = rcu_dereference(conf->disks[i].replacement); -+ smp_mb(); /* Ensure that if rrdev is NULL, rdev won't be */ -+ rdev = rcu_dereference(conf->disks[i].rdev); -+ if (!rdev) { -+ rdev = rrdev; -+ rrdev = NULL; -+ } -+ if (rw & WRITE) { -+ if (replace_only) -+ rdev = NULL; -+ if (rdev == rrdev) -+ /* We raced and saw duplicates */ -+ rrdev = NULL; -+ } else { -+ if (test_bit(R5_ReadRepl, &head_sh->dev[i].flags) && rrdev) -+ rdev = rrdev; -+ rrdev = NULL; -+ } -+ -+ if (rdev && test_bit(Faulty, &rdev->flags)) -+ rdev = NULL; -+ if (rdev) -+ atomic_inc(&rdev->nr_pending); -+ if (rrdev && test_bit(Faulty, &rrdev->flags)) -+ rrdev = NULL; -+ if (rrdev) -+ atomic_inc(&rrdev->nr_pending); -+ rcu_read_unlock(); -+ -+ /* We have already checked bad blocks for reads. Now -+ * need to check for writes. We never accept write errors -+ * on the replacement, so we don't to check rrdev. -+ */ -+ while ((rw & WRITE) && rdev && -+ test_bit(WriteErrorSeen, &rdev->flags)) { -+ sector_t first_bad; -+ int bad_sectors; -+ int bad = is_badblock(rdev, sh->sector, STRIPE_SECTORS, -+ &first_bad, &bad_sectors); -+ if (!bad) -+ break; -+ -+ if (bad < 0) { -+ set_bit(BlockedBadBlocks, &rdev->flags); -+ if (!conf->mddev->external && -+ conf->mddev->flags) { -+ /* It is very unlikely, but we might -+ * still need to write out the -+ * bad block log - better give it -+ * a chance*/ -+ md_check_recovery(conf->mddev); -+ } -+ /* -+ * Because md_wait_for_blocked_rdev -+ * will dec nr_pending, we must -+ * increment it first. -+ */ -+ atomic_inc(&rdev->nr_pending); -+ md_wait_for_blocked_rdev(rdev, conf->mddev); -+ } else { -+ /* Acknowledged bad block - skip the write */ -+ rdev_dec_pending(rdev, conf->mddev); -+ rdev = NULL; -+ } -+ } -+ -+ if (rdev) { -+ if (s->syncing || s->expanding || s->expanded -+ || s->replacing) -+ md_sync_acct(rdev->bdev, STRIPE_SECTORS); -+ -+ set_bit(STRIPE_IO_STARTED, &sh->state); -+ -+ bio_reset(bi); -+ bi->bi_bdev = rdev->bdev; -+ bi->bi_rw = rw; -+ bi->bi_end_io = (rw & WRITE) -+ ? raid5_end_write_request -+ : raid5_end_read_request; -+ bi->bi_private = sh; -+ -+ pr_debug("%s: for %llu schedule op %ld on disc %d\n", -+ __func__, (unsigned long long)sh->sector, -+ bi->bi_rw, i); -+ atomic_inc(&sh->count); -+ if (sh != head_sh) -+ atomic_inc(&head_sh->count); -+ if (use_new_offset(conf, sh)) -+ bi->bi_iter.bi_sector = (sh->sector -+ + rdev->new_data_offset); -+ else -+ bi->bi_iter.bi_sector = (sh->sector -+ + rdev->data_offset); -+ if (test_bit(R5_ReadNoMerge, &head_sh->dev[i].flags)) -+ bi->bi_rw |= REQ_NOMERGE; -+ -+ if (test_bit(R5_SkipCopy, &sh->dev[i].flags)) -+ WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags)); -+ sh->dev[i].vec.bv_page = sh->dev[i].page; -+ bi->bi_vcnt = 1; -+ bi->bi_io_vec[0].bv_len = STRIPE_SIZE; -+ bi->bi_io_vec[0].bv_offset = 0; -+ bi->bi_iter.bi_size = STRIPE_SIZE; -+ /* -+ * If this is discard request, set bi_vcnt 0. We don't -+ * want to confuse SCSI because SCSI will replace payload -+ */ -+ if (rw & REQ_DISCARD) -+ bi->bi_vcnt = 0; -+ if (rrdev) -+ set_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags); -+ -+ if (conf->mddev->gendisk) -+ trace_block_bio_remap(bdev_get_queue(bi->bi_bdev), -+ bi, disk_devt(conf->mddev->gendisk), -+ sh->dev[i].sector); -+ generic_make_request(bi); -+ } -+ if (rrdev) { -+ if (s->syncing || s->expanding || s->expanded -+ || s->replacing) -+ md_sync_acct(rrdev->bdev, STRIPE_SECTORS); -+ -+ set_bit(STRIPE_IO_STARTED, &sh->state); -+ -+ bio_reset(rbi); -+ rbi->bi_bdev = rrdev->bdev; -+ rbi->bi_rw = rw; -+ BUG_ON(!(rw & WRITE)); -+ rbi->bi_end_io = raid5_end_write_request; -+ rbi->bi_private = sh; -+ -+ pr_debug("%s: for %llu schedule op %ld on " -+ "replacement disc %d\n", -+ __func__, (unsigned long long)sh->sector, -+ rbi->bi_rw, i); -+ atomic_inc(&sh->count); -+ if (sh != head_sh) -+ atomic_inc(&head_sh->count); -+ if (use_new_offset(conf, sh)) -+ rbi->bi_iter.bi_sector = (sh->sector -+ + rrdev->new_data_offset); -+ else -+ rbi->bi_iter.bi_sector = (sh->sector -+ + rrdev->data_offset); -+ if (test_bit(R5_SkipCopy, &sh->dev[i].flags)) -+ WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags)); -+ sh->dev[i].rvec.bv_page = sh->dev[i].page; -+ rbi->bi_vcnt = 1; -+ rbi->bi_io_vec[0].bv_len = STRIPE_SIZE; -+ rbi->bi_io_vec[0].bv_offset = 0; -+ rbi->bi_iter.bi_size = STRIPE_SIZE; -+ /* -+ * If this is discard request, set bi_vcnt 0. We don't -+ * want to confuse SCSI because SCSI will replace payload -+ */ -+ if (rw & REQ_DISCARD) -+ rbi->bi_vcnt = 0; -+ if (conf->mddev->gendisk) -+ trace_block_bio_remap(bdev_get_queue(rbi->bi_bdev), -+ rbi, disk_devt(conf->mddev->gendisk), -+ sh->dev[i].sector); -+ generic_make_request(rbi); -+ } -+ if (!rdev && !rrdev) { -+ if (rw & WRITE) -+ set_bit(STRIPE_DEGRADED, &sh->state); -+ pr_debug("skip op %ld on disc %d for sector %llu\n", -+ bi->bi_rw, i, (unsigned long long)sh->sector); -+ clear_bit(R5_LOCKED, &sh->dev[i].flags); -+ set_bit(STRIPE_HANDLE, &sh->state); -+ } -+ -+ if (!head_sh->batch_head) -+ continue; -+ sh = list_first_entry(&sh->batch_list, struct stripe_head, -+ batch_list); -+ if (sh != head_sh) -+ goto again; -+ } -+} -+ -+static struct dma_async_tx_descriptor * -+async_copy_data(int frombio, struct bio *bio, struct page **page, -+ sector_t sector, struct dma_async_tx_descriptor *tx, -+ struct stripe_head *sh) ++/* ++ * simple_data_read - Wrapper read function for global state debugfs entries ++ * @filp: The active open file structure for the debugfs "file" ++ * @ubuf: The userspace provided buffer to read value into ++ * @cnt: The maximum number of bytes to read ++ * @ppos: The current "file" position ++ * @entry: The entry to read from ++ * ++ * This function provides a generic read implementation for the global state ++ * "data" structure debugfs filesystem entries. It would be nice to use ++ * simple_attr_read directly, but we need to make sure that the data.lock ++ * is held during the actual read. ++ */ ++static ssize_t simple_data_read(struct file *filp, char __user *ubuf, ++ size_t cnt, loff_t *ppos, const u64 *entry) +{ -+ struct bio_vec bvl; -+ struct bvec_iter iter; -+ struct page *bio_page; -+ int page_offset; -+ struct async_submit_ctl submit; -+ enum async_tx_flags flags = 0; -+ -+ if (bio->bi_iter.bi_sector >= sector) -+ page_offset = (signed)(bio->bi_iter.bi_sector - sector) * 512; -+ else -+ page_offset = (signed)(sector - bio->bi_iter.bi_sector) * -512; ++ char buf[U64STR_SIZE]; ++ u64 val = 0; ++ int len = 0; + -+ if (frombio) -+ flags |= ASYNC_TX_FENCE; -+ init_async_submit(&submit, flags, tx, NULL, NULL, NULL); ++ memset(buf, 0, sizeof(buf)); + -+ bio_for_each_segment(bvl, bio, iter) { -+ int len = bvl.bv_len; -+ int clen; -+ int b_offset = 0; ++ if (!entry) ++ return -EFAULT; + -+ if (page_offset < 0) { -+ b_offset = -page_offset; -+ page_offset += b_offset; -+ len -= b_offset; -+ } ++ mutex_lock(&data.lock); ++ val = *entry; ++ mutex_unlock(&data.lock); + -+ if (len > 0 && page_offset + len > STRIPE_SIZE) -+ clen = STRIPE_SIZE - page_offset; -+ else -+ clen = len; -+ -+ if (clen > 0) { -+ b_offset += bvl.bv_offset; -+ bio_page = bvl.bv_page; -+ if (frombio) { -+ if (sh->raid_conf->skip_copy && -+ b_offset == 0 && page_offset == 0 && -+ clen == STRIPE_SIZE) -+ *page = bio_page; -+ else -+ tx = async_memcpy(*page, bio_page, page_offset, -+ b_offset, clen, &submit); -+ } else -+ tx = async_memcpy(bio_page, *page, b_offset, -+ page_offset, clen, &submit); -+ } -+ /* chain the operations */ -+ submit.depend_tx = tx; ++ len = snprintf(buf, sizeof(buf), "%llu\n", (unsigned long long)val); + -+ if (clen < len) /* hit end of page */ -+ break; -+ page_offset += len; -+ } ++ return simple_read_from_buffer(ubuf, cnt, ppos, buf, len); + -+ return tx; +} + -+static void ops_complete_biofill(void *stripe_head_ref) ++/* ++ * simple_data_write - Wrapper write function for global state debugfs entries ++ * @filp: The active open file structure for the debugfs "file" ++ * @ubuf: The userspace provided buffer to write value from ++ * @cnt: The maximum number of bytes to write ++ * @ppos: The current "file" position ++ * @entry: The entry to write to ++ * ++ * This function provides a generic write implementation for the global state ++ * "data" structure debugfs filesystem entries. It would be nice to use ++ * simple_attr_write directly, but we need to make sure that the data.lock ++ * is held during the actual write. ++ */ ++static ssize_t simple_data_write(struct file *filp, const char __user *ubuf, ++ size_t cnt, loff_t *ppos, u64 *entry) +{ -+ struct stripe_head *sh = stripe_head_ref; -+ struct bio *return_bi = NULL; -+ int i; -+ -+ pr_debug("%s: stripe %llu\n", __func__, -+ (unsigned long long)sh->sector); -+ -+ /* clear completed biofills */ -+ for (i = sh->disks; i--; ) { -+ struct r5dev *dev = &sh->dev[i]; -+ -+ /* acknowledge completion of a biofill operation */ -+ /* and check if we need to reply to a read request, -+ * new R5_Wantfill requests are held off until -+ * !STRIPE_BIOFILL_RUN -+ */ -+ if (test_and_clear_bit(R5_Wantfill, &dev->flags)) { -+ struct bio *rbi, *rbi2; -+ -+ BUG_ON(!dev->read); -+ rbi = dev->read; -+ dev->read = NULL; -+ while (rbi && rbi->bi_iter.bi_sector < -+ dev->sector + STRIPE_SECTORS) { -+ rbi2 = r5_next_bio(rbi, dev->sector); -+ if (!raid5_dec_bi_active_stripes(rbi)) { -+ rbi->bi_next = return_bi; -+ return_bi = rbi; -+ } -+ rbi = rbi2; -+ } -+ } -+ } -+ clear_bit(STRIPE_BIOFILL_RUN, &sh->state); -+ -+ return_io(return_bi); ++ char buf[U64STR_SIZE]; ++ int csize = min(cnt, sizeof(buf)); ++ u64 val = 0; ++ int err = 0; + -+ set_bit(STRIPE_HANDLE, &sh->state); -+ release_stripe(sh); -+} ++ memset(buf, '\0', sizeof(buf)); ++ if (copy_from_user(buf, ubuf, csize)) ++ return -EFAULT; + -+static void ops_run_biofill(struct stripe_head *sh) -+{ -+ struct dma_async_tx_descriptor *tx = NULL; -+ struct async_submit_ctl submit; -+ int i; ++ buf[U64STR_SIZE-1] = '\0'; /* just in case */ ++ err = kstrtoull(buf, 10, &val); ++ if (err) ++ return -EINVAL; + -+ BUG_ON(sh->batch_head); -+ pr_debug("%s: stripe %llu\n", __func__, -+ (unsigned long long)sh->sector); -+ -+ for (i = sh->disks; i--; ) { -+ struct r5dev *dev = &sh->dev[i]; -+ if (test_bit(R5_Wantfill, &dev->flags)) { -+ struct bio *rbi; -+ spin_lock_irq(&sh->stripe_lock); -+ dev->read = rbi = dev->toread; -+ dev->toread = NULL; -+ spin_unlock_irq(&sh->stripe_lock); -+ while (rbi && rbi->bi_iter.bi_sector < -+ dev->sector + STRIPE_SECTORS) { -+ tx = async_copy_data(0, rbi, &dev->page, -+ dev->sector, tx, sh); -+ rbi = r5_next_bio(rbi, dev->sector); -+ } -+ } -+ } ++ mutex_lock(&data.lock); ++ *entry = val; ++ mutex_unlock(&data.lock); + -+ atomic_inc(&sh->count); -+ init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_biofill, sh, NULL); -+ async_trigger_callback(&submit); ++ return csize; +} + -+static void mark_target_uptodate(struct stripe_head *sh, int target) ++/** ++ * debug_count_fopen - Open function for "count" debugfs entry ++ * @inode: The in-kernel inode representation of the debugfs "file" ++ * @filp: The active open file structure for the debugfs "file" ++ * ++ * This function provides an open implementation for the "count" debugfs ++ * interface to the hardware latency detector. ++ */ ++static int debug_count_fopen(struct inode *inode, struct file *filp) +{ -+ struct r5dev *tgt; -+ -+ if (target < 0) -+ return; -+ -+ tgt = &sh->dev[target]; -+ set_bit(R5_UPTODATE, &tgt->flags); -+ BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags)); -+ clear_bit(R5_Wantcompute, &tgt->flags); ++ return 0; +} + -+static void ops_complete_compute(void *stripe_head_ref) ++/** ++ * debug_count_fread - Read function for "count" debugfs entry ++ * @filp: The active open file structure for the debugfs "file" ++ * @ubuf: The userspace provided buffer to read value into ++ * @cnt: The maximum number of bytes to read ++ * @ppos: The current "file" position ++ * ++ * This function provides a read implementation for the "count" debugfs ++ * interface to the hardware latency detector. Can be used to read the ++ * number of latency readings exceeding the configured threshold since ++ * the detector was last reset (e.g. by writing a zero into "count"). ++ */ ++static ssize_t debug_count_fread(struct file *filp, char __user *ubuf, ++ size_t cnt, loff_t *ppos) +{ -+ struct stripe_head *sh = stripe_head_ref; -+ -+ pr_debug("%s: stripe %llu\n", __func__, -+ (unsigned long long)sh->sector); -+ -+ /* mark the computed target(s) as uptodate */ -+ mark_target_uptodate(sh, sh->ops.target); -+ mark_target_uptodate(sh, sh->ops.target2); -+ -+ clear_bit(STRIPE_COMPUTE_RUN, &sh->state); -+ if (sh->check_state == check_state_compute_run) -+ sh->check_state = check_state_compute_result; -+ set_bit(STRIPE_HANDLE, &sh->state); -+ release_stripe(sh); ++ return simple_data_read(filp, ubuf, cnt, ppos, &data.count); +} + -+/* return a pointer to the address conversion region of the scribble buffer */ -+static addr_conv_t *to_addr_conv(struct stripe_head *sh, -+ struct raid5_percpu *percpu, int i) ++/** ++ * debug_count_fwrite - Write function for "count" debugfs entry ++ * @filp: The active open file structure for the debugfs "file" ++ * @ubuf: The user buffer that contains the value to write ++ * @cnt: The maximum number of bytes to write to "file" ++ * @ppos: The current position in the debugfs "file" ++ * ++ * This function provides a write implementation for the "count" debugfs ++ * interface to the hardware latency detector. Can be used to write a ++ * desired value, especially to zero the total count. ++ */ ++static ssize_t debug_count_fwrite(struct file *filp, ++ const char __user *ubuf, ++ size_t cnt, ++ loff_t *ppos) +{ -+ void *addr; -+ -+ addr = flex_array_get(percpu->scribble, i); -+ return addr + sizeof(struct page *) * (sh->disks + 2); ++ return simple_data_write(filp, ubuf, cnt, ppos, &data.count); +} + -+/* return a pointer to the address conversion region of the scribble buffer */ -+static struct page **to_addr_page(struct raid5_percpu *percpu, int i) ++/** ++ * debug_enable_fopen - Dummy open function for "enable" debugfs interface ++ * @inode: The in-kernel inode representation of the debugfs "file" ++ * @filp: The active open file structure for the debugfs "file" ++ * ++ * This function provides an open implementation for the "enable" debugfs ++ * interface to the hardware latency detector. ++ */ ++static int debug_enable_fopen(struct inode *inode, struct file *filp) +{ -+ void *addr; -+ -+ addr = flex_array_get(percpu->scribble, i); -+ return addr; ++ return 0; +} + -+static struct dma_async_tx_descriptor * -+ops_run_compute5(struct stripe_head *sh, struct raid5_percpu *percpu) ++/** ++ * debug_enable_fread - Read function for "enable" debugfs interface ++ * @filp: The active open file structure for the debugfs "file" ++ * @ubuf: The userspace provided buffer to read value into ++ * @cnt: The maximum number of bytes to read ++ * @ppos: The current "file" position ++ * ++ * This function provides a read implementation for the "enable" debugfs ++ * interface to the hardware latency detector. Can be used to determine ++ * whether the detector is currently enabled ("0\n" or "1\n" returned). ++ */ ++static ssize_t debug_enable_fread(struct file *filp, char __user *ubuf, ++ size_t cnt, loff_t *ppos) +{ -+ int disks = sh->disks; -+ struct page **xor_srcs = to_addr_page(percpu, 0); -+ int target = sh->ops.target; -+ struct r5dev *tgt = &sh->dev[target]; -+ struct page *xor_dest = tgt->page; -+ int count = 0; -+ struct dma_async_tx_descriptor *tx; -+ struct async_submit_ctl submit; -+ int i; -+ -+ BUG_ON(sh->batch_head); -+ -+ pr_debug("%s: stripe %llu block: %d\n", -+ __func__, (unsigned long long)sh->sector, target); -+ BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags)); -+ -+ for (i = disks; i--; ) -+ if (i != target) -+ xor_srcs[count++] = sh->dev[i].page; -+ -+ atomic_inc(&sh->count); ++ char buf[4]; + -+ init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, NULL, -+ ops_complete_compute, sh, to_addr_conv(sh, percpu, 0)); -+ if (unlikely(count == 1)) -+ tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit); -+ else -+ tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit); ++ if ((cnt < sizeof(buf)) || (*ppos)) ++ return 0; + -+ return tx; ++ buf[0] = enabled ? '1' : '0'; ++ buf[1] = '\n'; ++ buf[2] = '\0'; ++ if (copy_to_user(ubuf, buf, strlen(buf))) ++ return -EFAULT; ++ return *ppos = strlen(buf); +} + -+/* set_syndrome_sources - populate source buffers for gen_syndrome -+ * @srcs - (struct page *) array of size sh->disks -+ * @sh - stripe_head to parse ++/** ++ * debug_enable_fwrite - Write function for "enable" debugfs interface ++ * @filp: The active open file structure for the debugfs "file" ++ * @ubuf: The user buffer that contains the value to write ++ * @cnt: The maximum number of bytes to write to "file" ++ * @ppos: The current position in the debugfs "file" + * -+ * Populates srcs in proper layout order for the stripe and returns the -+ * 'count' of sources to be used in a call to async_gen_syndrome. The P -+ * destination buffer is recorded in srcs[count] and the Q destination -+ * is recorded in srcs[count+1]]. ++ * This function provides a write implementation for the "enable" debugfs ++ * interface to the hardware latency detector. Can be used to enable or ++ * disable the detector, which will have the side-effect of possibly ++ * also resetting the global stats and kicking off the measuring ++ * kthread (on an enable) or the converse (upon a disable). + */ -+static int set_syndrome_sources(struct page **srcs, -+ struct stripe_head *sh, -+ int srctype) ++static ssize_t debug_enable_fwrite(struct file *filp, ++ const char __user *ubuf, ++ size_t cnt, ++ loff_t *ppos) +{ -+ int disks = sh->disks; -+ int syndrome_disks = sh->ddf_layout ? disks : (disks - 2); -+ int d0_idx = raid6_d0(sh); -+ int count; -+ int i; -+ -+ for (i = 0; i < disks; i++) -+ srcs[i] = NULL; ++ char buf[4]; ++ int csize = min(cnt, sizeof(buf)); ++ long val = 0; ++ int err = 0; + -+ count = 0; -+ i = d0_idx; -+ do { -+ int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks); -+ struct r5dev *dev = &sh->dev[i]; -+ -+ if (i == sh->qd_idx || i == sh->pd_idx || -+ (srctype == SYNDROME_SRC_ALL) || -+ (srctype == SYNDROME_SRC_WANT_DRAIN && -+ test_bit(R5_Wantdrain, &dev->flags)) || -+ (srctype == SYNDROME_SRC_WRITTEN && -+ dev->written)) -+ srcs[slot] = sh->dev[i].page; -+ i = raid6_next_disk(i, disks); -+ } while (i != d0_idx); -+ -+ return syndrome_disks; -+} ++ memset(buf, '\0', sizeof(buf)); ++ if (copy_from_user(buf, ubuf, csize)) ++ return -EFAULT; + -+static struct dma_async_tx_descriptor * -+ops_run_compute6_1(struct stripe_head *sh, struct raid5_percpu *percpu) -+{ -+ int disks = sh->disks; -+ struct page **blocks = to_addr_page(percpu, 0); -+ int target; -+ int qd_idx = sh->qd_idx; -+ struct dma_async_tx_descriptor *tx; -+ struct async_submit_ctl submit; -+ struct r5dev *tgt; -+ struct page *dest; -+ int i; -+ int count; ++ buf[sizeof(buf)-1] = '\0'; /* just in case */ ++ err = kstrtoul(buf, 10, &val); ++ if (0 != err) ++ return -EINVAL; + -+ BUG_ON(sh->batch_head); -+ if (sh->ops.target < 0) -+ target = sh->ops.target2; -+ else if (sh->ops.target2 < 0) -+ target = sh->ops.target; -+ else -+ /* we should only have one valid target */ -+ BUG(); -+ BUG_ON(target < 0); -+ pr_debug("%s: stripe %llu block: %d\n", -+ __func__, (unsigned long long)sh->sector, target); -+ -+ tgt = &sh->dev[target]; -+ BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags)); -+ dest = tgt->page; -+ -+ atomic_inc(&sh->count); -+ -+ if (target == qd_idx) { -+ count = set_syndrome_sources(blocks, sh, SYNDROME_SRC_ALL); -+ blocks[count] = NULL; /* regenerating p is not necessary */ -+ BUG_ON(blocks[count+1] != dest); /* q should already be set */ -+ init_async_submit(&submit, ASYNC_TX_FENCE, NULL, -+ ops_complete_compute, sh, -+ to_addr_conv(sh, percpu, 0)); -+ tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit); ++ if (val) { ++ if (enabled) ++ goto unlock; ++ enabled = 1; ++ __reset_stats(); ++ if (start_kthread()) ++ return -EFAULT; + } else { -+ /* Compute any data- or p-drive using XOR */ -+ count = 0; -+ for (i = disks; i-- ; ) { -+ if (i == target || i == qd_idx) -+ continue; -+ blocks[count++] = sh->dev[i].page; ++ if (!enabled) ++ goto unlock; ++ enabled = 0; ++ err = stop_kthread(); ++ if (err) { ++ pr_err(BANNER "cannot stop kthread\n"); ++ return -EFAULT; + } -+ -+ init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, -+ NULL, ops_complete_compute, sh, -+ to_addr_conv(sh, percpu, 0)); -+ tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE, &submit); ++ wake_up(&data.wq); /* reader(s) should return */ + } -+ -+ return tx; ++unlock: ++ return csize; +} + -+static struct dma_async_tx_descriptor * -+ops_run_compute6_2(struct stripe_head *sh, struct raid5_percpu *percpu) ++/** ++ * debug_max_fopen - Open function for "max" debugfs entry ++ * @inode: The in-kernel inode representation of the debugfs "file" ++ * @filp: The active open file structure for the debugfs "file" ++ * ++ * This function provides an open implementation for the "max" debugfs ++ * interface to the hardware latency detector. ++ */ ++static int debug_max_fopen(struct inode *inode, struct file *filp) +{ -+ int i, count, disks = sh->disks; -+ int syndrome_disks = sh->ddf_layout ? disks : disks-2; -+ int d0_idx = raid6_d0(sh); -+ int faila = -1, failb = -1; -+ int target = sh->ops.target; -+ int target2 = sh->ops.target2; -+ struct r5dev *tgt = &sh->dev[target]; -+ struct r5dev *tgt2 = &sh->dev[target2]; -+ struct dma_async_tx_descriptor *tx; -+ struct page **blocks = to_addr_page(percpu, 0); -+ struct async_submit_ctl submit; -+ -+ BUG_ON(sh->batch_head); -+ pr_debug("%s: stripe %llu block1: %d block2: %d\n", -+ __func__, (unsigned long long)sh->sector, target, target2); -+ BUG_ON(target < 0 || target2 < 0); -+ BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags)); -+ BUG_ON(!test_bit(R5_Wantcompute, &tgt2->flags)); -+ -+ /* we need to open-code set_syndrome_sources to handle the -+ * slot number conversion for 'faila' and 'failb' -+ */ -+ for (i = 0; i < disks ; i++) -+ blocks[i] = NULL; -+ count = 0; -+ i = d0_idx; -+ do { -+ int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks); -+ -+ blocks[slot] = sh->dev[i].page; -+ -+ if (i == target) -+ faila = slot; -+ if (i == target2) -+ failb = slot; -+ i = raid6_next_disk(i, disks); -+ } while (i != d0_idx); -+ -+ BUG_ON(faila == failb); -+ if (failb < faila) -+ swap(faila, failb); -+ pr_debug("%s: stripe: %llu faila: %d failb: %d\n", -+ __func__, (unsigned long long)sh->sector, faila, failb); -+ -+ atomic_inc(&sh->count); -+ -+ if (failb == syndrome_disks+1) { -+ /* Q disk is one of the missing disks */ -+ if (faila == syndrome_disks) { -+ /* Missing P+Q, just recompute */ -+ init_async_submit(&submit, ASYNC_TX_FENCE, NULL, -+ ops_complete_compute, sh, -+ to_addr_conv(sh, percpu, 0)); -+ return async_gen_syndrome(blocks, 0, syndrome_disks+2, -+ STRIPE_SIZE, &submit); -+ } else { -+ struct page *dest; -+ int data_target; -+ int qd_idx = sh->qd_idx; -+ -+ /* Missing D+Q: recompute D from P, then recompute Q */ -+ if (target == qd_idx) -+ data_target = target2; -+ else -+ data_target = target; -+ -+ count = 0; -+ for (i = disks; i-- ; ) { -+ if (i == data_target || i == qd_idx) -+ continue; -+ blocks[count++] = sh->dev[i].page; -+ } -+ dest = sh->dev[data_target].page; -+ init_async_submit(&submit, -+ ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, -+ NULL, NULL, NULL, -+ to_addr_conv(sh, percpu, 0)); -+ tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE, -+ &submit); -+ -+ count = set_syndrome_sources(blocks, sh, SYNDROME_SRC_ALL); -+ init_async_submit(&submit, ASYNC_TX_FENCE, tx, -+ ops_complete_compute, sh, -+ to_addr_conv(sh, percpu, 0)); -+ return async_gen_syndrome(blocks, 0, count+2, -+ STRIPE_SIZE, &submit); -+ } -+ } else { -+ init_async_submit(&submit, ASYNC_TX_FENCE, NULL, -+ ops_complete_compute, sh, -+ to_addr_conv(sh, percpu, 0)); -+ if (failb == syndrome_disks) { -+ /* We're missing D+P. */ -+ return async_raid6_datap_recov(syndrome_disks+2, -+ STRIPE_SIZE, faila, -+ blocks, &submit); -+ } else { -+ /* We're missing D+D. */ -+ return async_raid6_2data_recov(syndrome_disks+2, -+ STRIPE_SIZE, faila, failb, -+ blocks, &submit); -+ } -+ } ++ return 0; +} + -+static void ops_complete_prexor(void *stripe_head_ref) ++/** ++ * debug_max_fread - Read function for "max" debugfs entry ++ * @filp: The active open file structure for the debugfs "file" ++ * @ubuf: The userspace provided buffer to read value into ++ * @cnt: The maximum number of bytes to read ++ * @ppos: The current "file" position ++ * ++ * This function provides a read implementation for the "max" debugfs ++ * interface to the hardware latency detector. Can be used to determine ++ * the maximum latency value observed since it was last reset. ++ */ ++static ssize_t debug_max_fread(struct file *filp, char __user *ubuf, ++ size_t cnt, loff_t *ppos) +{ -+ struct stripe_head *sh = stripe_head_ref; -+ -+ pr_debug("%s: stripe %llu\n", __func__, -+ (unsigned long long)sh->sector); ++ return simple_data_read(filp, ubuf, cnt, ppos, &data.max_sample); +} + -+static struct dma_async_tx_descriptor * -+ops_run_prexor5(struct stripe_head *sh, struct raid5_percpu *percpu, -+ struct dma_async_tx_descriptor *tx) ++/** ++ * debug_max_fwrite - Write function for "max" debugfs entry ++ * @filp: The active open file structure for the debugfs "file" ++ * @ubuf: The user buffer that contains the value to write ++ * @cnt: The maximum number of bytes to write to "file" ++ * @ppos: The current position in the debugfs "file" ++ * ++ * This function provides a write implementation for the "max" debugfs ++ * interface to the hardware latency detector. Can be used to reset the ++ * maximum or set it to some other desired value - if, then, subsequent ++ * measurements exceed this value, the maximum will be updated. ++ */ ++static ssize_t debug_max_fwrite(struct file *filp, ++ const char __user *ubuf, ++ size_t cnt, ++ loff_t *ppos) +{ -+ int disks = sh->disks; -+ struct page **xor_srcs = to_addr_page(percpu, 0); -+ int count = 0, pd_idx = sh->pd_idx, i; -+ struct async_submit_ctl submit; -+ -+ /* existing parity data subtracted */ -+ struct page *xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page; -+ -+ BUG_ON(sh->batch_head); -+ pr_debug("%s: stripe %llu\n", __func__, -+ (unsigned long long)sh->sector); -+ -+ for (i = disks; i--; ) { -+ struct r5dev *dev = &sh->dev[i]; -+ /* Only process blocks that are known to be uptodate */ -+ if (test_bit(R5_Wantdrain, &dev->flags)) -+ xor_srcs[count++] = dev->page; -+ } -+ -+ init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx, -+ ops_complete_prexor, sh, to_addr_conv(sh, percpu, 0)); -+ tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit); -+ -+ return tx; ++ return simple_data_write(filp, ubuf, cnt, ppos, &data.max_sample); +} + -+static struct dma_async_tx_descriptor * -+ops_run_prexor6(struct stripe_head *sh, struct raid5_percpu *percpu, -+ struct dma_async_tx_descriptor *tx) -+{ -+ struct page **blocks = to_addr_page(percpu, 0); -+ int count; -+ struct async_submit_ctl submit; -+ -+ pr_debug("%s: stripe %llu\n", __func__, -+ (unsigned long long)sh->sector); -+ -+ count = set_syndrome_sources(blocks, sh, SYNDROME_SRC_WANT_DRAIN); + -+ init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_PQ_XOR_DST, tx, -+ ops_complete_prexor, sh, to_addr_conv(sh, percpu, 0)); -+ tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit); -+ -+ return tx; ++/** ++ * debug_sample_fopen - An open function for "sample" debugfs interface ++ * @inode: The in-kernel inode representation of this debugfs "file" ++ * @filp: The active open file structure for the debugfs "file" ++ * ++ * This function handles opening the "sample" file within the hardware ++ * latency detector debugfs directory interface. This file is used to read ++ * raw samples from the global ring_buffer and allows the user to see a ++ * running latency history. Can be opened blocking or non-blocking, ++ * affecting whether it behaves as a buffer read pipe, or does not. ++ * Implements simple locking to prevent multiple simultaneous use. ++ */ ++static int debug_sample_fopen(struct inode *inode, struct file *filp) ++{ ++ if (!atomic_add_unless(&data.sample_open, 1, 1)) ++ return -EBUSY; ++ else ++ return 0; +} + -+static struct dma_async_tx_descriptor * -+ops_run_biodrain(struct stripe_head *sh, struct dma_async_tx_descriptor *tx) ++/** ++ * debug_sample_fread - A read function for "sample" debugfs interface ++ * @filp: The active open file structure for the debugfs "file" ++ * @ubuf: The user buffer that will contain the samples read ++ * @cnt: The maximum bytes to read from the debugfs "file" ++ * @ppos: The current position in the debugfs "file" ++ * ++ * This function handles reading from the "sample" file within the hardware ++ * latency detector debugfs directory interface. This file is used to read ++ * raw samples from the global ring_buffer and allows the user to see a ++ * running latency history. By default this will block pending a new ++ * value written into the sample buffer, unless there are already a ++ * number of value(s) waiting in the buffer, or the sample file was ++ * previously opened in a non-blocking mode of operation. ++ */ ++static ssize_t debug_sample_fread(struct file *filp, char __user *ubuf, ++ size_t cnt, loff_t *ppos) +{ -+ int disks = sh->disks; -+ int i; -+ struct stripe_head *head_sh = sh; ++ int len = 0; ++ char buf[64]; ++ struct sample *sample = NULL; + -+ pr_debug("%s: stripe %llu\n", __func__, -+ (unsigned long long)sh->sector); ++ if (!enabled) ++ return 0; + -+ for (i = disks; i--; ) { -+ struct r5dev *dev; -+ struct bio *chosen; ++ sample = kzalloc(sizeof(struct sample), GFP_KERNEL); ++ if (!sample) ++ return -ENOMEM; + -+ sh = head_sh; -+ if (test_and_clear_bit(R5_Wantdrain, &head_sh->dev[i].flags)) { -+ struct bio *wbi; ++ while (!buffer_get_sample(sample)) { + -+again: -+ dev = &sh->dev[i]; -+ spin_lock_irq(&sh->stripe_lock); -+ chosen = dev->towrite; -+ dev->towrite = NULL; -+ sh->overwrite_disks = 0; -+ BUG_ON(dev->written); -+ wbi = dev->written = chosen; -+ spin_unlock_irq(&sh->stripe_lock); -+ WARN_ON(dev->page != dev->orig_page); -+ -+ while (wbi && wbi->bi_iter.bi_sector < -+ dev->sector + STRIPE_SECTORS) { -+ if (wbi->bi_rw & REQ_FUA) -+ set_bit(R5_WantFUA, &dev->flags); -+ if (wbi->bi_rw & REQ_SYNC) -+ set_bit(R5_SyncIO, &dev->flags); -+ if (wbi->bi_rw & REQ_DISCARD) -+ set_bit(R5_Discard, &dev->flags); -+ else { -+ tx = async_copy_data(1, wbi, &dev->page, -+ dev->sector, tx, sh); -+ if (dev->page != dev->orig_page) { -+ set_bit(R5_SkipCopy, &dev->flags); -+ clear_bit(R5_UPTODATE, &dev->flags); -+ clear_bit(R5_OVERWRITE, &dev->flags); -+ } -+ } -+ wbi = r5_next_bio(wbi, dev->sector); -+ } ++ DEFINE_WAIT(wait); + -+ if (head_sh->batch_head) { -+ sh = list_first_entry(&sh->batch_list, -+ struct stripe_head, -+ batch_list); -+ if (sh == head_sh) -+ continue; -+ goto again; -+ } ++ if (filp->f_flags & O_NONBLOCK) { ++ len = -EAGAIN; ++ goto out; + } -+ } -+ -+ return tx; -+} + -+static void ops_complete_reconstruct(void *stripe_head_ref) -+{ -+ struct stripe_head *sh = stripe_head_ref; -+ int disks = sh->disks; -+ int pd_idx = sh->pd_idx; -+ int qd_idx = sh->qd_idx; -+ int i; -+ bool fua = false, sync = false, discard = false; -+ -+ pr_debug("%s: stripe %llu\n", __func__, -+ (unsigned long long)sh->sector); -+ -+ for (i = disks; i--; ) { -+ fua |= test_bit(R5_WantFUA, &sh->dev[i].flags); -+ sync |= test_bit(R5_SyncIO, &sh->dev[i].flags); -+ discard |= test_bit(R5_Discard, &sh->dev[i].flags); -+ } -+ -+ for (i = disks; i--; ) { -+ struct r5dev *dev = &sh->dev[i]; ++ prepare_to_wait(&data.wq, &wait, TASK_INTERRUPTIBLE); ++ schedule(); ++ finish_wait(&data.wq, &wait); + -+ if (dev->written || i == pd_idx || i == qd_idx) { -+ if (!discard && !test_bit(R5_SkipCopy, &dev->flags)) -+ set_bit(R5_UPTODATE, &dev->flags); -+ if (fua) -+ set_bit(R5_WantFUA, &dev->flags); -+ if (sync) -+ set_bit(R5_SyncIO, &dev->flags); ++ if (signal_pending(current)) { ++ len = -EINTR; ++ goto out; + } -+ } + -+ if (sh->reconstruct_state == reconstruct_state_drain_run) -+ sh->reconstruct_state = reconstruct_state_drain_result; -+ else if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) -+ sh->reconstruct_state = reconstruct_state_prexor_drain_result; -+ else { -+ BUG_ON(sh->reconstruct_state != reconstruct_state_run); -+ sh->reconstruct_state = reconstruct_state_result; ++ if (!enabled) { /* enable was toggled */ ++ len = 0; ++ goto out; ++ } + } + -+ set_bit(STRIPE_HANDLE, &sh->state); -+ release_stripe(sh); -+} -+ -+static void -+ops_run_reconstruct5(struct stripe_head *sh, struct raid5_percpu *percpu, -+ struct dma_async_tx_descriptor *tx) -+{ -+ int disks = sh->disks; -+ struct page **xor_srcs; -+ struct async_submit_ctl submit; -+ int count, pd_idx = sh->pd_idx, i; -+ struct page *xor_dest; -+ int prexor = 0; -+ unsigned long flags; -+ int j = 0; -+ struct stripe_head *head_sh = sh; -+ int last_stripe; ++ len = snprintf(buf, sizeof(buf), "%010lu.%010lu\t%llu\t%llu\n", ++ sample->timestamp.tv_sec, ++ sample->timestamp.tv_nsec, ++ sample->duration, ++ sample->outer_duration); + -+ pr_debug("%s: stripe %llu\n", __func__, -+ (unsigned long long)sh->sector); + -+ for (i = 0; i < sh->disks; i++) { -+ if (pd_idx == i) -+ continue; -+ if (!test_bit(R5_Discard, &sh->dev[i].flags)) -+ break; -+ } -+ if (i >= sh->disks) { -+ atomic_inc(&sh->count); -+ set_bit(R5_Discard, &sh->dev[pd_idx].flags); -+ ops_complete_reconstruct(sh); -+ return; -+ } -+again: -+ count = 0; -+ xor_srcs = to_addr_page(percpu, j); -+ /* check if prexor is active which means only process blocks -+ * that are part of a read-modify-write (written) -+ */ -+ if (head_sh->reconstruct_state == reconstruct_state_prexor_drain_run) { -+ prexor = 1; -+ xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page; -+ for (i = disks; i--; ) { -+ struct r5dev *dev = &sh->dev[i]; -+ if (head_sh->dev[i].written) -+ xor_srcs[count++] = dev->page; -+ } -+ } else { -+ xor_dest = sh->dev[pd_idx].page; -+ for (i = disks; i--; ) { -+ struct r5dev *dev = &sh->dev[i]; -+ if (i != pd_idx) -+ xor_srcs[count++] = dev->page; -+ } -+ } ++ /* handling partial reads is more trouble than it's worth */ ++ if (len > cnt) ++ goto out; + -+ /* 1/ if we prexor'd then the dest is reused as a source -+ * 2/ if we did not prexor then we are redoing the parity -+ * set ASYNC_TX_XOR_DROP_DST and ASYNC_TX_XOR_ZERO_DST -+ * for the synchronous xor case -+ */ -+ last_stripe = !head_sh->batch_head || -+ list_first_entry(&sh->batch_list, -+ struct stripe_head, batch_list) == head_sh; -+ if (last_stripe) { -+ flags = ASYNC_TX_ACK | -+ (prexor ? ASYNC_TX_XOR_DROP_DST : ASYNC_TX_XOR_ZERO_DST); -+ -+ atomic_inc(&head_sh->count); -+ init_async_submit(&submit, flags, tx, ops_complete_reconstruct, head_sh, -+ to_addr_conv(sh, percpu, j)); -+ } else { -+ flags = prexor ? ASYNC_TX_XOR_DROP_DST : ASYNC_TX_XOR_ZERO_DST; -+ init_async_submit(&submit, flags, tx, NULL, NULL, -+ to_addr_conv(sh, percpu, j)); -+ } ++ if (copy_to_user(ubuf, buf, len)) ++ len = -EFAULT; + -+ if (unlikely(count == 1)) -+ tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit); -+ else -+ tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit); -+ if (!last_stripe) { -+ j++; -+ sh = list_first_entry(&sh->batch_list, struct stripe_head, -+ batch_list); -+ goto again; -+ } ++out: ++ kfree(sample); ++ return len; +} + -+static void -+ops_run_reconstruct6(struct stripe_head *sh, struct raid5_percpu *percpu, -+ struct dma_async_tx_descriptor *tx) ++/** ++ * debug_sample_release - Release function for "sample" debugfs interface ++ * @inode: The in-kernel inode represenation of the debugfs "file" ++ * @filp: The active open file structure for the debugfs "file" ++ * ++ * This function completes the close of the debugfs interface "sample" file. ++ * Frees the sample_open "lock" so that other users may open the interface. ++ */ ++static int debug_sample_release(struct inode *inode, struct file *filp) +{ -+ struct async_submit_ctl submit; -+ struct page **blocks; -+ int count, i, j = 0; -+ struct stripe_head *head_sh = sh; -+ int last_stripe; -+ int synflags; -+ unsigned long txflags; -+ -+ pr_debug("%s: stripe %llu\n", __func__, (unsigned long long)sh->sector); -+ -+ for (i = 0; i < sh->disks; i++) { -+ if (sh->pd_idx == i || sh->qd_idx == i) -+ continue; -+ if (!test_bit(R5_Discard, &sh->dev[i].flags)) -+ break; -+ } -+ if (i >= sh->disks) { -+ atomic_inc(&sh->count); -+ set_bit(R5_Discard, &sh->dev[sh->pd_idx].flags); -+ set_bit(R5_Discard, &sh->dev[sh->qd_idx].flags); -+ ops_complete_reconstruct(sh); -+ return; -+ } -+ -+again: -+ blocks = to_addr_page(percpu, j); -+ -+ if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) { -+ synflags = SYNDROME_SRC_WRITTEN; -+ txflags = ASYNC_TX_ACK | ASYNC_TX_PQ_XOR_DST; -+ } else { -+ synflags = SYNDROME_SRC_ALL; -+ txflags = ASYNC_TX_ACK; -+ } -+ -+ count = set_syndrome_sources(blocks, sh, synflags); -+ last_stripe = !head_sh->batch_head || -+ list_first_entry(&sh->batch_list, -+ struct stripe_head, batch_list) == head_sh; ++ atomic_dec(&data.sample_open); + -+ if (last_stripe) { -+ atomic_inc(&head_sh->count); -+ init_async_submit(&submit, txflags, tx, ops_complete_reconstruct, -+ head_sh, to_addr_conv(sh, percpu, j)); -+ } else -+ init_async_submit(&submit, 0, tx, NULL, NULL, -+ to_addr_conv(sh, percpu, j)); -+ tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit); -+ if (!last_stripe) { -+ j++; -+ sh = list_first_entry(&sh->batch_list, struct stripe_head, -+ batch_list); -+ goto again; -+ } ++ return 0; +} + -+static void ops_complete_check(void *stripe_head_ref) ++/** ++ * debug_threshold_fopen - Open function for "threshold" debugfs entry ++ * @inode: The in-kernel inode representation of the debugfs "file" ++ * @filp: The active open file structure for the debugfs "file" ++ * ++ * This function provides an open implementation for the "threshold" debugfs ++ * interface to the hardware latency detector. ++ */ ++static int debug_threshold_fopen(struct inode *inode, struct file *filp) +{ -+ struct stripe_head *sh = stripe_head_ref; -+ -+ pr_debug("%s: stripe %llu\n", __func__, -+ (unsigned long long)sh->sector); -+ -+ sh->check_state = check_state_check_result; -+ set_bit(STRIPE_HANDLE, &sh->state); -+ release_stripe(sh); ++ return 0; +} + -+static void ops_run_check_p(struct stripe_head *sh, struct raid5_percpu *percpu) ++/** ++ * debug_threshold_fread - Read function for "threshold" debugfs entry ++ * @filp: The active open file structure for the debugfs "file" ++ * @ubuf: The userspace provided buffer to read value into ++ * @cnt: The maximum number of bytes to read ++ * @ppos: The current "file" position ++ * ++ * This function provides a read implementation for the "threshold" debugfs ++ * interface to the hardware latency detector. It can be used to determine ++ * the current threshold level at which a latency will be recorded in the ++ * global ring buffer, typically on the order of 10us. ++ */ ++static ssize_t debug_threshold_fread(struct file *filp, char __user *ubuf, ++ size_t cnt, loff_t *ppos) +{ -+ int disks = sh->disks; -+ int pd_idx = sh->pd_idx; -+ int qd_idx = sh->qd_idx; -+ struct page *xor_dest; -+ struct page **xor_srcs = to_addr_page(percpu, 0); -+ struct dma_async_tx_descriptor *tx; -+ struct async_submit_ctl submit; -+ int count; -+ int i; -+ -+ pr_debug("%s: stripe %llu\n", __func__, -+ (unsigned long long)sh->sector); -+ -+ BUG_ON(sh->batch_head); -+ count = 0; -+ xor_dest = sh->dev[pd_idx].page; -+ xor_srcs[count++] = xor_dest; -+ for (i = disks; i--; ) { -+ if (i == pd_idx || i == qd_idx) -+ continue; -+ xor_srcs[count++] = sh->dev[i].page; -+ } -+ -+ init_async_submit(&submit, 0, NULL, NULL, NULL, -+ to_addr_conv(sh, percpu, 0)); -+ tx = async_xor_val(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, -+ &sh->ops.zero_sum_result, &submit); -+ -+ atomic_inc(&sh->count); -+ init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_check, sh, NULL); -+ tx = async_trigger_callback(&submit); ++ return simple_data_read(filp, ubuf, cnt, ppos, &data.threshold); +} + -+static void ops_run_check_pq(struct stripe_head *sh, struct raid5_percpu *percpu, int checkp) ++/** ++ * debug_threshold_fwrite - Write function for "threshold" debugfs entry ++ * @filp: The active open file structure for the debugfs "file" ++ * @ubuf: The user buffer that contains the value to write ++ * @cnt: The maximum number of bytes to write to "file" ++ * @ppos: The current position in the debugfs "file" ++ * ++ * This function provides a write implementation for the "threshold" debugfs ++ * interface to the hardware latency detector. It can be used to configure ++ * the threshold level at which any subsequently detected latencies will ++ * be recorded into the global ring buffer. ++ */ ++static ssize_t debug_threshold_fwrite(struct file *filp, ++ const char __user *ubuf, ++ size_t cnt, ++ loff_t *ppos) +{ -+ struct page **srcs = to_addr_page(percpu, 0); -+ struct async_submit_ctl submit; -+ int count; ++ int ret; + -+ pr_debug("%s: stripe %llu checkp: %d\n", __func__, -+ (unsigned long long)sh->sector, checkp); -+ -+ BUG_ON(sh->batch_head); -+ count = set_syndrome_sources(srcs, sh, SYNDROME_SRC_ALL); -+ if (!checkp) -+ srcs[count] = NULL; -+ -+ atomic_inc(&sh->count); -+ init_async_submit(&submit, ASYNC_TX_ACK, NULL, ops_complete_check, -+ sh, to_addr_conv(sh, percpu, 0)); -+ async_syndrome_val(srcs, 0, count+2, STRIPE_SIZE, -+ &sh->ops.zero_sum_result, percpu->spare_page, &submit); -+} -+ -+static void raid_run_ops(struct stripe_head *sh, unsigned long ops_request) -+{ -+ int overlap_clear = 0, i, disks = sh->disks; -+ struct dma_async_tx_descriptor *tx = NULL; -+ struct r5conf *conf = sh->raid_conf; -+ int level = conf->level; -+ struct raid5_percpu *percpu; -+ unsigned long cpu; -+ -+ cpu = get_cpu(); -+ percpu = per_cpu_ptr(conf->percpu, cpu); -+ if (test_bit(STRIPE_OP_BIOFILL, &ops_request)) { -+ ops_run_biofill(sh); -+ overlap_clear++; -+ } -+ -+ if (test_bit(STRIPE_OP_COMPUTE_BLK, &ops_request)) { -+ if (level < 6) -+ tx = ops_run_compute5(sh, percpu); -+ else { -+ if (sh->ops.target2 < 0 || sh->ops.target < 0) -+ tx = ops_run_compute6_1(sh, percpu); -+ else -+ tx = ops_run_compute6_2(sh, percpu); -+ } -+ /* terminate the chain if reconstruct is not set to be run */ -+ if (tx && !test_bit(STRIPE_OP_RECONSTRUCT, &ops_request)) -+ async_tx_ack(tx); -+ } -+ -+ if (test_bit(STRIPE_OP_PREXOR, &ops_request)) { -+ if (level < 6) -+ tx = ops_run_prexor5(sh, percpu, tx); -+ else -+ tx = ops_run_prexor6(sh, percpu, tx); -+ } -+ -+ if (test_bit(STRIPE_OP_BIODRAIN, &ops_request)) { -+ tx = ops_run_biodrain(sh, tx); -+ overlap_clear++; -+ } -+ -+ if (test_bit(STRIPE_OP_RECONSTRUCT, &ops_request)) { -+ if (level < 6) -+ ops_run_reconstruct5(sh, percpu, tx); -+ else -+ ops_run_reconstruct6(sh, percpu, tx); -+ } -+ -+ if (test_bit(STRIPE_OP_CHECK, &ops_request)) { -+ if (sh->check_state == check_state_run) -+ ops_run_check_p(sh, percpu); -+ else if (sh->check_state == check_state_run_q) -+ ops_run_check_pq(sh, percpu, 0); -+ else if (sh->check_state == check_state_run_pq) -+ ops_run_check_pq(sh, percpu, 1); -+ else -+ BUG(); -+ } -+ -+ if (overlap_clear && !sh->batch_head) -+ for (i = disks; i--; ) { -+ struct r5dev *dev = &sh->dev[i]; -+ if (test_and_clear_bit(R5_Overlap, &dev->flags)) -+ wake_up(&sh->raid_conf->wait_for_overlap); -+ } -+ put_cpu(); -+} -+ -+static struct stripe_head *alloc_stripe(struct kmem_cache *sc, gfp_t gfp) -+{ -+ struct stripe_head *sh; -+ -+ sh = kmem_cache_zalloc(sc, gfp); -+ if (sh) { -+ spin_lock_init(&sh->stripe_lock); -+ spin_lock_init(&sh->batch_lock); -+ INIT_LIST_HEAD(&sh->batch_list); -+ INIT_LIST_HEAD(&sh->lru); -+ atomic_set(&sh->count, 1); -+ } -+ return sh; -+} -+static int grow_one_stripe(struct r5conf *conf, gfp_t gfp) -+{ -+ struct stripe_head *sh; -+ -+ sh = alloc_stripe(conf->slab_cache, gfp); -+ if (!sh) -+ return 0; -+ -+ sh->raid_conf = conf; ++ ret = simple_data_write(filp, ubuf, cnt, ppos, &data.threshold); + -+ if (grow_buffers(sh, gfp)) { -+ shrink_buffers(sh); -+ kmem_cache_free(conf->slab_cache, sh); -+ return 0; -+ } -+ sh->hash_lock_index = -+ conf->max_nr_stripes % NR_STRIPE_HASH_LOCKS; -+ /* we just created an active stripe so... */ -+ atomic_inc(&conf->active_stripes); ++ if (enabled) ++ wake_up_process(kthread); + -+ release_stripe(sh); -+ conf->max_nr_stripes++; -+ return 1; ++ return ret; +} + -+static int grow_stripes(struct r5conf *conf, int num) ++/** ++ * debug_width_fopen - Open function for "width" debugfs entry ++ * @inode: The in-kernel inode representation of the debugfs "file" ++ * @filp: The active open file structure for the debugfs "file" ++ * ++ * This function provides an open implementation for the "width" debugfs ++ * interface to the hardware latency detector. ++ */ ++static int debug_width_fopen(struct inode *inode, struct file *filp) +{ -+ struct kmem_cache *sc; -+ int devs = max(conf->raid_disks, conf->previous_raid_disks); -+ -+ if (conf->mddev->gendisk) -+ sprintf(conf->cache_name[0], -+ "raid%d-%s", conf->level, mdname(conf->mddev)); -+ else -+ sprintf(conf->cache_name[0], -+ "raid%d-%p", conf->level, conf->mddev); -+ sprintf(conf->cache_name[1], "%s-alt", conf->cache_name[0]); -+ -+ conf->active_name = 0; -+ sc = kmem_cache_create(conf->cache_name[conf->active_name], -+ sizeof(struct stripe_head)+(devs-1)*sizeof(struct r5dev), -+ 0, 0, NULL); -+ if (!sc) -+ return 1; -+ conf->slab_cache = sc; -+ conf->pool_size = devs; -+ while (num--) -+ if (!grow_one_stripe(conf, GFP_KERNEL)) -+ return 1; -+ + return 0; +} + +/** -+ * scribble_len - return the required size of the scribble region -+ * @num - total number of disks in the array -+ * -+ * The size must be enough to contain: -+ * 1/ a struct page pointer for each device in the array +2 -+ * 2/ room to convert each entry in (1) to its corresponding dma -+ * (dma_map_page()) or page (page_address()) address. ++ * debug_width_fread - Read function for "width" debugfs entry ++ * @filp: The active open file structure for the debugfs "file" ++ * @ubuf: The userspace provided buffer to read value into ++ * @cnt: The maximum number of bytes to read ++ * @ppos: The current "file" position + * -+ * Note: the +2 is for the destination buffers of the ddf/raid6 case where we -+ * calculate over all devices (not just the data blocks), using zeros in place -+ * of the P and Q blocks. ++ * This function provides a read implementation for the "width" debugfs ++ * interface to the hardware latency detector. It can be used to determine ++ * for how many us of the total window us we will actively sample for any ++ * hardware-induced latecy periods. Obviously, it is not possible to ++ * sample constantly and have the system respond to a sample reader, or, ++ * worse, without having the system appear to have gone out to lunch. + */ -+static struct flex_array *scribble_alloc(int num, int cnt, gfp_t flags) ++static ssize_t debug_width_fread(struct file *filp, char __user *ubuf, ++ size_t cnt, loff_t *ppos) +{ -+ struct flex_array *ret; -+ size_t len; -+ -+ len = sizeof(struct page *) * (num+2) + sizeof(addr_conv_t) * (num+2); -+ ret = flex_array_alloc(len, cnt, flags); -+ if (!ret) -+ return NULL; -+ /* always prealloc all elements, so no locking is required */ -+ if (flex_array_prealloc(ret, 0, cnt, flags)) { -+ flex_array_free(ret); -+ return NULL; -+ } -+ return ret; ++ return simple_data_read(filp, ubuf, cnt, ppos, &data.sample_width); +} + -+static int resize_chunks(struct r5conf *conf, int new_disks, int new_sectors) ++/** ++ * debug_width_fwrite - Write function for "width" debugfs entry ++ * @filp: The active open file structure for the debugfs "file" ++ * @ubuf: The user buffer that contains the value to write ++ * @cnt: The maximum number of bytes to write to "file" ++ * @ppos: The current position in the debugfs "file" ++ * ++ * This function provides a write implementation for the "width" debugfs ++ * interface to the hardware latency detector. It can be used to configure ++ * for how many us of the total window us we will actively sample for any ++ * hardware-induced latency periods. Obviously, it is not possible to ++ * sample constantly and have the system respond to a sample reader, or, ++ * worse, without having the system appear to have gone out to lunch. It ++ * is enforced that width is less that the total window size. ++ */ ++static ssize_t debug_width_fwrite(struct file *filp, ++ const char __user *ubuf, ++ size_t cnt, ++ loff_t *ppos) +{ -+ unsigned long cpu; ++ char buf[U64STR_SIZE]; ++ int csize = min(cnt, sizeof(buf)); ++ u64 val = 0; + int err = 0; + -+ mddev_suspend(conf->mddev); -+ get_online_cpus(); -+ for_each_present_cpu(cpu) { -+ struct raid5_percpu *percpu; -+ struct flex_array *scribble; -+ -+ percpu = per_cpu_ptr(conf->percpu, cpu); -+ scribble = scribble_alloc(new_disks, -+ new_sectors / STRIPE_SECTORS, -+ GFP_NOIO); -+ -+ if (scribble) { -+ flex_array_free(percpu->scribble); -+ percpu->scribble = scribble; -+ } else { -+ err = -ENOMEM; -+ break; -+ } -+ } -+ put_online_cpus(); -+ mddev_resume(conf->mddev); -+ return err; -+} -+ -+static int resize_stripes(struct r5conf *conf, int newsize) -+{ -+ /* Make all the stripes able to hold 'newsize' devices. -+ * New slots in each stripe get 'page' set to a new page. -+ * -+ * This happens in stages: -+ * 1/ create a new kmem_cache and allocate the required number of -+ * stripe_heads. -+ * 2/ gather all the old stripe_heads and transfer the pages across -+ * to the new stripe_heads. This will have the side effect of -+ * freezing the array as once all stripe_heads have been collected, -+ * no IO will be possible. Old stripe heads are freed once their -+ * pages have been transferred over, and the old kmem_cache is -+ * freed when all stripes are done. -+ * 3/ reallocate conf->disks to be suitable bigger. If this fails, -+ * we simple return a failre status - no need to clean anything up. -+ * 4/ allocate new pages for the new slots in the new stripe_heads. -+ * If this fails, we don't bother trying the shrink the -+ * stripe_heads down again, we just leave them as they are. -+ * As each stripe_head is processed the new one is released into -+ * active service. -+ * -+ * Once step2 is started, we cannot afford to wait for a write, -+ * so we use GFP_NOIO allocations. -+ */ -+ struct stripe_head *osh, *nsh; -+ LIST_HEAD(newstripes); -+ struct disk_info *ndisks; -+ int err; -+ struct kmem_cache *sc; -+ int i; -+ int hash, cnt; -+ -+ if (newsize <= conf->pool_size) -+ return 0; /* never bother to shrink */ -+ -+ err = md_allow_write(conf->mddev); -+ if (err) -+ return err; -+ -+ /* Step 1 */ -+ sc = kmem_cache_create(conf->cache_name[1-conf->active_name], -+ sizeof(struct stripe_head)+(newsize-1)*sizeof(struct r5dev), -+ 0, 0, NULL); -+ if (!sc) -+ return -ENOMEM; -+ -+ /* Need to ensure auto-resizing doesn't interfere */ -+ mutex_lock(&conf->cache_size_mutex); ++ memset(buf, '\0', sizeof(buf)); ++ if (copy_from_user(buf, ubuf, csize)) ++ return -EFAULT; + -+ for (i = conf->max_nr_stripes; i; i--) { -+ nsh = alloc_stripe(sc, GFP_KERNEL); -+ if (!nsh) -+ break; ++ buf[U64STR_SIZE-1] = '\0'; /* just in case */ ++ err = kstrtoull(buf, 10, &val); ++ if (0 != err) ++ return -EINVAL; + -+ nsh->raid_conf = conf; -+ list_add(&nsh->lru, &newstripes); -+ } -+ if (i) { -+ /* didn't get enough, give up */ -+ while (!list_empty(&newstripes)) { -+ nsh = list_entry(newstripes.next, struct stripe_head, lru); -+ list_del(&nsh->lru); -+ kmem_cache_free(sc, nsh); -+ } -+ kmem_cache_destroy(sc); -+ mutex_unlock(&conf->cache_size_mutex); -+ return -ENOMEM; -+ } -+ /* Step 2 - Must use GFP_NOIO now. -+ * OK, we have enough stripes, start collecting inactive -+ * stripes and copying them over -+ */ -+ hash = 0; -+ cnt = 0; -+ list_for_each_entry(nsh, &newstripes, lru) { -+ lock_device_hash_lock(conf, hash); -+ wait_event_cmd(conf->wait_for_stripe, -+ !list_empty(conf->inactive_list + hash), -+ unlock_device_hash_lock(conf, hash), -+ lock_device_hash_lock(conf, hash)); -+ osh = get_free_stripe(conf, hash); -+ unlock_device_hash_lock(conf, hash); -+ -+ for(i=0; ipool_size; i++) { -+ nsh->dev[i].page = osh->dev[i].page; -+ nsh->dev[i].orig_page = osh->dev[i].page; -+ } -+ nsh->hash_lock_index = hash; -+ kmem_cache_free(conf->slab_cache, osh); -+ cnt++; -+ if (cnt >= conf->max_nr_stripes / NR_STRIPE_HASH_LOCKS + -+ !!((conf->max_nr_stripes % NR_STRIPE_HASH_LOCKS) > hash)) { -+ hash++; -+ cnt = 0; -+ } ++ mutex_lock(&data.lock); ++ if (val < data.sample_window) ++ data.sample_width = val; ++ else { ++ mutex_unlock(&data.lock); ++ return -EINVAL; + } -+ kmem_cache_destroy(conf->slab_cache); ++ mutex_unlock(&data.lock); + -+ /* Step 3. -+ * At this point, we are holding all the stripes so the array -+ * is completely stalled, so now is a good time to resize -+ * conf->disks and the scribble region -+ */ -+ ndisks = kzalloc(newsize * sizeof(struct disk_info), GFP_NOIO); -+ if (ndisks) { -+ for (i=0; iraid_disks; i++) -+ ndisks[i] = conf->disks[i]; -+ kfree(conf->disks); -+ conf->disks = ndisks; -+ } else -+ err = -ENOMEM; -+ -+ mutex_unlock(&conf->cache_size_mutex); -+ /* Step 4, return new stripes to service */ -+ while(!list_empty(&newstripes)) { -+ nsh = list_entry(newstripes.next, struct stripe_head, lru); -+ list_del_init(&nsh->lru); -+ -+ for (i=conf->raid_disks; i < newsize; i++) -+ if (nsh->dev[i].page == NULL) { -+ struct page *p = alloc_page(GFP_NOIO); -+ nsh->dev[i].page = p; -+ nsh->dev[i].orig_page = p; -+ if (!p) -+ err = -ENOMEM; -+ } -+ release_stripe(nsh); -+ } -+ /* critical section pass, GFP_NOIO no longer needed */ ++ if (enabled) ++ wake_up_process(kthread); + -+ conf->slab_cache = sc; -+ conf->active_name = 1-conf->active_name; -+ if (!err) -+ conf->pool_size = newsize; -+ return err; ++ return csize; +} + -+static int drop_one_stripe(struct r5conf *conf) ++/** ++ * debug_window_fopen - Open function for "window" debugfs entry ++ * @inode: The in-kernel inode representation of the debugfs "file" ++ * @filp: The active open file structure for the debugfs "file" ++ * ++ * This function provides an open implementation for the "window" debugfs ++ * interface to the hardware latency detector. The window is the total time ++ * in us that will be considered one sample period. Conceptually, windows ++ * occur back-to-back and contain a sample width period during which ++ * actual sampling occurs. ++ */ ++static int debug_window_fopen(struct inode *inode, struct file *filp) +{ -+ struct stripe_head *sh; -+ int hash = (conf->max_nr_stripes - 1) & STRIPE_HASH_LOCKS_MASK; -+ -+ spin_lock_irq(conf->hash_locks + hash); -+ sh = get_free_stripe(conf, hash); -+ spin_unlock_irq(conf->hash_locks + hash); -+ if (!sh) -+ return 0; -+ BUG_ON(atomic_read(&sh->count)); -+ shrink_buffers(sh); -+ kmem_cache_free(conf->slab_cache, sh); -+ atomic_dec(&conf->active_stripes); -+ conf->max_nr_stripes--; -+ return 1; ++ return 0; +} + -+static void shrink_stripes(struct r5conf *conf) ++/** ++ * debug_window_fread - Read function for "window" debugfs entry ++ * @filp: The active open file structure for the debugfs "file" ++ * @ubuf: The userspace provided buffer to read value into ++ * @cnt: The maximum number of bytes to read ++ * @ppos: The current "file" position ++ * ++ * This function provides a read implementation for the "window" debugfs ++ * interface to the hardware latency detector. The window is the total time ++ * in us that will be considered one sample period. Conceptually, windows ++ * occur back-to-back and contain a sample width period during which ++ * actual sampling occurs. Can be used to read the total window size. ++ */ ++static ssize_t debug_window_fread(struct file *filp, char __user *ubuf, ++ size_t cnt, loff_t *ppos) +{ -+ while (conf->max_nr_stripes && -+ drop_one_stripe(conf)) -+ ; -+ -+ if (conf->slab_cache) -+ kmem_cache_destroy(conf->slab_cache); -+ conf->slab_cache = NULL; ++ return simple_data_read(filp, ubuf, cnt, ppos, &data.sample_window); +} + -+static void raid5_end_read_request(struct bio * bi, int error) ++/** ++ * debug_window_fwrite - Write function for "window" debugfs entry ++ * @filp: The active open file structure for the debugfs "file" ++ * @ubuf: The user buffer that contains the value to write ++ * @cnt: The maximum number of bytes to write to "file" ++ * @ppos: The current position in the debugfs "file" ++ * ++ * This function provides a write implementation for the "window" debufds ++ * interface to the hardware latency detetector. The window is the total time ++ * in us that will be considered one sample period. Conceptually, windows ++ * occur back-to-back and contain a sample width period during which ++ * actual sampling occurs. Can be used to write a new total window size. It ++ * is enfoced that any value written must be greater than the sample width ++ * size, or an error results. ++ */ ++static ssize_t debug_window_fwrite(struct file *filp, ++ const char __user *ubuf, ++ size_t cnt, ++ loff_t *ppos) +{ -+ struct stripe_head *sh = bi->bi_private; -+ struct r5conf *conf = sh->raid_conf; -+ int disks = sh->disks, i; -+ int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags); -+ char b[BDEVNAME_SIZE]; -+ struct md_rdev *rdev = NULL; -+ sector_t s; -+ -+ for (i=0 ; idev[i].req) -+ break; -+ -+ pr_debug("end_read_request %llu/%d, count: %d, uptodate %d.\n", -+ (unsigned long long)sh->sector, i, atomic_read(&sh->count), -+ uptodate); -+ if (i == disks) { -+ BUG(); -+ return; -+ } -+ if (test_bit(R5_ReadRepl, &sh->dev[i].flags)) -+ /* If replacement finished while this request was outstanding, -+ * 'replacement' might be NULL already. -+ * In that case it moved down to 'rdev'. -+ * rdev is not removed until all requests are finished. -+ */ -+ rdev = conf->disks[i].replacement; -+ if (!rdev) -+ rdev = conf->disks[i].rdev; ++ char buf[U64STR_SIZE]; ++ int csize = min(cnt, sizeof(buf)); ++ u64 val = 0; ++ int err = 0; + -+ if (use_new_offset(conf, sh)) -+ s = sh->sector + rdev->new_data_offset; -+ else -+ s = sh->sector + rdev->data_offset; -+ if (uptodate) { -+ set_bit(R5_UPTODATE, &sh->dev[i].flags); -+ if (test_bit(R5_ReadError, &sh->dev[i].flags)) { -+ /* Note that this cannot happen on a -+ * replacement device. We just fail those on -+ * any error -+ */ -+ printk_ratelimited( -+ KERN_INFO -+ "md/raid:%s: read error corrected" -+ " (%lu sectors at %llu on %s)\n", -+ mdname(conf->mddev), STRIPE_SECTORS, -+ (unsigned long long)s, -+ bdevname(rdev->bdev, b)); -+ atomic_add(STRIPE_SECTORS, &rdev->corrected_errors); -+ clear_bit(R5_ReadError, &sh->dev[i].flags); -+ clear_bit(R5_ReWrite, &sh->dev[i].flags); -+ } else if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags)) -+ clear_bit(R5_ReadNoMerge, &sh->dev[i].flags); -+ -+ if (atomic_read(&rdev->read_errors)) -+ atomic_set(&rdev->read_errors, 0); -+ } else { -+ const char *bdn = bdevname(rdev->bdev, b); -+ int retry = 0; -+ int set_bad = 0; -+ -+ clear_bit(R5_UPTODATE, &sh->dev[i].flags); -+ atomic_inc(&rdev->read_errors); -+ if (test_bit(R5_ReadRepl, &sh->dev[i].flags)) -+ printk_ratelimited( -+ KERN_WARNING -+ "md/raid:%s: read error on replacement device " -+ "(sector %llu on %s).\n", -+ mdname(conf->mddev), -+ (unsigned long long)s, -+ bdn); -+ else if (conf->mddev->degraded >= conf->max_degraded) { -+ set_bad = 1; -+ printk_ratelimited( -+ KERN_WARNING -+ "md/raid:%s: read error not correctable " -+ "(sector %llu on %s).\n", -+ mdname(conf->mddev), -+ (unsigned long long)s, -+ bdn); -+ } else if (test_bit(R5_ReWrite, &sh->dev[i].flags)) { -+ /* Oh, no!!! */ -+ set_bad = 1; -+ printk_ratelimited( -+ KERN_WARNING -+ "md/raid:%s: read error NOT corrected!! " -+ "(sector %llu on %s).\n", -+ mdname(conf->mddev), -+ (unsigned long long)s, -+ bdn); -+ } else if (atomic_read(&rdev->read_errors) -+ > conf->max_nr_stripes) -+ printk(KERN_WARNING -+ "md/raid:%s: Too many read errors, failing device %s.\n", -+ mdname(conf->mddev), bdn); -+ else -+ retry = 1; -+ if (set_bad && test_bit(In_sync, &rdev->flags) -+ && !test_bit(R5_ReadNoMerge, &sh->dev[i].flags)) -+ retry = 1; -+ if (retry) -+ if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags)) { -+ set_bit(R5_ReadError, &sh->dev[i].flags); -+ clear_bit(R5_ReadNoMerge, &sh->dev[i].flags); -+ } else -+ set_bit(R5_ReadNoMerge, &sh->dev[i].flags); -+ else { -+ clear_bit(R5_ReadError, &sh->dev[i].flags); -+ clear_bit(R5_ReWrite, &sh->dev[i].flags); -+ if (!(set_bad -+ && test_bit(In_sync, &rdev->flags) -+ && rdev_set_badblocks( -+ rdev, sh->sector, STRIPE_SECTORS, 0))) -+ md_error(conf->mddev, rdev); -+ } -+ } -+ rdev_dec_pending(rdev, conf->mddev); -+ clear_bit(R5_LOCKED, &sh->dev[i].flags); -+ set_bit(STRIPE_HANDLE, &sh->state); -+ release_stripe(sh); -+} ++ memset(buf, '\0', sizeof(buf)); ++ if (copy_from_user(buf, ubuf, csize)) ++ return -EFAULT; + -+static void raid5_end_write_request(struct bio *bi, int error) -+{ -+ struct stripe_head *sh = bi->bi_private; -+ struct r5conf *conf = sh->raid_conf; -+ int disks = sh->disks, i; -+ struct md_rdev *uninitialized_var(rdev); -+ int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags); -+ sector_t first_bad; -+ int bad_sectors; -+ int replacement = 0; -+ -+ for (i = 0 ; i < disks; i++) { -+ if (bi == &sh->dev[i].req) { -+ rdev = conf->disks[i].rdev; -+ break; -+ } -+ if (bi == &sh->dev[i].rreq) { -+ rdev = conf->disks[i].replacement; -+ if (rdev) -+ replacement = 1; -+ else -+ /* rdev was removed and 'replacement' -+ * replaced it. rdev is not removed -+ * until all requests are finished. -+ */ -+ rdev = conf->disks[i].rdev; -+ break; -+ } -+ } -+ pr_debug("end_write_request %llu/%d, count %d, uptodate: %d.\n", -+ (unsigned long long)sh->sector, i, atomic_read(&sh->count), -+ uptodate); -+ if (i == disks) { -+ BUG(); -+ return; -+ } ++ buf[U64STR_SIZE-1] = '\0'; /* just in case */ ++ err = kstrtoull(buf, 10, &val); ++ if (0 != err) ++ return -EINVAL; + -+ if (replacement) { -+ if (!uptodate) -+ md_error(conf->mddev, rdev); -+ else if (is_badblock(rdev, sh->sector, -+ STRIPE_SECTORS, -+ &first_bad, &bad_sectors)) -+ set_bit(R5_MadeGoodRepl, &sh->dev[i].flags); -+ } else { -+ if (!uptodate) { -+ set_bit(STRIPE_DEGRADED, &sh->state); -+ set_bit(WriteErrorSeen, &rdev->flags); -+ set_bit(R5_WriteError, &sh->dev[i].flags); -+ if (!test_and_set_bit(WantReplacement, &rdev->flags)) -+ set_bit(MD_RECOVERY_NEEDED, -+ &rdev->mddev->recovery); -+ } else if (is_badblock(rdev, sh->sector, -+ STRIPE_SECTORS, -+ &first_bad, &bad_sectors)) { -+ set_bit(R5_MadeGood, &sh->dev[i].flags); -+ if (test_bit(R5_ReadError, &sh->dev[i].flags)) -+ /* That was a successful write so make -+ * sure it looks like we already did -+ * a re-write. -+ */ -+ set_bit(R5_ReWrite, &sh->dev[i].flags); -+ } ++ mutex_lock(&data.lock); ++ if (data.sample_width < val) ++ data.sample_window = val; ++ else { ++ mutex_unlock(&data.lock); ++ return -EINVAL; + } -+ rdev_dec_pending(rdev, conf->mddev); -+ -+ if (sh->batch_head && !uptodate && !replacement) -+ set_bit(STRIPE_BATCH_ERR, &sh->batch_head->state); -+ -+ if (!test_and_clear_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags)) -+ clear_bit(R5_LOCKED, &sh->dev[i].flags); -+ set_bit(STRIPE_HANDLE, &sh->state); -+ release_stripe(sh); ++ mutex_unlock(&data.lock); + -+ if (sh->batch_head && sh != sh->batch_head) -+ release_stripe(sh->batch_head); ++ return csize; +} + -+static sector_t compute_blocknr(struct stripe_head *sh, int i, int previous); -+ -+static void raid5_build_block(struct stripe_head *sh, int i, int previous) -+{ -+ struct r5dev *dev = &sh->dev[i]; -+ -+ bio_init(&dev->req); -+ dev->req.bi_io_vec = &dev->vec; -+ dev->req.bi_max_vecs = 1; -+ dev->req.bi_private = sh; ++/* ++ * Function pointers for the "count" debugfs file operations ++ */ ++static const struct file_operations count_fops = { ++ .open = debug_count_fopen, ++ .read = debug_count_fread, ++ .write = debug_count_fwrite, ++ .owner = THIS_MODULE, ++}; + -+ bio_init(&dev->rreq); -+ dev->rreq.bi_io_vec = &dev->rvec; -+ dev->rreq.bi_max_vecs = 1; -+ dev->rreq.bi_private = sh; ++/* ++ * Function pointers for the "enable" debugfs file operations ++ */ ++static const struct file_operations enable_fops = { ++ .open = debug_enable_fopen, ++ .read = debug_enable_fread, ++ .write = debug_enable_fwrite, ++ .owner = THIS_MODULE, ++}; + -+ dev->flags = 0; -+ dev->sector = compute_blocknr(sh, i, previous); -+} ++/* ++ * Function pointers for the "max" debugfs file operations ++ */ ++static const struct file_operations max_fops = { ++ .open = debug_max_fopen, ++ .read = debug_max_fread, ++ .write = debug_max_fwrite, ++ .owner = THIS_MODULE, ++}; + -+static void error(struct mddev *mddev, struct md_rdev *rdev) -+{ -+ char b[BDEVNAME_SIZE]; -+ struct r5conf *conf = mddev->private; -+ unsigned long flags; -+ pr_debug("raid456: error called\n"); -+ -+ spin_lock_irqsave(&conf->device_lock, flags); -+ clear_bit(In_sync, &rdev->flags); -+ mddev->degraded = calc_degraded(conf); -+ spin_unlock_irqrestore(&conf->device_lock, flags); -+ set_bit(MD_RECOVERY_INTR, &mddev->recovery); -+ -+ set_bit(Blocked, &rdev->flags); -+ set_bit(Faulty, &rdev->flags); -+ set_bit(MD_CHANGE_DEVS, &mddev->flags); -+ printk(KERN_ALERT -+ "md/raid:%s: Disk failure on %s, disabling device.\n" -+ "md/raid:%s: Operation continuing on %d devices.\n", -+ mdname(mddev), -+ bdevname(rdev->bdev, b), -+ mdname(mddev), -+ conf->raid_disks - mddev->degraded); -+} ++/* ++ * Function pointers for the "sample" debugfs file operations ++ */ ++static const struct file_operations sample_fops = { ++ .open = debug_sample_fopen, ++ .read = debug_sample_fread, ++ .release = debug_sample_release, ++ .owner = THIS_MODULE, ++}; + +/* -+ * Input: a 'big' sector number, -+ * Output: index of the data and parity disk, and the sector # in them. ++ * Function pointers for the "threshold" debugfs file operations + */ -+static sector_t raid5_compute_sector(struct r5conf *conf, sector_t r_sector, -+ int previous, int *dd_idx, -+ struct stripe_head *sh) -+{ -+ sector_t stripe, stripe2; -+ sector_t chunk_number; -+ unsigned int chunk_offset; -+ int pd_idx, qd_idx; -+ int ddf_layout = 0; -+ sector_t new_sector; -+ int algorithm = previous ? conf->prev_algo -+ : conf->algorithm; -+ int sectors_per_chunk = previous ? conf->prev_chunk_sectors -+ : conf->chunk_sectors; -+ int raid_disks = previous ? conf->previous_raid_disks -+ : conf->raid_disks; -+ int data_disks = raid_disks - conf->max_degraded; -+ -+ /* First compute the information on this sector */ ++static const struct file_operations threshold_fops = { ++ .open = debug_threshold_fopen, ++ .read = debug_threshold_fread, ++ .write = debug_threshold_fwrite, ++ .owner = THIS_MODULE, ++}; + -+ /* -+ * Compute the chunk number and the sector offset inside the chunk -+ */ -+ chunk_offset = sector_div(r_sector, sectors_per_chunk); -+ chunk_number = r_sector; ++/* ++ * Function pointers for the "width" debugfs file operations ++ */ ++static const struct file_operations width_fops = { ++ .open = debug_width_fopen, ++ .read = debug_width_fread, ++ .write = debug_width_fwrite, ++ .owner = THIS_MODULE, ++}; + -+ /* -+ * Compute the stripe number -+ */ -+ stripe = chunk_number; -+ *dd_idx = sector_div(stripe, data_disks); -+ stripe2 = stripe; -+ /* -+ * Select the parity disk based on the user selected algorithm. -+ */ -+ pd_idx = qd_idx = -1; -+ switch(conf->level) { -+ case 4: -+ pd_idx = data_disks; -+ break; -+ case 5: -+ switch (algorithm) { -+ case ALGORITHM_LEFT_ASYMMETRIC: -+ pd_idx = data_disks - sector_div(stripe2, raid_disks); -+ if (*dd_idx >= pd_idx) -+ (*dd_idx)++; -+ break; -+ case ALGORITHM_RIGHT_ASYMMETRIC: -+ pd_idx = sector_div(stripe2, raid_disks); -+ if (*dd_idx >= pd_idx) -+ (*dd_idx)++; -+ break; -+ case ALGORITHM_LEFT_SYMMETRIC: -+ pd_idx = data_disks - sector_div(stripe2, raid_disks); -+ *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks; -+ break; -+ case ALGORITHM_RIGHT_SYMMETRIC: -+ pd_idx = sector_div(stripe2, raid_disks); -+ *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks; -+ break; -+ case ALGORITHM_PARITY_0: -+ pd_idx = 0; -+ (*dd_idx)++; -+ break; -+ case ALGORITHM_PARITY_N: -+ pd_idx = data_disks; -+ break; -+ default: -+ BUG(); -+ } -+ break; -+ case 6: -+ -+ switch (algorithm) { -+ case ALGORITHM_LEFT_ASYMMETRIC: -+ pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks); -+ qd_idx = pd_idx + 1; -+ if (pd_idx == raid_disks-1) { -+ (*dd_idx)++; /* Q D D D P */ -+ qd_idx = 0; -+ } else if (*dd_idx >= pd_idx) -+ (*dd_idx) += 2; /* D D P Q D */ -+ break; -+ case ALGORITHM_RIGHT_ASYMMETRIC: -+ pd_idx = sector_div(stripe2, raid_disks); -+ qd_idx = pd_idx + 1; -+ if (pd_idx == raid_disks-1) { -+ (*dd_idx)++; /* Q D D D P */ -+ qd_idx = 0; -+ } else if (*dd_idx >= pd_idx) -+ (*dd_idx) += 2; /* D D P Q D */ -+ break; -+ case ALGORITHM_LEFT_SYMMETRIC: -+ pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks); -+ qd_idx = (pd_idx + 1) % raid_disks; -+ *dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks; -+ break; -+ case ALGORITHM_RIGHT_SYMMETRIC: -+ pd_idx = sector_div(stripe2, raid_disks); -+ qd_idx = (pd_idx + 1) % raid_disks; -+ *dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks; -+ break; ++/* ++ * Function pointers for the "window" debugfs file operations ++ */ ++static const struct file_operations window_fops = { ++ .open = debug_window_fopen, ++ .read = debug_window_fread, ++ .write = debug_window_fwrite, ++ .owner = THIS_MODULE, ++}; + -+ case ALGORITHM_PARITY_0: -+ pd_idx = 0; -+ qd_idx = 1; -+ (*dd_idx) += 2; -+ break; -+ case ALGORITHM_PARITY_N: -+ pd_idx = data_disks; -+ qd_idx = data_disks + 1; -+ break; ++/** ++ * init_debugfs - A function to initialize the debugfs interface files ++ * ++ * This function creates entries in debugfs for "hwlat_detector", including ++ * files to read values from the detector, current samples, and the ++ * maximum sample that has been captured since the hardware latency ++ * dectector was started. ++ */ ++static int init_debugfs(void) ++{ ++ int ret = -ENOMEM; + -+ case ALGORITHM_ROTATING_ZERO_RESTART: -+ /* Exactly the same as RIGHT_ASYMMETRIC, but or -+ * of blocks for computing Q is different. -+ */ -+ pd_idx = sector_div(stripe2, raid_disks); -+ qd_idx = pd_idx + 1; -+ if (pd_idx == raid_disks-1) { -+ (*dd_idx)++; /* Q D D D P */ -+ qd_idx = 0; -+ } else if (*dd_idx >= pd_idx) -+ (*dd_idx) += 2; /* D D P Q D */ -+ ddf_layout = 1; -+ break; ++ debug_dir = debugfs_create_dir(DRVNAME, NULL); ++ if (!debug_dir) ++ goto err_debug_dir; + -+ case ALGORITHM_ROTATING_N_RESTART: -+ /* Same a left_asymmetric, by first stripe is -+ * D D D P Q rather than -+ * Q D D D P -+ */ -+ stripe2 += 1; -+ pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks); -+ qd_idx = pd_idx + 1; -+ if (pd_idx == raid_disks-1) { -+ (*dd_idx)++; /* Q D D D P */ -+ qd_idx = 0; -+ } else if (*dd_idx >= pd_idx) -+ (*dd_idx) += 2; /* D D P Q D */ -+ ddf_layout = 1; -+ break; ++ debug_sample = debugfs_create_file("sample", 0444, ++ debug_dir, NULL, ++ &sample_fops); ++ if (!debug_sample) ++ goto err_sample; + -+ case ALGORITHM_ROTATING_N_CONTINUE: -+ /* Same as left_symmetric but Q is before P */ -+ pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks); -+ qd_idx = (pd_idx + raid_disks - 1) % raid_disks; -+ *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks; -+ ddf_layout = 1; -+ break; ++ debug_count = debugfs_create_file("count", 0444, ++ debug_dir, NULL, ++ &count_fops); ++ if (!debug_count) ++ goto err_count; + -+ case ALGORITHM_LEFT_ASYMMETRIC_6: -+ /* RAID5 left_asymmetric, with Q on last device */ -+ pd_idx = data_disks - sector_div(stripe2, raid_disks-1); -+ if (*dd_idx >= pd_idx) -+ (*dd_idx)++; -+ qd_idx = raid_disks - 1; -+ break; ++ debug_max = debugfs_create_file("max", 0444, ++ debug_dir, NULL, ++ &max_fops); ++ if (!debug_max) ++ goto err_max; + -+ case ALGORITHM_RIGHT_ASYMMETRIC_6: -+ pd_idx = sector_div(stripe2, raid_disks-1); -+ if (*dd_idx >= pd_idx) -+ (*dd_idx)++; -+ qd_idx = raid_disks - 1; -+ break; ++ debug_sample_window = debugfs_create_file("window", 0644, ++ debug_dir, NULL, ++ &window_fops); ++ if (!debug_sample_window) ++ goto err_window; + -+ case ALGORITHM_LEFT_SYMMETRIC_6: -+ pd_idx = data_disks - sector_div(stripe2, raid_disks-1); -+ *dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1); -+ qd_idx = raid_disks - 1; -+ break; ++ debug_sample_width = debugfs_create_file("width", 0644, ++ debug_dir, NULL, ++ &width_fops); ++ if (!debug_sample_width) ++ goto err_width; + -+ case ALGORITHM_RIGHT_SYMMETRIC_6: -+ pd_idx = sector_div(stripe2, raid_disks-1); -+ *dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1); -+ qd_idx = raid_disks - 1; -+ break; ++ debug_threshold = debugfs_create_file("threshold", 0644, ++ debug_dir, NULL, ++ &threshold_fops); ++ if (!debug_threshold) ++ goto err_threshold; + -+ case ALGORITHM_PARITY_0_6: -+ pd_idx = 0; -+ (*dd_idx)++; -+ qd_idx = raid_disks - 1; -+ break; ++ debug_enable = debugfs_create_file("enable", 0644, ++ debug_dir, &enabled, ++ &enable_fops); ++ if (!debug_enable) ++ goto err_enable; + -+ default: -+ BUG(); -+ } -+ break; ++ else { ++ ret = 0; ++ goto out; + } + -+ if (sh) { -+ sh->pd_idx = pd_idx; -+ sh->qd_idx = qd_idx; -+ sh->ddf_layout = ddf_layout; -+ } -+ /* -+ * Finally, compute the new sector number -+ */ -+ new_sector = (sector_t)stripe * sectors_per_chunk + chunk_offset; -+ return new_sector; ++err_enable: ++ debugfs_remove(debug_threshold); ++err_threshold: ++ debugfs_remove(debug_sample_width); ++err_width: ++ debugfs_remove(debug_sample_window); ++err_window: ++ debugfs_remove(debug_max); ++err_max: ++ debugfs_remove(debug_count); ++err_count: ++ debugfs_remove(debug_sample); ++err_sample: ++ debugfs_remove(debug_dir); ++err_debug_dir: ++out: ++ return ret; +} + -+static sector_t compute_blocknr(struct stripe_head *sh, int i, int previous) ++/** ++ * free_debugfs - A function to cleanup the debugfs file interface ++ */ ++static void free_debugfs(void) +{ -+ struct r5conf *conf = sh->raid_conf; -+ int raid_disks = sh->disks; -+ int data_disks = raid_disks - conf->max_degraded; -+ sector_t new_sector = sh->sector, check; -+ int sectors_per_chunk = previous ? conf->prev_chunk_sectors -+ : conf->chunk_sectors; -+ int algorithm = previous ? conf->prev_algo -+ : conf->algorithm; -+ sector_t stripe; -+ int chunk_offset; -+ sector_t chunk_number; -+ int dummy1, dd_idx = i; -+ sector_t r_sector; -+ struct stripe_head sh2; -+ -+ chunk_offset = sector_div(new_sector, sectors_per_chunk); -+ stripe = new_sector; -+ -+ if (i == sh->pd_idx) -+ return 0; -+ switch(conf->level) { -+ case 4: break; -+ case 5: -+ switch (algorithm) { -+ case ALGORITHM_LEFT_ASYMMETRIC: -+ case ALGORITHM_RIGHT_ASYMMETRIC: -+ if (i > sh->pd_idx) -+ i--; -+ break; -+ case ALGORITHM_LEFT_SYMMETRIC: -+ case ALGORITHM_RIGHT_SYMMETRIC: -+ if (i < sh->pd_idx) -+ i += raid_disks; -+ i -= (sh->pd_idx + 1); -+ break; -+ case ALGORITHM_PARITY_0: -+ i -= 1; -+ break; -+ case ALGORITHM_PARITY_N: -+ break; -+ default: -+ BUG(); -+ } -+ break; -+ case 6: -+ if (i == sh->qd_idx) -+ return 0; /* It is the Q disk */ -+ switch (algorithm) { -+ case ALGORITHM_LEFT_ASYMMETRIC: -+ case ALGORITHM_RIGHT_ASYMMETRIC: -+ case ALGORITHM_ROTATING_ZERO_RESTART: -+ case ALGORITHM_ROTATING_N_RESTART: -+ if (sh->pd_idx == raid_disks-1) -+ i--; /* Q D D D P */ -+ else if (i > sh->pd_idx) -+ i -= 2; /* D D P Q D */ -+ break; -+ case ALGORITHM_LEFT_SYMMETRIC: -+ case ALGORITHM_RIGHT_SYMMETRIC: -+ if (sh->pd_idx == raid_disks-1) -+ i--; /* Q D D D P */ -+ else { -+ /* D D P Q D */ -+ if (i < sh->pd_idx) -+ i += raid_disks; -+ i -= (sh->pd_idx + 2); -+ } -+ break; -+ case ALGORITHM_PARITY_0: -+ i -= 2; -+ break; -+ case ALGORITHM_PARITY_N: -+ break; -+ case ALGORITHM_ROTATING_N_CONTINUE: -+ /* Like left_symmetric, but P is before Q */ -+ if (sh->pd_idx == 0) -+ i--; /* P D D D Q */ -+ else { -+ /* D D Q P D */ -+ if (i < sh->pd_idx) -+ i += raid_disks; -+ i -= (sh->pd_idx + 1); -+ } -+ break; -+ case ALGORITHM_LEFT_ASYMMETRIC_6: -+ case ALGORITHM_RIGHT_ASYMMETRIC_6: -+ if (i > sh->pd_idx) -+ i--; -+ break; -+ case ALGORITHM_LEFT_SYMMETRIC_6: -+ case ALGORITHM_RIGHT_SYMMETRIC_6: -+ if (i < sh->pd_idx) -+ i += data_disks + 1; -+ i -= (sh->pd_idx + 1); -+ break; -+ case ALGORITHM_PARITY_0_6: -+ i -= 1; -+ break; -+ default: -+ BUG(); -+ } -+ break; -+ } -+ -+ chunk_number = stripe * data_disks + i; -+ r_sector = chunk_number * sectors_per_chunk + chunk_offset; -+ -+ check = raid5_compute_sector(conf, r_sector, -+ previous, &dummy1, &sh2); -+ if (check != sh->sector || dummy1 != dd_idx || sh2.pd_idx != sh->pd_idx -+ || sh2.qd_idx != sh->qd_idx) { -+ printk(KERN_ERR "md/raid:%s: compute_blocknr: map not correct\n", -+ mdname(conf->mddev)); -+ return 0; -+ } -+ return r_sector; ++ /* could also use a debugfs_remove_recursive */ ++ debugfs_remove(debug_enable); ++ debugfs_remove(debug_threshold); ++ debugfs_remove(debug_sample_width); ++ debugfs_remove(debug_sample_window); ++ debugfs_remove(debug_max); ++ debugfs_remove(debug_count); ++ debugfs_remove(debug_sample); ++ debugfs_remove(debug_dir); +} + -+static void -+schedule_reconstruction(struct stripe_head *sh, struct stripe_head_state *s, -+ int rcw, int expand) ++/** ++ * detector_init - Standard module initialization code ++ */ ++static int detector_init(void) +{ -+ int i, pd_idx = sh->pd_idx, qd_idx = sh->qd_idx, disks = sh->disks; -+ struct r5conf *conf = sh->raid_conf; -+ int level = conf->level; -+ -+ if (rcw) { -+ -+ for (i = disks; i--; ) { -+ struct r5dev *dev = &sh->dev[i]; -+ -+ if (dev->towrite) { -+ set_bit(R5_LOCKED, &dev->flags); -+ set_bit(R5_Wantdrain, &dev->flags); -+ if (!expand) -+ clear_bit(R5_UPTODATE, &dev->flags); -+ s->locked++; -+ } -+ } -+ /* if we are not expanding this is a proper write request, and -+ * there will be bios with new data to be drained into the -+ * stripe cache -+ */ -+ if (!expand) { -+ if (!s->locked) -+ /* False alarm, nothing to do */ -+ return; -+ sh->reconstruct_state = reconstruct_state_drain_run; -+ set_bit(STRIPE_OP_BIODRAIN, &s->ops_request); -+ } else -+ sh->reconstruct_state = reconstruct_state_run; ++ int ret = -ENOMEM; + -+ set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request); ++ pr_info(BANNER "version %s\n", VERSION); + -+ if (s->locked + conf->max_degraded == disks) -+ if (!test_and_set_bit(STRIPE_FULL_WRITE, &sh->state)) -+ atomic_inc(&conf->pending_full_writes); -+ } else { -+ BUG_ON(!(test_bit(R5_UPTODATE, &sh->dev[pd_idx].flags) || -+ test_bit(R5_Wantcompute, &sh->dev[pd_idx].flags))); -+ BUG_ON(level == 6 && -+ (!(test_bit(R5_UPTODATE, &sh->dev[qd_idx].flags) || -+ test_bit(R5_Wantcompute, &sh->dev[qd_idx].flags)))); -+ -+ for (i = disks; i--; ) { -+ struct r5dev *dev = &sh->dev[i]; -+ if (i == pd_idx || i == qd_idx) -+ continue; ++ ret = init_stats(); ++ if (0 != ret) ++ goto out; + -+ if (dev->towrite && -+ (test_bit(R5_UPTODATE, &dev->flags) || -+ test_bit(R5_Wantcompute, &dev->flags))) { -+ set_bit(R5_Wantdrain, &dev->flags); -+ set_bit(R5_LOCKED, &dev->flags); -+ clear_bit(R5_UPTODATE, &dev->flags); -+ s->locked++; -+ } -+ } -+ if (!s->locked) -+ /* False alarm - nothing to do */ -+ return; -+ sh->reconstruct_state = reconstruct_state_prexor_drain_run; -+ set_bit(STRIPE_OP_PREXOR, &s->ops_request); -+ set_bit(STRIPE_OP_BIODRAIN, &s->ops_request); -+ set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request); -+ } ++ ret = init_debugfs(); ++ if (0 != ret) ++ goto err_stats; + -+ /* keep the parity disk(s) locked while asynchronous operations -+ * are in flight -+ */ -+ set_bit(R5_LOCKED, &sh->dev[pd_idx].flags); -+ clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags); -+ s->locked++; ++ if (enabled) ++ ret = start_kthread(); + -+ if (level == 6) { -+ int qd_idx = sh->qd_idx; -+ struct r5dev *dev = &sh->dev[qd_idx]; ++ goto out; + -+ set_bit(R5_LOCKED, &dev->flags); -+ clear_bit(R5_UPTODATE, &dev->flags); -+ s->locked++; -+ } ++err_stats: ++ ring_buffer_free(ring_buffer); ++out: ++ return ret; + -+ pr_debug("%s: stripe %llu locked: %d ops_request: %lx\n", -+ __func__, (unsigned long long)sh->sector, -+ s->locked, s->ops_request); +} + -+/* -+ * Each stripe/dev can have one or more bion attached. -+ * toread/towrite point to the first in a chain. -+ * The bi_next chain must be in order. ++/** ++ * detector_exit - Standard module cleanup code + */ -+static int add_stripe_bio(struct stripe_head *sh, struct bio *bi, int dd_idx, -+ int forwrite, int previous) ++static void detector_exit(void) +{ -+ struct bio **bip; -+ struct r5conf *conf = sh->raid_conf; -+ int firstwrite=0; ++ int err; + -+ pr_debug("adding bi b#%llu to stripe s#%llu\n", -+ (unsigned long long)bi->bi_iter.bi_sector, -+ (unsigned long long)sh->sector); ++ if (enabled) { ++ enabled = 0; ++ err = stop_kthread(); ++ if (err) ++ pr_err(BANNER "cannot stop kthread\n"); ++ } + -+ /* -+ * If several bio share a stripe. The bio bi_phys_segments acts as a -+ * reference count to avoid race. The reference count should already be -+ * increased before this function is called (for example, in -+ * make_request()), so other bio sharing this stripe will not free the -+ * stripe. If a stripe is owned by one stripe, the stripe lock will -+ * protect it. -+ */ -+ spin_lock_irq(&sh->stripe_lock); -+ /* Don't allow new IO added to stripes in batch list */ -+ if (sh->batch_head) -+ goto overlap; -+ if (forwrite) { -+ bip = &sh->dev[dd_idx].towrite; -+ if (*bip == NULL) -+ firstwrite = 1; -+ } else -+ bip = &sh->dev[dd_idx].toread; -+ while (*bip && (*bip)->bi_iter.bi_sector < bi->bi_iter.bi_sector) { -+ if (bio_end_sector(*bip) > bi->bi_iter.bi_sector) -+ goto overlap; -+ bip = & (*bip)->bi_next; -+ } -+ if (*bip && (*bip)->bi_iter.bi_sector < bio_end_sector(bi)) -+ goto overlap; -+ -+ if (!forwrite || previous) -+ clear_bit(STRIPE_BATCH_READY, &sh->state); -+ -+ BUG_ON(*bip && bi->bi_next && (*bip) != bi->bi_next); -+ if (*bip) -+ bi->bi_next = *bip; -+ *bip = bi; -+ raid5_inc_bi_active_stripes(bi); -+ -+ if (forwrite) { -+ /* check if page is covered */ -+ sector_t sector = sh->dev[dd_idx].sector; -+ for (bi=sh->dev[dd_idx].towrite; -+ sector < sh->dev[dd_idx].sector + STRIPE_SECTORS && -+ bi && bi->bi_iter.bi_sector <= sector; -+ bi = r5_next_bio(bi, sh->dev[dd_idx].sector)) { -+ if (bio_end_sector(bi) >= sector) -+ sector = bio_end_sector(bi); -+ } -+ if (sector >= sh->dev[dd_idx].sector + STRIPE_SECTORS) -+ if (!test_and_set_bit(R5_OVERWRITE, &sh->dev[dd_idx].flags)) -+ sh->overwrite_disks++; -+ } -+ -+ pr_debug("added bi b#%llu to stripe s#%llu, disk %d.\n", -+ (unsigned long long)(*bip)->bi_iter.bi_sector, -+ (unsigned long long)sh->sector, dd_idx); -+ -+ if (conf->mddev->bitmap && firstwrite) { -+ /* Cannot hold spinlock over bitmap_startwrite, -+ * but must ensure this isn't added to a batch until -+ * we have added to the bitmap and set bm_seq. -+ * So set STRIPE_BITMAP_PENDING to prevent -+ * batching. -+ * If multiple add_stripe_bio() calls race here they -+ * much all set STRIPE_BITMAP_PENDING. So only the first one -+ * to complete "bitmap_startwrite" gets to set -+ * STRIPE_BIT_DELAY. This is important as once a stripe -+ * is added to a batch, STRIPE_BIT_DELAY cannot be changed -+ * any more. -+ */ -+ set_bit(STRIPE_BITMAP_PENDING, &sh->state); -+ spin_unlock_irq(&sh->stripe_lock); -+ bitmap_startwrite(conf->mddev->bitmap, sh->sector, -+ STRIPE_SECTORS, 0); -+ spin_lock_irq(&sh->stripe_lock); -+ clear_bit(STRIPE_BITMAP_PENDING, &sh->state); -+ if (!sh->batch_head) { -+ sh->bm_seq = conf->seq_flush+1; -+ set_bit(STRIPE_BIT_DELAY, &sh->state); -+ } -+ } -+ spin_unlock_irq(&sh->stripe_lock); -+ -+ if (stripe_can_batch(sh)) -+ stripe_add_to_batch_list(conf, sh); -+ return 1; -+ -+ overlap: -+ set_bit(R5_Overlap, &sh->dev[dd_idx].flags); -+ spin_unlock_irq(&sh->stripe_lock); -+ return 0; -+} -+ -+static void end_reshape(struct r5conf *conf); -+ -+static void stripe_set_idx(sector_t stripe, struct r5conf *conf, int previous, -+ struct stripe_head *sh) -+{ -+ int sectors_per_chunk = -+ previous ? conf->prev_chunk_sectors : conf->chunk_sectors; -+ int dd_idx; -+ int chunk_offset = sector_div(stripe, sectors_per_chunk); -+ int disks = previous ? conf->previous_raid_disks : conf->raid_disks; -+ -+ raid5_compute_sector(conf, -+ stripe * (disks - conf->max_degraded) -+ *sectors_per_chunk + chunk_offset, -+ previous, -+ &dd_idx, sh); -+} -+ -+static void -+handle_failed_stripe(struct r5conf *conf, struct stripe_head *sh, -+ struct stripe_head_state *s, int disks, -+ struct bio **return_bi) -+{ -+ int i; -+ BUG_ON(sh->batch_head); -+ for (i = disks; i--; ) { -+ struct bio *bi; -+ int bitmap_end = 0; -+ -+ if (test_bit(R5_ReadError, &sh->dev[i].flags)) { -+ struct md_rdev *rdev; -+ rcu_read_lock(); -+ rdev = rcu_dereference(conf->disks[i].rdev); -+ if (rdev && test_bit(In_sync, &rdev->flags)) -+ atomic_inc(&rdev->nr_pending); -+ else -+ rdev = NULL; -+ rcu_read_unlock(); -+ if (rdev) { -+ if (!rdev_set_badblocks( -+ rdev, -+ sh->sector, -+ STRIPE_SECTORS, 0)) -+ md_error(conf->mddev, rdev); -+ rdev_dec_pending(rdev, conf->mddev); -+ } -+ } -+ spin_lock_irq(&sh->stripe_lock); -+ /* fail all writes first */ -+ bi = sh->dev[i].towrite; -+ sh->dev[i].towrite = NULL; -+ sh->overwrite_disks = 0; -+ spin_unlock_irq(&sh->stripe_lock); -+ if (bi) -+ bitmap_end = 1; -+ -+ if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags)) -+ wake_up(&conf->wait_for_overlap); -+ -+ while (bi && bi->bi_iter.bi_sector < -+ sh->dev[i].sector + STRIPE_SECTORS) { -+ struct bio *nextbi = r5_next_bio(bi, sh->dev[i].sector); -+ clear_bit(BIO_UPTODATE, &bi->bi_flags); -+ if (!raid5_dec_bi_active_stripes(bi)) { -+ md_write_end(conf->mddev); -+ bi->bi_next = *return_bi; -+ *return_bi = bi; -+ } -+ bi = nextbi; -+ } -+ if (bitmap_end) -+ bitmap_endwrite(conf->mddev->bitmap, sh->sector, -+ STRIPE_SECTORS, 0, 0); -+ bitmap_end = 0; -+ /* and fail all 'written' */ -+ bi = sh->dev[i].written; -+ sh->dev[i].written = NULL; -+ if (test_and_clear_bit(R5_SkipCopy, &sh->dev[i].flags)) { -+ WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags)); -+ sh->dev[i].page = sh->dev[i].orig_page; -+ } -+ -+ if (bi) bitmap_end = 1; -+ while (bi && bi->bi_iter.bi_sector < -+ sh->dev[i].sector + STRIPE_SECTORS) { -+ struct bio *bi2 = r5_next_bio(bi, sh->dev[i].sector); -+ clear_bit(BIO_UPTODATE, &bi->bi_flags); -+ if (!raid5_dec_bi_active_stripes(bi)) { -+ md_write_end(conf->mddev); -+ bi->bi_next = *return_bi; -+ *return_bi = bi; -+ } -+ bi = bi2; -+ } -+ -+ /* fail any reads if this device is non-operational and -+ * the data has not reached the cache yet. -+ */ -+ if (!test_bit(R5_Wantfill, &sh->dev[i].flags) && -+ (!test_bit(R5_Insync, &sh->dev[i].flags) || -+ test_bit(R5_ReadError, &sh->dev[i].flags))) { -+ spin_lock_irq(&sh->stripe_lock); -+ bi = sh->dev[i].toread; -+ sh->dev[i].toread = NULL; -+ spin_unlock_irq(&sh->stripe_lock); -+ if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags)) -+ wake_up(&conf->wait_for_overlap); -+ while (bi && bi->bi_iter.bi_sector < -+ sh->dev[i].sector + STRIPE_SECTORS) { -+ struct bio *nextbi = -+ r5_next_bio(bi, sh->dev[i].sector); -+ clear_bit(BIO_UPTODATE, &bi->bi_flags); -+ if (!raid5_dec_bi_active_stripes(bi)) { -+ bi->bi_next = *return_bi; -+ *return_bi = bi; -+ } -+ bi = nextbi; -+ } -+ } -+ if (bitmap_end) -+ bitmap_endwrite(conf->mddev->bitmap, sh->sector, -+ STRIPE_SECTORS, 0, 0); -+ /* If we were in the middle of a write the parity block might -+ * still be locked - so just clear all R5_LOCKED flags -+ */ -+ clear_bit(R5_LOCKED, &sh->dev[i].flags); -+ } -+ -+ if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state)) -+ if (atomic_dec_and_test(&conf->pending_full_writes)) -+ md_wakeup_thread(conf->mddev->thread); -+} -+ -+static void -+handle_failed_sync(struct r5conf *conf, struct stripe_head *sh, -+ struct stripe_head_state *s) -+{ -+ int abort = 0; -+ int i; -+ -+ BUG_ON(sh->batch_head); -+ clear_bit(STRIPE_SYNCING, &sh->state); -+ if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags)) -+ wake_up(&conf->wait_for_overlap); -+ s->syncing = 0; -+ s->replacing = 0; -+ /* There is nothing more to do for sync/check/repair. -+ * Don't even need to abort as that is handled elsewhere -+ * if needed, and not always wanted e.g. if there is a known -+ * bad block here. -+ * For recover/replace we need to record a bad block on all -+ * non-sync devices, or abort the recovery -+ */ -+ if (test_bit(MD_RECOVERY_RECOVER, &conf->mddev->recovery)) { -+ /* During recovery devices cannot be removed, so -+ * locking and refcounting of rdevs is not needed -+ */ -+ for (i = 0; i < conf->raid_disks; i++) { -+ struct md_rdev *rdev = conf->disks[i].rdev; -+ if (rdev -+ && !test_bit(Faulty, &rdev->flags) -+ && !test_bit(In_sync, &rdev->flags) -+ && !rdev_set_badblocks(rdev, sh->sector, -+ STRIPE_SECTORS, 0)) -+ abort = 1; -+ rdev = conf->disks[i].replacement; -+ if (rdev -+ && !test_bit(Faulty, &rdev->flags) -+ && !test_bit(In_sync, &rdev->flags) -+ && !rdev_set_badblocks(rdev, sh->sector, -+ STRIPE_SECTORS, 0)) -+ abort = 1; -+ } -+ if (abort) -+ conf->recovery_disabled = -+ conf->mddev->recovery_disabled; -+ } -+ md_done_sync(conf->mddev, STRIPE_SECTORS, !abort); -+} -+ -+static int want_replace(struct stripe_head *sh, int disk_idx) -+{ -+ struct md_rdev *rdev; -+ int rv = 0; -+ /* Doing recovery so rcu locking not required */ -+ rdev = sh->raid_conf->disks[disk_idx].replacement; -+ if (rdev -+ && !test_bit(Faulty, &rdev->flags) -+ && !test_bit(In_sync, &rdev->flags) -+ && (rdev->recovery_offset <= sh->sector -+ || rdev->mddev->recovery_cp <= sh->sector)) -+ rv = 1; -+ -+ return rv; -+} -+ -+/* fetch_block - checks the given member device to see if its data needs -+ * to be read or computed to satisfy a request. -+ * -+ * Returns 1 when no more member devices need to be checked, otherwise returns -+ * 0 to tell the loop in handle_stripe_fill to continue -+ */ -+ -+static int need_this_block(struct stripe_head *sh, struct stripe_head_state *s, -+ int disk_idx, int disks) -+{ -+ struct r5dev *dev = &sh->dev[disk_idx]; -+ struct r5dev *fdev[2] = { &sh->dev[s->failed_num[0]], -+ &sh->dev[s->failed_num[1]] }; -+ int i; -+ -+ -+ if (test_bit(R5_LOCKED, &dev->flags) || -+ test_bit(R5_UPTODATE, &dev->flags)) -+ /* No point reading this as we already have it or have -+ * decided to get it. -+ */ -+ return 0; -+ -+ if (dev->toread || -+ (dev->towrite && !test_bit(R5_OVERWRITE, &dev->flags))) -+ /* We need this block to directly satisfy a request */ -+ return 1; -+ -+ if (s->syncing || s->expanding || -+ (s->replacing && want_replace(sh, disk_idx))) -+ /* When syncing, or expanding we read everything. -+ * When replacing, we need the replaced block. -+ */ -+ return 1; -+ -+ if ((s->failed >= 1 && fdev[0]->toread) || -+ (s->failed >= 2 && fdev[1]->toread)) -+ /* If we want to read from a failed device, then -+ * we need to actually read every other device. -+ */ -+ return 1; -+ -+ /* Sometimes neither read-modify-write nor reconstruct-write -+ * cycles can work. In those cases we read every block we -+ * can. Then the parity-update is certain to have enough to -+ * work with. -+ * This can only be a problem when we need to write something, -+ * and some device has failed. If either of those tests -+ * fail we need look no further. -+ */ -+ if (!s->failed || !s->to_write) -+ return 0; -+ -+ if (test_bit(R5_Insync, &dev->flags) && -+ !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) -+ /* Pre-reads at not permitted until after short delay -+ * to gather multiple requests. However if this -+ * device is no Insync, the block could only be be computed -+ * and there is no need to delay that. -+ */ -+ return 0; -+ -+ for (i = 0; i < s->failed; i++) { -+ if (fdev[i]->towrite && -+ !test_bit(R5_UPTODATE, &fdev[i]->flags) && -+ !test_bit(R5_OVERWRITE, &fdev[i]->flags)) -+ /* If we have a partial write to a failed -+ * device, then we will need to reconstruct -+ * the content of that device, so all other -+ * devices must be read. -+ */ -+ return 1; -+ } -+ -+ /* If we are forced to do a reconstruct-write, either because -+ * the current RAID6 implementation only supports that, or -+ * or because parity cannot be trusted and we are currently -+ * recovering it, there is extra need to be careful. -+ * If one of the devices that we would need to read, because -+ * it is not being overwritten (and maybe not written at all) -+ * is missing/faulty, then we need to read everything we can. -+ */ -+ if (sh->raid_conf->level != 6 && -+ sh->sector < sh->raid_conf->mddev->recovery_cp) -+ /* reconstruct-write isn't being forced */ -+ return 0; -+ for (i = 0; i < s->failed; i++) { -+ if (s->failed_num[i] != sh->pd_idx && -+ s->failed_num[i] != sh->qd_idx && -+ !test_bit(R5_UPTODATE, &fdev[i]->flags) && -+ !test_bit(R5_OVERWRITE, &fdev[i]->flags)) -+ return 1; -+ } -+ -+ return 0; -+} -+ -+static int fetch_block(struct stripe_head *sh, struct stripe_head_state *s, -+ int disk_idx, int disks) -+{ -+ struct r5dev *dev = &sh->dev[disk_idx]; -+ -+ /* is the data in this block needed, and can we get it? */ -+ if (need_this_block(sh, s, disk_idx, disks)) { -+ /* we would like to get this block, possibly by computing it, -+ * otherwise read it if the backing disk is insync -+ */ -+ BUG_ON(test_bit(R5_Wantcompute, &dev->flags)); -+ BUG_ON(test_bit(R5_Wantread, &dev->flags)); -+ BUG_ON(sh->batch_head); -+ if ((s->uptodate == disks - 1) && -+ (s->failed && (disk_idx == s->failed_num[0] || -+ disk_idx == s->failed_num[1]))) { -+ /* have disk failed, and we're requested to fetch it; -+ * do compute it -+ */ -+ pr_debug("Computing stripe %llu block %d\n", -+ (unsigned long long)sh->sector, disk_idx); -+ set_bit(STRIPE_COMPUTE_RUN, &sh->state); -+ set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request); -+ set_bit(R5_Wantcompute, &dev->flags); -+ sh->ops.target = disk_idx; -+ sh->ops.target2 = -1; /* no 2nd target */ -+ s->req_compute = 1; -+ /* Careful: from this point on 'uptodate' is in the eye -+ * of raid_run_ops which services 'compute' operations -+ * before writes. R5_Wantcompute flags a block that will -+ * be R5_UPTODATE by the time it is needed for a -+ * subsequent operation. -+ */ -+ s->uptodate++; -+ return 1; -+ } else if (s->uptodate == disks-2 && s->failed >= 2) { -+ /* Computing 2-failure is *very* expensive; only -+ * do it if failed >= 2 -+ */ -+ int other; -+ for (other = disks; other--; ) { -+ if (other == disk_idx) -+ continue; -+ if (!test_bit(R5_UPTODATE, -+ &sh->dev[other].flags)) -+ break; -+ } -+ BUG_ON(other < 0); -+ pr_debug("Computing stripe %llu blocks %d,%d\n", -+ (unsigned long long)sh->sector, -+ disk_idx, other); -+ set_bit(STRIPE_COMPUTE_RUN, &sh->state); -+ set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request); -+ set_bit(R5_Wantcompute, &sh->dev[disk_idx].flags); -+ set_bit(R5_Wantcompute, &sh->dev[other].flags); -+ sh->ops.target = disk_idx; -+ sh->ops.target2 = other; -+ s->uptodate += 2; -+ s->req_compute = 1; -+ return 1; -+ } else if (test_bit(R5_Insync, &dev->flags)) { -+ set_bit(R5_LOCKED, &dev->flags); -+ set_bit(R5_Wantread, &dev->flags); -+ s->locked++; -+ pr_debug("Reading block %d (sync=%d)\n", -+ disk_idx, s->syncing); -+ } -+ } -+ -+ return 0; -+} -+ -+/** -+ * handle_stripe_fill - read or compute data to satisfy pending requests. -+ */ -+static void handle_stripe_fill(struct stripe_head *sh, -+ struct stripe_head_state *s, -+ int disks) -+{ -+ int i; -+ -+ /* look for blocks to read/compute, skip this if a compute -+ * is already in flight, or if the stripe contents are in the -+ * midst of changing due to a write -+ */ -+ if (!test_bit(STRIPE_COMPUTE_RUN, &sh->state) && !sh->check_state && -+ !sh->reconstruct_state) -+ for (i = disks; i--; ) -+ if (fetch_block(sh, s, i, disks)) -+ break; -+ set_bit(STRIPE_HANDLE, &sh->state); -+} -+ -+static void break_stripe_batch_list(struct stripe_head *head_sh, -+ unsigned long handle_flags); -+/* handle_stripe_clean_event -+ * any written block on an uptodate or failed drive can be returned. -+ * Note that if we 'wrote' to a failed drive, it will be UPTODATE, but -+ * never LOCKED, so we don't need to test 'failed' directly. -+ */ -+static void handle_stripe_clean_event(struct r5conf *conf, -+ struct stripe_head *sh, int disks, struct bio **return_bi) -+{ -+ int i; -+ struct r5dev *dev; -+ int discard_pending = 0; -+ struct stripe_head *head_sh = sh; -+ bool do_endio = false; -+ -+ for (i = disks; i--; ) -+ if (sh->dev[i].written) { -+ dev = &sh->dev[i]; -+ if (!test_bit(R5_LOCKED, &dev->flags) && -+ (test_bit(R5_UPTODATE, &dev->flags) || -+ test_bit(R5_Discard, &dev->flags) || -+ test_bit(R5_SkipCopy, &dev->flags))) { -+ /* We can return any write requests */ -+ struct bio *wbi, *wbi2; -+ pr_debug("Return write for disc %d\n", i); -+ if (test_and_clear_bit(R5_Discard, &dev->flags)) -+ clear_bit(R5_UPTODATE, &dev->flags); -+ if (test_and_clear_bit(R5_SkipCopy, &dev->flags)) { -+ WARN_ON(test_bit(R5_UPTODATE, &dev->flags)); -+ } -+ do_endio = true; -+ -+returnbi: -+ dev->page = dev->orig_page; -+ wbi = dev->written; -+ dev->written = NULL; -+ while (wbi && wbi->bi_iter.bi_sector < -+ dev->sector + STRIPE_SECTORS) { -+ wbi2 = r5_next_bio(wbi, dev->sector); -+ if (!raid5_dec_bi_active_stripes(wbi)) { -+ md_write_end(conf->mddev); -+ wbi->bi_next = *return_bi; -+ *return_bi = wbi; -+ } -+ wbi = wbi2; -+ } -+ bitmap_endwrite(conf->mddev->bitmap, sh->sector, -+ STRIPE_SECTORS, -+ !test_bit(STRIPE_DEGRADED, &sh->state), -+ 0); -+ if (head_sh->batch_head) { -+ sh = list_first_entry(&sh->batch_list, -+ struct stripe_head, -+ batch_list); -+ if (sh != head_sh) { -+ dev = &sh->dev[i]; -+ goto returnbi; -+ } -+ } -+ sh = head_sh; -+ dev = &sh->dev[i]; -+ } else if (test_bit(R5_Discard, &dev->flags)) -+ discard_pending = 1; -+ WARN_ON(test_bit(R5_SkipCopy, &dev->flags)); -+ WARN_ON(dev->page != dev->orig_page); -+ } -+ if (!discard_pending && -+ test_bit(R5_Discard, &sh->dev[sh->pd_idx].flags)) { -+ clear_bit(R5_Discard, &sh->dev[sh->pd_idx].flags); -+ clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags); -+ if (sh->qd_idx >= 0) { -+ clear_bit(R5_Discard, &sh->dev[sh->qd_idx].flags); -+ clear_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags); -+ } -+ /* now that discard is done we can proceed with any sync */ -+ clear_bit(STRIPE_DISCARD, &sh->state); -+ /* -+ * SCSI discard will change some bio fields and the stripe has -+ * no updated data, so remove it from hash list and the stripe -+ * will be reinitialized -+ */ -+ spin_lock_irq(&conf->device_lock); -+unhash: -+ remove_hash(sh); -+ if (head_sh->batch_head) { -+ sh = list_first_entry(&sh->batch_list, -+ struct stripe_head, batch_list); -+ if (sh != head_sh) -+ goto unhash; -+ } -+ spin_unlock_irq(&conf->device_lock); -+ sh = head_sh; -+ -+ if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state)) -+ set_bit(STRIPE_HANDLE, &sh->state); -+ -+ } -+ -+ if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state)) -+ if (atomic_dec_and_test(&conf->pending_full_writes)) -+ md_wakeup_thread(conf->mddev->thread); -+ -+ if (head_sh->batch_head && do_endio) -+ break_stripe_batch_list(head_sh, STRIPE_EXPAND_SYNC_FLAGS); -+} -+ -+static void handle_stripe_dirtying(struct r5conf *conf, -+ struct stripe_head *sh, -+ struct stripe_head_state *s, -+ int disks) -+{ -+ int rmw = 0, rcw = 0, i; -+ sector_t recovery_cp = conf->mddev->recovery_cp; -+ -+ /* Check whether resync is now happening or should start. -+ * If yes, then the array is dirty (after unclean shutdown or -+ * initial creation), so parity in some stripes might be inconsistent. -+ * In this case, we need to always do reconstruct-write, to ensure -+ * that in case of drive failure or read-error correction, we -+ * generate correct data from the parity. -+ */ -+ if (conf->rmw_level == PARITY_DISABLE_RMW || -+ (recovery_cp < MaxSector && sh->sector >= recovery_cp && -+ s->failed == 0)) { -+ /* Calculate the real rcw later - for now make it -+ * look like rcw is cheaper -+ */ -+ rcw = 1; rmw = 2; -+ pr_debug("force RCW rmw_level=%u, recovery_cp=%llu sh->sector=%llu\n", -+ conf->rmw_level, (unsigned long long)recovery_cp, -+ (unsigned long long)sh->sector); -+ } else for (i = disks; i--; ) { -+ /* would I have to read this buffer for read_modify_write */ -+ struct r5dev *dev = &sh->dev[i]; -+ if ((dev->towrite || i == sh->pd_idx || i == sh->qd_idx) && -+ !test_bit(R5_LOCKED, &dev->flags) && -+ !(test_bit(R5_UPTODATE, &dev->flags) || -+ test_bit(R5_Wantcompute, &dev->flags))) { -+ if (test_bit(R5_Insync, &dev->flags)) -+ rmw++; -+ else -+ rmw += 2*disks; /* cannot read it */ -+ } -+ /* Would I have to read this buffer for reconstruct_write */ -+ if (!test_bit(R5_OVERWRITE, &dev->flags) && -+ i != sh->pd_idx && i != sh->qd_idx && -+ !test_bit(R5_LOCKED, &dev->flags) && -+ !(test_bit(R5_UPTODATE, &dev->flags) || -+ test_bit(R5_Wantcompute, &dev->flags))) { -+ if (test_bit(R5_Insync, &dev->flags)) -+ rcw++; -+ else -+ rcw += 2*disks; -+ } -+ } -+ pr_debug("for sector %llu, rmw=%d rcw=%d\n", -+ (unsigned long long)sh->sector, rmw, rcw); -+ set_bit(STRIPE_HANDLE, &sh->state); -+ if ((rmw < rcw || (rmw == rcw && conf->rmw_level == PARITY_ENABLE_RMW)) && rmw > 0) { -+ /* prefer read-modify-write, but need to get some data */ -+ if (conf->mddev->queue) -+ blk_add_trace_msg(conf->mddev->queue, -+ "raid5 rmw %llu %d", -+ (unsigned long long)sh->sector, rmw); -+ for (i = disks; i--; ) { -+ struct r5dev *dev = &sh->dev[i]; -+ if ((dev->towrite || i == sh->pd_idx || i == sh->qd_idx) && -+ !test_bit(R5_LOCKED, &dev->flags) && -+ !(test_bit(R5_UPTODATE, &dev->flags) || -+ test_bit(R5_Wantcompute, &dev->flags)) && -+ test_bit(R5_Insync, &dev->flags)) { -+ if (test_bit(STRIPE_PREREAD_ACTIVE, -+ &sh->state)) { -+ pr_debug("Read_old block %d for r-m-w\n", -+ i); -+ set_bit(R5_LOCKED, &dev->flags); -+ set_bit(R5_Wantread, &dev->flags); -+ s->locked++; -+ } else { -+ set_bit(STRIPE_DELAYED, &sh->state); -+ set_bit(STRIPE_HANDLE, &sh->state); -+ } -+ } -+ } -+ } -+ if ((rcw < rmw || (rcw == rmw && conf->rmw_level != PARITY_ENABLE_RMW)) && rcw > 0) { -+ /* want reconstruct write, but need to get some data */ -+ int qread =0; -+ rcw = 0; -+ for (i = disks; i--; ) { -+ struct r5dev *dev = &sh->dev[i]; -+ if (!test_bit(R5_OVERWRITE, &dev->flags) && -+ i != sh->pd_idx && i != sh->qd_idx && -+ !test_bit(R5_LOCKED, &dev->flags) && -+ !(test_bit(R5_UPTODATE, &dev->flags) || -+ test_bit(R5_Wantcompute, &dev->flags))) { -+ rcw++; -+ if (test_bit(R5_Insync, &dev->flags) && -+ test_bit(STRIPE_PREREAD_ACTIVE, -+ &sh->state)) { -+ pr_debug("Read_old block " -+ "%d for Reconstruct\n", i); -+ set_bit(R5_LOCKED, &dev->flags); -+ set_bit(R5_Wantread, &dev->flags); -+ s->locked++; -+ qread++; -+ } else { -+ set_bit(STRIPE_DELAYED, &sh->state); -+ set_bit(STRIPE_HANDLE, &sh->state); -+ } -+ } -+ } -+ if (rcw && conf->mddev->queue) -+ blk_add_trace_msg(conf->mddev->queue, "raid5 rcw %llu %d %d %d", -+ (unsigned long long)sh->sector, -+ rcw, qread, test_bit(STRIPE_DELAYED, &sh->state)); -+ } -+ -+ if (rcw > disks && rmw > disks && -+ !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) -+ set_bit(STRIPE_DELAYED, &sh->state); -+ -+ /* now if nothing is locked, and if we have enough data, -+ * we can start a write request -+ */ -+ /* since handle_stripe can be called at any time we need to handle the -+ * case where a compute block operation has been submitted and then a -+ * subsequent call wants to start a write request. raid_run_ops only -+ * handles the case where compute block and reconstruct are requested -+ * simultaneously. If this is not the case then new writes need to be -+ * held off until the compute completes. -+ */ -+ if ((s->req_compute || !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) && -+ (s->locked == 0 && (rcw == 0 || rmw == 0) && -+ !test_bit(STRIPE_BIT_DELAY, &sh->state))) -+ schedule_reconstruction(sh, s, rcw == 0, 0); -+} -+ -+static void handle_parity_checks5(struct r5conf *conf, struct stripe_head *sh, -+ struct stripe_head_state *s, int disks) -+{ -+ struct r5dev *dev = NULL; -+ -+ BUG_ON(sh->batch_head); -+ set_bit(STRIPE_HANDLE, &sh->state); -+ -+ switch (sh->check_state) { -+ case check_state_idle: -+ /* start a new check operation if there are no failures */ -+ if (s->failed == 0) { -+ BUG_ON(s->uptodate != disks); -+ sh->check_state = check_state_run; -+ set_bit(STRIPE_OP_CHECK, &s->ops_request); -+ clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags); -+ s->uptodate--; -+ break; -+ } -+ dev = &sh->dev[s->failed_num[0]]; -+ /* fall through */ -+ case check_state_compute_result: -+ sh->check_state = check_state_idle; -+ if (!dev) -+ dev = &sh->dev[sh->pd_idx]; -+ -+ /* check that a write has not made the stripe insync */ -+ if (test_bit(STRIPE_INSYNC, &sh->state)) -+ break; -+ -+ /* either failed parity check, or recovery is happening */ -+ BUG_ON(!test_bit(R5_UPTODATE, &dev->flags)); -+ BUG_ON(s->uptodate != disks); -+ -+ set_bit(R5_LOCKED, &dev->flags); -+ s->locked++; -+ set_bit(R5_Wantwrite, &dev->flags); -+ -+ clear_bit(STRIPE_DEGRADED, &sh->state); -+ set_bit(STRIPE_INSYNC, &sh->state); -+ break; -+ case check_state_run: -+ break; /* we will be called again upon completion */ -+ case check_state_check_result: -+ sh->check_state = check_state_idle; -+ -+ /* if a failure occurred during the check operation, leave -+ * STRIPE_INSYNC not set and let the stripe be handled again -+ */ -+ if (s->failed) -+ break; ++ free_debugfs(); ++ ring_buffer_free(ring_buffer); /* free up the ring buffer */ + -+ /* handle a successful check operation, if parity is correct -+ * we are done. Otherwise update the mismatch count and repair -+ * parity if !MD_RECOVERY_CHECK -+ */ -+ if ((sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) == 0) -+ /* parity is correct (on disc, -+ * not in buffer any more) -+ */ -+ set_bit(STRIPE_INSYNC, &sh->state); -+ else { -+ atomic64_add(STRIPE_SECTORS, &conf->mddev->resync_mismatches); -+ if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery)) -+ /* don't try to repair!! */ -+ set_bit(STRIPE_INSYNC, &sh->state); -+ else { -+ sh->check_state = check_state_compute_run; -+ set_bit(STRIPE_COMPUTE_RUN, &sh->state); -+ set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request); -+ set_bit(R5_Wantcompute, -+ &sh->dev[sh->pd_idx].flags); -+ sh->ops.target = sh->pd_idx; -+ sh->ops.target2 = -1; -+ s->uptodate++; -+ } -+ } -+ break; -+ case check_state_compute_run: -+ break; -+ default: -+ printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n", -+ __func__, sh->check_state, -+ (unsigned long long) sh->sector); -+ BUG(); -+ } +} + -+static void handle_parity_checks6(struct r5conf *conf, struct stripe_head *sh, -+ struct stripe_head_state *s, -+ int disks) -+{ -+ int pd_idx = sh->pd_idx; -+ int qd_idx = sh->qd_idx; -+ struct r5dev *dev; -+ -+ BUG_ON(sh->batch_head); -+ set_bit(STRIPE_HANDLE, &sh->state); -+ -+ BUG_ON(s->failed > 2); -+ -+ /* Want to check and possibly repair P and Q. -+ * However there could be one 'failed' device, in which -+ * case we can only check one of them, possibly using the -+ * other to generate missing data -+ */ -+ -+ switch (sh->check_state) { -+ case check_state_idle: -+ /* start a new check operation if there are < 2 failures */ -+ if (s->failed == s->q_failed) { -+ /* The only possible failed device holds Q, so it -+ * makes sense to check P (If anything else were failed, -+ * we would have used P to recreate it). -+ */ -+ sh->check_state = check_state_run; -+ } -+ if (!s->q_failed && s->failed < 2) { -+ /* Q is not failed, and we didn't use it to generate -+ * anything, so it makes sense to check it -+ */ -+ if (sh->check_state == check_state_run) -+ sh->check_state = check_state_run_pq; -+ else -+ sh->check_state = check_state_run_q; -+ } -+ -+ /* discard potentially stale zero_sum_result */ -+ sh->ops.zero_sum_result = 0; -+ -+ if (sh->check_state == check_state_run) { -+ /* async_xor_zero_sum destroys the contents of P */ -+ clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags); -+ s->uptodate--; -+ } -+ if (sh->check_state >= check_state_run && -+ sh->check_state <= check_state_run_pq) { -+ /* async_syndrome_zero_sum preserves P and Q, so -+ * no need to mark them !uptodate here -+ */ -+ set_bit(STRIPE_OP_CHECK, &s->ops_request); -+ break; -+ } -+ -+ /* we have 2-disk failure */ -+ BUG_ON(s->failed != 2); -+ /* fall through */ -+ case check_state_compute_result: -+ sh->check_state = check_state_idle; -+ -+ /* check that a write has not made the stripe insync */ -+ if (test_bit(STRIPE_INSYNC, &sh->state)) -+ break; ++module_init(detector_init); ++module_exit(detector_exit); +diff -Nur linux-4.1.10.orig/drivers/misc/Kconfig linux-4.1.10/drivers/misc/Kconfig +--- linux-4.1.10.orig/drivers/misc/Kconfig 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/misc/Kconfig 2015-10-12 22:33:32.252678339 +0200 +@@ -54,6 +54,7 @@ + config ATMEL_TCLIB + bool "Atmel AT32/AT91 Timer/Counter Library" + depends on (AVR32 || ARCH_AT91) ++ default y if PREEMPT_RT_FULL + help + Select this if you want a library to allocate the Timer/Counter + blocks found on many Atmel processors. This facilitates using +@@ -69,8 +70,7 @@ + are combined to make a single 32-bit timer. + + When GENERIC_CLOCKEVENTS is defined, the third timer channel +- may be used as a clock event device supporting oneshot mode +- (delays of up to two seconds) based on the 32 KiHz clock. ++ may be used as a clock event device supporting oneshot mode. + + config ATMEL_TCB_CLKSRC_BLOCK + int +@@ -84,6 +84,15 @@ + TC can be used for other purposes, such as PWM generation and + interval timing. + ++config ATMEL_TCB_CLKSRC_USE_SLOW_CLOCK ++ bool "TC Block use 32 KiHz clock" ++ depends on ATMEL_TCB_CLKSRC ++ default y if !PREEMPT_RT_FULL ++ help ++ Select this to use 32 KiHz base clock rate as TC block clock ++ source for clock events. + -+ /* now write out any block on a failed drive, -+ * or P or Q if they were recomputed -+ */ -+ BUG_ON(s->uptodate < disks - 1); /* We don't need Q to recover */ -+ if (s->failed == 2) { -+ dev = &sh->dev[s->failed_num[1]]; -+ s->locked++; -+ set_bit(R5_LOCKED, &dev->flags); -+ set_bit(R5_Wantwrite, &dev->flags); -+ } -+ if (s->failed >= 1) { -+ dev = &sh->dev[s->failed_num[0]]; -+ s->locked++; -+ set_bit(R5_LOCKED, &dev->flags); -+ set_bit(R5_Wantwrite, &dev->flags); -+ } -+ if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) { -+ dev = &sh->dev[pd_idx]; -+ s->locked++; -+ set_bit(R5_LOCKED, &dev->flags); -+ set_bit(R5_Wantwrite, &dev->flags); -+ } -+ if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) { -+ dev = &sh->dev[qd_idx]; -+ s->locked++; -+ set_bit(R5_LOCKED, &dev->flags); -+ set_bit(R5_Wantwrite, &dev->flags); -+ } -+ clear_bit(STRIPE_DEGRADED, &sh->state); + -+ set_bit(STRIPE_INSYNC, &sh->state); -+ break; -+ case check_state_run: -+ case check_state_run_q: -+ case check_state_run_pq: -+ break; /* we will be called again upon completion */ -+ case check_state_check_result: -+ sh->check_state = check_state_idle; -+ -+ /* handle a successful check operation, if parity is correct -+ * we are done. Otherwise update the mismatch count and repair -+ * parity if !MD_RECOVERY_CHECK -+ */ -+ if (sh->ops.zero_sum_result == 0) { -+ /* both parities are correct */ -+ if (!s->failed) -+ set_bit(STRIPE_INSYNC, &sh->state); -+ else { -+ /* in contrast to the raid5 case we can validate -+ * parity, but still have a failure to write -+ * back -+ */ -+ sh->check_state = check_state_compute_result; -+ /* Returning at this point means that we may go -+ * off and bring p and/or q uptodate again so -+ * we make sure to check zero_sum_result again -+ * to verify if p or q need writeback -+ */ -+ } -+ } else { -+ atomic64_add(STRIPE_SECTORS, &conf->mddev->resync_mismatches); -+ if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery)) -+ /* don't try to repair!! */ -+ set_bit(STRIPE_INSYNC, &sh->state); -+ else { -+ int *target = &sh->ops.target; -+ -+ sh->ops.target = -1; -+ sh->ops.target2 = -1; -+ sh->check_state = check_state_compute_run; -+ set_bit(STRIPE_COMPUTE_RUN, &sh->state); -+ set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request); -+ if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) { -+ set_bit(R5_Wantcompute, -+ &sh->dev[pd_idx].flags); -+ *target = pd_idx; -+ target = &sh->ops.target2; -+ s->uptodate++; -+ } -+ if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) { -+ set_bit(R5_Wantcompute, -+ &sh->dev[qd_idx].flags); -+ *target = qd_idx; -+ s->uptodate++; -+ } -+ } -+ } -+ break; -+ case check_state_compute_run: -+ break; -+ default: -+ printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n", -+ __func__, sh->check_state, -+ (unsigned long long) sh->sector); -+ BUG(); -+ } -+} + config DUMMY_IRQ + tristate "Dummy IRQ handler" + default n +@@ -113,6 +122,35 @@ + for information on the specific driver level and support statement + for your IBM server. + ++config HWLAT_DETECTOR ++ tristate "Testing module to detect hardware-induced latencies" ++ depends on DEBUG_FS ++ depends on RING_BUFFER ++ default m ++ ---help--- ++ A simple hardware latency detector. Use this module to detect ++ large latencies introduced by the behavior of the underlying ++ system firmware external to Linux. We do this using periodic ++ use of stop_machine to grab all available CPUs and measure ++ for unexplainable gaps in the CPU timestamp counter(s). By ++ default, the module is not enabled until the "enable" file ++ within the "hwlat_detector" debugfs directory is toggled. + -+static void handle_stripe_expansion(struct r5conf *conf, struct stripe_head *sh) -+{ -+ int i; ++ This module is often used to detect SMI (System Management ++ Interrupts) on x86 systems, though is not x86 specific. To ++ this end, we default to using a sample window of 1 second, ++ during which we will sample for 0.5 seconds. If an SMI or ++ similar event occurs during that time, it is recorded ++ into an 8K samples global ring buffer until retreived. + -+ /* We have read all the blocks in this stripe and now we need to -+ * copy some of them into a target stripe for expand. -+ */ -+ struct dma_async_tx_descriptor *tx = NULL; -+ BUG_ON(sh->batch_head); -+ clear_bit(STRIPE_EXPAND_SOURCE, &sh->state); -+ for (i = 0; i < sh->disks; i++) -+ if (i != sh->pd_idx && i != sh->qd_idx) { -+ int dd_idx, j; -+ struct stripe_head *sh2; -+ struct async_submit_ctl submit; -+ -+ sector_t bn = compute_blocknr(sh, i, 1); -+ sector_t s = raid5_compute_sector(conf, bn, 0, -+ &dd_idx, NULL); -+ sh2 = get_active_stripe(conf, s, 0, 1, 1); -+ if (sh2 == NULL) -+ /* so far only the early blocks of this stripe -+ * have been requested. When later blocks -+ * get requested, we will try again -+ */ -+ continue; -+ if (!test_bit(STRIPE_EXPANDING, &sh2->state) || -+ test_bit(R5_Expanded, &sh2->dev[dd_idx].flags)) { -+ /* must have already done this block */ -+ release_stripe(sh2); -+ continue; -+ } ++ WARNING: This software should never be enabled (it can be built ++ but should not be turned on after it is loaded) in a production ++ environment where high latencies are a concern since the ++ sampling mechanism actually introduces latencies for ++ regular tasks while the CPU(s) are being held. + -+ /* place all the copies on one channel */ -+ init_async_submit(&submit, 0, tx, NULL, NULL, NULL); -+ tx = async_memcpy(sh2->dev[dd_idx].page, -+ sh->dev[i].page, 0, 0, STRIPE_SIZE, -+ &submit); -+ -+ set_bit(R5_Expanded, &sh2->dev[dd_idx].flags); -+ set_bit(R5_UPTODATE, &sh2->dev[dd_idx].flags); -+ for (j = 0; j < conf->raid_disks; j++) -+ if (j != sh2->pd_idx && -+ j != sh2->qd_idx && -+ !test_bit(R5_Expanded, &sh2->dev[j].flags)) -+ break; -+ if (j == conf->raid_disks) { -+ set_bit(STRIPE_EXPAND_READY, &sh2->state); -+ set_bit(STRIPE_HANDLE, &sh2->state); -+ } -+ release_stripe(sh2); ++ If unsure, say N + -+ } -+ /* done submitting copies, wait for them to complete */ -+ async_tx_quiesce(&tx); -+} + config PHANTOM + tristate "Sensable PHANToM (PCI)" + depends on PCI +diff -Nur linux-4.1.10.orig/drivers/misc/Makefile linux-4.1.10/drivers/misc/Makefile +--- linux-4.1.10.orig/drivers/misc/Makefile 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/misc/Makefile 2015-10-12 22:33:32.252678339 +0200 +@@ -38,6 +38,7 @@ + obj-$(CONFIG_HMC6352) += hmc6352.o + obj-y += eeprom/ + obj-y += cb710/ ++obj-$(CONFIG_HWLAT_DETECTOR) += hwlat_detector.o + obj-$(CONFIG_SPEAR13XX_PCIE_GADGET) += spear13xx_pcie_gadget.o + obj-$(CONFIG_VMWARE_BALLOON) += vmw_balloon.o + obj-$(CONFIG_ARM_CHARLCD) += arm-charlcd.o +diff -Nur linux-4.1.10.orig/drivers/mmc/host/mmci.c linux-4.1.10/drivers/mmc/host/mmci.c +--- linux-4.1.10.orig/drivers/mmc/host/mmci.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/mmc/host/mmci.c 2015-10-12 22:33:32.252678339 +0200 +@@ -1155,15 +1155,12 @@ + struct sg_mapping_iter *sg_miter = &host->sg_miter; + struct variant_data *variant = host->variant; + void __iomem *base = host->base; +- unsigned long flags; + u32 status; + + status = readl(base + MMCISTATUS); + + dev_dbg(mmc_dev(host->mmc), "irq1 (pio) %08x\n", status); + +- local_irq_save(flags); +- + do { + unsigned int remain, len; + char *buffer; +@@ -1203,8 +1200,6 @@ + + sg_miter_stop(sg_miter); + +- local_irq_restore(flags); +- + /* + * If we have less than the fifo 'half-full' threshold to transfer, + * trigger a PIO interrupt as soon as any data is available. +diff -Nur linux-4.1.10.orig/drivers/net/ethernet/3com/3c59x.c linux-4.1.10/drivers/net/ethernet/3com/3c59x.c +--- linux-4.1.10.orig/drivers/net/ethernet/3com/3c59x.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/net/ethernet/3com/3c59x.c 2015-10-12 22:33:32.252678339 +0200 +@@ -842,9 +842,9 @@ + { + struct vortex_private *vp = netdev_priv(dev); + unsigned long flags; +- local_irq_save(flags); ++ local_irq_save_nort(flags); + (vp->full_bus_master_rx ? boomerang_interrupt:vortex_interrupt)(dev->irq,dev); +- local_irq_restore(flags); ++ local_irq_restore_nort(flags); + } + #endif + +@@ -1916,12 +1916,12 @@ + * Block interrupts because vortex_interrupt does a bare spin_lock() + */ + unsigned long flags; +- local_irq_save(flags); ++ local_irq_save_nort(flags); + if (vp->full_bus_master_tx) + boomerang_interrupt(dev->irq, dev); + else + vortex_interrupt(dev->irq, dev); +- local_irq_restore(flags); ++ local_irq_restore_nort(flags); + } + } + +diff -Nur linux-4.1.10.orig/drivers/net/ethernet/atheros/atl1c/atl1c_main.c linux-4.1.10/drivers/net/ethernet/atheros/atl1c/atl1c_main.c +--- linux-4.1.10.orig/drivers/net/ethernet/atheros/atl1c/atl1c_main.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/net/ethernet/atheros/atl1c/atl1c_main.c 2015-10-12 22:33:32.252678339 +0200 +@@ -2213,11 +2213,7 @@ + } + + tpd_req = atl1c_cal_tpd_req(skb); +- if (!spin_trylock_irqsave(&adapter->tx_lock, flags)) { +- if (netif_msg_pktdata(adapter)) +- dev_info(&adapter->pdev->dev, "tx locked\n"); +- return NETDEV_TX_LOCKED; +- } ++ spin_lock_irqsave(&adapter->tx_lock, flags); + + if (atl1c_tpd_avail(adapter, type) < tpd_req) { + /* no enough descriptor, just stop queue */ +diff -Nur linux-4.1.10.orig/drivers/net/ethernet/atheros/atl1e/atl1e_main.c linux-4.1.10/drivers/net/ethernet/atheros/atl1e/atl1e_main.c +--- linux-4.1.10.orig/drivers/net/ethernet/atheros/atl1e/atl1e_main.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/net/ethernet/atheros/atl1e/atl1e_main.c 2015-10-12 22:33:32.256678075 +0200 +@@ -1880,8 +1880,7 @@ + return NETDEV_TX_OK; + } + tpd_req = atl1e_cal_tdp_req(skb); +- if (!spin_trylock_irqsave(&adapter->tx_lock, flags)) +- return NETDEV_TX_LOCKED; ++ spin_lock_irqsave(&adapter->tx_lock, flags); + + if (atl1e_tpd_avail(adapter) < tpd_req) { + /* no enough descriptor, just stop queue */ +diff -Nur linux-4.1.10.orig/drivers/net/ethernet/chelsio/cxgb/sge.c linux-4.1.10/drivers/net/ethernet/chelsio/cxgb/sge.c +--- linux-4.1.10.orig/drivers/net/ethernet/chelsio/cxgb/sge.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/net/ethernet/chelsio/cxgb/sge.c 2015-10-12 22:33:32.256678075 +0200 +@@ -1664,8 +1664,7 @@ + struct cmdQ *q = &sge->cmdQ[qid]; + unsigned int credits, pidx, genbit, count, use_sched_skb = 0; + +- if (!spin_trylock(&q->lock)) +- return NETDEV_TX_LOCKED; ++ spin_lock(&q->lock); + + reclaim_completed_tx(sge, q); + +diff -Nur linux-4.1.10.orig/drivers/net/ethernet/freescale/gianfar.c linux-4.1.10/drivers/net/ethernet/freescale/gianfar.c +--- linux-4.1.10.orig/drivers/net/ethernet/freescale/gianfar.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/net/ethernet/freescale/gianfar.c 2015-10-12 22:33:32.256678075 +0200 +@@ -1540,7 +1540,7 @@ + + if (netif_running(ndev)) { + +- local_irq_save(flags); ++ local_irq_save_nort(flags); + lock_tx_qs(priv); + + gfar_halt_nodisable(priv); +@@ -1556,7 +1556,7 @@ + gfar_write(®s->maccfg1, tempval); + + unlock_tx_qs(priv); +- local_irq_restore(flags); ++ local_irq_restore_nort(flags); + + disable_napi(priv); + +@@ -1598,7 +1598,7 @@ + /* Disable Magic Packet mode, in case something + * else woke us up. + */ +- local_irq_save(flags); ++ local_irq_save_nort(flags); + lock_tx_qs(priv); + + tempval = gfar_read(®s->maccfg2); +@@ -1608,7 +1608,7 @@ + gfar_start(priv); + + unlock_tx_qs(priv); +- local_irq_restore(flags); ++ local_irq_restore_nort(flags); + + netif_device_attach(ndev); + +@@ -3418,14 +3418,14 @@ + dev->stats.tx_dropped++; + atomic64_inc(&priv->extra_stats.tx_underrun); + +- local_irq_save(flags); ++ local_irq_save_nort(flags); + lock_tx_qs(priv); + + /* Reactivate the Tx Queues */ + gfar_write(®s->tstat, gfargrp->tstat); + + unlock_tx_qs(priv); +- local_irq_restore(flags); ++ local_irq_restore_nort(flags); + } + netif_dbg(priv, tx_err, dev, "Transmit Error\n"); + } +diff -Nur linux-4.1.10.orig/drivers/net/ethernet/neterion/s2io.c linux-4.1.10/drivers/net/ethernet/neterion/s2io.c +--- linux-4.1.10.orig/drivers/net/ethernet/neterion/s2io.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/net/ethernet/neterion/s2io.c 2015-10-12 22:33:32.256678075 +0200 +@@ -4084,12 +4084,7 @@ + [skb->priority & (MAX_TX_FIFOS - 1)]; + fifo = &mac_control->fifos[queue]; + +- if (do_spin_lock) +- spin_lock_irqsave(&fifo->tx_lock, flags); +- else { +- if (unlikely(!spin_trylock_irqsave(&fifo->tx_lock, flags))) +- return NETDEV_TX_LOCKED; +- } ++ spin_lock_irqsave(&fifo->tx_lock, flags); + + if (sp->config.multiq) { + if (__netif_subqueue_stopped(dev, fifo->fifo_no)) { +diff -Nur linux-4.1.10.orig/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c linux-4.1.10/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c +--- linux-4.1.10.orig/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c 2015-10-12 22:33:32.256678075 +0200 +@@ -2137,10 +2137,8 @@ + struct pch_gbe_tx_ring *tx_ring = adapter->tx_ring; + unsigned long flags; + +- if (!spin_trylock_irqsave(&tx_ring->tx_lock, flags)) { +- /* Collision - tell upper layer to requeue */ +- return NETDEV_TX_LOCKED; +- } ++ spin_lock_irqsave(&tx_ring->tx_lock, flags); + -+/* -+ * handle_stripe - do things to a stripe. -+ * -+ * We lock the stripe by setting STRIPE_ACTIVE and then examine the -+ * state of various bits to see what needs to be done. -+ * Possible results: -+ * return some read requests which now have data -+ * return some write requests which are safely on storage -+ * schedule a read on some buffers -+ * schedule a write of some buffers -+ * return confirmation of parity correctness -+ * -+ */ + if (unlikely(!PCH_GBE_DESC_UNUSED(tx_ring))) { + netif_stop_queue(netdev); + spin_unlock_irqrestore(&tx_ring->tx_lock, flags); +diff -Nur linux-4.1.10.orig/drivers/net/ethernet/realtek/8139too.c linux-4.1.10/drivers/net/ethernet/realtek/8139too.c +--- linux-4.1.10.orig/drivers/net/ethernet/realtek/8139too.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/net/ethernet/realtek/8139too.c 2015-10-12 22:33:32.260677811 +0200 +@@ -2229,7 +2229,7 @@ + struct rtl8139_private *tp = netdev_priv(dev); + const int irq = tp->pci_dev->irq; + +- disable_irq(irq); ++ disable_irq_nosync(irq); + rtl8139_interrupt(irq, dev); + enable_irq(irq); + } +diff -Nur linux-4.1.10.orig/drivers/net/ethernet/tehuti/tehuti.c linux-4.1.10/drivers/net/ethernet/tehuti/tehuti.c +--- linux-4.1.10.orig/drivers/net/ethernet/tehuti/tehuti.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/net/ethernet/tehuti/tehuti.c 2015-10-12 22:33:32.260677811 +0200 +@@ -1629,13 +1629,8 @@ + unsigned long flags; + + ENTER; +- local_irq_save(flags); +- if (!spin_trylock(&priv->tx_lock)) { +- local_irq_restore(flags); +- DBG("%s[%s]: TX locked, returning NETDEV_TX_LOCKED\n", +- BDX_DRV_NAME, ndev->name); +- return NETDEV_TX_LOCKED; +- } + -+static void analyse_stripe(struct stripe_head *sh, struct stripe_head_state *s) -+{ -+ struct r5conf *conf = sh->raid_conf; -+ int disks = sh->disks; -+ struct r5dev *dev; -+ int i; -+ int do_recovery = 0; -+ -+ memset(s, 0, sizeof(*s)); -+ -+ s->expanding = test_bit(STRIPE_EXPAND_SOURCE, &sh->state) && !sh->batch_head; -+ s->expanded = test_bit(STRIPE_EXPAND_READY, &sh->state) && !sh->batch_head; -+ s->failed_num[0] = -1; -+ s->failed_num[1] = -1; -+ -+ /* Now to look around and see what can be done */ -+ rcu_read_lock(); -+ for (i=disks; i--; ) { -+ struct md_rdev *rdev; -+ sector_t first_bad; -+ int bad_sectors; -+ int is_bad = 0; -+ -+ dev = &sh->dev[i]; -+ -+ pr_debug("check %d: state 0x%lx read %p write %p written %p\n", -+ i, dev->flags, -+ dev->toread, dev->towrite, dev->written); -+ /* maybe we can reply to a read -+ * -+ * new wantfill requests are only permitted while -+ * ops_complete_biofill is guaranteed to be inactive -+ */ -+ if (test_bit(R5_UPTODATE, &dev->flags) && dev->toread && -+ !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) -+ set_bit(R5_Wantfill, &dev->flags); -+ -+ /* now count some things */ -+ if (test_bit(R5_LOCKED, &dev->flags)) -+ s->locked++; -+ if (test_bit(R5_UPTODATE, &dev->flags)) -+ s->uptodate++; -+ if (test_bit(R5_Wantcompute, &dev->flags)) { -+ s->compute++; -+ BUG_ON(s->compute > 2); -+ } -+ -+ if (test_bit(R5_Wantfill, &dev->flags)) -+ s->to_fill++; -+ else if (dev->toread) -+ s->to_read++; -+ if (dev->towrite) { -+ s->to_write++; -+ if (!test_bit(R5_OVERWRITE, &dev->flags)) -+ s->non_overwrite++; -+ } -+ if (dev->written) -+ s->written++; -+ /* Prefer to use the replacement for reads, but only -+ * if it is recovered enough and has no bad blocks. -+ */ -+ rdev = rcu_dereference(conf->disks[i].replacement); -+ if (rdev && !test_bit(Faulty, &rdev->flags) && -+ rdev->recovery_offset >= sh->sector + STRIPE_SECTORS && -+ !is_badblock(rdev, sh->sector, STRIPE_SECTORS, -+ &first_bad, &bad_sectors)) -+ set_bit(R5_ReadRepl, &dev->flags); -+ else { -+ if (rdev) -+ set_bit(R5_NeedReplace, &dev->flags); -+ rdev = rcu_dereference(conf->disks[i].rdev); -+ clear_bit(R5_ReadRepl, &dev->flags); -+ } -+ if (rdev && test_bit(Faulty, &rdev->flags)) -+ rdev = NULL; -+ if (rdev) { -+ is_bad = is_badblock(rdev, sh->sector, STRIPE_SECTORS, -+ &first_bad, &bad_sectors); -+ if (s->blocked_rdev == NULL -+ && (test_bit(Blocked, &rdev->flags) -+ || is_bad < 0)) { -+ if (is_bad < 0) -+ set_bit(BlockedBadBlocks, -+ &rdev->flags); -+ s->blocked_rdev = rdev; -+ atomic_inc(&rdev->nr_pending); -+ } -+ } -+ clear_bit(R5_Insync, &dev->flags); -+ if (!rdev) -+ /* Not in-sync */; -+ else if (is_bad) { -+ /* also not in-sync */ -+ if (!test_bit(WriteErrorSeen, &rdev->flags) && -+ test_bit(R5_UPTODATE, &dev->flags)) { -+ /* treat as in-sync, but with a read error -+ * which we can now try to correct -+ */ -+ set_bit(R5_Insync, &dev->flags); -+ set_bit(R5_ReadError, &dev->flags); -+ } -+ } else if (test_bit(In_sync, &rdev->flags)) -+ set_bit(R5_Insync, &dev->flags); -+ else if (sh->sector + STRIPE_SECTORS <= rdev->recovery_offset) -+ /* in sync if before recovery_offset */ -+ set_bit(R5_Insync, &dev->flags); -+ else if (test_bit(R5_UPTODATE, &dev->flags) && -+ test_bit(R5_Expanded, &dev->flags)) -+ /* If we've reshaped into here, we assume it is Insync. -+ * We will shortly update recovery_offset to make -+ * it official. -+ */ -+ set_bit(R5_Insync, &dev->flags); -+ -+ if (test_bit(R5_WriteError, &dev->flags)) { -+ /* This flag does not apply to '.replacement' -+ * only to .rdev, so make sure to check that*/ -+ struct md_rdev *rdev2 = rcu_dereference( -+ conf->disks[i].rdev); -+ if (rdev2 == rdev) -+ clear_bit(R5_Insync, &dev->flags); -+ if (rdev2 && !test_bit(Faulty, &rdev2->flags)) { -+ s->handle_bad_blocks = 1; -+ atomic_inc(&rdev2->nr_pending); -+ } else -+ clear_bit(R5_WriteError, &dev->flags); -+ } -+ if (test_bit(R5_MadeGood, &dev->flags)) { -+ /* This flag does not apply to '.replacement' -+ * only to .rdev, so make sure to check that*/ -+ struct md_rdev *rdev2 = rcu_dereference( -+ conf->disks[i].rdev); -+ if (rdev2 && !test_bit(Faulty, &rdev2->flags)) { -+ s->handle_bad_blocks = 1; -+ atomic_inc(&rdev2->nr_pending); -+ } else -+ clear_bit(R5_MadeGood, &dev->flags); -+ } -+ if (test_bit(R5_MadeGoodRepl, &dev->flags)) { -+ struct md_rdev *rdev2 = rcu_dereference( -+ conf->disks[i].replacement); -+ if (rdev2 && !test_bit(Faulty, &rdev2->flags)) { -+ s->handle_bad_blocks = 1; -+ atomic_inc(&rdev2->nr_pending); -+ } else -+ clear_bit(R5_MadeGoodRepl, &dev->flags); -+ } -+ if (!test_bit(R5_Insync, &dev->flags)) { -+ /* The ReadError flag will just be confusing now */ -+ clear_bit(R5_ReadError, &dev->flags); -+ clear_bit(R5_ReWrite, &dev->flags); -+ } -+ if (test_bit(R5_ReadError, &dev->flags)) -+ clear_bit(R5_Insync, &dev->flags); -+ if (!test_bit(R5_Insync, &dev->flags)) { -+ if (s->failed < 2) -+ s->failed_num[s->failed] = i; -+ s->failed++; -+ if (rdev && !test_bit(Faulty, &rdev->flags)) -+ do_recovery = 1; -+ } -+ } -+ if (test_bit(STRIPE_SYNCING, &sh->state)) { -+ /* If there is a failed device being replaced, -+ * we must be recovering. -+ * else if we are after recovery_cp, we must be syncing -+ * else if MD_RECOVERY_REQUESTED is set, we also are syncing. -+ * else we can only be replacing -+ * sync and recovery both need to read all devices, and so -+ * use the same flag. -+ */ -+ if (do_recovery || -+ sh->sector >= conf->mddev->recovery_cp || -+ test_bit(MD_RECOVERY_REQUESTED, &(conf->mddev->recovery))) -+ s->syncing = 1; -+ else -+ s->replacing = 1; -+ } -+ rcu_read_unlock(); -+} -+ -+static int clear_batch_ready(struct stripe_head *sh) -+{ -+ /* Return '1' if this is a member of batch, or -+ * '0' if it is a lone stripe or a head which can now be -+ * handled. -+ */ -+ struct stripe_head *tmp; -+ if (!test_and_clear_bit(STRIPE_BATCH_READY, &sh->state)) -+ return (sh->batch_head && sh->batch_head != sh); -+ spin_lock(&sh->stripe_lock); -+ if (!sh->batch_head) { -+ spin_unlock(&sh->stripe_lock); -+ return 0; -+ } -+ -+ /* -+ * this stripe could be added to a batch list before we check -+ * BATCH_READY, skips it -+ */ -+ if (sh->batch_head != sh) { -+ spin_unlock(&sh->stripe_lock); -+ return 1; -+ } -+ spin_lock(&sh->batch_lock); -+ list_for_each_entry(tmp, &sh->batch_list, batch_list) -+ clear_bit(STRIPE_BATCH_READY, &tmp->state); -+ spin_unlock(&sh->batch_lock); -+ spin_unlock(&sh->stripe_lock); -+ -+ /* -+ * BATCH_READY is cleared, no new stripes can be added. -+ * batch_list can be accessed without lock -+ */ -+ return 0; ++ spin_lock_irqsave(&priv->tx_lock, flags); + + /* build tx descriptor */ + BDX_ASSERT(f->m.wptr >= f->m.memsz); /* started with valid wptr */ +diff -Nur linux-4.1.10.orig/drivers/net/rionet.c linux-4.1.10/drivers/net/rionet.c +--- linux-4.1.10.orig/drivers/net/rionet.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/net/rionet.c 2015-10-12 22:33:32.260677811 +0200 +@@ -174,11 +174,7 @@ + unsigned long flags; + int add_num = 1; + +- local_irq_save(flags); +- if (!spin_trylock(&rnet->tx_lock)) { +- local_irq_restore(flags); +- return NETDEV_TX_LOCKED; +- } ++ spin_lock_irqsave(&rnet->tx_lock, flags); + + if (is_multicast_ether_addr(eth->h_dest)) + add_num = nets[rnet->mport->id].nact; +diff -Nur linux-4.1.10.orig/drivers/net/wireless/orinoco/orinoco_usb.c linux-4.1.10/drivers/net/wireless/orinoco/orinoco_usb.c +--- linux-4.1.10.orig/drivers/net/wireless/orinoco/orinoco_usb.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/net/wireless/orinoco/orinoco_usb.c 2015-10-12 22:33:32.260677811 +0200 +@@ -697,7 +697,7 @@ + while (!ctx->done.done && msecs--) + udelay(1000); + } else { +- wait_event_interruptible(ctx->done.wait, ++ swait_event_interruptible(ctx->done.wait, + ctx->done.done); + } + break; +diff -Nur linux-4.1.10.orig/drivers/pci/access.c linux-4.1.10/drivers/pci/access.c +--- linux-4.1.10.orig/drivers/pci/access.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/pci/access.c 2015-10-12 22:33:32.260677811 +0200 +@@ -580,7 +580,7 @@ + WARN_ON(!dev->block_cfg_access); + + dev->block_cfg_access = 0; +- wake_up_all(&pci_cfg_wait); ++ wake_up_all_locked(&pci_cfg_wait); + raw_spin_unlock_irqrestore(&pci_lock, flags); + } + EXPORT_SYMBOL_GPL(pci_cfg_access_unlock); +diff -Nur linux-4.1.10.orig/drivers/scsi/fcoe/fcoe.c linux-4.1.10/drivers/scsi/fcoe/fcoe.c +--- linux-4.1.10.orig/drivers/scsi/fcoe/fcoe.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/scsi/fcoe/fcoe.c 2015-10-12 22:33:32.260677811 +0200 +@@ -1287,7 +1287,7 @@ + struct sk_buff *skb; + #ifdef CONFIG_SMP + struct fcoe_percpu_s *p0; +- unsigned targ_cpu = get_cpu(); ++ unsigned targ_cpu = get_cpu_light(); + #endif /* CONFIG_SMP */ + + FCOE_DBG("Destroying receive thread for CPU %d\n", cpu); +@@ -1343,7 +1343,7 @@ + kfree_skb(skb); + spin_unlock_bh(&p->fcoe_rx_list.lock); + } +- put_cpu(); ++ put_cpu_light(); + #else + /* + * This a non-SMP scenario where the singular Rx thread is +@@ -1567,11 +1567,11 @@ + static int fcoe_alloc_paged_crc_eof(struct sk_buff *skb, int tlen) + { + struct fcoe_percpu_s *fps; +- int rc; ++ int rc, cpu = get_cpu_light(); + +- fps = &get_cpu_var(fcoe_percpu); ++ fps = &per_cpu(fcoe_percpu, cpu); + rc = fcoe_get_paged_crc_eof(skb, tlen, fps); +- put_cpu_var(fcoe_percpu); ++ put_cpu_light(); + + return rc; + } +@@ -1767,11 +1767,11 @@ + return 0; + } + +- stats = per_cpu_ptr(lport->stats, get_cpu()); ++ stats = per_cpu_ptr(lport->stats, get_cpu_light()); + stats->InvalidCRCCount++; + if (stats->InvalidCRCCount < 5) + printk(KERN_WARNING "fcoe: dropping frame with CRC error\n"); +- put_cpu(); ++ put_cpu_light(); + return -EINVAL; + } + +@@ -1847,13 +1847,13 @@ + goto drop; + + if (!fcoe_filter_frames(lport, fp)) { +- put_cpu(); ++ put_cpu_light(); + fc_exch_recv(lport, fp); + return; + } + drop: + stats->ErrorFrames++; +- put_cpu(); ++ put_cpu_light(); + kfree_skb(skb); + } + +diff -Nur linux-4.1.10.orig/drivers/scsi/fcoe/fcoe_ctlr.c linux-4.1.10/drivers/scsi/fcoe/fcoe_ctlr.c +--- linux-4.1.10.orig/drivers/scsi/fcoe/fcoe_ctlr.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/scsi/fcoe/fcoe_ctlr.c 2015-10-12 22:33:32.260677811 +0200 +@@ -831,7 +831,7 @@ + + INIT_LIST_HEAD(&del_list); + +- stats = per_cpu_ptr(fip->lp->stats, get_cpu()); ++ stats = per_cpu_ptr(fip->lp->stats, get_cpu_light()); + + list_for_each_entry_safe(fcf, next, &fip->fcfs, list) { + deadline = fcf->time + fcf->fka_period + fcf->fka_period / 2; +@@ -867,7 +867,7 @@ + sel_time = fcf->time; + } + } +- put_cpu(); ++ put_cpu_light(); + + list_for_each_entry_safe(fcf, next, &del_list, list) { + /* Removes fcf from current list */ +diff -Nur linux-4.1.10.orig/drivers/scsi/libfc/fc_exch.c linux-4.1.10/drivers/scsi/libfc/fc_exch.c +--- linux-4.1.10.orig/drivers/scsi/libfc/fc_exch.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/scsi/libfc/fc_exch.c 2015-10-12 22:33:32.260677811 +0200 +@@ -814,10 +814,10 @@ + } + memset(ep, 0, sizeof(*ep)); + +- cpu = get_cpu(); ++ cpu = get_cpu_light(); + pool = per_cpu_ptr(mp->pool, cpu); + spin_lock_bh(&pool->lock); +- put_cpu(); ++ put_cpu_light(); + + /* peek cache of free slot */ + if (pool->left != FC_XID_UNKNOWN) { +diff -Nur linux-4.1.10.orig/drivers/scsi/libsas/sas_ata.c linux-4.1.10/drivers/scsi/libsas/sas_ata.c +--- linux-4.1.10.orig/drivers/scsi/libsas/sas_ata.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/scsi/libsas/sas_ata.c 2015-10-12 22:33:32.264677547 +0200 +@@ -190,7 +190,7 @@ + /* TODO: audit callers to ensure they are ready for qc_issue to + * unconditionally re-enable interrupts + */ +- local_irq_save(flags); ++ local_irq_save_nort(flags); + spin_unlock(ap->lock); + + /* If the device fell off, no sense in issuing commands */ +@@ -255,7 +255,7 @@ + + out: + spin_lock(ap->lock); +- local_irq_restore(flags); ++ local_irq_restore_nort(flags); + return ret; + } + +diff -Nur linux-4.1.10.orig/drivers/scsi/qla2xxx/qla_inline.h linux-4.1.10/drivers/scsi/qla2xxx/qla_inline.h +--- linux-4.1.10.orig/drivers/scsi/qla2xxx/qla_inline.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/scsi/qla2xxx/qla_inline.h 2015-10-12 22:33:32.264677547 +0200 +@@ -59,12 +59,12 @@ + { + unsigned long flags; + struct qla_hw_data *ha = rsp->hw; +- local_irq_save(flags); ++ local_irq_save_nort(flags); + if (IS_P3P_TYPE(ha)) + qla82xx_poll(0, rsp); + else + ha->isp_ops->intr_handler(0, rsp); +- local_irq_restore(flags); ++ local_irq_restore_nort(flags); + } + + static inline uint8_t * +diff -Nur linux-4.1.10.orig/drivers/thermal/x86_pkg_temp_thermal.c linux-4.1.10/drivers/thermal/x86_pkg_temp_thermal.c +--- linux-4.1.10.orig/drivers/thermal/x86_pkg_temp_thermal.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/thermal/x86_pkg_temp_thermal.c 2015-10-12 22:33:32.264677547 +0200 +@@ -29,6 +29,7 @@ + #include + #include + #include ++#include + #include + #include + +@@ -352,7 +353,7 @@ + } + } + +-static int pkg_temp_thermal_platform_thermal_notify(__u64 msr_val) ++static void platform_thermal_notify_work(struct swork_event *event) + { + unsigned long flags; + int cpu = smp_processor_id(); +@@ -369,7 +370,7 @@ + pkg_work_scheduled[phy_id]) { + disable_pkg_thres_interrupt(); + spin_unlock_irqrestore(&pkg_work_lock, flags); +- return -EINVAL; ++ return; + } + pkg_work_scheduled[phy_id] = 1; + spin_unlock_irqrestore(&pkg_work_lock, flags); +@@ -378,9 +379,48 @@ + schedule_delayed_work_on(cpu, + &per_cpu(pkg_temp_thermal_threshold_work, cpu), + msecs_to_jiffies(notify_delay_ms)); +} + -+static void break_stripe_batch_list(struct stripe_head *head_sh, -+ unsigned long handle_flags) -+{ -+ struct stripe_head *sh, *next; -+ int i; -+ int do_wakeup = 0; -+ -+ list_for_each_entry_safe(sh, next, &head_sh->batch_list, batch_list) { -+ -+ list_del_init(&sh->batch_list); -+ -+ WARN_ON_ONCE(sh->state & ((1 << STRIPE_ACTIVE) | -+ (1 << STRIPE_SYNCING) | -+ (1 << STRIPE_REPLACED) | -+ (1 << STRIPE_PREREAD_ACTIVE) | -+ (1 << STRIPE_DELAYED) | -+ (1 << STRIPE_BIT_DELAY) | -+ (1 << STRIPE_FULL_WRITE) | -+ (1 << STRIPE_BIOFILL_RUN) | -+ (1 << STRIPE_COMPUTE_RUN) | -+ (1 << STRIPE_OPS_REQ_PENDING) | -+ (1 << STRIPE_DISCARD) | -+ (1 << STRIPE_BATCH_READY) | -+ (1 << STRIPE_BATCH_ERR) | -+ (1 << STRIPE_BITMAP_PENDING))); -+ WARN_ON_ONCE(head_sh->state & ((1 << STRIPE_DISCARD) | -+ (1 << STRIPE_REPLACED))); -+ -+ set_mask_bits(&sh->state, ~(STRIPE_EXPAND_SYNC_FLAGS | -+ (1 << STRIPE_DEGRADED)), -+ head_sh->state & (1 << STRIPE_INSYNC)); -+ -+ sh->check_state = head_sh->check_state; -+ sh->reconstruct_state = head_sh->reconstruct_state; -+ for (i = 0; i < sh->disks; i++) { -+ if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags)) -+ do_wakeup = 1; -+ sh->dev[i].flags = head_sh->dev[i].flags & -+ (~((1 << R5_WriteError) | (1 << R5_Overlap))); -+ } -+ spin_lock_irq(&sh->stripe_lock); -+ sh->batch_head = NULL; -+ spin_unlock_irq(&sh->stripe_lock); -+ if (handle_flags == 0 || -+ sh->state & handle_flags) -+ set_bit(STRIPE_HANDLE, &sh->state); -+ release_stripe(sh); -+ } -+ spin_lock_irq(&head_sh->stripe_lock); -+ head_sh->batch_head = NULL; -+ spin_unlock_irq(&head_sh->stripe_lock); -+ for (i = 0; i < head_sh->disks; i++) -+ if (test_and_clear_bit(R5_Overlap, &head_sh->dev[i].flags)) -+ do_wakeup = 1; -+ if (head_sh->state & handle_flags) -+ set_bit(STRIPE_HANDLE, &head_sh->state); -+ -+ if (do_wakeup) -+ wake_up(&head_sh->raid_conf->wait_for_overlap); -+} ++#ifdef CONFIG_PREEMPT_RT_FULL ++static struct swork_event notify_work; + -+static void handle_stripe(struct stripe_head *sh) ++static int thermal_notify_work_init(void) +{ -+ struct stripe_head_state s; -+ struct r5conf *conf = sh->raid_conf; -+ int i; -+ int prexor; -+ int disks = sh->disks; -+ struct r5dev *pdev, *qdev; -+ -+ clear_bit(STRIPE_HANDLE, &sh->state); -+ if (test_and_set_bit_lock(STRIPE_ACTIVE, &sh->state)) { -+ /* already being handled, ensure it gets handled -+ * again when current action finishes */ -+ set_bit(STRIPE_HANDLE, &sh->state); -+ return; -+ } -+ -+ if (clear_batch_ready(sh) ) { -+ clear_bit_unlock(STRIPE_ACTIVE, &sh->state); -+ return; -+ } -+ -+ if (test_and_clear_bit(STRIPE_BATCH_ERR, &sh->state)) -+ break_stripe_batch_list(sh, 0); -+ -+ if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state) && !sh->batch_head) { -+ spin_lock(&sh->stripe_lock); -+ /* Cannot process 'sync' concurrently with 'discard' */ -+ if (!test_bit(STRIPE_DISCARD, &sh->state) && -+ test_and_clear_bit(STRIPE_SYNC_REQUESTED, &sh->state)) { -+ set_bit(STRIPE_SYNCING, &sh->state); -+ clear_bit(STRIPE_INSYNC, &sh->state); -+ clear_bit(STRIPE_REPLACED, &sh->state); -+ } -+ spin_unlock(&sh->stripe_lock); -+ } -+ clear_bit(STRIPE_DELAYED, &sh->state); -+ -+ pr_debug("handling stripe %llu, state=%#lx cnt=%d, " -+ "pd_idx=%d, qd_idx=%d\n, check:%d, reconstruct:%d\n", -+ (unsigned long long)sh->sector, sh->state, -+ atomic_read(&sh->count), sh->pd_idx, sh->qd_idx, -+ sh->check_state, sh->reconstruct_state); -+ -+ analyse_stripe(sh, &s); -+ -+ if (s.handle_bad_blocks) { -+ set_bit(STRIPE_HANDLE, &sh->state); -+ goto finish; -+ } -+ -+ if (unlikely(s.blocked_rdev)) { -+ if (s.syncing || s.expanding || s.expanded || -+ s.replacing || s.to_write || s.written) { -+ set_bit(STRIPE_HANDLE, &sh->state); -+ goto finish; -+ } -+ /* There is nothing for the blocked_rdev to block */ -+ rdev_dec_pending(s.blocked_rdev, conf->mddev); -+ s.blocked_rdev = NULL; -+ } -+ -+ if (s.to_fill && !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) { -+ set_bit(STRIPE_OP_BIOFILL, &s.ops_request); -+ set_bit(STRIPE_BIOFILL_RUN, &sh->state); -+ } -+ -+ pr_debug("locked=%d uptodate=%d to_read=%d" -+ " to_write=%d failed=%d failed_num=%d,%d\n", -+ s.locked, s.uptodate, s.to_read, s.to_write, s.failed, -+ s.failed_num[0], s.failed_num[1]); -+ /* check if the array has lost more than max_degraded devices and, -+ * if so, some requests might need to be failed. -+ */ -+ if (s.failed > conf->max_degraded) { -+ sh->check_state = 0; -+ sh->reconstruct_state = 0; -+ break_stripe_batch_list(sh, 0); -+ if (s.to_read+s.to_write+s.written) -+ handle_failed_stripe(conf, sh, &s, disks, &s.return_bi); -+ if (s.syncing + s.replacing) -+ handle_failed_sync(conf, sh, &s); -+ } -+ -+ /* Now we check to see if any write operations have recently -+ * completed -+ */ -+ prexor = 0; -+ if (sh->reconstruct_state == reconstruct_state_prexor_drain_result) -+ prexor = 1; -+ if (sh->reconstruct_state == reconstruct_state_drain_result || -+ sh->reconstruct_state == reconstruct_state_prexor_drain_result) { -+ sh->reconstruct_state = reconstruct_state_idle; -+ -+ /* All the 'written' buffers and the parity block are ready to -+ * be written back to disk -+ */ -+ BUG_ON(!test_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags) && -+ !test_bit(R5_Discard, &sh->dev[sh->pd_idx].flags)); -+ BUG_ON(sh->qd_idx >= 0 && -+ !test_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags) && -+ !test_bit(R5_Discard, &sh->dev[sh->qd_idx].flags)); -+ for (i = disks; i--; ) { -+ struct r5dev *dev = &sh->dev[i]; -+ if (test_bit(R5_LOCKED, &dev->flags) && -+ (i == sh->pd_idx || i == sh->qd_idx || -+ dev->written)) { -+ pr_debug("Writing block %d\n", i); -+ set_bit(R5_Wantwrite, &dev->flags); -+ if (prexor) -+ continue; -+ if (s.failed > 1) -+ continue; -+ if (!test_bit(R5_Insync, &dev->flags) || -+ ((i == sh->pd_idx || i == sh->qd_idx) && -+ s.failed == 0)) -+ set_bit(STRIPE_INSYNC, &sh->state); -+ } -+ } -+ if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) -+ s.dec_preread_active = 1; -+ } -+ -+ /* -+ * might be able to return some write requests if the parity blocks -+ * are safe, or on a failed drive -+ */ -+ pdev = &sh->dev[sh->pd_idx]; -+ s.p_failed = (s.failed >= 1 && s.failed_num[0] == sh->pd_idx) -+ || (s.failed >= 2 && s.failed_num[1] == sh->pd_idx); -+ qdev = &sh->dev[sh->qd_idx]; -+ s.q_failed = (s.failed >= 1 && s.failed_num[0] == sh->qd_idx) -+ || (s.failed >= 2 && s.failed_num[1] == sh->qd_idx) -+ || conf->level < 6; -+ -+ if (s.written && -+ (s.p_failed || ((test_bit(R5_Insync, &pdev->flags) -+ && !test_bit(R5_LOCKED, &pdev->flags) -+ && (test_bit(R5_UPTODATE, &pdev->flags) || -+ test_bit(R5_Discard, &pdev->flags))))) && -+ (s.q_failed || ((test_bit(R5_Insync, &qdev->flags) -+ && !test_bit(R5_LOCKED, &qdev->flags) -+ && (test_bit(R5_UPTODATE, &qdev->flags) || -+ test_bit(R5_Discard, &qdev->flags)))))) -+ handle_stripe_clean_event(conf, sh, disks, &s.return_bi); -+ -+ /* Now we might consider reading some blocks, either to check/generate -+ * parity, or to satisfy requests -+ * or to load a block that is being partially written. -+ */ -+ if (s.to_read || s.non_overwrite -+ || (conf->level == 6 && s.to_write && s.failed) -+ || (s.syncing && (s.uptodate + s.compute < disks)) -+ || s.replacing -+ || s.expanding) -+ handle_stripe_fill(sh, &s, disks); -+ -+ /* Now to consider new write requests and what else, if anything -+ * should be read. We do not handle new writes when: -+ * 1/ A 'write' operation (copy+xor) is already in flight. -+ * 2/ A 'check' operation is in flight, as it may clobber the parity -+ * block. -+ */ -+ if (s.to_write && !sh->reconstruct_state && !sh->check_state) -+ handle_stripe_dirtying(conf, sh, &s, disks); -+ -+ /* maybe we need to check and possibly fix the parity for this stripe -+ * Any reads will already have been scheduled, so we just see if enough -+ * data is available. The parity check is held off while parity -+ * dependent operations are in flight. -+ */ -+ if (sh->check_state || -+ (s.syncing && s.locked == 0 && -+ !test_bit(STRIPE_COMPUTE_RUN, &sh->state) && -+ !test_bit(STRIPE_INSYNC, &sh->state))) { -+ if (conf->level == 6) -+ handle_parity_checks6(conf, sh, &s, disks); -+ else -+ handle_parity_checks5(conf, sh, &s, disks); -+ } -+ -+ if ((s.replacing || s.syncing) && s.locked == 0 -+ && !test_bit(STRIPE_COMPUTE_RUN, &sh->state) -+ && !test_bit(STRIPE_REPLACED, &sh->state)) { -+ /* Write out to replacement devices where possible */ -+ for (i = 0; i < conf->raid_disks; i++) -+ if (test_bit(R5_NeedReplace, &sh->dev[i].flags)) { -+ WARN_ON(!test_bit(R5_UPTODATE, &sh->dev[i].flags)); -+ set_bit(R5_WantReplace, &sh->dev[i].flags); -+ set_bit(R5_LOCKED, &sh->dev[i].flags); -+ s.locked++; -+ } -+ if (s.replacing) -+ set_bit(STRIPE_INSYNC, &sh->state); -+ set_bit(STRIPE_REPLACED, &sh->state); -+ } -+ if ((s.syncing || s.replacing) && s.locked == 0 && -+ !test_bit(STRIPE_COMPUTE_RUN, &sh->state) && -+ test_bit(STRIPE_INSYNC, &sh->state)) { -+ md_done_sync(conf->mddev, STRIPE_SECTORS, 1); -+ clear_bit(STRIPE_SYNCING, &sh->state); -+ if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags)) -+ wake_up(&conf->wait_for_overlap); -+ } -+ -+ /* If the failed drives are just a ReadError, then we might need -+ * to progress the repair/check process -+ */ -+ if (s.failed <= conf->max_degraded && !conf->mddev->ro) -+ for (i = 0; i < s.failed; i++) { -+ struct r5dev *dev = &sh->dev[s.failed_num[i]]; -+ if (test_bit(R5_ReadError, &dev->flags) -+ && !test_bit(R5_LOCKED, &dev->flags) -+ && test_bit(R5_UPTODATE, &dev->flags) -+ ) { -+ if (!test_bit(R5_ReWrite, &dev->flags)) { -+ set_bit(R5_Wantwrite, &dev->flags); -+ set_bit(R5_ReWrite, &dev->flags); -+ set_bit(R5_LOCKED, &dev->flags); -+ s.locked++; -+ } else { -+ /* let's read it back */ -+ set_bit(R5_Wantread, &dev->flags); -+ set_bit(R5_LOCKED, &dev->flags); -+ s.locked++; -+ } -+ } -+ } -+ -+ /* Finish reconstruct operations initiated by the expansion process */ -+ if (sh->reconstruct_state == reconstruct_state_result) { -+ struct stripe_head *sh_src -+ = get_active_stripe(conf, sh->sector, 1, 1, 1); -+ if (sh_src && test_bit(STRIPE_EXPAND_SOURCE, &sh_src->state)) { -+ /* sh cannot be written until sh_src has been read. -+ * so arrange for sh to be delayed a little -+ */ -+ set_bit(STRIPE_DELAYED, &sh->state); -+ set_bit(STRIPE_HANDLE, &sh->state); -+ if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, -+ &sh_src->state)) -+ atomic_inc(&conf->preread_active_stripes); -+ release_stripe(sh_src); -+ goto finish; -+ } -+ if (sh_src) -+ release_stripe(sh_src); -+ -+ sh->reconstruct_state = reconstruct_state_idle; -+ clear_bit(STRIPE_EXPANDING, &sh->state); -+ for (i = conf->raid_disks; i--; ) { -+ set_bit(R5_Wantwrite, &sh->dev[i].flags); -+ set_bit(R5_LOCKED, &sh->dev[i].flags); -+ s.locked++; -+ } -+ } -+ -+ if (s.expanded && test_bit(STRIPE_EXPANDING, &sh->state) && -+ !sh->reconstruct_state) { -+ /* Need to write out all blocks after computing parity */ -+ sh->disks = conf->raid_disks; -+ stripe_set_idx(sh->sector, conf, 0, sh); -+ schedule_reconstruction(sh, &s, 1, 1); -+ } else if (s.expanded && !sh->reconstruct_state && s.locked == 0) { -+ clear_bit(STRIPE_EXPAND_READY, &sh->state); -+ atomic_dec(&conf->reshape_stripes); -+ wake_up(&conf->wait_for_overlap); -+ md_done_sync(conf->mddev, STRIPE_SECTORS, 1); -+ } -+ -+ if (s.expanding && s.locked == 0 && -+ !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) -+ handle_stripe_expansion(conf, sh); -+ -+finish: -+ /* wait for this device to become unblocked */ -+ if (unlikely(s.blocked_rdev)) { -+ if (conf->mddev->external) -+ md_wait_for_blocked_rdev(s.blocked_rdev, -+ conf->mddev); -+ else -+ /* Internal metadata will immediately -+ * be written by raid5d, so we don't -+ * need to wait here. -+ */ -+ rdev_dec_pending(s.blocked_rdev, -+ conf->mddev); -+ } -+ -+ if (s.handle_bad_blocks) -+ for (i = disks; i--; ) { -+ struct md_rdev *rdev; -+ struct r5dev *dev = &sh->dev[i]; -+ if (test_and_clear_bit(R5_WriteError, &dev->flags)) { -+ /* We own a safe reference to the rdev */ -+ rdev = conf->disks[i].rdev; -+ if (!rdev_set_badblocks(rdev, sh->sector, -+ STRIPE_SECTORS, 0)) -+ md_error(conf->mddev, rdev); -+ rdev_dec_pending(rdev, conf->mddev); -+ } -+ if (test_and_clear_bit(R5_MadeGood, &dev->flags)) { -+ rdev = conf->disks[i].rdev; -+ rdev_clear_badblocks(rdev, sh->sector, -+ STRIPE_SECTORS, 0); -+ rdev_dec_pending(rdev, conf->mddev); -+ } -+ if (test_and_clear_bit(R5_MadeGoodRepl, &dev->flags)) { -+ rdev = conf->disks[i].replacement; -+ if (!rdev) -+ /* rdev have been moved down */ -+ rdev = conf->disks[i].rdev; -+ rdev_clear_badblocks(rdev, sh->sector, -+ STRIPE_SECTORS, 0); -+ rdev_dec_pending(rdev, conf->mddev); -+ } -+ } -+ -+ if (s.ops_request) -+ raid_run_ops(sh, s.ops_request); -+ -+ ops_run_io(sh, &s); -+ -+ if (s.dec_preread_active) { -+ /* We delay this until after ops_run_io so that if make_request -+ * is waiting on a flush, it won't continue until the writes -+ * have actually been submitted. -+ */ -+ atomic_dec(&conf->preread_active_stripes); -+ if (atomic_read(&conf->preread_active_stripes) < -+ IO_THRESHOLD) -+ md_wakeup_thread(conf->mddev->thread); -+ } -+ -+ return_io(s.return_bi); -+ -+ clear_bit_unlock(STRIPE_ACTIVE, &sh->state); -+} ++ int err; + -+static void raid5_activate_delayed(struct r5conf *conf) -+{ -+ if (atomic_read(&conf->preread_active_stripes) < IO_THRESHOLD) { -+ while (!list_empty(&conf->delayed_list)) { -+ struct list_head *l = conf->delayed_list.next; -+ struct stripe_head *sh; -+ sh = list_entry(l, struct stripe_head, lru); -+ list_del_init(l); -+ clear_bit(STRIPE_DELAYED, &sh->state); -+ if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) -+ atomic_inc(&conf->preread_active_stripes); -+ list_add_tail(&sh->lru, &conf->hold_list); -+ raid5_wakeup_stripe_thread(sh); -+ } -+ } -+} ++ err = swork_get(); ++ if (err) ++ return err; + -+static void activate_bit_delay(struct r5conf *conf, -+ struct list_head *temp_inactive_list) ++ INIT_SWORK(¬ify_work, platform_thermal_notify_work); + return 0; + } + ++static void thermal_notify_work_cleanup(void) +{ -+ /* device_lock is held */ -+ struct list_head head; -+ list_add(&head, &conf->bitmap_list); -+ list_del_init(&conf->bitmap_list); -+ while (!list_empty(&head)) { -+ struct stripe_head *sh = list_entry(head.next, struct stripe_head, lru); -+ int hash; -+ list_del_init(&sh->lru); -+ atomic_inc(&sh->count); -+ hash = sh->hash_lock_index; -+ __release_stripe(conf, sh, &temp_inactive_list[hash]); -+ } ++ swork_put(); +} + -+static int raid5_congested(struct mddev *mddev, int bits) ++static int pkg_temp_thermal_platform_thermal_notify(__u64 msr_val) +{ -+ struct r5conf *conf = mddev->private; -+ -+ /* No difference between reads and writes. Just check -+ * how busy the stripe_cache is -+ */ -+ -+ if (test_bit(R5_INACTIVE_BLOCKED, &conf->cache_state)) -+ return 1; -+ if (conf->quiesce) -+ return 1; -+ if (atomic_read(&conf->empty_inactive_list_nr)) -+ return 1; -+ ++ swork_queue(¬ify_work); + return 0; +} + -+/* We want read requests to align with chunks where possible, -+ * but write requests don't need to. -+ */ -+static int raid5_mergeable_bvec(struct mddev *mddev, -+ struct bvec_merge_data *bvm, -+ struct bio_vec *biovec) -+{ -+ sector_t sector = bvm->bi_sector + get_start_sect(bvm->bi_bdev); -+ int max; -+ unsigned int chunk_sectors = mddev->chunk_sectors; -+ unsigned int bio_sectors = bvm->bi_size >> 9; ++#else /* !CONFIG_PREEMPT_RT_FULL */ + -+ /* -+ * always allow writes to be mergeable, read as well if array -+ * is degraded as we'll go through stripe cache anyway. -+ */ -+ if ((bvm->bi_rw & 1) == WRITE || mddev->degraded) -+ return biovec->bv_len; -+ -+ if (mddev->new_chunk_sectors < mddev->chunk_sectors) -+ chunk_sectors = mddev->new_chunk_sectors; -+ max = (chunk_sectors - ((sector & (chunk_sectors - 1)) + bio_sectors)) << 9; -+ if (max < 0) max = 0; -+ if (max <= biovec->bv_len && bio_sectors == 0) -+ return biovec->bv_len; -+ else -+ return max; -+} ++static int thermal_notify_work_init(void) { return 0; } + -+static int in_chunk_boundary(struct mddev *mddev, struct bio *bio) -+{ -+ sector_t sector = bio->bi_iter.bi_sector + get_start_sect(bio->bi_bdev); -+ unsigned int chunk_sectors = mddev->chunk_sectors; -+ unsigned int bio_sectors = bio_sectors(bio); -+ -+ if (mddev->new_chunk_sectors < mddev->chunk_sectors) -+ chunk_sectors = mddev->new_chunk_sectors; -+ return chunk_sectors >= -+ ((sector & (chunk_sectors - 1)) + bio_sectors); -+} ++static void thermal_notify_work_cleanup(void) { } + -+/* -+ * add bio to the retry LIFO ( in O(1) ... we are in interrupt ) -+ * later sampled by raid5d. -+ */ -+static void add_bio_to_retry(struct bio *bi,struct r5conf *conf) ++static int pkg_temp_thermal_platform_thermal_notify(__u64 msr_val) +{ -+ unsigned long flags; -+ -+ spin_lock_irqsave(&conf->device_lock, flags); -+ -+ bi->bi_next = conf->retry_read_aligned_list; -+ conf->retry_read_aligned_list = bi; ++ platform_thermal_notify_work(NULL); + -+ spin_unlock_irqrestore(&conf->device_lock, flags); -+ md_wakeup_thread(conf->mddev->thread); ++ return 0; +} ++#endif /* CONFIG_PREEMPT_RT_FULL */ + -+static struct bio *remove_bio_from_retry(struct r5conf *conf) -+{ -+ struct bio *bi; -+ -+ bi = conf->retry_read_aligned; -+ if (bi) { -+ conf->retry_read_aligned = NULL; -+ return bi; -+ } -+ bi = conf->retry_read_aligned_list; -+ if(bi) { -+ conf->retry_read_aligned_list = bi->bi_next; -+ bi->bi_next = NULL; -+ /* -+ * this sets the active strip count to 1 and the processed -+ * strip count to zero (upper 8 bits) -+ */ -+ raid5_set_bi_stripes(bi, 1); /* biased count of active stripes */ -+ } -+ -+ return bi; -+} + static int find_siblings_cpu(int cpu) + { + int i; +@@ -584,6 +624,9 @@ + if (!x86_match_cpu(pkg_temp_thermal_ids)) + return -ENODEV; + ++ if (!thermal_notify_work_init()) ++ return -ENODEV; + + spin_lock_init(&pkg_work_lock); + platform_thermal_package_notify = + pkg_temp_thermal_platform_thermal_notify; +@@ -608,7 +651,7 @@ + kfree(pkg_work_scheduled); + platform_thermal_package_notify = NULL; + platform_thermal_package_rate_control = NULL; +- ++ thermal_notify_work_cleanup(); + return -ENODEV; + } + +@@ -633,6 +676,7 @@ + mutex_unlock(&phy_dev_list_mutex); + platform_thermal_package_notify = NULL; + platform_thermal_package_rate_control = NULL; ++ thermal_notify_work_cleanup(); + for_each_online_cpu(i) + cancel_delayed_work_sync( + &per_cpu(pkg_temp_thermal_threshold_work, i)); +diff -Nur linux-4.1.10.orig/drivers/tty/serial/8250/8250_core.c linux-4.1.10/drivers/tty/serial/8250/8250_core.c +--- linux-4.1.10.orig/drivers/tty/serial/8250/8250_core.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/tty/serial/8250/8250_core.c 2015-10-12 22:33:32.264677547 +0200 +@@ -36,6 +36,7 @@ + #include + #include + #include ++#include + #include + #include + #ifdef CONFIG_SPARC +@@ -80,7 +81,16 @@ + #define DEBUG_INTR(fmt...) do { } while (0) + #endif + +-#define PASS_LIMIT 512 +/* -+ * The "raid5_align_endio" should check if the read succeeded and if it -+ * did, call bio_endio on the original bio (having bio_put the new bio -+ * first). -+ * If the read failed.. ++ * On -rt we can have a more delays, and legitimately ++ * so - so don't drop work spuriously and spam the ++ * syslog: + */ -+static void raid5_align_endio(struct bio *bi, int error) -+{ -+ struct bio* raid_bi = bi->bi_private; -+ struct mddev *mddev; -+ struct r5conf *conf; -+ int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags); -+ struct md_rdev *rdev; -+ -+ bio_put(bi); -+ -+ rdev = (void*)raid_bi->bi_next; -+ raid_bi->bi_next = NULL; -+ mddev = rdev->mddev; -+ conf = mddev->private; -+ -+ rdev_dec_pending(rdev, conf->mddev); -+ -+ if (!error && uptodate) { -+ trace_block_bio_complete(bdev_get_queue(raid_bi->bi_bdev), -+ raid_bi, 0); -+ bio_endio(raid_bi, 0); -+ if (atomic_dec_and_test(&conf->active_aligned_reads)) -+ wake_up(&conf->wait_for_stripe); -+ return; -+ } -+ -+ pr_debug("raid5_align_endio : io error...handing IO for a retry\n"); -+ -+ add_bio_to_retry(raid_bi, conf); -+} -+ -+static int bio_fits_rdev(struct bio *bi) -+{ -+ struct request_queue *q = bdev_get_queue(bi->bi_bdev); -+ -+ if (bio_sectors(bi) > queue_max_sectors(q)) -+ return 0; -+ blk_recount_segments(q, bi); -+ if (bi->bi_phys_segments > queue_max_segments(q)) -+ return 0; -+ -+ if (q->merge_bvec_fn) -+ /* it's too hard to apply the merge_bvec_fn at this stage, -+ * just just give up -+ */ -+ return 0; -+ -+ return 1; -+} -+ -+static int chunk_aligned_read(struct mddev *mddev, struct bio * raid_bio) -+{ -+ struct r5conf *conf = mddev->private; -+ int dd_idx; -+ struct bio* align_bi; -+ struct md_rdev *rdev; -+ sector_t end_sector; -+ -+ if (!in_chunk_boundary(mddev, raid_bio)) { -+ pr_debug("chunk_aligned_read : non aligned\n"); -+ return 0; -+ } -+ /* -+ * use bio_clone_mddev to make a copy of the bio -+ */ -+ align_bi = bio_clone_mddev(raid_bio, GFP_NOIO, mddev); -+ if (!align_bi) -+ return 0; -+ /* -+ * set bi_end_io to a new function, and set bi_private to the -+ * original bio. -+ */ -+ align_bi->bi_end_io = raid5_align_endio; -+ align_bi->bi_private = raid_bio; ++#ifdef CONFIG_PREEMPT_RT_FULL ++# define PASS_LIMIT 1000000 ++#else ++# define PASS_LIMIT 512 ++#endif + + #define BOTH_EMPTY (UART_LSR_TEMT | UART_LSR_THRE) + +@@ -3364,7 +3374,7 @@ + + if (port->sysrq) + locked = 0; +- else if (oops_in_progress) ++ else if (oops_in_progress || in_kdb_printk()) + locked = spin_trylock_irqsave(&port->lock, flags); + else + spin_lock_irqsave(&port->lock, flags); +diff -Nur linux-4.1.10.orig/drivers/tty/serial/amba-pl011.c linux-4.1.10/drivers/tty/serial/amba-pl011.c +--- linux-4.1.10.orig/drivers/tty/serial/amba-pl011.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/tty/serial/amba-pl011.c 2015-10-12 22:33:32.264677547 +0200 +@@ -2000,13 +2000,19 @@ + + clk_enable(uap->clk); + +- local_irq_save(flags); + /* -+ * compute position -+ */ -+ align_bi->bi_iter.bi_sector = -+ raid5_compute_sector(conf, raid_bio->bi_iter.bi_sector, -+ 0, &dd_idx, NULL); -+ -+ end_sector = bio_end_sector(align_bi); -+ rcu_read_lock(); -+ rdev = rcu_dereference(conf->disks[dd_idx].replacement); -+ if (!rdev || test_bit(Faulty, &rdev->flags) || -+ rdev->recovery_offset < end_sector) { -+ rdev = rcu_dereference(conf->disks[dd_idx].rdev); -+ if (rdev && -+ (test_bit(Faulty, &rdev->flags) || -+ !(test_bit(In_sync, &rdev->flags) || -+ rdev->recovery_offset >= end_sector))) -+ rdev = NULL; -+ } -+ if (rdev) { -+ sector_t first_bad; -+ int bad_sectors; -+ -+ atomic_inc(&rdev->nr_pending); -+ rcu_read_unlock(); -+ raid_bio->bi_next = (void*)rdev; -+ align_bi->bi_bdev = rdev->bdev; -+ __clear_bit(BIO_SEG_VALID, &align_bi->bi_flags); -+ -+ if (!bio_fits_rdev(align_bi) || -+ is_badblock(rdev, align_bi->bi_iter.bi_sector, -+ bio_sectors(align_bi), -+ &first_bad, &bad_sectors)) { -+ /* too big in some way, or has a known bad block */ -+ bio_put(align_bi); -+ rdev_dec_pending(rdev, mddev); -+ return 0; -+ } -+ -+ /* No reshape active, so we can trust rdev->data_offset */ -+ align_bi->bi_iter.bi_sector += rdev->data_offset; -+ -+ spin_lock_irq(&conf->device_lock); -+ wait_event_lock_irq(conf->wait_for_stripe, -+ conf->quiesce == 0, -+ conf->device_lock); -+ atomic_inc(&conf->active_aligned_reads); -+ spin_unlock_irq(&conf->device_lock); -+ -+ if (mddev->gendisk) -+ trace_block_bio_remap(bdev_get_queue(align_bi->bi_bdev), -+ align_bi, disk_devt(mddev->gendisk), -+ raid_bio->bi_iter.bi_sector); -+ generic_make_request(align_bi); -+ return 1; -+ } else { -+ rcu_read_unlock(); -+ bio_put(align_bi); -+ return 0; -+ } -+} -+ -+/* __get_priority_stripe - get the next stripe to process -+ * -+ * Full stripe writes are allowed to pass preread active stripes up until -+ * the bypass_threshold is exceeded. In general the bypass_count -+ * increments when the handle_list is handled before the hold_list; however, it -+ * will not be incremented when STRIPE_IO_STARTED is sampled set signifying a -+ * stripe with in flight i/o. The bypass_count will be reset when the -+ * head of the hold_list has changed, i.e. the head was promoted to the -+ * handle_list. -+ */ -+static struct stripe_head *__get_priority_stripe(struct r5conf *conf, int group) -+{ -+ struct stripe_head *sh = NULL, *tmp; -+ struct list_head *handle_list = NULL; -+ struct r5worker_group *wg = NULL; -+ -+ if (conf->worker_cnt_per_group == 0) { -+ handle_list = &conf->handle_list; -+ } else if (group != ANY_GROUP) { -+ handle_list = &conf->worker_groups[group].handle_list; -+ wg = &conf->worker_groups[group]; -+ } else { -+ int i; -+ for (i = 0; i < conf->group_cnt; i++) { -+ handle_list = &conf->worker_groups[i].handle_list; -+ wg = &conf->worker_groups[i]; -+ if (!list_empty(handle_list)) -+ break; -+ } -+ } -+ -+ pr_debug("%s: handle: %s hold: %s full_writes: %d bypass_count: %d\n", -+ __func__, -+ list_empty(handle_list) ? "empty" : "busy", -+ list_empty(&conf->hold_list) ? "empty" : "busy", -+ atomic_read(&conf->pending_full_writes), conf->bypass_count); -+ -+ if (!list_empty(handle_list)) { -+ sh = list_entry(handle_list->next, typeof(*sh), lru); -+ -+ if (list_empty(&conf->hold_list)) -+ conf->bypass_count = 0; -+ else if (!test_bit(STRIPE_IO_STARTED, &sh->state)) { -+ if (conf->hold_list.next == conf->last_hold) -+ conf->bypass_count++; -+ else { -+ conf->last_hold = conf->hold_list.next; -+ conf->bypass_count -= conf->bypass_threshold; -+ if (conf->bypass_count < 0) -+ conf->bypass_count = 0; -+ } -+ } -+ } else if (!list_empty(&conf->hold_list) && -+ ((conf->bypass_threshold && -+ conf->bypass_count > conf->bypass_threshold) || -+ atomic_read(&conf->pending_full_writes) == 0)) { -+ -+ list_for_each_entry(tmp, &conf->hold_list, lru) { -+ if (conf->worker_cnt_per_group == 0 || -+ group == ANY_GROUP || -+ !cpu_online(tmp->cpu) || -+ cpu_to_group(tmp->cpu) == group) { -+ sh = tmp; -+ break; -+ } -+ } -+ -+ if (sh) { -+ conf->bypass_count -= conf->bypass_threshold; -+ if (conf->bypass_count < 0) -+ conf->bypass_count = 0; -+ } -+ wg = NULL; -+ } -+ -+ if (!sh) -+ return NULL; -+ -+ if (wg) { -+ wg->stripes_cnt--; -+ sh->group = NULL; -+ } -+ list_del_init(&sh->lru); -+ BUG_ON(atomic_inc_return(&sh->count) != 1); -+ return sh; -+} -+ -+struct raid5_plug_cb { -+ struct blk_plug_cb cb; -+ struct list_head list; -+ struct list_head temp_inactive_list[NR_STRIPE_HASH_LOCKS]; -+}; -+ -+static void raid5_unplug(struct blk_plug_cb *blk_cb, bool from_schedule) -+{ -+ struct raid5_plug_cb *cb = container_of( -+ blk_cb, struct raid5_plug_cb, cb); -+ struct stripe_head *sh; -+ struct mddev *mddev = cb->cb.data; -+ struct r5conf *conf = mddev->private; -+ int cnt = 0; -+ int hash; -+ -+ if (cb->list.next && !list_empty(&cb->list)) { -+ spin_lock_irq(&conf->device_lock); -+ while (!list_empty(&cb->list)) { -+ sh = list_first_entry(&cb->list, struct stripe_head, lru); -+ list_del_init(&sh->lru); -+ /* -+ * avoid race release_stripe_plug() sees -+ * STRIPE_ON_UNPLUG_LIST clear but the stripe -+ * is still in our list -+ */ -+ smp_mb__before_atomic(); -+ clear_bit(STRIPE_ON_UNPLUG_LIST, &sh->state); -+ /* -+ * STRIPE_ON_RELEASE_LIST could be set here. In that -+ * case, the count is always > 1 here -+ */ -+ hash = sh->hash_lock_index; -+ __release_stripe(conf, sh, &cb->temp_inactive_list[hash]); -+ cnt++; -+ } -+ spin_unlock_irq(&conf->device_lock); -+ } -+ release_inactive_stripe_list(conf, cb->temp_inactive_list, -+ NR_STRIPE_HASH_LOCKS); -+ if (mddev->queue) -+ trace_block_unplug(mddev->queue, cnt, !from_schedule); -+ kfree(cb); -+} -+ -+static void release_stripe_plug(struct mddev *mddev, -+ struct stripe_head *sh) ++ * local_irq_save(flags); ++ * ++ * This local_irq_save() is nonsense. If we come in via sysrq ++ * handling then interrupts are already disabled. Aside of ++ * that the port.sysrq check is racy on SMP regardless. ++ */ + if (uap->port.sysrq) + locked = 0; + else if (oops_in_progress) +- locked = spin_trylock(&uap->port.lock); ++ locked = spin_trylock_irqsave(&uap->port.lock, flags); + else +- spin_lock(&uap->port.lock); ++ spin_lock_irqsave(&uap->port.lock, flags); + + /* + * First save the CR then disable the interrupts +@@ -2028,8 +2034,7 @@ + writew(old_cr, uap->port.membase + UART011_CR); + + if (locked) +- spin_unlock(&uap->port.lock); +- local_irq_restore(flags); ++ spin_unlock_irqrestore(&uap->port.lock, flags); + + clk_disable(uap->clk); + } +diff -Nur linux-4.1.10.orig/drivers/tty/serial/omap-serial.c linux-4.1.10/drivers/tty/serial/omap-serial.c +--- linux-4.1.10.orig/drivers/tty/serial/omap-serial.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/tty/serial/omap-serial.c 2015-10-12 22:33:32.264677547 +0200 +@@ -1282,13 +1282,10 @@ + + pm_runtime_get_sync(up->dev); + +- local_irq_save(flags); +- if (up->port.sysrq) +- locked = 0; +- else if (oops_in_progress) +- locked = spin_trylock(&up->port.lock); ++ if (up->port.sysrq || oops_in_progress) ++ locked = spin_trylock_irqsave(&up->port.lock, flags); + else +- spin_lock(&up->port.lock); ++ spin_lock_irqsave(&up->port.lock, flags); + + /* + * First save the IER then disable the interrupts +@@ -1317,8 +1314,7 @@ + pm_runtime_mark_last_busy(up->dev); + pm_runtime_put_autosuspend(up->dev); + if (locked) +- spin_unlock(&up->port.lock); +- local_irq_restore(flags); ++ spin_unlock_irqrestore(&up->port.lock, flags); + } + + static int __init +diff -Nur linux-4.1.10.orig/drivers/usb/core/hcd.c linux-4.1.10/drivers/usb/core/hcd.c +--- linux-4.1.10.orig/drivers/usb/core/hcd.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/usb/core/hcd.c 2015-10-12 22:33:32.264677547 +0200 +@@ -1684,9 +1684,9 @@ + * and no one may trigger the above deadlock situation when + * running complete() in tasklet. + */ +- local_irq_save(flags); ++ local_irq_save_nort(flags); + urb->complete(urb); +- local_irq_restore(flags); ++ local_irq_restore_nort(flags); + + usb_anchor_resume_wakeups(anchor); + atomic_dec(&urb->use_count); +diff -Nur linux-4.1.10.orig/drivers/usb/gadget/function/f_fs.c linux-4.1.10/drivers/usb/gadget/function/f_fs.c +--- linux-4.1.10.orig/drivers/usb/gadget/function/f_fs.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/usb/gadget/function/f_fs.c 2015-10-12 22:33:32.264677547 +0200 +@@ -1405,7 +1405,7 @@ + pr_info("%s(): freeing\n", __func__); + ffs_data_clear(ffs); + BUG_ON(waitqueue_active(&ffs->ev.waitq) || +- waitqueue_active(&ffs->ep0req_completion.wait)); ++ swaitqueue_active(&ffs->ep0req_completion.wait)); + kfree(ffs->dev_name); + kfree(ffs); + } +diff -Nur linux-4.1.10.orig/drivers/usb/gadget/legacy/inode.c linux-4.1.10/drivers/usb/gadget/legacy/inode.c +--- linux-4.1.10.orig/drivers/usb/gadget/legacy/inode.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/usb/gadget/legacy/inode.c 2015-10-12 22:33:32.264677547 +0200 +@@ -345,7 +345,7 @@ + spin_unlock_irq (&epdata->dev->lock); + + if (likely (value == 0)) { +- value = wait_event_interruptible (done.wait, done.done); ++ value = swait_event_interruptible (done.wait, done.done); + if (value != 0) { + spin_lock_irq (&epdata->dev->lock); + if (likely (epdata->ep != NULL)) { +@@ -354,7 +354,7 @@ + usb_ep_dequeue (epdata->ep, epdata->req); + spin_unlock_irq (&epdata->dev->lock); + +- wait_event (done.wait, done.done); ++ swait_event (done.wait, done.done); + if (epdata->status == -ECONNRESET) + epdata->status = -EINTR; + } else { +diff -Nur linux-4.1.10.orig/fs/aio.c linux-4.1.10/fs/aio.c +--- linux-4.1.10.orig/fs/aio.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/fs/aio.c 2015-10-12 22:33:32.264677547 +0200 +@@ -40,6 +40,7 @@ + #include + #include + #include ++#include + + #include + #include +@@ -115,7 +116,7 @@ + struct page **ring_pages; + long nr_pages; + +- struct work_struct free_work; ++ struct swork_event free_work; + + /* + * signals when all in-flight requests are done +@@ -253,6 +254,7 @@ + .mount = aio_mount, + .kill_sb = kill_anon_super, + }; ++ BUG_ON(swork_get()); + aio_mnt = kern_mount(&aio_fs); + if (IS_ERR(aio_mnt)) + panic("Failed to create aio fs mount."); +@@ -559,9 +561,9 @@ + return cancel(&kiocb->common); + } + +-static void free_ioctx(struct work_struct *work) ++static void free_ioctx(struct swork_event *sev) + { +- struct kioctx *ctx = container_of(work, struct kioctx, free_work); ++ struct kioctx *ctx = container_of(sev, struct kioctx, free_work); + + pr_debug("freeing %p\n", ctx); + +@@ -580,8 +582,8 @@ + if (ctx->rq_wait && atomic_dec_and_test(&ctx->rq_wait->count)) + complete(&ctx->rq_wait->comp); + +- INIT_WORK(&ctx->free_work, free_ioctx); +- schedule_work(&ctx->free_work); ++ INIT_SWORK(&ctx->free_work, free_ioctx); ++ swork_queue(&ctx->free_work); + } + + /* +@@ -589,9 +591,9 @@ + * and ctx->users has dropped to 0, so we know no more kiocbs can be submitted - + * now it's safe to cancel any that need to be. + */ +-static void free_ioctx_users(struct percpu_ref *ref) ++static void free_ioctx_users_work(struct swork_event *sev) + { +- struct kioctx *ctx = container_of(ref, struct kioctx, users); ++ struct kioctx *ctx = container_of(sev, struct kioctx, free_work); + struct aio_kiocb *req; + + spin_lock_irq(&ctx->ctx_lock); +@@ -610,6 +612,14 @@ + percpu_ref_put(&ctx->reqs); + } + ++static void free_ioctx_users(struct percpu_ref *ref) +{ -+ struct blk_plug_cb *blk_cb = blk_check_plugged( -+ raid5_unplug, mddev, -+ sizeof(struct raid5_plug_cb)); -+ struct raid5_plug_cb *cb; -+ -+ if (!blk_cb) { -+ release_stripe(sh); -+ return; -+ } ++ struct kioctx *ctx = container_of(ref, struct kioctx, users); + -+ cb = container_of(blk_cb, struct raid5_plug_cb, cb); ++ INIT_SWORK(&ctx->free_work, free_ioctx_users_work); ++ swork_queue(&ctx->free_work); ++} + -+ if (cb->list.next == NULL) { -+ int i; -+ INIT_LIST_HEAD(&cb->list); -+ for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++) -+ INIT_LIST_HEAD(cb->temp_inactive_list + i); + static int ioctx_add_table(struct kioctx *ctx, struct mm_struct *mm) + { + unsigned i, new_nr; +diff -Nur linux-4.1.10.orig/fs/autofs4/autofs_i.h linux-4.1.10/fs/autofs4/autofs_i.h +--- linux-4.1.10.orig/fs/autofs4/autofs_i.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/fs/autofs4/autofs_i.h 2015-10-12 22:33:32.268677282 +0200 +@@ -34,6 +34,7 @@ + #include + #include + #include ++#include + #include + #include + +diff -Nur linux-4.1.10.orig/fs/autofs4/expire.c linux-4.1.10/fs/autofs4/expire.c +--- linux-4.1.10.orig/fs/autofs4/expire.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/fs/autofs4/expire.c 2015-10-12 22:33:32.268677282 +0200 +@@ -150,7 +150,7 @@ + parent = p->d_parent; + if (!spin_trylock(&parent->d_lock)) { + spin_unlock(&p->d_lock); +- cpu_relax(); ++ cpu_chill(); + goto relock; + } + spin_unlock(&p->d_lock); +diff -Nur linux-4.1.10.orig/fs/buffer.c linux-4.1.10/fs/buffer.c +--- linux-4.1.10.orig/fs/buffer.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/fs/buffer.c 2015-10-12 22:33:32.268677282 +0200 +@@ -301,8 +301,7 @@ + * decide that the page is now completely done. + */ + first = page_buffers(page); +- local_irq_save(flags); +- bit_spin_lock(BH_Uptodate_Lock, &first->b_state); ++ flags = bh_uptodate_lock_irqsave(first); + clear_buffer_async_read(bh); + unlock_buffer(bh); + tmp = bh; +@@ -315,8 +314,7 @@ + } + tmp = tmp->b_this_page; + } while (tmp != bh); +- bit_spin_unlock(BH_Uptodate_Lock, &first->b_state); +- local_irq_restore(flags); ++ bh_uptodate_unlock_irqrestore(first, flags); + + /* + * If none of the buffers had errors and they are all +@@ -328,9 +326,7 @@ + return; + + still_busy: +- bit_spin_unlock(BH_Uptodate_Lock, &first->b_state); +- local_irq_restore(flags); +- return; ++ bh_uptodate_unlock_irqrestore(first, flags); + } + + /* +@@ -358,8 +354,7 @@ + } + + first = page_buffers(page); +- local_irq_save(flags); +- bit_spin_lock(BH_Uptodate_Lock, &first->b_state); ++ flags = bh_uptodate_lock_irqsave(first); + + clear_buffer_async_write(bh); + unlock_buffer(bh); +@@ -371,15 +366,12 @@ + } + tmp = tmp->b_this_page; + } +- bit_spin_unlock(BH_Uptodate_Lock, &first->b_state); +- local_irq_restore(flags); ++ bh_uptodate_unlock_irqrestore(first, flags); + end_page_writeback(page); + return; + + still_busy: +- bit_spin_unlock(BH_Uptodate_Lock, &first->b_state); +- local_irq_restore(flags); +- return; ++ bh_uptodate_unlock_irqrestore(first, flags); + } + EXPORT_SYMBOL(end_buffer_async_write); + +@@ -3325,6 +3317,7 @@ + struct buffer_head *ret = kmem_cache_zalloc(bh_cachep, gfp_flags); + if (ret) { + INIT_LIST_HEAD(&ret->b_assoc_buffers); ++ buffer_head_init_locks(ret); + preempt_disable(); + __this_cpu_inc(bh_accounting.nr); + recalc_bh_state(); +diff -Nur linux-4.1.10.orig/fs/dcache.c linux-4.1.10/fs/dcache.c +--- linux-4.1.10.orig/fs/dcache.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/fs/dcache.c 2015-10-12 22:33:32.268677282 +0200 +@@ -19,6 +19,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -589,7 +590,7 @@ + + failed: + spin_unlock(&dentry->d_lock); +- cpu_relax(); ++ cpu_chill(); + return dentry; /* try again with same dentry */ + } + +@@ -2395,7 +2396,7 @@ + if (dentry->d_lockref.count == 1) { + if (!spin_trylock(&inode->i_lock)) { + spin_unlock(&dentry->d_lock); +- cpu_relax(); ++ cpu_chill(); + goto again; + } + dentry->d_flags &= ~DCACHE_CANT_MOUNT; +diff -Nur linux-4.1.10.orig/fs/eventpoll.c linux-4.1.10/fs/eventpoll.c +--- linux-4.1.10.orig/fs/eventpoll.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/fs/eventpoll.c 2015-10-12 22:33:32.268677282 +0200 +@@ -505,12 +505,12 @@ + */ + static void ep_poll_safewake(wait_queue_head_t *wq) + { +- int this_cpu = get_cpu(); ++ int this_cpu = get_cpu_light(); + + ep_call_nested(&poll_safewake_ncalls, EP_MAX_NESTS, + ep_poll_wakeup_proc, NULL, wq, (void *) (long) this_cpu); + +- put_cpu(); ++ put_cpu_light(); + } + + static void ep_remove_wait_queue(struct eppoll_entry *pwq) +diff -Nur linux-4.1.10.orig/fs/exec.c linux-4.1.10/fs/exec.c +--- linux-4.1.10.orig/fs/exec.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/fs/exec.c 2015-10-12 22:33:32.268677282 +0200 +@@ -859,12 +859,14 @@ + } + } + task_lock(tsk); ++ preempt_disable_rt(); + active_mm = tsk->active_mm; + tsk->mm = mm; + tsk->active_mm = mm; + activate_mm(active_mm, mm); + tsk->mm->vmacache_seqnum = 0; + vmacache_flush(tsk); ++ preempt_enable_rt(); + task_unlock(tsk); + if (old_mm) { + up_read(&old_mm->mmap_sem); +diff -Nur linux-4.1.10.orig/fs/jbd/checkpoint.c linux-4.1.10/fs/jbd/checkpoint.c +--- linux-4.1.10.orig/fs/jbd/checkpoint.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/fs/jbd/checkpoint.c 2015-10-12 22:33:32.268677282 +0200 +@@ -129,6 +129,8 @@ + if (journal->j_flags & JFS_ABORT) + return; + spin_unlock(&journal->j_state_lock); ++ if (current->plug) ++ io_schedule(); + mutex_lock(&journal->j_checkpoint_mutex); + + /* +diff -Nur linux-4.1.10.orig/fs/jbd2/checkpoint.c linux-4.1.10/fs/jbd2/checkpoint.c +--- linux-4.1.10.orig/fs/jbd2/checkpoint.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/fs/jbd2/checkpoint.c 2015-10-12 22:33:32.268677282 +0200 +@@ -116,6 +116,8 @@ + nblocks = jbd2_space_needed(journal); + while (jbd2_log_space_left(journal) < nblocks) { + write_unlock(&journal->j_state_lock); ++ if (current->plug) ++ io_schedule(); + mutex_lock(&journal->j_checkpoint_mutex); + + /* +diff -Nur linux-4.1.10.orig/fs/namespace.c linux-4.1.10/fs/namespace.c +--- linux-4.1.10.orig/fs/namespace.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/fs/namespace.c 2015-10-12 22:33:32.268677282 +0200 +@@ -14,6 +14,7 @@ + #include + #include + #include ++#include + #include + #include + #include /* init_rootfs */ +@@ -353,8 +354,11 @@ + * incremented count after it has set MNT_WRITE_HOLD. + */ + smp_mb(); +- while (ACCESS_ONCE(mnt->mnt.mnt_flags) & MNT_WRITE_HOLD) +- cpu_relax(); ++ while (ACCESS_ONCE(mnt->mnt.mnt_flags) & MNT_WRITE_HOLD) { ++ preempt_enable(); ++ cpu_chill(); ++ preempt_disable(); + } + /* + * After the slowpath clears MNT_WRITE_HOLD, mnt_is_readonly will + * be set to match its requirements. So we must not load that until +diff -Nur linux-4.1.10.orig/fs/ntfs/aops.c linux-4.1.10/fs/ntfs/aops.c +--- linux-4.1.10.orig/fs/ntfs/aops.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/fs/ntfs/aops.c 2015-10-12 22:33:32.268677282 +0200 +@@ -107,8 +107,7 @@ + "0x%llx.", (unsigned long long)bh->b_blocknr); + } + first = page_buffers(page); +- local_irq_save(flags); +- bit_spin_lock(BH_Uptodate_Lock, &first->b_state); ++ flags = bh_uptodate_lock_irqsave(first); + clear_buffer_async_read(bh); + unlock_buffer(bh); + tmp = bh; +@@ -123,8 +122,7 @@ + } + tmp = tmp->b_this_page; + } while (tmp != bh); +- bit_spin_unlock(BH_Uptodate_Lock, &first->b_state); +- local_irq_restore(flags); ++ bh_uptodate_unlock_irqrestore(first, flags); + /* + * If none of the buffers had errors then we can set the page uptodate, + * but we first have to perform the post read mst fixups, if the +@@ -145,13 +143,13 @@ + recs = PAGE_CACHE_SIZE / rec_size; + /* Should have been verified before we got here... */ + BUG_ON(!recs); +- local_irq_save(flags); ++ local_irq_save_nort(flags); + kaddr = kmap_atomic(page); + for (i = 0; i < recs; i++) + post_read_mst_fixup((NTFS_RECORD*)(kaddr + + i * rec_size), rec_size); + kunmap_atomic(kaddr); +- local_irq_restore(flags); ++ local_irq_restore_nort(flags); + flush_dcache_page(page); + if (likely(page_uptodate && !PageError(page))) + SetPageUptodate(page); +@@ -159,9 +157,7 @@ + unlock_page(page); + return; + still_busy: +- bit_spin_unlock(BH_Uptodate_Lock, &first->b_state); +- local_irq_restore(flags); +- return; ++ bh_uptodate_unlock_irqrestore(first, flags); + } + + /** +diff -Nur linux-4.1.10.orig/fs/timerfd.c linux-4.1.10/fs/timerfd.c +--- linux-4.1.10.orig/fs/timerfd.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/fs/timerfd.c 2015-10-12 22:33:32.268677282 +0200 +@@ -450,7 +450,10 @@ + break; + } + spin_unlock_irq(&ctx->wqh.lock); +- cpu_relax(); ++ if (isalarm(ctx)) ++ hrtimer_wait_for_timer(&ctx->t.alarm.timer); ++ else ++ hrtimer_wait_for_timer(&ctx->t.tmr); + } + + /* +diff -Nur linux-4.1.10.orig/fs/xfs/xfs_inode.c linux-4.1.10/fs/xfs/xfs_inode.c +--- linux-4.1.10.orig/fs/xfs/xfs_inode.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/fs/xfs/xfs_inode.c 2015-10-12 22:33:32.272677018 +0200 +@@ -164,7 +164,7 @@ + (XFS_MMAPLOCK_SHARED | XFS_MMAPLOCK_EXCL)); + ASSERT((lock_flags & (XFS_ILOCK_SHARED | XFS_ILOCK_EXCL)) != + (XFS_ILOCK_SHARED | XFS_ILOCK_EXCL)); +- ASSERT((lock_flags & ~(XFS_LOCK_MASK | XFS_LOCK_DEP_MASK)) == 0); ++ ASSERT((lock_flags & ~(XFS_LOCK_MASK | XFS_LOCK_SUBCLASS_MASK)) == 0); + + if (lock_flags & XFS_IOLOCK_EXCL) + mrupdate_nested(&ip->i_iolock, XFS_IOLOCK_DEP(lock_flags)); +@@ -212,7 +212,7 @@ + (XFS_MMAPLOCK_SHARED | XFS_MMAPLOCK_EXCL)); + ASSERT((lock_flags & (XFS_ILOCK_SHARED | XFS_ILOCK_EXCL)) != + (XFS_ILOCK_SHARED | XFS_ILOCK_EXCL)); +- ASSERT((lock_flags & ~(XFS_LOCK_MASK | XFS_LOCK_DEP_MASK)) == 0); ++ ASSERT((lock_flags & ~(XFS_LOCK_MASK | XFS_LOCK_SUBCLASS_MASK)) == 0); + + if (lock_flags & XFS_IOLOCK_EXCL) { + if (!mrtryupdate(&ip->i_iolock)) +@@ -281,7 +281,7 @@ + (XFS_MMAPLOCK_SHARED | XFS_MMAPLOCK_EXCL)); + ASSERT((lock_flags & (XFS_ILOCK_SHARED | XFS_ILOCK_EXCL)) != + (XFS_ILOCK_SHARED | XFS_ILOCK_EXCL)); +- ASSERT((lock_flags & ~(XFS_LOCK_MASK | XFS_LOCK_DEP_MASK)) == 0); ++ ASSERT((lock_flags & ~(XFS_LOCK_MASK | XFS_LOCK_SUBCLASS_MASK)) == 0); + ASSERT(lock_flags != 0); + + if (lock_flags & XFS_IOLOCK_EXCL) +@@ -364,30 +364,38 @@ + + /* + * Bump the subclass so xfs_lock_inodes() acquires each lock with a different +- * value. This shouldn't be called for page fault locking, but we also need to +- * ensure we don't overrun the number of lockdep subclasses for the iolock or +- * mmaplock as that is limited to 12 by the mmap lock lockdep annotations. ++ * value. This can be called for any type of inode lock combination, including ++ * parent locking. Care must be taken to ensure we don't overrun the subclass ++ * storage fields in the class mask we build. + */ + static inline int + xfs_lock_inumorder(int lock_mode, int subclass) + { ++ int class = 0; + -+ if (!test_and_set_bit(STRIPE_ON_UNPLUG_LIST, &sh->state)) -+ list_add_tail(&sh->lru, &cb->list); -+ else -+ release_stripe(sh); -+} -+ -+static void make_discard_request(struct mddev *mddev, struct bio *bi) -+{ -+ struct r5conf *conf = mddev->private; -+ sector_t logical_sector, last_sector; -+ struct stripe_head *sh; -+ int remaining; -+ int stripe_sectors; -+ -+ if (mddev->reshape_position != MaxSector) -+ /* Skip discard while reshape is happening */ -+ return; ++ ASSERT(!(lock_mode & (XFS_ILOCK_PARENT | XFS_ILOCK_RTBITMAP | ++ XFS_ILOCK_RTSUM))); + -+ logical_sector = bi->bi_iter.bi_sector & ~((sector_t)STRIPE_SECTORS-1); -+ last_sector = bi->bi_iter.bi_sector + (bi->bi_iter.bi_size>>9); -+ -+ bi->bi_next = NULL; -+ bi->bi_phys_segments = 1; /* over-loaded to count active stripes */ -+ -+ stripe_sectors = conf->chunk_sectors * -+ (conf->raid_disks - conf->max_degraded); -+ logical_sector = DIV_ROUND_UP_SECTOR_T(logical_sector, -+ stripe_sectors); -+ sector_div(last_sector, stripe_sectors); -+ -+ logical_sector *= conf->chunk_sectors; -+ last_sector *= conf->chunk_sectors; -+ -+ for (; logical_sector < last_sector; -+ logical_sector += STRIPE_SECTORS) { -+ DEFINE_WAIT(w); -+ int d; -+ again: -+ sh = get_active_stripe(conf, logical_sector, 0, 0, 0); -+ prepare_to_wait(&conf->wait_for_overlap, &w, -+ TASK_UNINTERRUPTIBLE); -+ set_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags); -+ if (test_bit(STRIPE_SYNCING, &sh->state)) { -+ release_stripe(sh); -+ schedule(); -+ goto again; -+ } -+ clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags); -+ spin_lock_irq(&sh->stripe_lock); -+ for (d = 0; d < conf->raid_disks; d++) { -+ if (d == sh->pd_idx || d == sh->qd_idx) -+ continue; -+ if (sh->dev[d].towrite || sh->dev[d].toread) { -+ set_bit(R5_Overlap, &sh->dev[d].flags); -+ spin_unlock_irq(&sh->stripe_lock); -+ release_stripe(sh); -+ schedule(); -+ goto again; -+ } -+ } -+ set_bit(STRIPE_DISCARD, &sh->state); -+ finish_wait(&conf->wait_for_overlap, &w); -+ sh->overwrite_disks = 0; -+ for (d = 0; d < conf->raid_disks; d++) { -+ if (d == sh->pd_idx || d == sh->qd_idx) -+ continue; -+ sh->dev[d].towrite = bi; -+ set_bit(R5_OVERWRITE, &sh->dev[d].flags); -+ raid5_inc_bi_active_stripes(bi); -+ sh->overwrite_disks++; -+ } -+ spin_unlock_irq(&sh->stripe_lock); -+ if (conf->mddev->bitmap) { -+ for (d = 0; -+ d < conf->raid_disks - conf->max_degraded; -+ d++) -+ bitmap_startwrite(mddev->bitmap, -+ sh->sector, -+ STRIPE_SECTORS, -+ 0); -+ sh->bm_seq = conf->seq_flush + 1; -+ set_bit(STRIPE_BIT_DELAY, &sh->state); -+ } -+ -+ set_bit(STRIPE_HANDLE, &sh->state); -+ clear_bit(STRIPE_DELAYED, &sh->state); -+ if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) -+ atomic_inc(&conf->preread_active_stripes); -+ release_stripe_plug(mddev, sh); -+ } -+ -+ remaining = raid5_dec_bi_active_stripes(bi); -+ if (remaining == 0) { -+ md_write_end(mddev); -+ bio_endio(bi, 0); -+ } -+} -+ -+static void make_request(struct mddev *mddev, struct bio * bi) -+{ -+ struct r5conf *conf = mddev->private; -+ int dd_idx; -+ sector_t new_sector; -+ sector_t logical_sector, last_sector; -+ struct stripe_head *sh; -+ const int rw = bio_data_dir(bi); -+ int remaining; -+ DEFINE_WAIT(w); -+ bool do_prepare; -+ -+ if (unlikely(bi->bi_rw & REQ_FLUSH)) { -+ md_flush_request(mddev, bi); -+ return; + if (lock_mode & (XFS_IOLOCK_SHARED|XFS_IOLOCK_EXCL)) { +- ASSERT(subclass + XFS_LOCK_INUMORDER < +- (1 << (XFS_MMAPLOCK_SHIFT - XFS_IOLOCK_SHIFT))); +- lock_mode |= (subclass + XFS_LOCK_INUMORDER) << XFS_IOLOCK_SHIFT; ++ ASSERT(subclass <= XFS_IOLOCK_MAX_SUBCLASS); ++ ASSERT(subclass + XFS_IOLOCK_PARENT_VAL < ++ MAX_LOCKDEP_SUBCLASSES); ++ class += subclass << XFS_IOLOCK_SHIFT; ++ if (lock_mode & XFS_IOLOCK_PARENT) ++ class += XFS_IOLOCK_PARENT_VAL << XFS_IOLOCK_SHIFT; + } + + if (lock_mode & (XFS_MMAPLOCK_SHARED|XFS_MMAPLOCK_EXCL)) { +- ASSERT(subclass + XFS_LOCK_INUMORDER < +- (1 << (XFS_ILOCK_SHIFT - XFS_MMAPLOCK_SHIFT))); +- lock_mode |= (subclass + XFS_LOCK_INUMORDER) << +- XFS_MMAPLOCK_SHIFT; ++ ASSERT(subclass <= XFS_MMAPLOCK_MAX_SUBCLASS); ++ class += subclass << XFS_MMAPLOCK_SHIFT; + } + +- if (lock_mode & (XFS_ILOCK_SHARED|XFS_ILOCK_EXCL)) +- lock_mode |= (subclass + XFS_LOCK_INUMORDER) << XFS_ILOCK_SHIFT; ++ if (lock_mode & (XFS_ILOCK_SHARED|XFS_ILOCK_EXCL)) { ++ ASSERT(subclass <= XFS_ILOCK_MAX_SUBCLASS); ++ class += subclass << XFS_ILOCK_SHIFT; + } -+ -+ md_write_start(mddev, bi); -+ + +- return lock_mode; ++ return (lock_mode & ~XFS_LOCK_SUBCLASS_MASK) | class; + } + + /* +@@ -399,6 +407,11 @@ + * transaction (such as truncate). This can result in deadlock since the long + * running trans might need to wait for the inode we just locked in order to + * push the tail and free space in the log. ++ * ++ * xfs_lock_inodes() can only be used to lock one type of lock at a time - ++ * the iolock, the mmaplock or the ilock, but not more than one at a time. If we ++ * lock more than one at a time, lockdep will report false positives saying we ++ * have violated locking orders. + */ + void + xfs_lock_inodes( +@@ -409,8 +422,29 @@ + int attempts = 0, i, j, try_lock; + xfs_log_item_t *lp; + +- /* currently supports between 2 and 5 inodes */ + /* -+ * If array is degraded, better not do chunk aligned read because -+ * later we might have to read it again in order to reconstruct -+ * data on failed drives. -+ */ -+ if (rw == READ && mddev->degraded == 0 && -+ mddev->reshape_position == MaxSector && -+ chunk_aligned_read(mddev,bi)) -+ return; -+ -+ if (unlikely(bi->bi_rw & REQ_DISCARD)) { -+ make_discard_request(mddev, bi); -+ return; -+ } -+ -+ logical_sector = bi->bi_iter.bi_sector & ~((sector_t)STRIPE_SECTORS-1); -+ last_sector = bio_end_sector(bi); -+ bi->bi_next = NULL; -+ bi->bi_phys_segments = 1; /* over-loaded to count active stripes */ -+ -+ prepare_to_wait(&conf->wait_for_overlap, &w, TASK_UNINTERRUPTIBLE); -+ for (;logical_sector < last_sector; logical_sector += STRIPE_SECTORS) { -+ int previous; -+ int seq; -+ -+ do_prepare = false; -+ retry: -+ seq = read_seqcount_begin(&conf->gen_lock); -+ previous = 0; -+ if (do_prepare) -+ prepare_to_wait(&conf->wait_for_overlap, &w, -+ TASK_UNINTERRUPTIBLE); -+ if (unlikely(conf->reshape_progress != MaxSector)) { -+ /* spinlock is needed as reshape_progress may be -+ * 64bit on a 32bit platform, and so it might be -+ * possible to see a half-updated value -+ * Of course reshape_progress could change after -+ * the lock is dropped, so once we get a reference -+ * to the stripe that we think it is, we will have -+ * to check again. -+ */ -+ spin_lock_irq(&conf->device_lock); -+ if (mddev->reshape_backwards -+ ? logical_sector < conf->reshape_progress -+ : logical_sector >= conf->reshape_progress) { -+ previous = 1; -+ } else { -+ if (mddev->reshape_backwards -+ ? logical_sector < conf->reshape_safe -+ : logical_sector >= conf->reshape_safe) { -+ spin_unlock_irq(&conf->device_lock); -+ schedule(); -+ do_prepare = true; -+ goto retry; -+ } -+ } -+ spin_unlock_irq(&conf->device_lock); -+ } -+ -+ new_sector = raid5_compute_sector(conf, logical_sector, -+ previous, -+ &dd_idx, NULL); -+ pr_debug("raid456: make_request, sector %llu logical %llu\n", -+ (unsigned long long)new_sector, -+ (unsigned long long)logical_sector); -+ -+ sh = get_active_stripe(conf, new_sector, previous, -+ (bi->bi_rw&RWA_MASK), 0); -+ if (sh) { -+ if (unlikely(previous)) { -+ /* expansion might have moved on while waiting for a -+ * stripe, so we must do the range check again. -+ * Expansion could still move past after this -+ * test, but as we are holding a reference to -+ * 'sh', we know that if that happens, -+ * STRIPE_EXPANDING will get set and the expansion -+ * won't proceed until we finish with the stripe. -+ */ -+ int must_retry = 0; -+ spin_lock_irq(&conf->device_lock); -+ if (mddev->reshape_backwards -+ ? logical_sector >= conf->reshape_progress -+ : logical_sector < conf->reshape_progress) -+ /* mismatch, need to try again */ -+ must_retry = 1; -+ spin_unlock_irq(&conf->device_lock); -+ if (must_retry) { -+ release_stripe(sh); -+ schedule(); -+ do_prepare = true; -+ goto retry; -+ } -+ } -+ if (read_seqcount_retry(&conf->gen_lock, seq)) { -+ /* Might have got the wrong stripe_head -+ * by accident -+ */ -+ release_stripe(sh); -+ goto retry; -+ } -+ -+ if (rw == WRITE && -+ logical_sector >= mddev->suspend_lo && -+ logical_sector < mddev->suspend_hi) { -+ release_stripe(sh); -+ /* As the suspend_* range is controlled by -+ * userspace, we want an interruptible -+ * wait. -+ */ -+ flush_signals(current); -+ prepare_to_wait(&conf->wait_for_overlap, -+ &w, TASK_INTERRUPTIBLE); -+ if (logical_sector >= mddev->suspend_lo && -+ logical_sector < mddev->suspend_hi) { -+ schedule(); -+ do_prepare = true; -+ } -+ goto retry; -+ } -+ -+ if (test_bit(STRIPE_EXPANDING, &sh->state) || -+ !add_stripe_bio(sh, bi, dd_idx, rw, previous)) { -+ /* Stripe is busy expanding or -+ * add failed due to overlap. Flush everything -+ * and wait a while -+ */ -+ md_wakeup_thread(mddev->thread); -+ release_stripe(sh); -+ schedule(); -+ do_prepare = true; -+ goto retry; -+ } -+ set_bit(STRIPE_HANDLE, &sh->state); -+ clear_bit(STRIPE_DELAYED, &sh->state); -+ if ((!sh->batch_head || sh == sh->batch_head) && -+ (bi->bi_rw & REQ_SYNC) && -+ !test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) -+ atomic_inc(&conf->preread_active_stripes); -+ release_stripe_plug(mddev, sh); -+ } else { -+ /* cannot get stripe for read-ahead, just give-up */ -+ clear_bit(BIO_UPTODATE, &bi->bi_flags); -+ break; -+ } -+ } -+ finish_wait(&conf->wait_for_overlap, &w); -+ -+ remaining = raid5_dec_bi_active_stripes(bi); -+ if (remaining == 0) { -+ -+ if ( rw == WRITE ) -+ md_write_end(mddev); -+ -+ trace_block_bio_complete(bdev_get_queue(bi->bi_bdev), -+ bi, 0); -+ bio_endio(bi, 0); -+ } -+} -+ -+static sector_t raid5_size(struct mddev *mddev, sector_t sectors, int raid_disks); -+ -+static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, int *skipped) -+{ -+ /* reshaping is quite different to recovery/resync so it is -+ * handled quite separately ... here. -+ * -+ * On each call to sync_request, we gather one chunk worth of -+ * destination stripes and flag them as expanding. -+ * Then we find all the source stripes and request reads. -+ * As the reads complete, handle_stripe will copy the data -+ * into the destination stripe and release that stripe. -+ */ -+ struct r5conf *conf = mddev->private; -+ struct stripe_head *sh; -+ sector_t first_sector, last_sector; -+ int raid_disks = conf->previous_raid_disks; -+ int data_disks = raid_disks - conf->max_degraded; -+ int new_data_disks = conf->raid_disks - conf->max_degraded; -+ int i; -+ int dd_idx; -+ sector_t writepos, readpos, safepos; -+ sector_t stripe_addr; -+ int reshape_sectors; -+ struct list_head stripes; -+ -+ if (sector_nr == 0) { -+ /* If restarting in the middle, skip the initial sectors */ -+ if (mddev->reshape_backwards && -+ conf->reshape_progress < raid5_size(mddev, 0, 0)) { -+ sector_nr = raid5_size(mddev, 0, 0) -+ - conf->reshape_progress; -+ } else if (!mddev->reshape_backwards && -+ conf->reshape_progress > 0) -+ sector_nr = conf->reshape_progress; -+ sector_div(sector_nr, new_data_disks); -+ if (sector_nr) { -+ mddev->curr_resync_completed = sector_nr; -+ sysfs_notify(&mddev->kobj, NULL, "sync_completed"); -+ *skipped = 1; -+ return sector_nr; -+ } -+ } -+ -+ /* We need to process a full chunk at a time. -+ * If old and new chunk sizes differ, we need to process the -+ * largest of these -+ */ -+ if (mddev->new_chunk_sectors > mddev->chunk_sectors) -+ reshape_sectors = mddev->new_chunk_sectors; -+ else -+ reshape_sectors = mddev->chunk_sectors; -+ -+ /* We update the metadata at least every 10 seconds, or when -+ * the data about to be copied would over-write the source of -+ * the data at the front of the range. i.e. one new_stripe -+ * along from reshape_progress new_maps to after where -+ * reshape_safe old_maps to -+ */ -+ writepos = conf->reshape_progress; -+ sector_div(writepos, new_data_disks); -+ readpos = conf->reshape_progress; -+ sector_div(readpos, data_disks); -+ safepos = conf->reshape_safe; -+ sector_div(safepos, data_disks); -+ if (mddev->reshape_backwards) { -+ writepos -= min_t(sector_t, reshape_sectors, writepos); -+ readpos += reshape_sectors; -+ safepos += reshape_sectors; -+ } else { -+ writepos += reshape_sectors; -+ readpos -= min_t(sector_t, reshape_sectors, readpos); -+ safepos -= min_t(sector_t, reshape_sectors, safepos); -+ } -+ -+ /* Having calculated the 'writepos' possibly use it -+ * to set 'stripe_addr' which is where we will write to. -+ */ -+ if (mddev->reshape_backwards) { -+ BUG_ON(conf->reshape_progress == 0); -+ stripe_addr = writepos; -+ BUG_ON((mddev->dev_sectors & -+ ~((sector_t)reshape_sectors - 1)) -+ - reshape_sectors - stripe_addr -+ != sector_nr); -+ } else { -+ BUG_ON(writepos != sector_nr + reshape_sectors); -+ stripe_addr = sector_nr; -+ } -+ -+ /* 'writepos' is the most advanced device address we might write. -+ * 'readpos' is the least advanced device address we might read. -+ * 'safepos' is the least address recorded in the metadata as having -+ * been reshaped. -+ * If there is a min_offset_diff, these are adjusted either by -+ * increasing the safepos/readpos if diff is negative, or -+ * increasing writepos if diff is positive. -+ * If 'readpos' is then behind 'writepos', there is no way that we can -+ * ensure safety in the face of a crash - that must be done by userspace -+ * making a backup of the data. So in that case there is no particular -+ * rush to update metadata. -+ * Otherwise if 'safepos' is behind 'writepos', then we really need to -+ * update the metadata to advance 'safepos' to match 'readpos' so that -+ * we can be safe in the event of a crash. -+ * So we insist on updating metadata if safepos is behind writepos and -+ * readpos is beyond writepos. -+ * In any case, update the metadata every 10 seconds. -+ * Maybe that number should be configurable, but I'm not sure it is -+ * worth it.... maybe it could be a multiple of safemode_delay??? -+ */ -+ if (conf->min_offset_diff < 0) { -+ safepos += -conf->min_offset_diff; -+ readpos += -conf->min_offset_diff; -+ } else -+ writepos += conf->min_offset_diff; -+ -+ if ((mddev->reshape_backwards -+ ? (safepos > writepos && readpos < writepos) -+ : (safepos < writepos && readpos > writepos)) || -+ time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) { -+ /* Cannot proceed until we've updated the superblock... */ -+ wait_event(conf->wait_for_overlap, -+ atomic_read(&conf->reshape_stripes)==0 -+ || test_bit(MD_RECOVERY_INTR, &mddev->recovery)); -+ if (atomic_read(&conf->reshape_stripes) != 0) -+ return 0; -+ mddev->reshape_position = conf->reshape_progress; -+ mddev->curr_resync_completed = sector_nr; -+ conf->reshape_checkpoint = jiffies; -+ set_bit(MD_CHANGE_DEVS, &mddev->flags); -+ md_wakeup_thread(mddev->thread); -+ wait_event(mddev->sb_wait, mddev->flags == 0 || -+ test_bit(MD_RECOVERY_INTR, &mddev->recovery)); -+ if (test_bit(MD_RECOVERY_INTR, &mddev->recovery)) -+ return 0; -+ spin_lock_irq(&conf->device_lock); -+ conf->reshape_safe = mddev->reshape_position; -+ spin_unlock_irq(&conf->device_lock); -+ wake_up(&conf->wait_for_overlap); -+ sysfs_notify(&mddev->kobj, NULL, "sync_completed"); -+ } -+ -+ INIT_LIST_HEAD(&stripes); -+ for (i = 0; i < reshape_sectors; i += STRIPE_SECTORS) { -+ int j; -+ int skipped_disk = 0; -+ sh = get_active_stripe(conf, stripe_addr+i, 0, 0, 1); -+ set_bit(STRIPE_EXPANDING, &sh->state); -+ atomic_inc(&conf->reshape_stripes); -+ /* If any of this stripe is beyond the end of the old -+ * array, then we need to zero those blocks -+ */ -+ for (j=sh->disks; j--;) { -+ sector_t s; -+ if (j == sh->pd_idx) -+ continue; -+ if (conf->level == 6 && -+ j == sh->qd_idx) -+ continue; -+ s = compute_blocknr(sh, j, 0); -+ if (s < raid5_size(mddev, 0, 0)) { -+ skipped_disk = 1; -+ continue; -+ } -+ memset(page_address(sh->dev[j].page), 0, STRIPE_SIZE); -+ set_bit(R5_Expanded, &sh->dev[j].flags); -+ set_bit(R5_UPTODATE, &sh->dev[j].flags); -+ } -+ if (!skipped_disk) { -+ set_bit(STRIPE_EXPAND_READY, &sh->state); -+ set_bit(STRIPE_HANDLE, &sh->state); -+ } -+ list_add(&sh->lru, &stripes); -+ } -+ spin_lock_irq(&conf->device_lock); -+ if (mddev->reshape_backwards) -+ conf->reshape_progress -= reshape_sectors * new_data_disks; -+ else -+ conf->reshape_progress += reshape_sectors * new_data_disks; -+ spin_unlock_irq(&conf->device_lock); -+ /* Ok, those stripe are ready. We can start scheduling -+ * reads on the source stripes. -+ * The source stripes are determined by mapping the first and last -+ * block on the destination stripes. -+ */ -+ first_sector = -+ raid5_compute_sector(conf, stripe_addr*(new_data_disks), -+ 1, &dd_idx, NULL); -+ last_sector = -+ raid5_compute_sector(conf, ((stripe_addr+reshape_sectors) -+ * new_data_disks - 1), -+ 1, &dd_idx, NULL); -+ if (last_sector >= mddev->dev_sectors) -+ last_sector = mddev->dev_sectors - 1; -+ while (first_sector <= last_sector) { -+ sh = get_active_stripe(conf, first_sector, 1, 0, 1); -+ set_bit(STRIPE_EXPAND_SOURCE, &sh->state); -+ set_bit(STRIPE_HANDLE, &sh->state); -+ release_stripe(sh); -+ first_sector += STRIPE_SECTORS; -+ } -+ /* Now that the sources are clearly marked, we can release -+ * the destination stripes -+ */ -+ while (!list_empty(&stripes)) { -+ sh = list_entry(stripes.next, struct stripe_head, lru); -+ list_del_init(&sh->lru); -+ release_stripe(sh); -+ } -+ /* If this takes us to the resync_max point where we have to pause, -+ * then we need to write out the superblock. -+ */ -+ sector_nr += reshape_sectors; -+ if ((sector_nr - mddev->curr_resync_completed) * 2 -+ >= mddev->resync_max - mddev->curr_resync_completed) { -+ /* Cannot proceed until we've updated the superblock... */ -+ wait_event(conf->wait_for_overlap, -+ atomic_read(&conf->reshape_stripes) == 0 -+ || test_bit(MD_RECOVERY_INTR, &mddev->recovery)); -+ if (atomic_read(&conf->reshape_stripes) != 0) -+ goto ret; -+ mddev->reshape_position = conf->reshape_progress; -+ mddev->curr_resync_completed = sector_nr; -+ conf->reshape_checkpoint = jiffies; -+ set_bit(MD_CHANGE_DEVS, &mddev->flags); -+ md_wakeup_thread(mddev->thread); -+ wait_event(mddev->sb_wait, -+ !test_bit(MD_CHANGE_DEVS, &mddev->flags) -+ || test_bit(MD_RECOVERY_INTR, &mddev->recovery)); -+ if (test_bit(MD_RECOVERY_INTR, &mddev->recovery)) -+ goto ret; -+ spin_lock_irq(&conf->device_lock); -+ conf->reshape_safe = mddev->reshape_position; -+ spin_unlock_irq(&conf->device_lock); -+ wake_up(&conf->wait_for_overlap); -+ sysfs_notify(&mddev->kobj, NULL, "sync_completed"); -+ } -+ret: -+ return reshape_sectors; -+} -+ -+static inline sector_t sync_request(struct mddev *mddev, sector_t sector_nr, int *skipped) -+{ -+ struct r5conf *conf = mddev->private; -+ struct stripe_head *sh; -+ sector_t max_sector = mddev->dev_sectors; -+ sector_t sync_blocks; -+ int still_degraded = 0; -+ int i; -+ -+ if (sector_nr >= max_sector) { -+ /* just being told to finish up .. nothing much to do */ -+ -+ if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) { -+ end_reshape(conf); -+ return 0; -+ } -+ -+ if (mddev->curr_resync < max_sector) /* aborted */ -+ bitmap_end_sync(mddev->bitmap, mddev->curr_resync, -+ &sync_blocks, 1); -+ else /* completed sync */ -+ conf->fullsync = 0; -+ bitmap_close_sync(mddev->bitmap); -+ -+ return 0; -+ } -+ -+ /* Allow raid5_quiesce to complete */ -+ wait_event(conf->wait_for_overlap, conf->quiesce != 2); -+ -+ if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) -+ return reshape_request(mddev, sector_nr, skipped); -+ -+ /* No need to check resync_max as we never do more than one -+ * stripe, and as resync_max will always be on a chunk boundary, -+ * if the check in md_do_sync didn't fire, there is no chance -+ * of overstepping resync_max here -+ */ -+ -+ /* if there is too many failed drives and we are trying -+ * to resync, then assert that we are finished, because there is -+ * nothing we can do. ++ * Currently supports between 2 and 5 inodes with exclusive locking. We ++ * support an arbitrary depth of locking here, but absolute limits on ++ * inodes depend on the the type of locking and the limits placed by ++ * lockdep annotations in xfs_lock_inumorder. These are all checked by ++ * the asserts. + */ -+ if (mddev->degraded >= conf->max_degraded && -+ test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) { -+ sector_t rv = mddev->dev_sectors - sector_nr; -+ *skipped = 1; -+ return rv; -+ } -+ if (!test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) && -+ !conf->fullsync && -+ !bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, 1) && -+ sync_blocks >= STRIPE_SECTORS) { -+ /* we can skip this block, and probably more */ -+ sync_blocks /= STRIPE_SECTORS; -+ *skipped = 1; -+ return sync_blocks * STRIPE_SECTORS; /* keep things rounded to whole stripes */ -+ } + ASSERT(ips && inodes >= 2 && inodes <= 5); ++ ASSERT(lock_mode & (XFS_IOLOCK_EXCL | XFS_MMAPLOCK_EXCL | ++ XFS_ILOCK_EXCL)); ++ ASSERT(!(lock_mode & (XFS_IOLOCK_SHARED | XFS_MMAPLOCK_SHARED | ++ XFS_ILOCK_SHARED))); ++ ASSERT(!(lock_mode & XFS_IOLOCK_EXCL) || ++ inodes <= XFS_IOLOCK_MAX_SUBCLASS + 1); ++ ASSERT(!(lock_mode & XFS_MMAPLOCK_EXCL) || ++ inodes <= XFS_MMAPLOCK_MAX_SUBCLASS + 1); ++ ASSERT(!(lock_mode & XFS_ILOCK_EXCL) || ++ inodes <= XFS_ILOCK_MAX_SUBCLASS + 1); + -+ bitmap_cond_end_sync(mddev->bitmap, sector_nr); ++ if (lock_mode & XFS_IOLOCK_EXCL) { ++ ASSERT(!(lock_mode & (XFS_MMAPLOCK_EXCL | XFS_ILOCK_EXCL))); ++ } else if (lock_mode & XFS_MMAPLOCK_EXCL) ++ ASSERT(!(lock_mode & XFS_ILOCK_EXCL)); + + try_lock = 0; + i = 0; +diff -Nur linux-4.1.10.orig/fs/xfs/xfs_inode.h linux-4.1.10/fs/xfs/xfs_inode.h +--- linux-4.1.10.orig/fs/xfs/xfs_inode.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/fs/xfs/xfs_inode.h 2015-10-12 22:33:32.272677018 +0200 +@@ -284,9 +284,9 @@ + * Flags for lockdep annotations. + * + * XFS_LOCK_PARENT - for directory operations that require locking a +- * parent directory inode and a child entry inode. The parent gets locked +- * with this flag so it gets a lockdep subclass of 1 and the child entry +- * lock will have a lockdep subclass of 0. ++ * parent directory inode and a child entry inode. IOLOCK requires nesting, ++ * MMAPLOCK does not support this class, ILOCK requires a single subclass ++ * to differentiate parent from child. + * + * XFS_LOCK_RTBITMAP/XFS_LOCK_RTSUM - the realtime device bitmap and summary + * inodes do not participate in the normal lock order, and thus have their +@@ -295,30 +295,63 @@ + * XFS_LOCK_INUMORDER - for locking several inodes at the some time + * with xfs_lock_inodes(). This flag is used as the starting subclass + * and each subsequent lock acquired will increment the subclass by one. +- * So the first lock acquired will have a lockdep subclass of 4, the +- * second lock will have a lockdep subclass of 5, and so on. It is +- * the responsibility of the class builder to shift this to the correct +- * portion of the lock_mode lockdep mask. ++ * However, MAX_LOCKDEP_SUBCLASSES == 8, which means we are greatly ++ * limited to the subclasses we can represent via nesting. We need at least ++ * 5 inodes nest depth for the ILOCK through rename, and we also have to support ++ * XFS_ILOCK_PARENT, which gives 6 subclasses. Then we have XFS_ILOCK_RTBITMAP ++ * and XFS_ILOCK_RTSUM, which are another 2 unique subclasses, so that's all ++ * 8 subclasses supported by lockdep. ++ * ++ * This also means we have to number the sub-classes in the lowest bits of ++ * the mask we keep, and we have to ensure we never exceed 3 bits of lockdep ++ * mask and we can't use bit-masking to build the subclasses. What a mess. ++ * ++ * Bit layout: ++ * ++ * Bit Lock Region ++ * 16-19 XFS_IOLOCK_SHIFT dependencies ++ * 20-23 XFS_MMAPLOCK_SHIFT dependencies ++ * 24-31 XFS_ILOCK_SHIFT dependencies ++ * ++ * IOLOCK values ++ * ++ * 0-3 subclass value ++ * 4-7 PARENT subclass values ++ * ++ * MMAPLOCK values ++ * ++ * 0-3 subclass value ++ * 4-7 unused ++ * ++ * ILOCK values ++ * 0-4 subclass values ++ * 5 PARENT subclass (not nestable) ++ * 6 RTBITMAP subclass (not nestable) ++ * 7 RTSUM subclass (not nestable) ++ * + */ +-#define XFS_LOCK_PARENT 1 +-#define XFS_LOCK_RTBITMAP 2 +-#define XFS_LOCK_RTSUM 3 +-#define XFS_LOCK_INUMORDER 4 +- +-#define XFS_IOLOCK_SHIFT 16 +-#define XFS_IOLOCK_PARENT (XFS_LOCK_PARENT << XFS_IOLOCK_SHIFT) ++#define XFS_IOLOCK_SHIFT 16 ++#define XFS_IOLOCK_PARENT_VAL 4 ++#define XFS_IOLOCK_MAX_SUBCLASS (XFS_IOLOCK_PARENT_VAL - 1) ++#define XFS_IOLOCK_DEP_MASK 0x000f0000 ++#define XFS_IOLOCK_PARENT (XFS_IOLOCK_PARENT_VAL << XFS_IOLOCK_SHIFT) + +-#define XFS_MMAPLOCK_SHIFT 20 ++#define XFS_MMAPLOCK_SHIFT 20 ++#define XFS_MMAPLOCK_NUMORDER 0 ++#define XFS_MMAPLOCK_MAX_SUBCLASS 3 ++#define XFS_MMAPLOCK_DEP_MASK 0x00f00000 + +-#define XFS_ILOCK_SHIFT 24 +-#define XFS_ILOCK_PARENT (XFS_LOCK_PARENT << XFS_ILOCK_SHIFT) +-#define XFS_ILOCK_RTBITMAP (XFS_LOCK_RTBITMAP << XFS_ILOCK_SHIFT) +-#define XFS_ILOCK_RTSUM (XFS_LOCK_RTSUM << XFS_ILOCK_SHIFT) ++#define XFS_ILOCK_SHIFT 24 ++#define XFS_ILOCK_PARENT_VAL 5 ++#define XFS_ILOCK_MAX_SUBCLASS (XFS_ILOCK_PARENT_VAL - 1) ++#define XFS_ILOCK_RTBITMAP_VAL 6 ++#define XFS_ILOCK_RTSUM_VAL 7 ++#define XFS_ILOCK_DEP_MASK 0xff000000 ++#define XFS_ILOCK_PARENT (XFS_ILOCK_PARENT_VAL << XFS_ILOCK_SHIFT) ++#define XFS_ILOCK_RTBITMAP (XFS_ILOCK_RTBITMAP_VAL << XFS_ILOCK_SHIFT) ++#define XFS_ILOCK_RTSUM (XFS_ILOCK_RTSUM_VAL << XFS_ILOCK_SHIFT) + +-#define XFS_IOLOCK_DEP_MASK 0x000f0000 +-#define XFS_MMAPLOCK_DEP_MASK 0x00f00000 +-#define XFS_ILOCK_DEP_MASK 0xff000000 +-#define XFS_LOCK_DEP_MASK (XFS_IOLOCK_DEP_MASK | \ ++#define XFS_LOCK_SUBCLASS_MASK (XFS_IOLOCK_DEP_MASK | \ + XFS_MMAPLOCK_DEP_MASK | \ + XFS_ILOCK_DEP_MASK) + +diff -Nur linux-4.1.10.orig/include/acpi/platform/aclinux.h linux-4.1.10/include/acpi/platform/aclinux.h +--- linux-4.1.10.orig/include/acpi/platform/aclinux.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/acpi/platform/aclinux.h 2015-10-12 22:33:32.272677018 +0200 +@@ -123,6 +123,7 @@ + + #define acpi_cache_t struct kmem_cache + #define acpi_spinlock spinlock_t * ++#define acpi_raw_spinlock raw_spinlock_t * + #define acpi_cpu_flags unsigned long + + /* Use native linux version of acpi_os_allocate_zeroed */ +@@ -141,6 +142,20 @@ + #define ACPI_USE_ALTERNATE_PROTOTYPE_acpi_os_get_thread_id + #define ACPI_USE_ALTERNATE_PROTOTYPE_acpi_os_create_lock + ++#define acpi_os_create_raw_lock(__handle) \ ++({ \ ++ raw_spinlock_t *lock = ACPI_ALLOCATE(sizeof(*lock)); \ ++ \ ++ if (lock) { \ ++ *(__handle) = lock; \ ++ raw_spin_lock_init(*(__handle)); \ ++ } \ ++ lock ? AE_OK : AE_NO_MEMORY; \ ++ }) + -+ sh = get_active_stripe(conf, sector_nr, 0, 1, 0); -+ if (sh == NULL) { -+ sh = get_active_stripe(conf, sector_nr, 0, 0, 0); -+ /* make sure we don't swamp the stripe cache if someone else -+ * is trying to get access -+ */ -+ schedule_timeout_uninterruptible(1); -+ } -+ /* Need to check if array will still be degraded after recovery/resync -+ * Note in case of > 1 drive failures it's possible we're rebuilding -+ * one drive while leaving another faulty drive in array. -+ */ -+ rcu_read_lock(); -+ for (i = 0; i < conf->raid_disks; i++) { -+ struct md_rdev *rdev = ACCESS_ONCE(conf->disks[i].rdev); ++#define acpi_os_delete_raw_lock(__handle) kfree(__handle) + -+ if (rdev == NULL || test_bit(Faulty, &rdev->flags)) -+ still_degraded = 1; -+ } -+ rcu_read_unlock(); + -+ bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, still_degraded); + /* + * OSL interfaces used by debugger/disassembler + */ +diff -Nur linux-4.1.10.orig/include/asm-generic/bug.h linux-4.1.10/include/asm-generic/bug.h +--- linux-4.1.10.orig/include/asm-generic/bug.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/asm-generic/bug.h 2015-10-12 22:33:32.272677018 +0200 +@@ -206,6 +206,20 @@ + # define WARN_ON_SMP(x) ({0;}) + #endif + ++#ifdef CONFIG_PREEMPT_RT_BASE ++# define BUG_ON_RT(c) BUG_ON(c) ++# define BUG_ON_NONRT(c) do { } while (0) ++# define WARN_ON_RT(condition) WARN_ON(condition) ++# define WARN_ON_NONRT(condition) do { } while (0) ++# define WARN_ON_ONCE_NONRT(condition) do { } while (0) ++#else ++# define BUG_ON_RT(c) do { } while (0) ++# define BUG_ON_NONRT(c) BUG_ON(c) ++# define WARN_ON_RT(condition) do { } while (0) ++# define WARN_ON_NONRT(condition) WARN_ON(condition) ++# define WARN_ON_ONCE_NONRT(condition) WARN_ON_ONCE(condition) ++#endif + -+ set_bit(STRIPE_SYNC_REQUESTED, &sh->state); -+ set_bit(STRIPE_HANDLE, &sh->state); + #endif /* __ASSEMBLY__ */ + + #endif +diff -Nur linux-4.1.10.orig/include/asm-generic/futex.h linux-4.1.10/include/asm-generic/futex.h +--- linux-4.1.10.orig/include/asm-generic/futex.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/asm-generic/futex.h 2015-10-12 22:33:32.272677018 +0200 +@@ -8,8 +8,7 @@ + #ifndef CONFIG_SMP + /* + * The following implementation only for uniprocessor machines. +- * For UP, it's relies on the fact that pagefault_disable() also disables +- * preemption to ensure mutual exclusion. ++ * It relies on preempt_disable() ensuring mutual exclusion. + * + */ + +@@ -38,6 +37,7 @@ + if (encoded_op & (FUTEX_OP_OPARG_SHIFT << 28)) + oparg = 1 << oparg; + ++ preempt_disable(); + pagefault_disable(); + + ret = -EFAULT; +@@ -72,6 +72,7 @@ + + out_pagefault_enable: + pagefault_enable(); ++ preempt_enable(); + + if (ret == 0) { + switch (cmp) { +@@ -106,6 +107,7 @@ + { + u32 val; + ++ preempt_disable(); + if (unlikely(get_user(val, uaddr) != 0)) + return -EFAULT; + +@@ -113,6 +115,7 @@ + return -EFAULT; + + *uval = val; ++ preempt_enable(); + + return 0; + } +diff -Nur linux-4.1.10.orig/include/linux/blkdev.h linux-4.1.10/include/linux/blkdev.h +--- linux-4.1.10.orig/include/linux/blkdev.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/blkdev.h 2015-10-12 22:33:32.272677018 +0200 +@@ -101,6 +101,7 @@ + struct list_head queuelist; + union { + struct call_single_data csd; ++ struct work_struct work; + unsigned long fifo_time; + }; + +@@ -482,7 +483,7 @@ + struct throtl_data *td; + #endif + struct rcu_head rcu_head; +- wait_queue_head_t mq_freeze_wq; ++ struct swait_head mq_freeze_wq; + struct percpu_ref mq_usage_counter; + struct list_head all_q_node; + +diff -Nur linux-4.1.10.orig/include/linux/blk-mq.h linux-4.1.10/include/linux/blk-mq.h +--- linux-4.1.10.orig/include/linux/blk-mq.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/blk-mq.h 2015-10-12 22:33:32.272677018 +0200 +@@ -202,6 +202,7 @@ + + struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *, const int ctx_index); + struct blk_mq_hw_ctx *blk_mq_alloc_single_hw_queue(struct blk_mq_tag_set *, unsigned int, int); ++void __blk_mq_complete_request_remote_work(struct work_struct *work); + + int blk_mq_request_started(struct request *rq); + void blk_mq_start_request(struct request *rq); +diff -Nur linux-4.1.10.orig/include/linux/bottom_half.h linux-4.1.10/include/linux/bottom_half.h +--- linux-4.1.10.orig/include/linux/bottom_half.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/bottom_half.h 2015-10-12 22:33:32.272677018 +0200 +@@ -4,6 +4,17 @@ + #include + #include + ++#ifdef CONFIG_PREEMPT_RT_FULL + -+ release_stripe(sh); ++extern void local_bh_disable(void); ++extern void _local_bh_enable(void); ++extern void local_bh_enable(void); ++extern void local_bh_enable_ip(unsigned long ip); ++extern void __local_bh_disable_ip(unsigned long ip, unsigned int cnt); ++extern void __local_bh_enable_ip(unsigned long ip, unsigned int cnt); + -+ return STRIPE_SECTORS; -+} ++#else + -+static int retry_aligned_read(struct r5conf *conf, struct bio *raid_bio) + #ifdef CONFIG_TRACE_IRQFLAGS + extern void __local_bh_disable_ip(unsigned long ip, unsigned int cnt); + #else +@@ -31,5 +42,6 @@ + { + __local_bh_enable_ip(_THIS_IP_, SOFTIRQ_DISABLE_OFFSET); + } ++#endif + + #endif /* _LINUX_BH_H */ +diff -Nur linux-4.1.10.orig/include/linux/buffer_head.h linux-4.1.10/include/linux/buffer_head.h +--- linux-4.1.10.orig/include/linux/buffer_head.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/buffer_head.h 2015-10-12 22:33:32.272677018 +0200 +@@ -75,8 +75,52 @@ + struct address_space *b_assoc_map; /* mapping this buffer is + associated with */ + atomic_t b_count; /* users using this buffer_head */ ++#ifdef CONFIG_PREEMPT_RT_BASE ++ spinlock_t b_uptodate_lock; ++#if defined(CONFIG_JBD) || defined(CONFIG_JBD_MODULE) || \ ++ defined(CONFIG_JBD2) || defined(CONFIG_JBD2_MODULE) ++ spinlock_t b_state_lock; ++ spinlock_t b_journal_head_lock; ++#endif ++#endif + }; + ++static inline unsigned long bh_uptodate_lock_irqsave(struct buffer_head *bh) +{ -+ /* We may not be able to submit a whole bio at once as there -+ * may not be enough stripe_heads available. -+ * We cannot pre-allocate enough stripe_heads as we may need -+ * more than exist in the cache (if we allow ever large chunks). -+ * So we do one stripe head at a time and record in -+ * ->bi_hw_segments how many have been done. -+ * -+ * We *know* that this entire raid_bio is in one chunk, so -+ * it will be only one 'dd_idx' and only need one call to raid5_compute_sector. -+ */ -+ struct stripe_head *sh; -+ int dd_idx; -+ sector_t sector, logical_sector, last_sector; -+ int scnt = 0; -+ int remaining; -+ int handled = 0; -+ -+ logical_sector = raid_bio->bi_iter.bi_sector & -+ ~((sector_t)STRIPE_SECTORS-1); -+ sector = raid5_compute_sector(conf, logical_sector, -+ 0, &dd_idx, NULL); -+ last_sector = bio_end_sector(raid_bio); -+ -+ for (; logical_sector < last_sector; -+ logical_sector += STRIPE_SECTORS, -+ sector += STRIPE_SECTORS, -+ scnt++) { -+ -+ if (scnt < raid5_bi_processed_stripes(raid_bio)) -+ /* already done this stripe */ -+ continue; -+ -+ sh = get_active_stripe(conf, sector, 0, 1, 1); -+ -+ if (!sh) { -+ /* failed to get a stripe - must wait */ -+ raid5_set_bi_processed_stripes(raid_bio, scnt); -+ conf->retry_read_aligned = raid_bio; -+ return handled; -+ } -+ -+ if (!add_stripe_bio(sh, raid_bio, dd_idx, 0, 0)) { -+ release_stripe(sh); -+ raid5_set_bi_processed_stripes(raid_bio, scnt); -+ conf->retry_read_aligned = raid_bio; -+ return handled; -+ } ++ unsigned long flags; + -+ set_bit(R5_ReadNoMerge, &sh->dev[dd_idx].flags); -+ handle_stripe(sh); -+ release_stripe(sh); -+ handled++; -+ } -+ remaining = raid5_dec_bi_active_stripes(raid_bio); -+ if (remaining == 0) { -+ trace_block_bio_complete(bdev_get_queue(raid_bio->bi_bdev), -+ raid_bio, 0); -+ bio_endio(raid_bio, 0); -+ } -+ if (atomic_dec_and_test(&conf->active_aligned_reads)) -+ wake_up(&conf->wait_for_stripe); -+ return handled; ++#ifndef CONFIG_PREEMPT_RT_BASE ++ local_irq_save(flags); ++ bit_spin_lock(BH_Uptodate_Lock, &bh->b_state); ++#else ++ spin_lock_irqsave(&bh->b_uptodate_lock, flags); ++#endif ++ return flags; +} + -+static int handle_active_stripes(struct r5conf *conf, int group, -+ struct r5worker *worker, -+ struct list_head *temp_inactive_list) ++static inline void ++bh_uptodate_unlock_irqrestore(struct buffer_head *bh, unsigned long flags) +{ -+ struct stripe_head *batch[MAX_STRIPE_BATCH], *sh; -+ int i, batch_size = 0, hash; -+ bool release_inactive = false; -+ -+ while (batch_size < MAX_STRIPE_BATCH && -+ (sh = __get_priority_stripe(conf, group)) != NULL) -+ batch[batch_size++] = sh; -+ -+ if (batch_size == 0) { -+ for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++) -+ if (!list_empty(temp_inactive_list + i)) -+ break; -+ if (i == NR_STRIPE_HASH_LOCKS) -+ return batch_size; -+ release_inactive = true; -+ } -+ spin_unlock_irq(&conf->device_lock); -+ -+ release_inactive_stripe_list(conf, temp_inactive_list, -+ NR_STRIPE_HASH_LOCKS); -+ -+ if (release_inactive) { -+ spin_lock_irq(&conf->device_lock); -+ return 0; -+ } -+ -+ for (i = 0; i < batch_size; i++) -+ handle_stripe(batch[i]); -+ -+ cond_resched(); -+ -+ spin_lock_irq(&conf->device_lock); -+ for (i = 0; i < batch_size; i++) { -+ hash = batch[i]->hash_lock_index; -+ __release_stripe(conf, batch[i], &temp_inactive_list[hash]); -+ } -+ return batch_size; ++#ifndef CONFIG_PREEMPT_RT_BASE ++ bit_spin_unlock(BH_Uptodate_Lock, &bh->b_state); ++ local_irq_restore(flags); ++#else ++ spin_unlock_irqrestore(&bh->b_uptodate_lock, flags); ++#endif +} + -+static void raid5_do_work(struct work_struct *work) ++static inline void buffer_head_init_locks(struct buffer_head *bh) +{ -+ struct r5worker *worker = container_of(work, struct r5worker, work); -+ struct r5worker_group *group = worker->group; -+ struct r5conf *conf = group->conf; -+ int group_id = group - conf->worker_groups; -+ int handled; -+ struct blk_plug plug; -+ -+ pr_debug("+++ raid5worker active\n"); -+ -+ blk_start_plug(&plug); -+ handled = 0; -+ spin_lock_irq(&conf->device_lock); -+ while (1) { -+ int batch_size, released; -+ -+ released = release_stripe_list(conf, worker->temp_inactive_list); -+ -+ batch_size = handle_active_stripes(conf, group_id, worker, -+ worker->temp_inactive_list); -+ worker->working = false; -+ if (!batch_size && !released) -+ break; -+ handled += batch_size; -+ } -+ pr_debug("%d stripes handled\n", handled); -+ -+ spin_unlock_irq(&conf->device_lock); -+ blk_finish_plug(&plug); -+ -+ pr_debug("--- raid5worker inactive\n"); ++#ifdef CONFIG_PREEMPT_RT_BASE ++ spin_lock_init(&bh->b_uptodate_lock); ++#if defined(CONFIG_JBD) || defined(CONFIG_JBD_MODULE) || \ ++ defined(CONFIG_JBD2) || defined(CONFIG_JBD2_MODULE) ++ spin_lock_init(&bh->b_state_lock); ++ spin_lock_init(&bh->b_journal_head_lock); ++#endif ++#endif +} + -+/* -+ * This is our raid5 kernel thread. -+ * -+ * We scan the hash table for stripes which can be handled now. -+ * During the scan, completed stripes are saved for us by the interrupt -+ * handler, so that they will not have to wait for our next wakeup. -+ */ -+static void raid5d(struct md_thread *thread) -+{ -+ struct mddev *mddev = thread->mddev; -+ struct r5conf *conf = mddev->private; -+ int handled; -+ struct blk_plug plug; -+ -+ pr_debug("+++ raid5d active\n"); -+ -+ md_check_recovery(mddev); -+ -+ blk_start_plug(&plug); -+ handled = 0; -+ spin_lock_irq(&conf->device_lock); -+ while (1) { -+ struct bio *bio; -+ int batch_size, released; -+ -+ released = release_stripe_list(conf, conf->temp_inactive_list); -+ if (released) -+ clear_bit(R5_DID_ALLOC, &conf->cache_state); -+ -+ if ( -+ !list_empty(&conf->bitmap_list)) { -+ /* Now is a good time to flush some bitmap updates */ -+ conf->seq_flush++; -+ spin_unlock_irq(&conf->device_lock); -+ bitmap_unplug(mddev->bitmap); -+ spin_lock_irq(&conf->device_lock); -+ conf->seq_write = conf->seq_flush; -+ activate_bit_delay(conf, conf->temp_inactive_list); -+ } -+ raid5_activate_delayed(conf); -+ -+ while ((bio = remove_bio_from_retry(conf))) { -+ int ok; -+ spin_unlock_irq(&conf->device_lock); -+ ok = retry_aligned_read(conf, bio); -+ spin_lock_irq(&conf->device_lock); -+ if (!ok) -+ break; -+ handled++; -+ } -+ -+ batch_size = handle_active_stripes(conf, ANY_GROUP, NULL, -+ conf->temp_inactive_list); -+ if (!batch_size && !released) -+ break; -+ handled += batch_size; + /* + * macro tricks to expand the set_buffer_foo(), clear_buffer_foo() + * and buffer_foo() functions. +diff -Nur linux-4.1.10.orig/include/linux/cgroup.h linux-4.1.10/include/linux/cgroup.h +--- linux-4.1.10.orig/include/linux/cgroup.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/cgroup.h 2015-10-12 22:33:32.272677018 +0200 +@@ -22,6 +22,7 @@ + #include + #include + #include ++#include + + #ifdef CONFIG_CGROUPS + +@@ -91,6 +92,7 @@ + /* percpu_ref killing and RCU release */ + struct rcu_head rcu_head; + struct work_struct destroy_work; ++ struct swork_event destroy_swork; + }; + + /* bits in struct cgroup_subsys_state flags field */ +diff -Nur linux-4.1.10.orig/include/linux/completion.h linux-4.1.10/include/linux/completion.h +--- linux-4.1.10.orig/include/linux/completion.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/completion.h 2015-10-12 22:33:32.272677018 +0200 +@@ -7,8 +7,7 @@ + * Atomic wait-for-completion handler data structures. + * See kernel/sched/completion.c for details. + */ +- +-#include ++#include + + /* + * struct completion - structure used to maintain state for a "completion" +@@ -24,11 +23,11 @@ + */ + struct completion { + unsigned int done; +- wait_queue_head_t wait; ++ struct swait_head wait; + }; + + #define COMPLETION_INITIALIZER(work) \ +- { 0, __WAIT_QUEUE_HEAD_INITIALIZER((work).wait) } ++ { 0, SWAIT_HEAD_INITIALIZER((work).wait) } + + #define COMPLETION_INITIALIZER_ONSTACK(work) \ + ({ init_completion(&work); work; }) +@@ -73,7 +72,7 @@ + static inline void init_completion(struct completion *x) + { + x->done = 0; +- init_waitqueue_head(&x->wait); ++ init_swait_head(&x->wait); + } + + /** +diff -Nur linux-4.1.10.orig/include/linux/cpu.h linux-4.1.10/include/linux/cpu.h +--- linux-4.1.10.orig/include/linux/cpu.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/cpu.h 2015-10-12 22:33:32.272677018 +0200 +@@ -231,6 +231,8 @@ + extern void put_online_cpus(void); + extern void cpu_hotplug_disable(void); + extern void cpu_hotplug_enable(void); ++extern void pin_current_cpu(void); ++extern void unpin_current_cpu(void); + #define hotcpu_notifier(fn, pri) cpu_notifier(fn, pri) + #define __hotcpu_notifier(fn, pri) __cpu_notifier(fn, pri) + #define register_hotcpu_notifier(nb) register_cpu_notifier(nb) +@@ -249,6 +251,8 @@ + #define put_online_cpus() do { } while (0) + #define cpu_hotplug_disable() do { } while (0) + #define cpu_hotplug_enable() do { } while (0) ++static inline void pin_current_cpu(void) { } ++static inline void unpin_current_cpu(void) { } + #define hotcpu_notifier(fn, pri) do { (void)(fn); } while (0) + #define __hotcpu_notifier(fn, pri) do { (void)(fn); } while (0) + /* These aren't inline functions due to a GCC bug. */ +diff -Nur linux-4.1.10.orig/include/linux/delay.h linux-4.1.10/include/linux/delay.h +--- linux-4.1.10.orig/include/linux/delay.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/delay.h 2015-10-12 22:33:32.272677018 +0200 +@@ -52,4 +52,10 @@ + msleep(seconds * 1000); + } + ++#ifdef CONFIG_PREEMPT_RT_FULL ++extern void cpu_chill(void); ++#else ++# define cpu_chill() cpu_relax() ++#endif + -+ if (mddev->flags & ~(1<device_lock); -+ md_check_recovery(mddev); -+ spin_lock_irq(&conf->device_lock); -+ } -+ } -+ pr_debug("%d stripes handled\n", handled); -+ -+ spin_unlock_irq(&conf->device_lock); -+ if (test_and_clear_bit(R5_ALLOC_MORE, &conf->cache_state) && -+ mutex_trylock(&conf->cache_size_mutex)) { -+ grow_one_stripe(conf, __GFP_NOWARN); -+ /* Set flag even if allocation failed. This helps -+ * slow down allocation requests when mem is short -+ */ -+ set_bit(R5_DID_ALLOC, &conf->cache_state); -+ mutex_unlock(&conf->cache_size_mutex); -+ } + #endif /* defined(_LINUX_DELAY_H) */ +diff -Nur linux-4.1.10.orig/include/linux/ftrace_event.h linux-4.1.10/include/linux/ftrace_event.h +--- linux-4.1.10.orig/include/linux/ftrace_event.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/ftrace_event.h 2015-10-12 22:33:32.272677018 +0200 +@@ -66,6 +66,9 @@ + unsigned char flags; + unsigned char preempt_count; + int pid; ++ unsigned short migrate_disable; ++ unsigned short padding; ++ unsigned char preempt_lazy_count; + }; + + #define FTRACE_MAX_EVENT \ +diff -Nur linux-4.1.10.orig/include/linux/highmem.h linux-4.1.10/include/linux/highmem.h +--- linux-4.1.10.orig/include/linux/highmem.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/highmem.h 2015-10-12 22:33:32.272677018 +0200 +@@ -7,6 +7,7 @@ + #include + #include + #include ++#include + + #include + +@@ -65,6 +66,7 @@ + + static inline void *kmap_atomic(struct page *page) + { ++ preempt_disable_nort(); + pagefault_disable(); + return page_address(page); + } +@@ -73,6 +75,7 @@ + static inline void __kunmap_atomic(void *addr) + { + pagefault_enable(); ++ preempt_enable_nort(); + } + + #define kmap_atomic_pfn(pfn) kmap_atomic(pfn_to_page(pfn)) +@@ -85,32 +88,51 @@ + + #if defined(CONFIG_HIGHMEM) || defined(CONFIG_X86_32) + ++#ifndef CONFIG_PREEMPT_RT_FULL + DECLARE_PER_CPU(int, __kmap_atomic_idx); ++#endif + + static inline int kmap_atomic_idx_push(void) + { ++#ifndef CONFIG_PREEMPT_RT_FULL + int idx = __this_cpu_inc_return(__kmap_atomic_idx) - 1; + +-#ifdef CONFIG_DEBUG_HIGHMEM ++# ifdef CONFIG_DEBUG_HIGHMEM + WARN_ON_ONCE(in_irq() && !irqs_disabled()); + BUG_ON(idx >= KM_TYPE_NR); +-#endif ++# endif + return idx; ++#else ++ current->kmap_idx++; ++ BUG_ON(current->kmap_idx > KM_TYPE_NR); ++ return current->kmap_idx - 1; ++#endif + } + + static inline int kmap_atomic_idx(void) + { ++#ifndef CONFIG_PREEMPT_RT_FULL + return __this_cpu_read(__kmap_atomic_idx) - 1; ++#else ++ return current->kmap_idx - 1; ++#endif + } + + static inline void kmap_atomic_idx_pop(void) + { +-#ifdef CONFIG_DEBUG_HIGHMEM ++#ifndef CONFIG_PREEMPT_RT_FULL ++# ifdef CONFIG_DEBUG_HIGHMEM + int idx = __this_cpu_dec_return(__kmap_atomic_idx); + + BUG_ON(idx < 0); +-#else ++# else + __this_cpu_dec(__kmap_atomic_idx); ++# endif ++#else ++ current->kmap_idx--; ++# ifdef CONFIG_DEBUG_HIGHMEM ++ BUG_ON(current->kmap_idx < 0); ++# endif + #endif + } + +diff -Nur linux-4.1.10.orig/include/linux/hrtimer.h linux-4.1.10/include/linux/hrtimer.h +--- linux-4.1.10.orig/include/linux/hrtimer.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/hrtimer.h 2015-10-12 22:33:32.272677018 +0200 +@@ -111,6 +111,11 @@ + enum hrtimer_restart (*function)(struct hrtimer *); + struct hrtimer_clock_base *base; + unsigned long state; ++ struct list_head cb_entry; ++ int irqsafe; ++#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST ++ ktime_t praecox; ++#endif + #ifdef CONFIG_TIMER_STATS + int start_pid; + void *start_site; +@@ -147,6 +152,7 @@ + int index; + clockid_t clockid; + struct timerqueue_head active; ++ struct list_head expired; + ktime_t resolution; + ktime_t (*get_time)(void); + ktime_t softirq_time; +@@ -194,6 +200,9 @@ + unsigned long nr_hangs; + ktime_t max_hang_time; + #endif ++#ifdef CONFIG_PREEMPT_RT_BASE ++ wait_queue_head_t wait; ++#endif + struct hrtimer_clock_base clock_base[HRTIMER_MAX_CLOCK_BASES]; + }; + +@@ -381,6 +390,13 @@ + return hrtimer_start_expires(timer, HRTIMER_MODE_ABS); + } + ++/* Softirq preemption could deadlock timer removal */ ++#ifdef CONFIG_PREEMPT_RT_BASE ++ extern void hrtimer_wait_for_timer(const struct hrtimer *timer); ++#else ++# define hrtimer_wait_for_timer(timer) do { cpu_relax(); } while (0) ++#endif + -+ async_tx_issue_pending_all(); -+ blk_finish_plug(&plug); + /* Query timers: */ + extern ktime_t hrtimer_get_remaining(const struct hrtimer *timer); + extern int hrtimer_get_res(const clockid_t which_clock, struct timespec *tp); +diff -Nur linux-4.1.10.orig/include/linux/idr.h linux-4.1.10/include/linux/idr.h +--- linux-4.1.10.orig/include/linux/idr.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/idr.h 2015-10-12 22:33:32.272677018 +0200 +@@ -95,10 +95,14 @@ + * Each idr_preload() should be matched with an invocation of this + * function. See idr_preload() for details. + */ ++#ifdef CONFIG_PREEMPT_RT_FULL ++void idr_preload_end(void); ++#else + static inline void idr_preload_end(void) + { + preempt_enable(); + } ++#endif + + /** + * idr_find - return pointer for given id +diff -Nur linux-4.1.10.orig/include/linux/init_task.h linux-4.1.10/include/linux/init_task.h +--- linux-4.1.10.orig/include/linux/init_task.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/init_task.h 2015-10-12 22:33:32.272677018 +0200 +@@ -147,9 +147,16 @@ + # define INIT_PERF_EVENTS(tsk) + #endif + ++#ifdef CONFIG_PREEMPT_RT_BASE ++# define INIT_TIMER_LIST .posix_timer_list = NULL, ++#else ++# define INIT_TIMER_LIST ++#endif + -+ pr_debug("--- raid5d inactive\n"); -+} + #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN + # define INIT_VTIME(tsk) \ +- .vtime_seqlock = __SEQLOCK_UNLOCKED(tsk.vtime_seqlock), \ ++ .vtime_lock = __RAW_SPIN_LOCK_UNLOCKED(tsk.vtime_lock), \ ++ .vtime_seq = SEQCNT_ZERO(tsk.vtime_seq), \ + .vtime_snap = 0, \ + .vtime_snap_whence = VTIME_SYS, + #else +@@ -238,6 +245,7 @@ + .cpu_timers = INIT_CPU_TIMERS(tsk.cpu_timers), \ + .pi_lock = __RAW_SPIN_LOCK_UNLOCKED(tsk.pi_lock), \ + .timer_slack_ns = 50000, /* 50 usec default slack */ \ ++ INIT_TIMER_LIST \ + .pids = { \ + [PIDTYPE_PID] = INIT_PID_LINK(PIDTYPE_PID), \ + [PIDTYPE_PGID] = INIT_PID_LINK(PIDTYPE_PGID), \ +diff -Nur linux-4.1.10.orig/include/linux/interrupt.h linux-4.1.10/include/linux/interrupt.h +--- linux-4.1.10.orig/include/linux/interrupt.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/interrupt.h 2015-10-12 22:33:32.272677018 +0200 +@@ -61,6 +61,7 @@ + * interrupt handler after suspending interrupts. For system + * wakeup devices users need to implement wakeup detection in + * their interrupt handlers. ++ * IRQF_NO_SOFTIRQ_CALL - Do not process softirqs in the irq thread context (RT) + */ + #define IRQF_SHARED 0x00000080 + #define IRQF_PROBE_SHARED 0x00000100 +@@ -74,6 +75,7 @@ + #define IRQF_NO_THREAD 0x00010000 + #define IRQF_EARLY_RESUME 0x00020000 + #define IRQF_COND_SUSPEND 0x00040000 ++#define IRQF_NO_SOFTIRQ_CALL 0x00080000 + + #define IRQF_TIMER (__IRQF_TIMER | IRQF_NO_SUSPEND | IRQF_NO_THREAD) + +@@ -102,6 +104,7 @@ + * @flags: flags (see IRQF_* above) + * @thread_fn: interrupt handler function for threaded interrupts + * @thread: thread pointer for threaded interrupts ++ * @secondary: pointer to secondary irqaction (force threading) + * @thread_flags: flags related to @thread + * @thread_mask: bitmask for keeping track of @thread activity + * @dir: pointer to the proc/irq/NN/name entry +@@ -113,6 +116,7 @@ + struct irqaction *next; + irq_handler_t thread_fn; + struct task_struct *thread; ++ struct irqaction *secondary; + unsigned int irq; + unsigned int flags; + unsigned long thread_flags; +@@ -184,7 +188,7 @@ + #ifdef CONFIG_LOCKDEP + # define local_irq_enable_in_hardirq() do { } while (0) + #else +-# define local_irq_enable_in_hardirq() local_irq_enable() ++# define local_irq_enable_in_hardirq() local_irq_enable_nort() + #endif + + extern void disable_irq_nosync(unsigned int irq); +@@ -215,6 +219,7 @@ + unsigned int irq; + struct kref kref; + struct work_struct work; ++ struct list_head list; + void (*notify)(struct irq_affinity_notify *, const cpumask_t *mask); + void (*release)(struct kref *ref); + }; +@@ -377,9 +382,13 @@ + bool state); + + #ifdef CONFIG_IRQ_FORCED_THREADING ++# ifndef CONFIG_PREEMPT_RT_BASE + extern bool force_irqthreads; ++# else ++# define force_irqthreads (true) ++# endif + #else +-#define force_irqthreads (0) ++#define force_irqthreads (false) + #endif + + #ifndef __ARCH_SET_SOFTIRQ_PENDING +@@ -435,9 +444,10 @@ + void (*action)(struct softirq_action *); + }; + ++#ifndef CONFIG_PREEMPT_RT_FULL + asmlinkage void do_softirq(void); + asmlinkage void __do_softirq(void); +- ++static inline void thread_do_softirq(void) { do_softirq(); } + #ifdef __ARCH_HAS_DO_SOFTIRQ + void do_softirq_own_stack(void); + #else +@@ -446,6 +456,9 @@ + __do_softirq(); + } + #endif ++#else ++extern void thread_do_softirq(void); ++#endif + + extern void open_softirq(int nr, void (*action)(struct softirq_action *)); + extern void softirq_init(void); +@@ -453,6 +466,7 @@ + + extern void raise_softirq_irqoff(unsigned int nr); + extern void raise_softirq(unsigned int nr); ++extern void softirq_check_pending_idle(void); + + DECLARE_PER_CPU(struct task_struct *, ksoftirqd); + +@@ -474,8 +488,9 @@ + to be executed on some cpu at least once after this. + * If the tasklet is already scheduled, but its execution is still not + started, it will be executed only once. +- * If this tasklet is already running on another CPU (or schedule is called +- from tasklet itself), it is rescheduled for later. ++ * If this tasklet is already running on another CPU, it is rescheduled ++ for later. ++ * Schedule must not be called from the tasklet itself (a lockup occurs) + * Tasklet is strictly serialized wrt itself, but not + wrt another tasklets. If client needs some intertask synchronization, + he makes it with spinlocks. +@@ -500,27 +515,36 @@ + enum + { + TASKLET_STATE_SCHED, /* Tasklet is scheduled for execution */ +- TASKLET_STATE_RUN /* Tasklet is running (SMP only) */ ++ TASKLET_STATE_RUN, /* Tasklet is running (SMP only) */ ++ TASKLET_STATE_PENDING /* Tasklet is pending */ + }; + +-#ifdef CONFIG_SMP ++#define TASKLET_STATEF_SCHED (1 << TASKLET_STATE_SCHED) ++#define TASKLET_STATEF_RUN (1 << TASKLET_STATE_RUN) ++#define TASKLET_STATEF_PENDING (1 << TASKLET_STATE_PENDING) + -+static ssize_t -+raid5_show_stripe_cache_size(struct mddev *mddev, char *page) ++#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT_FULL) + static inline int tasklet_trylock(struct tasklet_struct *t) + { + return !test_and_set_bit(TASKLET_STATE_RUN, &(t)->state); + } + ++static inline int tasklet_tryunlock(struct tasklet_struct *t) +{ -+ struct r5conf *conf; -+ int ret = 0; -+ spin_lock(&mddev->lock); -+ conf = mddev->private; -+ if (conf) -+ ret = sprintf(page, "%d\n", conf->min_nr_stripes); -+ spin_unlock(&mddev->lock); -+ return ret; ++ return cmpxchg(&t->state, TASKLET_STATEF_RUN, 0) == TASKLET_STATEF_RUN; +} + -+int -+raid5_set_cache_size(struct mddev *mddev, int size) -+{ -+ struct r5conf *conf = mddev->private; -+ int err; -+ -+ if (size <= 16 || size > 32768) -+ return -EINVAL; -+ -+ conf->min_nr_stripes = size; -+ mutex_lock(&conf->cache_size_mutex); -+ while (size < conf->max_nr_stripes && -+ drop_one_stripe(conf)) -+ ; -+ mutex_unlock(&conf->cache_size_mutex); -+ -+ -+ err = md_allow_write(mddev); -+ if (err) -+ return err; + static inline void tasklet_unlock(struct tasklet_struct *t) + { + smp_mb__before_atomic(); + clear_bit(TASKLET_STATE_RUN, &(t)->state); + } + +-static inline void tasklet_unlock_wait(struct tasklet_struct *t) +-{ +- while (test_bit(TASKLET_STATE_RUN, &(t)->state)) { barrier(); } +-} ++extern void tasklet_unlock_wait(struct tasklet_struct *t); + -+ mutex_lock(&conf->cache_size_mutex); -+ while (size > conf->max_nr_stripes) -+ if (!grow_one_stripe(conf, GFP_KERNEL)) -+ break; -+ mutex_unlock(&conf->cache_size_mutex); + #else + #define tasklet_trylock(t) 1 ++#define tasklet_tryunlock(t) 1 + #define tasklet_unlock_wait(t) do { } while (0) + #define tasklet_unlock(t) do { } while (0) + #endif +@@ -569,12 +593,7 @@ + smp_mb(); + } + +-static inline void tasklet_enable(struct tasklet_struct *t) +-{ +- smp_mb__before_atomic(); +- atomic_dec(&t->count); +-} +- ++extern void tasklet_enable(struct tasklet_struct *t); + extern void tasklet_kill(struct tasklet_struct *t); + extern void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu); + extern void tasklet_init(struct tasklet_struct *t, +@@ -605,6 +624,12 @@ + tasklet_kill(&ttimer->tasklet); + } + ++#ifdef CONFIG_PREEMPT_RT_FULL ++extern void softirq_early_init(void); ++#else ++static inline void softirq_early_init(void) { } ++#endif + -+ return 0; -+} -+EXPORT_SYMBOL(raid5_set_cache_size); + /* + * Autoprobing for irqs: + * +diff -Nur linux-4.1.10.orig/include/linux/io-mapping.h linux-4.1.10/include/linux/io-mapping.h +--- linux-4.1.10.orig/include/linux/io-mapping.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/io-mapping.h 2015-10-12 22:33:32.272677018 +0200 +@@ -141,6 +141,7 @@ + io_mapping_map_atomic_wc(struct io_mapping *mapping, + unsigned long offset) + { ++ preempt_disable(); + pagefault_disable(); + return ((char __force __iomem *) mapping) + offset; + } +@@ -149,6 +150,7 @@ + io_mapping_unmap_atomic(void __iomem *vaddr) + { + pagefault_enable(); ++ preempt_enable(); + } + + /* Non-atomic map/unmap */ +diff -Nur linux-4.1.10.orig/include/linux/irqdesc.h linux-4.1.10/include/linux/irqdesc.h +--- linux-4.1.10.orig/include/linux/irqdesc.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/irqdesc.h 2015-10-12 22:33:32.272677018 +0200 +@@ -63,6 +63,7 @@ + unsigned int irqs_unhandled; + atomic_t threads_handled; + int threads_handled_last; ++ u64 random_ip; + raw_spinlock_t lock; + struct cpumask *percpu_enabled; + #ifdef CONFIG_SMP +diff -Nur linux-4.1.10.orig/include/linux/irqflags.h linux-4.1.10/include/linux/irqflags.h +--- linux-4.1.10.orig/include/linux/irqflags.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/irqflags.h 2015-10-12 22:33:32.272677018 +0200 +@@ -25,8 +25,6 @@ + # define trace_softirqs_enabled(p) ((p)->softirqs_enabled) + # define trace_hardirq_enter() do { current->hardirq_context++; } while (0) + # define trace_hardirq_exit() do { current->hardirq_context--; } while (0) +-# define lockdep_softirq_enter() do { current->softirq_context++; } while (0) +-# define lockdep_softirq_exit() do { current->softirq_context--; } while (0) + # define INIT_TRACE_IRQFLAGS .softirqs_enabled = 1, + #else + # define trace_hardirqs_on() do { } while (0) +@@ -39,9 +37,15 @@ + # define trace_softirqs_enabled(p) 0 + # define trace_hardirq_enter() do { } while (0) + # define trace_hardirq_exit() do { } while (0) ++# define INIT_TRACE_IRQFLAGS ++#endif + -+static ssize_t -+raid5_store_stripe_cache_size(struct mddev *mddev, const char *page, size_t len) -+{ -+ struct r5conf *conf; -+ unsigned long new; -+ int err; ++#if defined(CONFIG_TRACE_IRQFLAGS) && !defined(CONFIG_PREEMPT_RT_FULL) ++# define lockdep_softirq_enter() do { current->softirq_context++; } while (0) ++# define lockdep_softirq_exit() do { current->softirq_context--; } while (0) ++#else + # define lockdep_softirq_enter() do { } while (0) + # define lockdep_softirq_exit() do { } while (0) +-# define INIT_TRACE_IRQFLAGS + #endif + + #if defined(CONFIG_IRQSOFF_TRACER) || \ +@@ -148,4 +152,23 @@ + + #define irqs_disabled_flags(flags) raw_irqs_disabled_flags(flags) + ++/* ++ * local_irq* variants depending on RT/!RT ++ */ ++#ifdef CONFIG_PREEMPT_RT_FULL ++# define local_irq_disable_nort() do { } while (0) ++# define local_irq_enable_nort() do { } while (0) ++# define local_irq_save_nort(flags) local_save_flags(flags) ++# define local_irq_restore_nort(flags) (void)(flags) ++# define local_irq_disable_rt() local_irq_disable() ++# define local_irq_enable_rt() local_irq_enable() ++#else ++# define local_irq_disable_nort() local_irq_disable() ++# define local_irq_enable_nort() local_irq_enable() ++# define local_irq_save_nort(flags) local_irq_save(flags) ++# define local_irq_restore_nort(flags) local_irq_restore(flags) ++# define local_irq_disable_rt() do { } while (0) ++# define local_irq_enable_rt() do { } while (0) ++#endif + -+ if (len >= PAGE_SIZE) -+ return -EINVAL; -+ if (kstrtoul(page, 10, &new)) -+ return -EINVAL; -+ err = mddev_lock(mddev); -+ if (err) -+ return err; -+ conf = mddev->private; -+ if (!conf) -+ err = -ENODEV; -+ else -+ err = raid5_set_cache_size(mddev, new); -+ mddev_unlock(mddev); + #endif +diff -Nur linux-4.1.10.orig/include/linux/irq.h linux-4.1.10/include/linux/irq.h +--- linux-4.1.10.orig/include/linux/irq.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/irq.h 2015-10-12 22:33:32.276676754 +0200 +@@ -72,6 +72,7 @@ + * IRQ_IS_POLLED - Always polled by another interrupt. Exclude + * it from the spurious interrupt detection + * mechanism and from core side polling. ++ * IRQ_NO_SOFTIRQ_CALL - No softirq processing in the irq thread context (RT) + */ + enum { + IRQ_TYPE_NONE = 0x00000000, +@@ -97,13 +98,14 @@ + IRQ_NOTHREAD = (1 << 16), + IRQ_PER_CPU_DEVID = (1 << 17), + IRQ_IS_POLLED = (1 << 18), ++ IRQ_NO_SOFTIRQ_CALL = (1 << 19), + }; + + #define IRQF_MODIFY_MASK \ + (IRQ_TYPE_SENSE_MASK | IRQ_NOPROBE | IRQ_NOREQUEST | \ + IRQ_NOAUTOEN | IRQ_MOVE_PCNTXT | IRQ_LEVEL | IRQ_NO_BALANCING | \ + IRQ_PER_CPU | IRQ_NESTED_THREAD | IRQ_NOTHREAD | IRQ_PER_CPU_DEVID | \ +- IRQ_IS_POLLED) ++ IRQ_IS_POLLED | IRQ_NO_SOFTIRQ_CALL) + + #define IRQ_NO_BALANCING_MASK (IRQ_PER_CPU | IRQ_NO_BALANCING) + +diff -Nur linux-4.1.10.orig/include/linux/irq_work.h linux-4.1.10/include/linux/irq_work.h +--- linux-4.1.10.orig/include/linux/irq_work.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/irq_work.h 2015-10-12 22:33:32.276676754 +0200 +@@ -16,6 +16,7 @@ + #define IRQ_WORK_BUSY 2UL + #define IRQ_WORK_FLAGS 3UL + #define IRQ_WORK_LAZY 4UL /* Doesn't want IPI, wait for tick */ ++#define IRQ_WORK_HARD_IRQ 8UL /* Run hard IRQ context, even on RT */ + + struct irq_work { + unsigned long flags; +diff -Nur linux-4.1.10.orig/include/linux/jbd_common.h linux-4.1.10/include/linux/jbd_common.h +--- linux-4.1.10.orig/include/linux/jbd_common.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/jbd_common.h 2015-10-12 22:33:32.276676754 +0200 +@@ -15,32 +15,56 @@ + + static inline void jbd_lock_bh_state(struct buffer_head *bh) + { ++#ifndef CONFIG_PREEMPT_RT_BASE + bit_spin_lock(BH_State, &bh->b_state); ++#else ++ spin_lock(&bh->b_state_lock); ++#endif + } + + static inline int jbd_trylock_bh_state(struct buffer_head *bh) + { ++#ifndef CONFIG_PREEMPT_RT_BASE + return bit_spin_trylock(BH_State, &bh->b_state); ++#else ++ return spin_trylock(&bh->b_state_lock); ++#endif + } + + static inline int jbd_is_locked_bh_state(struct buffer_head *bh) + { ++#ifndef CONFIG_PREEMPT_RT_BASE + return bit_spin_is_locked(BH_State, &bh->b_state); ++#else ++ return spin_is_locked(&bh->b_state_lock); ++#endif + } + + static inline void jbd_unlock_bh_state(struct buffer_head *bh) + { ++#ifndef CONFIG_PREEMPT_RT_BASE + bit_spin_unlock(BH_State, &bh->b_state); ++#else ++ spin_unlock(&bh->b_state_lock); ++#endif + } + + static inline void jbd_lock_bh_journal_head(struct buffer_head *bh) + { ++#ifndef CONFIG_PREEMPT_RT_BASE + bit_spin_lock(BH_JournalHead, &bh->b_state); ++#else ++ spin_lock(&bh->b_journal_head_lock); ++#endif + } + + static inline void jbd_unlock_bh_journal_head(struct buffer_head *bh) + { ++#ifndef CONFIG_PREEMPT_RT_BASE + bit_spin_unlock(BH_JournalHead, &bh->b_state); ++#else ++ spin_unlock(&bh->b_journal_head_lock); ++#endif + } + + #endif +diff -Nur linux-4.1.10.orig/include/linux/kdb.h linux-4.1.10/include/linux/kdb.h +--- linux-4.1.10.orig/include/linux/kdb.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/kdb.h 2015-10-12 22:33:32.276676754 +0200 +@@ -167,6 +167,7 @@ + extern __printf(1, 2) int kdb_printf(const char *, ...); + typedef __printf(1, 2) int (*kdb_printf_t)(const char *, ...); + ++#define in_kdb_printk() (kdb_trap_printk) + extern void kdb_init(int level); + + /* Access to kdb specific polling devices */ +@@ -201,6 +202,7 @@ + extern int kdb_unregister(char *); + #else /* ! CONFIG_KGDB_KDB */ + static inline __printf(1, 2) int kdb_printf(const char *fmt, ...) { return 0; } ++#define in_kdb_printk() (0) + static inline void kdb_init(int level) {} + static inline int kdb_register(char *cmd, kdb_func_t func, char *usage, + char *help, short minlen) { return 0; } +diff -Nur linux-4.1.10.orig/include/linux/kernel.h linux-4.1.10/include/linux/kernel.h +--- linux-4.1.10.orig/include/linux/kernel.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/kernel.h 2015-10-12 22:33:32.276676754 +0200 +@@ -188,6 +188,9 @@ + */ + # define might_sleep() \ + do { __might_sleep(__FILE__, __LINE__, 0); might_resched(); } while (0) + -+ return err ?: len; -+} ++# define might_sleep_no_state_check() \ ++ do { ___might_sleep(__FILE__, __LINE__, 0); might_resched(); } while (0) + # define sched_annotate_sleep() (current->task_state_change = 0) + #else + static inline void ___might_sleep(const char *file, int line, +@@ -195,6 +198,7 @@ + static inline void __might_sleep(const char *file, int line, + int preempt_offset) { } + # define might_sleep() do { might_resched(); } while (0) ++# define might_sleep_no_state_check() do { might_resched(); } while (0) + # define sched_annotate_sleep() do { } while (0) + #endif + +@@ -244,7 +248,8 @@ + + #if defined(CONFIG_MMU) && \ + (defined(CONFIG_PROVE_LOCKING) || defined(CONFIG_DEBUG_ATOMIC_SLEEP)) +-void might_fault(void); ++#define might_fault() __might_fault(__FILE__, __LINE__) ++void __might_fault(const char *file, int line); + #else + static inline void might_fault(void) { } + #endif +@@ -466,6 +471,7 @@ + SYSTEM_HALT, + SYSTEM_POWER_OFF, + SYSTEM_RESTART, ++ SYSTEM_SUSPEND, + } system_state; + + #define TAINT_PROPRIETARY_MODULE 0 +diff -Nur linux-4.1.10.orig/include/linux/kvm_host.h linux-4.1.10/include/linux/kvm_host.h +--- linux-4.1.10.orig/include/linux/kvm_host.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/kvm_host.h 2015-10-12 22:33:32.276676754 +0200 +@@ -230,7 +230,7 @@ + + int fpu_active; + int guest_fpu_loaded, guest_xcr0_loaded; +- wait_queue_head_t wq; ++ struct swait_head wq; + struct pid *pid; + int sigset_active; + sigset_t sigset; +@@ -690,7 +690,7 @@ + } + #endif + +-static inline wait_queue_head_t *kvm_arch_vcpu_wq(struct kvm_vcpu *vcpu) ++static inline struct swait_head *kvm_arch_vcpu_wq(struct kvm_vcpu *vcpu) + { + #ifdef __KVM_HAVE_ARCH_WQP + return vcpu->arch.wqp; +diff -Nur linux-4.1.10.orig/include/linux/lglock.h linux-4.1.10/include/linux/lglock.h +--- linux-4.1.10.orig/include/linux/lglock.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/lglock.h 2015-10-12 22:33:32.276676754 +0200 +@@ -34,22 +34,39 @@ + #endif + + struct lglock { ++#ifndef CONFIG_PREEMPT_RT_FULL + arch_spinlock_t __percpu *lock; ++#else ++ struct rt_mutex __percpu *lock; ++#endif + #ifdef CONFIG_DEBUG_LOCK_ALLOC + struct lock_class_key lock_key; + struct lockdep_map lock_dep_map; + #endif + }; + +-#define DEFINE_LGLOCK(name) \ ++#ifndef CONFIG_PREEMPT_RT_FULL ++# define DEFINE_LGLOCK(name) \ + static DEFINE_PER_CPU(arch_spinlock_t, name ## _lock) \ + = __ARCH_SPIN_LOCK_UNLOCKED; \ + struct lglock name = { .lock = &name ## _lock } + +-#define DEFINE_STATIC_LGLOCK(name) \ ++# define DEFINE_STATIC_LGLOCK(name) \ + static DEFINE_PER_CPU(arch_spinlock_t, name ## _lock) \ + = __ARCH_SPIN_LOCK_UNLOCKED; \ + static struct lglock name = { .lock = &name ## _lock } ++#else + -+static struct md_sysfs_entry -+raid5_stripecache_size = __ATTR(stripe_cache_size, S_IRUGO | S_IWUSR, -+ raid5_show_stripe_cache_size, -+ raid5_store_stripe_cache_size); ++# define DEFINE_LGLOCK(name) \ ++ static DEFINE_PER_CPU(struct rt_mutex, name ## _lock) \ ++ = __RT_MUTEX_INITIALIZER( name ## _lock); \ ++ struct lglock name = { .lock = &name ## _lock } + -+static ssize_t -+raid5_show_rmw_level(struct mddev *mddev, char *page) -+{ -+ struct r5conf *conf = mddev->private; -+ if (conf) -+ return sprintf(page, "%d\n", conf->rmw_level); -+ else -+ return 0; -+} ++# define DEFINE_STATIC_LGLOCK(name) \ ++ static DEFINE_PER_CPU(struct rt_mutex, name ## _lock) \ ++ = __RT_MUTEX_INITIALIZER( name ## _lock); \ ++ static struct lglock name = { .lock = &name ## _lock } ++#endif + + void lg_lock_init(struct lglock *lg, char *name); + void lg_local_lock(struct lglock *lg); +@@ -59,6 +76,12 @@ + void lg_global_lock(struct lglock *lg); + void lg_global_unlock(struct lglock *lg); + ++#ifndef CONFIG_PREEMPT_RT_FULL ++#define lg_global_trylock_relax(name) lg_global_lock(name) ++#else ++void lg_global_trylock_relax(struct lglock *lg); ++#endif + -+static ssize_t -+raid5_store_rmw_level(struct mddev *mddev, const char *page, size_t len) -+{ -+ struct r5conf *conf = mddev->private; -+ unsigned long new; + #else + /* When !CONFIG_SMP, map lglock to spinlock */ + #define lglock spinlock +diff -Nur linux-4.1.10.orig/include/linux/list_bl.h linux-4.1.10/include/linux/list_bl.h +--- linux-4.1.10.orig/include/linux/list_bl.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/list_bl.h 2015-10-12 22:33:32.276676754 +0200 +@@ -2,6 +2,7 @@ + #define _LINUX_LIST_BL_H + + #include ++#include + #include + + /* +@@ -32,13 +33,22 @@ + + struct hlist_bl_head { + struct hlist_bl_node *first; ++#ifdef CONFIG_PREEMPT_RT_BASE ++ raw_spinlock_t lock; ++#endif + }; + + struct hlist_bl_node { + struct hlist_bl_node *next, **pprev; + }; +-#define INIT_HLIST_BL_HEAD(ptr) \ +- ((ptr)->first = NULL) + -+ if (!conf) -+ return -ENODEV; ++static inline void INIT_HLIST_BL_HEAD(struct hlist_bl_head *h) ++{ ++ h->first = NULL; ++#ifdef CONFIG_PREEMPT_RT_BASE ++ raw_spin_lock_init(&h->lock); ++#endif ++} + + static inline void INIT_HLIST_BL_NODE(struct hlist_bl_node *h) + { +@@ -117,12 +127,26 @@ + + static inline void hlist_bl_lock(struct hlist_bl_head *b) + { ++#ifndef CONFIG_PREEMPT_RT_BASE + bit_spin_lock(0, (unsigned long *)b); ++#else ++ raw_spin_lock(&b->lock); ++#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK) ++ __set_bit(0, (unsigned long *)b); ++#endif ++#endif + } + + static inline void hlist_bl_unlock(struct hlist_bl_head *b) + { ++#ifndef CONFIG_PREEMPT_RT_BASE + __bit_spin_unlock(0, (unsigned long *)b); ++#else ++#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK) ++ __clear_bit(0, (unsigned long *)b); ++#endif ++ raw_spin_unlock(&b->lock); ++#endif + } + + static inline bool hlist_bl_is_locked(struct hlist_bl_head *b) +diff -Nur linux-4.1.10.orig/include/linux/locallock.h linux-4.1.10/include/linux/locallock.h +--- linux-4.1.10.orig/include/linux/locallock.h 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/include/linux/locallock.h 2015-10-12 22:33:32.276676754 +0200 +@@ -0,0 +1,270 @@ ++#ifndef _LINUX_LOCALLOCK_H ++#define _LINUX_LOCALLOCK_H + -+ if (len >= PAGE_SIZE) -+ return -EINVAL; ++#include ++#include + -+ if (kstrtoul(page, 10, &new)) -+ return -EINVAL; ++#ifdef CONFIG_PREEMPT_RT_BASE + -+ if (new != PARITY_DISABLE_RMW && !raid6_call.xor_syndrome) -+ return -EINVAL; ++#ifdef CONFIG_DEBUG_SPINLOCK ++# define LL_WARN(cond) WARN_ON(cond) ++#else ++# define LL_WARN(cond) do { } while (0) ++#endif + -+ if (new != PARITY_DISABLE_RMW && -+ new != PARITY_ENABLE_RMW && -+ new != PARITY_PREFER_RMW) -+ return -EINVAL; ++/* ++ * per cpu lock based substitute for local_irq_*() ++ */ ++struct local_irq_lock { ++ spinlock_t lock; ++ struct task_struct *owner; ++ int nestcnt; ++ unsigned long flags; ++}; + -+ conf->rmw_level = new; -+ return len; -+} ++#define DEFINE_LOCAL_IRQ_LOCK(lvar) \ ++ DEFINE_PER_CPU(struct local_irq_lock, lvar) = { \ ++ .lock = __SPIN_LOCK_UNLOCKED((lvar).lock) } + -+static struct md_sysfs_entry -+raid5_rmw_level = __ATTR(rmw_level, S_IRUGO | S_IWUSR, -+ raid5_show_rmw_level, -+ raid5_store_rmw_level); ++#define DECLARE_LOCAL_IRQ_LOCK(lvar) \ ++ DECLARE_PER_CPU(struct local_irq_lock, lvar) + ++#define local_irq_lock_init(lvar) \ ++ do { \ ++ int __cpu; \ ++ for_each_possible_cpu(__cpu) \ ++ spin_lock_init(&per_cpu(lvar, __cpu).lock); \ ++ } while (0) + -+static ssize_t -+raid5_show_preread_threshold(struct mddev *mddev, char *page) -+{ -+ struct r5conf *conf; -+ int ret = 0; -+ spin_lock(&mddev->lock); -+ conf = mddev->private; -+ if (conf) -+ ret = sprintf(page, "%d\n", conf->bypass_threshold); -+ spin_unlock(&mddev->lock); -+ return ret; -+} ++/* ++ * spin_lock|trylock|unlock_local flavour that does not migrate disable ++ * used for __local_lock|trylock|unlock where get_local_var/put_local_var ++ * already takes care of the migrate_disable/enable ++ * for CONFIG_PREEMPT_BASE map to the normal spin_* calls. ++ */ ++#ifdef CONFIG_PREEMPT_RT_FULL ++# define spin_lock_local(lock) rt_spin_lock(lock) ++# define spin_trylock_local(lock) rt_spin_trylock(lock) ++# define spin_unlock_local(lock) rt_spin_unlock(lock) ++#else ++# define spin_lock_local(lock) spin_lock(lock) ++# define spin_trylock_local(lock) spin_trylock(lock) ++# define spin_unlock_local(lock) spin_unlock(lock) ++#endif + -+static ssize_t -+raid5_store_preread_threshold(struct mddev *mddev, const char *page, size_t len) ++static inline void __local_lock(struct local_irq_lock *lv) +{ -+ struct r5conf *conf; -+ unsigned long new; -+ int err; -+ -+ if (len >= PAGE_SIZE) -+ return -EINVAL; -+ if (kstrtoul(page, 10, &new)) -+ return -EINVAL; -+ -+ err = mddev_lock(mddev); -+ if (err) -+ return err; -+ conf = mddev->private; -+ if (!conf) -+ err = -ENODEV; -+ else if (new > conf->min_nr_stripes) -+ err = -EINVAL; -+ else -+ conf->bypass_threshold = new; -+ mddev_unlock(mddev); -+ return err ?: len; ++ if (lv->owner != current) { ++ spin_lock_local(&lv->lock); ++ LL_WARN(lv->owner); ++ LL_WARN(lv->nestcnt); ++ lv->owner = current; ++ } ++ lv->nestcnt++; +} + -+static struct md_sysfs_entry -+raid5_preread_bypass_threshold = __ATTR(preread_bypass_threshold, -+ S_IRUGO | S_IWUSR, -+ raid5_show_preread_threshold, -+ raid5_store_preread_threshold); -+ -+static ssize_t -+raid5_show_skip_copy(struct mddev *mddev, char *page) -+{ -+ struct r5conf *conf; -+ int ret = 0; -+ spin_lock(&mddev->lock); -+ conf = mddev->private; -+ if (conf) -+ ret = sprintf(page, "%d\n", conf->skip_copy); -+ spin_unlock(&mddev->lock); -+ return ret; -+} ++#define local_lock(lvar) \ ++ do { __local_lock(&get_local_var(lvar)); } while (0) + -+static ssize_t -+raid5_store_skip_copy(struct mddev *mddev, const char *page, size_t len) ++static inline int __local_trylock(struct local_irq_lock *lv) +{ -+ struct r5conf *conf; -+ unsigned long new; -+ int err; -+ -+ if (len >= PAGE_SIZE) -+ return -EINVAL; -+ if (kstrtoul(page, 10, &new)) -+ return -EINVAL; -+ new = !!new; -+ -+ err = mddev_lock(mddev); -+ if (err) -+ return err; -+ conf = mddev->private; -+ if (!conf) -+ err = -ENODEV; -+ else if (new != conf->skip_copy) { -+ mddev_suspend(mddev); -+ conf->skip_copy = new; -+ if (new) -+ mddev->queue->backing_dev_info.capabilities |= -+ BDI_CAP_STABLE_WRITES; -+ else -+ mddev->queue->backing_dev_info.capabilities &= -+ ~BDI_CAP_STABLE_WRITES; -+ mddev_resume(mddev); ++ if (lv->owner != current && spin_trylock_local(&lv->lock)) { ++ LL_WARN(lv->owner); ++ LL_WARN(lv->nestcnt); ++ lv->owner = current; ++ lv->nestcnt = 1; ++ return 1; + } -+ mddev_unlock(mddev); -+ return err ?: len; ++ return 0; +} + -+static struct md_sysfs_entry -+raid5_skip_copy = __ATTR(skip_copy, S_IRUGO | S_IWUSR, -+ raid5_show_skip_copy, -+ raid5_store_skip_copy); ++#define local_trylock(lvar) \ ++ ({ \ ++ int __locked; \ ++ __locked = __local_trylock(&get_local_var(lvar)); \ ++ if (!__locked) \ ++ put_local_var(lvar); \ ++ __locked; \ ++ }) + -+static ssize_t -+stripe_cache_active_show(struct mddev *mddev, char *page) ++static inline void __local_unlock(struct local_irq_lock *lv) +{ -+ struct r5conf *conf = mddev->private; -+ if (conf) -+ return sprintf(page, "%d\n", atomic_read(&conf->active_stripes)); -+ else -+ return 0; ++ LL_WARN(lv->nestcnt == 0); ++ LL_WARN(lv->owner != current); ++ if (--lv->nestcnt) ++ return; ++ ++ lv->owner = NULL; ++ spin_unlock_local(&lv->lock); +} + -+static struct md_sysfs_entry -+raid5_stripecache_active = __ATTR_RO(stripe_cache_active); ++#define local_unlock(lvar) \ ++ do { \ ++ __local_unlock(this_cpu_ptr(&lvar)); \ ++ put_local_var(lvar); \ ++ } while (0) + -+static ssize_t -+raid5_show_group_thread_cnt(struct mddev *mddev, char *page) ++static inline void __local_lock_irq(struct local_irq_lock *lv) +{ -+ struct r5conf *conf; -+ int ret = 0; -+ spin_lock(&mddev->lock); -+ conf = mddev->private; -+ if (conf) -+ ret = sprintf(page, "%d\n", conf->worker_cnt_per_group); -+ spin_unlock(&mddev->lock); -+ return ret; ++ spin_lock_irqsave(&lv->lock, lv->flags); ++ LL_WARN(lv->owner); ++ LL_WARN(lv->nestcnt); ++ lv->owner = current; ++ lv->nestcnt = 1; +} + -+static int alloc_thread_groups(struct r5conf *conf, int cnt, -+ int *group_cnt, -+ int *worker_cnt_per_group, -+ struct r5worker_group **worker_groups); -+static ssize_t -+raid5_store_group_thread_cnt(struct mddev *mddev, const char *page, size_t len) -+{ -+ struct r5conf *conf; -+ unsigned long new; -+ int err; -+ struct r5worker_group *new_groups, *old_groups; -+ int group_cnt, worker_cnt_per_group; -+ -+ if (len >= PAGE_SIZE) -+ return -EINVAL; -+ if (kstrtoul(page, 10, &new)) -+ return -EINVAL; ++#define local_lock_irq(lvar) \ ++ do { __local_lock_irq(&get_local_var(lvar)); } while (0) + -+ err = mddev_lock(mddev); -+ if (err) -+ return err; -+ conf = mddev->private; -+ if (!conf) -+ err = -ENODEV; -+ else if (new != conf->worker_cnt_per_group) { -+ mddev_suspend(mddev); -+ -+ old_groups = conf->worker_groups; -+ if (old_groups) -+ flush_workqueue(raid5_wq); -+ -+ err = alloc_thread_groups(conf, new, -+ &group_cnt, &worker_cnt_per_group, -+ &new_groups); -+ if (!err) { -+ spin_lock_irq(&conf->device_lock); -+ conf->group_cnt = group_cnt; -+ conf->worker_cnt_per_group = worker_cnt_per_group; -+ conf->worker_groups = new_groups; -+ spin_unlock_irq(&conf->device_lock); -+ -+ if (old_groups) -+ kfree(old_groups[0].workers); -+ kfree(old_groups); -+ } -+ mddev_resume(mddev); -+ } -+ mddev_unlock(mddev); ++#define local_lock_irq_on(lvar, cpu) \ ++ do { __local_lock_irq(&per_cpu(lvar, cpu)); } while (0) + -+ return err ?: len; ++static inline void __local_unlock_irq(struct local_irq_lock *lv) ++{ ++ LL_WARN(!lv->nestcnt); ++ LL_WARN(lv->owner != current); ++ lv->owner = NULL; ++ lv->nestcnt = 0; ++ spin_unlock_irq(&lv->lock); +} + -+static struct md_sysfs_entry -+raid5_group_thread_cnt = __ATTR(group_thread_cnt, S_IRUGO | S_IWUSR, -+ raid5_show_group_thread_cnt, -+ raid5_store_group_thread_cnt); -+ -+static struct attribute *raid5_attrs[] = { -+ &raid5_stripecache_size.attr, -+ &raid5_stripecache_active.attr, -+ &raid5_preread_bypass_threshold.attr, -+ &raid5_group_thread_cnt.attr, -+ &raid5_skip_copy.attr, -+ &raid5_rmw_level.attr, -+ NULL, -+}; -+static struct attribute_group raid5_attrs_group = { -+ .name = NULL, -+ .attrs = raid5_attrs, -+}; ++#define local_unlock_irq(lvar) \ ++ do { \ ++ __local_unlock_irq(this_cpu_ptr(&lvar)); \ ++ put_local_var(lvar); \ ++ } while (0) ++ ++#define local_unlock_irq_on(lvar, cpu) \ ++ do { \ ++ __local_unlock_irq(&per_cpu(lvar, cpu)); \ ++ } while (0) + -+static int alloc_thread_groups(struct r5conf *conf, int cnt, -+ int *group_cnt, -+ int *worker_cnt_per_group, -+ struct r5worker_group **worker_groups) ++static inline int __local_lock_irqsave(struct local_irq_lock *lv) +{ -+ int i, j, k; -+ ssize_t size; -+ struct r5worker *workers; -+ -+ *worker_cnt_per_group = cnt; -+ if (cnt == 0) { -+ *group_cnt = 0; -+ *worker_groups = NULL; ++ if (lv->owner != current) { ++ __local_lock_irq(lv); + return 0; ++ } else { ++ lv->nestcnt++; ++ return 1; + } -+ *group_cnt = num_possible_nodes(); -+ size = sizeof(struct r5worker) * cnt; -+ workers = kzalloc(size * *group_cnt, GFP_NOIO); -+ *worker_groups = kzalloc(sizeof(struct r5worker_group) * -+ *group_cnt, GFP_NOIO); -+ if (!*worker_groups || !workers) { -+ kfree(workers); -+ kfree(*worker_groups); -+ return -ENOMEM; -+ } ++} + -+ for (i = 0; i < *group_cnt; i++) { -+ struct r5worker_group *group; ++#define local_lock_irqsave(lvar, _flags) \ ++ do { \ ++ if (__local_lock_irqsave(&get_local_var(lvar))) \ ++ put_local_var(lvar); \ ++ _flags = __this_cpu_read(lvar.flags); \ ++ } while (0) + -+ group = &(*worker_groups)[i]; -+ INIT_LIST_HEAD(&group->handle_list); -+ group->conf = conf; -+ group->workers = workers + i * cnt; -+ -+ for (j = 0; j < cnt; j++) { -+ struct r5worker *worker = group->workers + j; -+ worker->group = group; -+ INIT_WORK(&worker->work, raid5_do_work); -+ -+ for (k = 0; k < NR_STRIPE_HASH_LOCKS; k++) -+ INIT_LIST_HEAD(worker->temp_inactive_list + k); -+ } -+ } -+ -+ return 0; -+} ++#define local_lock_irqsave_on(lvar, _flags, cpu) \ ++ do { \ ++ __local_lock_irqsave(&per_cpu(lvar, cpu)); \ ++ _flags = per_cpu(lvar, cpu).flags; \ ++ } while (0) + -+static void free_thread_groups(struct r5conf *conf) ++static inline int __local_unlock_irqrestore(struct local_irq_lock *lv, ++ unsigned long flags) +{ -+ if (conf->worker_groups) -+ kfree(conf->worker_groups[0].workers); -+ kfree(conf->worker_groups); -+ conf->worker_groups = NULL; ++ LL_WARN(!lv->nestcnt); ++ LL_WARN(lv->owner != current); ++ if (--lv->nestcnt) ++ return 0; ++ ++ lv->owner = NULL; ++ spin_unlock_irqrestore(&lv->lock, lv->flags); ++ return 1; +} + -+static sector_t -+raid5_size(struct mddev *mddev, sector_t sectors, int raid_disks) -+{ -+ struct r5conf *conf = mddev->private; ++#define local_unlock_irqrestore(lvar, flags) \ ++ do { \ ++ if (__local_unlock_irqrestore(this_cpu_ptr(&lvar), flags)) \ ++ put_local_var(lvar); \ ++ } while (0) + -+ if (!sectors) -+ sectors = mddev->dev_sectors; -+ if (!raid_disks) -+ /* size is defined by the smallest of previous and new size */ -+ raid_disks = min(conf->raid_disks, conf->previous_raid_disks); ++#define local_unlock_irqrestore_on(lvar, flags, cpu) \ ++ do { \ ++ __local_unlock_irqrestore(&per_cpu(lvar, cpu), flags); \ ++ } while (0) + -+ sectors &= ~((sector_t)mddev->chunk_sectors - 1); -+ sectors &= ~((sector_t)mddev->new_chunk_sectors - 1); -+ return sectors * (raid_disks - conf->max_degraded); -+} ++#define local_spin_trylock_irq(lvar, lock) \ ++ ({ \ ++ int __locked; \ ++ local_lock_irq(lvar); \ ++ __locked = spin_trylock(lock); \ ++ if (!__locked) \ ++ local_unlock_irq(lvar); \ ++ __locked; \ ++ }) + -+static void free_scratch_buffer(struct r5conf *conf, struct raid5_percpu *percpu) -+{ -+ safe_put_page(percpu->spare_page); -+ if (percpu->scribble) -+ flex_array_free(percpu->scribble); -+ percpu->spare_page = NULL; -+ percpu->scribble = NULL; -+} ++#define local_spin_lock_irq(lvar, lock) \ ++ do { \ ++ local_lock_irq(lvar); \ ++ spin_lock(lock); \ ++ } while (0) + -+static int alloc_scratch_buffer(struct r5conf *conf, struct raid5_percpu *percpu) -+{ -+ if (conf->level == 6 && !percpu->spare_page) -+ percpu->spare_page = alloc_page(GFP_KERNEL); -+ if (!percpu->scribble) -+ percpu->scribble = scribble_alloc(max(conf->raid_disks, -+ conf->previous_raid_disks), -+ max(conf->chunk_sectors, -+ conf->prev_chunk_sectors) -+ / STRIPE_SECTORS, -+ GFP_KERNEL); -+ -+ if (!percpu->scribble || (conf->level == 6 && !percpu->spare_page)) { -+ free_scratch_buffer(conf, percpu); -+ return -ENOMEM; -+ } ++#define local_spin_unlock_irq(lvar, lock) \ ++ do { \ ++ spin_unlock(lock); \ ++ local_unlock_irq(lvar); \ ++ } while (0) + -+ return 0; -+} ++#define local_spin_lock_irqsave(lvar, lock, flags) \ ++ do { \ ++ local_lock_irqsave(lvar, flags); \ ++ spin_lock(lock); \ ++ } while (0) + -+static void raid5_free_percpu(struct r5conf *conf) -+{ -+ unsigned long cpu; ++#define local_spin_unlock_irqrestore(lvar, lock, flags) \ ++ do { \ ++ spin_unlock(lock); \ ++ local_unlock_irqrestore(lvar, flags); \ ++ } while (0) + -+ if (!conf->percpu) -+ return; ++#define get_locked_var(lvar, var) \ ++ (*({ \ ++ local_lock(lvar); \ ++ this_cpu_ptr(&var); \ ++ })) + -+#ifdef CONFIG_HOTPLUG_CPU -+ unregister_cpu_notifier(&conf->cpu_notify); -+#endif ++#define put_locked_var(lvar, var) local_unlock(lvar); + -+ get_online_cpus(); -+ for_each_possible_cpu(cpu) -+ free_scratch_buffer(conf, per_cpu_ptr(conf->percpu, cpu)); -+ put_online_cpus(); ++#define local_lock_cpu(lvar) \ ++ ({ \ ++ local_lock(lvar); \ ++ smp_processor_id(); \ ++ }) + -+ free_percpu(conf->percpu); -+} ++#define local_unlock_cpu(lvar) local_unlock(lvar) + -+static void free_conf(struct r5conf *conf) -+{ -+ if (conf->shrinker.seeks) -+ unregister_shrinker(&conf->shrinker); -+ free_thread_groups(conf); -+ shrink_stripes(conf); -+ raid5_free_percpu(conf); -+ kfree(conf->disks); -+ kfree(conf->stripe_hashtbl); -+ kfree(conf); -+} ++#else /* PREEMPT_RT_BASE */ + -+#ifdef CONFIG_HOTPLUG_CPU -+static int raid456_cpu_notify(struct notifier_block *nfb, unsigned long action, -+ void *hcpu) -+{ -+ struct r5conf *conf = container_of(nfb, struct r5conf, cpu_notify); -+ long cpu = (long)hcpu; -+ struct raid5_percpu *percpu = per_cpu_ptr(conf->percpu, cpu); ++#define DEFINE_LOCAL_IRQ_LOCK(lvar) __typeof__(const int) lvar ++#define DECLARE_LOCAL_IRQ_LOCK(lvar) extern __typeof__(const int) lvar + -+ switch (action) { -+ case CPU_UP_PREPARE: -+ case CPU_UP_PREPARE_FROZEN: -+ if (alloc_scratch_buffer(conf, percpu)) { -+ pr_err("%s: failed memory allocation for cpu%ld\n", -+ __func__, cpu); -+ return notifier_from_errno(-ENOMEM); -+ } -+ break; -+ case CPU_DEAD: -+ case CPU_DEAD_FROZEN: -+ free_scratch_buffer(conf, per_cpu_ptr(conf->percpu, cpu)); -+ break; -+ default: -+ break; -+ } -+ return NOTIFY_OK; -+} -+#endif ++static inline void local_irq_lock_init(int lvar) { } + -+static int raid5_alloc_percpu(struct r5conf *conf) -+{ -+ unsigned long cpu; -+ int err = 0; ++#define local_lock(lvar) preempt_disable() ++#define local_unlock(lvar) preempt_enable() ++#define local_lock_irq(lvar) local_irq_disable() ++#define local_unlock_irq(lvar) local_irq_enable() ++#define local_lock_irqsave(lvar, flags) local_irq_save(flags) ++#define local_unlock_irqrestore(lvar, flags) local_irq_restore(flags) + -+ conf->percpu = alloc_percpu(struct raid5_percpu); -+ if (!conf->percpu) -+ return -ENOMEM; ++#define local_spin_trylock_irq(lvar, lock) spin_trylock_irq(lock) ++#define local_spin_lock_irq(lvar, lock) spin_lock_irq(lock) ++#define local_spin_unlock_irq(lvar, lock) spin_unlock_irq(lock) ++#define local_spin_lock_irqsave(lvar, lock, flags) \ ++ spin_lock_irqsave(lock, flags) ++#define local_spin_unlock_irqrestore(lvar, lock, flags) \ ++ spin_unlock_irqrestore(lock, flags) + -+#ifdef CONFIG_HOTPLUG_CPU -+ conf->cpu_notify.notifier_call = raid456_cpu_notify; -+ conf->cpu_notify.priority = 0; -+ err = register_cpu_notifier(&conf->cpu_notify); -+ if (err) -+ return err; -+#endif ++#define get_locked_var(lvar, var) get_cpu_var(var) ++#define put_locked_var(lvar, var) put_cpu_var(var) + -+ get_online_cpus(); -+ for_each_present_cpu(cpu) { -+ err = alloc_scratch_buffer(conf, per_cpu_ptr(conf->percpu, cpu)); -+ if (err) { -+ pr_err("%s: failed memory allocation for cpu%ld\n", -+ __func__, cpu); -+ break; -+ } -+ } -+ put_online_cpus(); ++#define local_lock_cpu(lvar) get_cpu() ++#define local_unlock_cpu(lvar) put_cpu() + -+ return err; -+} ++#endif + -+static unsigned long raid5_cache_scan(struct shrinker *shrink, -+ struct shrink_control *sc) -+{ -+ struct r5conf *conf = container_of(shrink, struct r5conf, shrinker); -+ unsigned long ret = SHRINK_STOP; -+ -+ if (mutex_trylock(&conf->cache_size_mutex)) { -+ ret= 0; -+ while (ret < sc->nr_to_scan && -+ conf->max_nr_stripes > conf->min_nr_stripes) { -+ if (drop_one_stripe(conf) == 0) { -+ ret = SHRINK_STOP; -+ break; -+ } -+ ret++; -+ } -+ mutex_unlock(&conf->cache_size_mutex); -+ } -+ return ret; -+} ++#endif +diff -Nur linux-4.1.10.orig/include/linux/mm_types.h linux-4.1.10/include/linux/mm_types.h +--- linux-4.1.10.orig/include/linux/mm_types.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/mm_types.h 2015-10-12 22:33:32.276676754 +0200 +@@ -11,6 +11,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -453,6 +454,9 @@ + bool tlb_flush_pending; + #endif + struct uprobes_state uprobes_state; ++#ifdef CONFIG_PREEMPT_RT_BASE ++ struct rcu_head delayed_drop; ++#endif + #ifdef CONFIG_X86_INTEL_MPX + /* address of the bounds directory */ + void __user *bd_addr; +diff -Nur linux-4.1.10.orig/include/linux/mutex.h linux-4.1.10/include/linux/mutex.h +--- linux-4.1.10.orig/include/linux/mutex.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/mutex.h 2015-10-12 22:33:32.276676754 +0200 +@@ -19,6 +19,17 @@ + #include + #include + ++#ifdef CONFIG_DEBUG_LOCK_ALLOC ++# define __DEP_MAP_MUTEX_INITIALIZER(lockname) \ ++ , .dep_map = { .name = #lockname } ++#else ++# define __DEP_MAP_MUTEX_INITIALIZER(lockname) ++#endif + -+static unsigned long raid5_cache_count(struct shrinker *shrink, -+ struct shrink_control *sc) -+{ -+ struct r5conf *conf = container_of(shrink, struct r5conf, shrinker); ++#ifdef CONFIG_PREEMPT_RT_FULL ++# include ++#else + -+ if (conf->max_nr_stripes < conf->min_nr_stripes) -+ /* unlikely, but not impossible */ -+ return 0; -+ return conf->max_nr_stripes - conf->min_nr_stripes; -+} + /* + * Simple, straightforward mutexes with strict semantics: + * +@@ -99,13 +110,6 @@ + static inline void mutex_destroy(struct mutex *lock) {} + #endif + +-#ifdef CONFIG_DEBUG_LOCK_ALLOC +-# define __DEP_MAP_MUTEX_INITIALIZER(lockname) \ +- , .dep_map = { .name = #lockname } +-#else +-# define __DEP_MAP_MUTEX_INITIALIZER(lockname) +-#endif +- + #define __MUTEX_INITIALIZER(lockname) \ + { .count = ATOMIC_INIT(1) \ + , .wait_lock = __SPIN_LOCK_UNLOCKED(lockname.wait_lock) \ +@@ -173,6 +177,8 @@ + extern int mutex_trylock(struct mutex *lock); + extern void mutex_unlock(struct mutex *lock); + ++#endif /* !PREEMPT_RT_FULL */ + -+static struct r5conf *setup_conf(struct mddev *mddev) -+{ -+ struct r5conf *conf; -+ int raid_disk, memory, max_disks; -+ struct md_rdev *rdev; -+ struct disk_info *disk; -+ char pers_name[6]; -+ int i; -+ int group_cnt, worker_cnt_per_group; -+ struct r5worker_group *new_group; -+ -+ if (mddev->new_level != 5 -+ && mddev->new_level != 4 -+ && mddev->new_level != 6) { -+ printk(KERN_ERR "md/raid:%s: raid level not set to 4/5/6 (%d)\n", -+ mdname(mddev), mddev->new_level); -+ return ERR_PTR(-EIO); -+ } -+ if ((mddev->new_level == 5 -+ && !algorithm_valid_raid5(mddev->new_layout)) || -+ (mddev->new_level == 6 -+ && !algorithm_valid_raid6(mddev->new_layout))) { -+ printk(KERN_ERR "md/raid:%s: layout %d not supported\n", -+ mdname(mddev), mddev->new_layout); -+ return ERR_PTR(-EIO); -+ } -+ if (mddev->new_level == 6 && mddev->raid_disks < 4) { -+ printk(KERN_ERR "md/raid:%s: not enough configured devices (%d, minimum 4)\n", -+ mdname(mddev), mddev->raid_disks); -+ return ERR_PTR(-EINVAL); -+ } + extern int atomic_dec_and_mutex_lock(atomic_t *cnt, struct mutex *lock); + + #endif /* __LINUX_MUTEX_H */ +diff -Nur linux-4.1.10.orig/include/linux/mutex_rt.h linux-4.1.10/include/linux/mutex_rt.h +--- linux-4.1.10.orig/include/linux/mutex_rt.h 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/include/linux/mutex_rt.h 2015-10-12 22:33:32.276676754 +0200 +@@ -0,0 +1,84 @@ ++#ifndef __LINUX_MUTEX_RT_H ++#define __LINUX_MUTEX_RT_H + -+ if (!mddev->new_chunk_sectors || -+ (mddev->new_chunk_sectors << 9) % PAGE_SIZE || -+ !is_power_of_2(mddev->new_chunk_sectors)) { -+ printk(KERN_ERR "md/raid:%s: invalid chunk size %d\n", -+ mdname(mddev), mddev->new_chunk_sectors << 9); -+ return ERR_PTR(-EINVAL); -+ } ++#ifndef __LINUX_MUTEX_H ++#error "Please include mutex.h" ++#endif + -+ conf = kzalloc(sizeof(struct r5conf), GFP_KERNEL); -+ if (conf == NULL) -+ goto abort; -+ /* Don't enable multi-threading by default*/ -+ if (!alloc_thread_groups(conf, 0, &group_cnt, &worker_cnt_per_group, -+ &new_group)) { -+ conf->group_cnt = group_cnt; -+ conf->worker_cnt_per_group = worker_cnt_per_group; -+ conf->worker_groups = new_group; -+ } else -+ goto abort; -+ spin_lock_init(&conf->device_lock); -+ seqcount_init(&conf->gen_lock); -+ mutex_init(&conf->cache_size_mutex); -+ init_waitqueue_head(&conf->wait_for_stripe); -+ init_waitqueue_head(&conf->wait_for_overlap); -+ INIT_LIST_HEAD(&conf->handle_list); -+ INIT_LIST_HEAD(&conf->hold_list); -+ INIT_LIST_HEAD(&conf->delayed_list); -+ INIT_LIST_HEAD(&conf->bitmap_list); -+ init_llist_head(&conf->released_stripes); -+ atomic_set(&conf->active_stripes, 0); -+ atomic_set(&conf->preread_active_stripes, 0); -+ atomic_set(&conf->active_aligned_reads, 0); -+ conf->bypass_threshold = BYPASS_THRESHOLD; -+ conf->recovery_disabled = mddev->recovery_disabled - 1; -+ -+ conf->raid_disks = mddev->raid_disks; -+ if (mddev->reshape_position == MaxSector) -+ conf->previous_raid_disks = mddev->raid_disks; -+ else -+ conf->previous_raid_disks = mddev->raid_disks - mddev->delta_disks; -+ max_disks = max(conf->raid_disks, conf->previous_raid_disks); ++#include + -+ conf->disks = kzalloc(max_disks * sizeof(struct disk_info), -+ GFP_KERNEL); -+ if (!conf->disks) -+ goto abort; ++/* FIXME: Just for __lockfunc */ ++#include + -+ conf->mddev = mddev; ++struct mutex { ++ struct rt_mutex lock; ++#ifdef CONFIG_DEBUG_LOCK_ALLOC ++ struct lockdep_map dep_map; ++#endif ++}; + -+ if ((conf->stripe_hashtbl = kzalloc(PAGE_SIZE, GFP_KERNEL)) == NULL) -+ goto abort; ++#define __MUTEX_INITIALIZER(mutexname) \ ++ { \ ++ .lock = __RT_MUTEX_INITIALIZER(mutexname.lock) \ ++ __DEP_MAP_MUTEX_INITIALIZER(mutexname) \ ++ } + -+ /* We init hash_locks[0] separately to that it can be used -+ * as the reference lock in the spin_lock_nest_lock() call -+ * in lock_all_device_hash_locks_irq in order to convince -+ * lockdep that we know what we are doing. -+ */ -+ spin_lock_init(conf->hash_locks); -+ for (i = 1; i < NR_STRIPE_HASH_LOCKS; i++) -+ spin_lock_init(conf->hash_locks + i); ++#define DEFINE_MUTEX(mutexname) \ ++ struct mutex mutexname = __MUTEX_INITIALIZER(mutexname) + -+ for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++) -+ INIT_LIST_HEAD(conf->inactive_list + i); ++extern void __mutex_do_init(struct mutex *lock, const char *name, struct lock_class_key *key); ++extern void __lockfunc _mutex_lock(struct mutex *lock); ++extern int __lockfunc _mutex_lock_interruptible(struct mutex *lock); ++extern int __lockfunc _mutex_lock_killable(struct mutex *lock); ++extern void __lockfunc _mutex_lock_nested(struct mutex *lock, int subclass); ++extern void __lockfunc _mutex_lock_nest_lock(struct mutex *lock, struct lockdep_map *nest_lock); ++extern int __lockfunc _mutex_lock_interruptible_nested(struct mutex *lock, int subclass); ++extern int __lockfunc _mutex_lock_killable_nested(struct mutex *lock, int subclass); ++extern int __lockfunc _mutex_trylock(struct mutex *lock); ++extern void __lockfunc _mutex_unlock(struct mutex *lock); + -+ for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++) -+ INIT_LIST_HEAD(conf->temp_inactive_list + i); ++#define mutex_is_locked(l) rt_mutex_is_locked(&(l)->lock) ++#define mutex_lock(l) _mutex_lock(l) ++#define mutex_lock_interruptible(l) _mutex_lock_interruptible(l) ++#define mutex_lock_killable(l) _mutex_lock_killable(l) ++#define mutex_trylock(l) _mutex_trylock(l) ++#define mutex_unlock(l) _mutex_unlock(l) ++#define mutex_destroy(l) rt_mutex_destroy(&(l)->lock) + -+ conf->level = mddev->new_level; -+ conf->chunk_sectors = mddev->new_chunk_sectors; -+ if (raid5_alloc_percpu(conf) != 0) -+ goto abort; ++#ifdef CONFIG_DEBUG_LOCK_ALLOC ++# define mutex_lock_nested(l, s) _mutex_lock_nested(l, s) ++# define mutex_lock_interruptible_nested(l, s) \ ++ _mutex_lock_interruptible_nested(l, s) ++# define mutex_lock_killable_nested(l, s) \ ++ _mutex_lock_killable_nested(l, s) + -+ pr_debug("raid456: run(%s) called.\n", mdname(mddev)); ++# define mutex_lock_nest_lock(lock, nest_lock) \ ++do { \ ++ typecheck(struct lockdep_map *, &(nest_lock)->dep_map); \ ++ _mutex_lock_nest_lock(lock, &(nest_lock)->dep_map); \ ++} while (0) + -+ rdev_for_each(rdev, mddev) { -+ raid_disk = rdev->raid_disk; -+ if (raid_disk >= max_disks -+ || raid_disk < 0) -+ continue; -+ disk = conf->disks + raid_disk; ++#else ++# define mutex_lock_nested(l, s) _mutex_lock(l) ++# define mutex_lock_interruptible_nested(l, s) \ ++ _mutex_lock_interruptible(l) ++# define mutex_lock_killable_nested(l, s) \ ++ _mutex_lock_killable(l) ++# define mutex_lock_nest_lock(lock, nest_lock) mutex_lock(lock) ++#endif + -+ if (test_bit(Replacement, &rdev->flags)) { -+ if (disk->replacement) -+ goto abort; -+ disk->replacement = rdev; -+ } else { -+ if (disk->rdev) -+ goto abort; -+ disk->rdev = rdev; -+ } ++# define mutex_init(mutex) \ ++do { \ ++ static struct lock_class_key __key; \ ++ \ ++ rt_mutex_init(&(mutex)->lock); \ ++ __mutex_do_init((mutex), #mutex, &__key); \ ++} while (0) + -+ if (test_bit(In_sync, &rdev->flags)) { -+ char b[BDEVNAME_SIZE]; -+ printk(KERN_INFO "md/raid:%s: device %s operational as raid" -+ " disk %d\n", -+ mdname(mddev), bdevname(rdev->bdev, b), raid_disk); -+ } else if (rdev->saved_raid_disk != raid_disk) -+ /* Cannot rely on bitmap to complete recovery */ -+ conf->fullsync = 1; -+ } ++# define __mutex_init(mutex, name, key) \ ++do { \ ++ rt_mutex_init(&(mutex)->lock); \ ++ __mutex_do_init((mutex), name, key); \ ++} while (0) + -+ conf->level = mddev->new_level; -+ if (conf->level == 6) { -+ conf->max_degraded = 2; -+ if (raid6_call.xor_syndrome) -+ conf->rmw_level = PARITY_ENABLE_RMW; -+ else -+ conf->rmw_level = PARITY_DISABLE_RMW; -+ } else { -+ conf->max_degraded = 1; -+ conf->rmw_level = PARITY_ENABLE_RMW; -+ } -+ conf->algorithm = mddev->new_layout; -+ conf->reshape_progress = mddev->reshape_position; -+ if (conf->reshape_progress != MaxSector) { -+ conf->prev_chunk_sectors = mddev->chunk_sectors; -+ conf->prev_algo = mddev->layout; -+ } ++#endif +diff -Nur linux-4.1.10.orig/include/linux/netdevice.h linux-4.1.10/include/linux/netdevice.h +--- linux-4.1.10.orig/include/linux/netdevice.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/netdevice.h 2015-10-12 22:33:32.276676754 +0200 +@@ -2469,6 +2469,7 @@ + unsigned int dropped; + struct sk_buff_head input_pkt_queue; + struct napi_struct backlog; ++ struct sk_buff_head tofree_queue; + + }; + +diff -Nur linux-4.1.10.orig/include/linux/netfilter/x_tables.h linux-4.1.10/include/linux/netfilter/x_tables.h +--- linux-4.1.10.orig/include/linux/netfilter/x_tables.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/netfilter/x_tables.h 2015-10-12 22:33:32.276676754 +0200 +@@ -3,6 +3,7 @@ + + + #include ++#include + #include + + /** +@@ -282,6 +283,8 @@ + */ + DECLARE_PER_CPU(seqcount_t, xt_recseq); + ++DECLARE_LOCAL_IRQ_LOCK(xt_write_lock); + -+ conf->min_nr_stripes = NR_STRIPES; -+ memory = conf->min_nr_stripes * (sizeof(struct stripe_head) + -+ max_disks * ((sizeof(struct bio) + PAGE_SIZE))) / 1024; -+ atomic_set(&conf->empty_inactive_list_nr, NR_STRIPE_HASH_LOCKS); -+ if (grow_stripes(conf, conf->min_nr_stripes)) { -+ printk(KERN_ERR -+ "md/raid:%s: couldn't allocate %dkB for buffers\n", -+ mdname(mddev), memory); -+ goto abort; -+ } else -+ printk(KERN_INFO "md/raid:%s: allocated %dkB\n", -+ mdname(mddev), memory); -+ /* -+ * Losing a stripe head costs more than the time to refill it, -+ * it reduces the queue depth and so can hurt throughput. -+ * So set it rather large, scaled by number of devices. -+ */ -+ conf->shrinker.seeks = DEFAULT_SEEKS * conf->raid_disks * 4; -+ conf->shrinker.scan_objects = raid5_cache_scan; -+ conf->shrinker.count_objects = raid5_cache_count; -+ conf->shrinker.batch = 128; -+ conf->shrinker.flags = 0; -+ register_shrinker(&conf->shrinker); -+ -+ sprintf(pers_name, "raid%d", mddev->new_level); -+ conf->thread = md_register_thread(raid5d, mddev, pers_name); -+ if (!conf->thread) { -+ printk(KERN_ERR -+ "md/raid:%s: couldn't allocate thread.\n", -+ mdname(mddev)); -+ goto abort; -+ } + /** + * xt_write_recseq_begin - start of a write section + * +@@ -296,6 +299,9 @@ + { + unsigned int addend; + ++ /* RT protection */ ++ local_lock(xt_write_lock); + -+ return conf; + /* + * Low order bit of sequence is set if we already + * called xt_write_recseq_begin(). +@@ -326,6 +332,7 @@ + /* this is kind of a write_seqcount_end(), but addend is 0 or 1 */ + smp_wmb(); + __this_cpu_add(xt_recseq.sequence, addend); ++ local_unlock(xt_write_lock); + } + + /* +diff -Nur linux-4.1.10.orig/include/linux/notifier.h linux-4.1.10/include/linux/notifier.h +--- linux-4.1.10.orig/include/linux/notifier.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/notifier.h 2015-10-12 22:33:32.276676754 +0200 +@@ -6,7 +6,7 @@ + * + * Alan Cox + */ +- + -+ abort: -+ if (conf) { -+ free_conf(conf); -+ return ERR_PTR(-EIO); -+ } else -+ return ERR_PTR(-ENOMEM); -+} + #ifndef _LINUX_NOTIFIER_H + #define _LINUX_NOTIFIER_H + #include +@@ -42,9 +42,7 @@ + * in srcu_notifier_call_chain(): no cache bounces and no memory barriers. + * As compensation, srcu_notifier_chain_unregister() is rather expensive. + * SRCU notifier chains should be used when the chain will be called very +- * often but notifier_blocks will seldom be removed. Also, SRCU notifier +- * chains are slightly more difficult to use because they require special +- * runtime initialization. ++ * often but notifier_blocks will seldom be removed. + */ + + typedef int (*notifier_fn_t)(struct notifier_block *nb, +@@ -88,7 +86,7 @@ + (name)->head = NULL; \ + } while (0) + +-/* srcu_notifier_heads must be initialized and cleaned up dynamically */ ++/* srcu_notifier_heads must be cleaned up dynamically */ + extern void srcu_init_notifier_head(struct srcu_notifier_head *nh); + #define srcu_cleanup_notifier_head(name) \ + cleanup_srcu_struct(&(name)->srcu); +@@ -101,7 +99,13 @@ + .head = NULL } + #define RAW_NOTIFIER_INIT(name) { \ + .head = NULL } +-/* srcu_notifier_heads cannot be initialized statically */ + -+static int only_parity(int raid_disk, int algo, int raid_disks, int max_degraded) -+{ -+ switch (algo) { -+ case ALGORITHM_PARITY_0: -+ if (raid_disk < max_degraded) -+ return 1; -+ break; -+ case ALGORITHM_PARITY_N: -+ if (raid_disk >= raid_disks - max_degraded) -+ return 1; -+ break; -+ case ALGORITHM_PARITY_0_6: -+ if (raid_disk == 0 || -+ raid_disk == raid_disks - 1) -+ return 1; -+ break; -+ case ALGORITHM_LEFT_ASYMMETRIC_6: -+ case ALGORITHM_RIGHT_ASYMMETRIC_6: -+ case ALGORITHM_LEFT_SYMMETRIC_6: -+ case ALGORITHM_RIGHT_SYMMETRIC_6: -+ if (raid_disk == raid_disks - 1) -+ return 1; ++#define SRCU_NOTIFIER_INIT(name, pcpu) \ ++ { \ ++ .mutex = __MUTEX_INITIALIZER(name.mutex), \ ++ .head = NULL, \ ++ .srcu = __SRCU_STRUCT_INIT(name.srcu, pcpu), \ + } -+ return 0; -+} + + #define ATOMIC_NOTIFIER_HEAD(name) \ + struct atomic_notifier_head name = \ +@@ -113,6 +117,18 @@ + struct raw_notifier_head name = \ + RAW_NOTIFIER_INIT(name) + ++#define _SRCU_NOTIFIER_HEAD(name, mod) \ ++ static DEFINE_PER_CPU(struct srcu_struct_array, \ ++ name##_head_srcu_array); \ ++ mod struct srcu_notifier_head name = \ ++ SRCU_NOTIFIER_INIT(name, name##_head_srcu_array) + -+static int run(struct mddev *mddev) -+{ -+ struct r5conf *conf; -+ int working_disks = 0; -+ int dirty_parity_disks = 0; -+ struct md_rdev *rdev; -+ sector_t reshape_offset = 0; -+ int i; -+ long long min_offset_diff = 0; -+ int first = 1; ++#define SRCU_NOTIFIER_HEAD(name) \ ++ _SRCU_NOTIFIER_HEAD(name, ) + -+ if (mddev->recovery_cp != MaxSector) -+ printk(KERN_NOTICE "md/raid:%s: not clean" -+ " -- starting background reconstruction\n", -+ mdname(mddev)); ++#define SRCU_NOTIFIER_HEAD_STATIC(name) \ ++ _SRCU_NOTIFIER_HEAD(name, static) + -+ rdev_for_each(rdev, mddev) { -+ long long diff; -+ if (rdev->raid_disk < 0) -+ continue; -+ diff = (rdev->new_data_offset - rdev->data_offset); -+ if (first) { -+ min_offset_diff = diff; -+ first = 0; -+ } else if (mddev->reshape_backwards && -+ diff < min_offset_diff) -+ min_offset_diff = diff; -+ else if (!mddev->reshape_backwards && -+ diff > min_offset_diff) -+ min_offset_diff = diff; -+ } + #ifdef __KERNEL__ + + extern int atomic_notifier_chain_register(struct atomic_notifier_head *nh, +@@ -182,12 +198,12 @@ + + /* + * Declared notifiers so far. I can imagine quite a few more chains +- * over time (eg laptop power reset chains, reboot chain (to clean ++ * over time (eg laptop power reset chains, reboot chain (to clean + * device units up), device [un]mount chain, module load/unload chain, +- * low memory chain, screenblank chain (for plug in modular screenblankers) ++ * low memory chain, screenblank chain (for plug in modular screenblankers) + * VC switch chains (for loadable kernel svgalib VC switch helpers) etc... + */ +- + -+ if (mddev->reshape_position != MaxSector) { -+ /* Check that we can continue the reshape. -+ * Difficulties arise if the stripe we would write to -+ * next is at or after the stripe we would read from next. -+ * For a reshape that changes the number of devices, this -+ * is only possible for a very short time, and mdadm makes -+ * sure that time appears to have past before assembling -+ * the array. So we fail if that time hasn't passed. -+ * For a reshape that keeps the number of devices the same -+ * mdadm must be monitoring the reshape can keeping the -+ * critical areas read-only and backed up. It will start -+ * the array in read-only mode, so we check for that. -+ */ -+ sector_t here_new, here_old; -+ int old_disks; -+ int max_degraded = (mddev->level == 6 ? 2 : 1); -+ -+ if (mddev->new_level != mddev->level) { -+ printk(KERN_ERR "md/raid:%s: unsupported reshape " -+ "required - aborting.\n", -+ mdname(mddev)); -+ return -EINVAL; -+ } -+ old_disks = mddev->raid_disks - mddev->delta_disks; -+ /* reshape_position must be on a new-stripe boundary, and one -+ * further up in new geometry must map after here in old -+ * geometry. -+ */ -+ here_new = mddev->reshape_position; -+ if (sector_div(here_new, mddev->new_chunk_sectors * -+ (mddev->raid_disks - max_degraded))) { -+ printk(KERN_ERR "md/raid:%s: reshape_position not " -+ "on a stripe boundary\n", mdname(mddev)); -+ return -EINVAL; -+ } -+ reshape_offset = here_new * mddev->new_chunk_sectors; -+ /* here_new is the stripe we will write to */ -+ here_old = mddev->reshape_position; -+ sector_div(here_old, mddev->chunk_sectors * -+ (old_disks-max_degraded)); -+ /* here_old is the first stripe that we might need to read -+ * from */ -+ if (mddev->delta_disks == 0) { -+ if ((here_new * mddev->new_chunk_sectors != -+ here_old * mddev->chunk_sectors)) { -+ printk(KERN_ERR "md/raid:%s: reshape position is" -+ " confused - aborting\n", mdname(mddev)); -+ return -EINVAL; -+ } -+ /* We cannot be sure it is safe to start an in-place -+ * reshape. It is only safe if user-space is monitoring -+ * and taking constant backups. -+ * mdadm always starts a situation like this in -+ * readonly mode so it can take control before -+ * allowing any writes. So just check for that. -+ */ -+ if (abs(min_offset_diff) >= mddev->chunk_sectors && -+ abs(min_offset_diff) >= mddev->new_chunk_sectors) -+ /* not really in-place - so OK */; -+ else if (mddev->ro == 0) { -+ printk(KERN_ERR "md/raid:%s: in-place reshape " -+ "must be started in read-only mode " -+ "- aborting\n", -+ mdname(mddev)); -+ return -EINVAL; -+ } -+ } else if (mddev->reshape_backwards -+ ? (here_new * mddev->new_chunk_sectors + min_offset_diff <= -+ here_old * mddev->chunk_sectors) -+ : (here_new * mddev->new_chunk_sectors >= -+ here_old * mddev->chunk_sectors + (-min_offset_diff))) { -+ /* Reading from the same stripe as writing to - bad */ -+ printk(KERN_ERR "md/raid:%s: reshape_position too early for " -+ "auto-recovery - aborting.\n", -+ mdname(mddev)); -+ return -EINVAL; -+ } -+ printk(KERN_INFO "md/raid:%s: reshape will continue\n", -+ mdname(mddev)); -+ /* OK, we should be able to continue; */ -+ } else { -+ BUG_ON(mddev->level != mddev->new_level); -+ BUG_ON(mddev->layout != mddev->new_layout); -+ BUG_ON(mddev->chunk_sectors != mddev->new_chunk_sectors); -+ BUG_ON(mddev->delta_disks != 0); -+ } + /* CPU notfiers are defined in include/linux/cpu.h. */ + + /* netdevice notifiers are defined in include/linux/netdevice.h */ +diff -Nur linux-4.1.10.orig/include/linux/percpu.h linux-4.1.10/include/linux/percpu.h +--- linux-4.1.10.orig/include/linux/percpu.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/percpu.h 2015-10-12 22:33:32.276676754 +0200 +@@ -24,6 +24,35 @@ + PERCPU_MODULE_RESERVE) + #endif + ++#ifdef CONFIG_PREEMPT_RT_FULL + -+ if (mddev->private == NULL) -+ conf = setup_conf(mddev); -+ else -+ conf = mddev->private; -+ -+ if (IS_ERR(conf)) -+ return PTR_ERR(conf); -+ -+ conf->min_offset_diff = min_offset_diff; -+ mddev->thread = conf->thread; -+ conf->thread = NULL; -+ mddev->private = conf; -+ -+ for (i = 0; i < conf->raid_disks && conf->previous_raid_disks; -+ i++) { -+ rdev = conf->disks[i].rdev; -+ if (!rdev && conf->disks[i].replacement) { -+ /* The replacement is all we have yet */ -+ rdev = conf->disks[i].replacement; -+ conf->disks[i].replacement = NULL; -+ clear_bit(Replacement, &rdev->flags); -+ conf->disks[i].rdev = rdev; -+ } -+ if (!rdev) -+ continue; -+ if (conf->disks[i].replacement && -+ conf->reshape_progress != MaxSector) { -+ /* replacements and reshape simply do not mix. */ -+ printk(KERN_ERR "md: cannot handle concurrent " -+ "replacement and reshape.\n"); -+ goto abort; -+ } -+ if (test_bit(In_sync, &rdev->flags)) { -+ working_disks++; -+ continue; -+ } -+ /* This disc is not fully in-sync. However if it -+ * just stored parity (beyond the recovery_offset), -+ * when we don't need to be concerned about the -+ * array being dirty. -+ * When reshape goes 'backwards', we never have -+ * partially completed devices, so we only need -+ * to worry about reshape going forwards. -+ */ -+ /* Hack because v0.91 doesn't store recovery_offset properly. */ -+ if (mddev->major_version == 0 && -+ mddev->minor_version > 90) -+ rdev->recovery_offset = reshape_offset; -+ -+ if (rdev->recovery_offset < reshape_offset) { -+ /* We need to check old and new layout */ -+ if (!only_parity(rdev->raid_disk, -+ conf->algorithm, -+ conf->raid_disks, -+ conf->max_degraded)) -+ continue; -+ } -+ if (!only_parity(rdev->raid_disk, -+ conf->prev_algo, -+ conf->previous_raid_disks, -+ conf->max_degraded)) -+ continue; -+ dirty_parity_disks++; -+ } ++#define get_local_var(var) (*({ \ ++ migrate_disable(); \ ++ this_cpu_ptr(&var); })) + -+ /* -+ * 0 for a fully functional array, 1 or 2 for a degraded array. -+ */ -+ mddev->degraded = calc_degraded(conf); ++#define put_local_var(var) do { \ ++ (void)&(var); \ ++ migrate_enable(); \ ++} while (0) + -+ if (has_failed(conf)) { -+ printk(KERN_ERR "md/raid:%s: not enough operational devices" -+ " (%d/%d failed)\n", -+ mdname(mddev), mddev->degraded, conf->raid_disks); -+ goto abort; -+ } ++# define get_local_ptr(var) ({ \ ++ migrate_disable(); \ ++ this_cpu_ptr(var); }) + -+ /* device size must be a multiple of chunk size */ -+ mddev->dev_sectors &= ~(mddev->chunk_sectors - 1); -+ mddev->resync_max_sectors = mddev->dev_sectors; -+ -+ if (mddev->degraded > dirty_parity_disks && -+ mddev->recovery_cp != MaxSector) { -+ if (mddev->ok_start_degraded) -+ printk(KERN_WARNING -+ "md/raid:%s: starting dirty degraded array" -+ " - data corruption possible.\n", -+ mdname(mddev)); -+ else { -+ printk(KERN_ERR -+ "md/raid:%s: cannot start dirty degraded array.\n", -+ mdname(mddev)); -+ goto abort; -+ } -+ } ++# define put_local_ptr(var) do { \ ++ (void)(var); \ ++ migrate_enable(); \ ++} while (0) + -+ if (mddev->degraded == 0) -+ printk(KERN_INFO "md/raid:%s: raid level %d active with %d out of %d" -+ " devices, algorithm %d\n", mdname(mddev), conf->level, -+ mddev->raid_disks-mddev->degraded, mddev->raid_disks, -+ mddev->new_layout); -+ else -+ printk(KERN_ALERT "md/raid:%s: raid level %d active with %d" -+ " out of %d devices, algorithm %d\n", -+ mdname(mddev), conf->level, -+ mddev->raid_disks - mddev->degraded, -+ mddev->raid_disks, mddev->new_layout); -+ -+ print_raid5_conf(conf); -+ -+ if (conf->reshape_progress != MaxSector) { -+ conf->reshape_safe = conf->reshape_progress; -+ atomic_set(&conf->reshape_stripes, 0); -+ clear_bit(MD_RECOVERY_SYNC, &mddev->recovery); -+ clear_bit(MD_RECOVERY_CHECK, &mddev->recovery); -+ set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery); -+ set_bit(MD_RECOVERY_RUNNING, &mddev->recovery); -+ mddev->sync_thread = md_register_thread(md_do_sync, mddev, -+ "reshape"); -+ } ++#else + -+ /* Ok, everything is just fine now */ -+ if (mddev->to_remove == &raid5_attrs_group) -+ mddev->to_remove = NULL; -+ else if (mddev->kobj.sd && -+ sysfs_create_group(&mddev->kobj, &raid5_attrs_group)) -+ printk(KERN_WARNING -+ "raid5: failed to create sysfs attributes for %s\n", -+ mdname(mddev)); -+ md_set_array_sectors(mddev, raid5_size(mddev, 0, 0)); -+ -+ if (mddev->queue) { -+ int chunk_size; -+ bool discard_supported = true; -+ /* read-ahead size must cover two whole stripes, which -+ * is 2 * (datadisks) * chunksize where 'n' is the -+ * number of raid devices -+ */ -+ int data_disks = conf->previous_raid_disks - conf->max_degraded; -+ int stripe = data_disks * -+ ((mddev->chunk_sectors << 9) / PAGE_SIZE); -+ if (mddev->queue->backing_dev_info.ra_pages < 2 * stripe) -+ mddev->queue->backing_dev_info.ra_pages = 2 * stripe; -+ -+ chunk_size = mddev->chunk_sectors << 9; -+ blk_queue_io_min(mddev->queue, chunk_size); -+ blk_queue_io_opt(mddev->queue, chunk_size * -+ (conf->raid_disks - conf->max_degraded)); -+ mddev->queue->limits.raid_partial_stripes_expensive = 1; -+ /* -+ * We can only discard a whole stripe. It doesn't make sense to -+ * discard data disk but write parity disk -+ */ -+ stripe = stripe * PAGE_SIZE; -+ /* Round up to power of 2, as discard handling -+ * currently assumes that */ -+ while ((stripe-1) & stripe) -+ stripe = (stripe | (stripe-1)) + 1; -+ mddev->queue->limits.discard_alignment = stripe; -+ mddev->queue->limits.discard_granularity = stripe; -+ /* -+ * unaligned part of discard request will be ignored, so can't -+ * guarantee discard_zeroes_data -+ */ -+ mddev->queue->limits.discard_zeroes_data = 0; ++#define get_local_var(var) get_cpu_var(var) ++#define put_local_var(var) put_cpu_var(var) ++#define get_local_ptr(var) get_cpu_ptr(var) ++#define put_local_ptr(var) put_cpu_ptr(var) + -+ blk_queue_max_write_same_sectors(mddev->queue, 0); ++#endif + -+ rdev_for_each(rdev, mddev) { -+ disk_stack_limits(mddev->gendisk, rdev->bdev, -+ rdev->data_offset << 9); -+ disk_stack_limits(mddev->gendisk, rdev->bdev, -+ rdev->new_data_offset << 9); -+ /* -+ * discard_zeroes_data is required, otherwise data -+ * could be lost. Consider a scenario: discard a stripe -+ * (the stripe could be inconsistent if -+ * discard_zeroes_data is 0); write one disk of the -+ * stripe (the stripe could be inconsistent again -+ * depending on which disks are used to calculate -+ * parity); the disk is broken; The stripe data of this -+ * disk is lost. -+ */ -+ if (!blk_queue_discard(bdev_get_queue(rdev->bdev)) || -+ !bdev_get_queue(rdev->bdev)-> -+ limits.discard_zeroes_data) -+ discard_supported = false; -+ /* Unfortunately, discard_zeroes_data is not currently -+ * a guarantee - just a hint. So we only allow DISCARD -+ * if the sysadmin has confirmed that only safe devices -+ * are in use by setting a module parameter. -+ */ -+ if (!devices_handle_discard_safely) { -+ if (discard_supported) { -+ pr_info("md/raid456: discard support disabled due to uncertainty.\n"); -+ pr_info("Set raid456.devices_handle_discard_safely=Y to override.\n"); -+ } -+ discard_supported = false; -+ } -+ } + /* minimum unit size, also is the maximum supported allocation size */ + #define PCPU_MIN_UNIT_SIZE PFN_ALIGN(32 << 10) + +diff -Nur linux-4.1.10.orig/include/linux/pid.h linux-4.1.10/include/linux/pid.h +--- linux-4.1.10.orig/include/linux/pid.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/pid.h 2015-10-12 22:33:32.276676754 +0200 +@@ -2,6 +2,7 @@ + #define _LINUX_PID_H + + #include ++#include + + enum pid_type + { +diff -Nur linux-4.1.10.orig/include/linux/preempt.h linux-4.1.10/include/linux/preempt.h +--- linux-4.1.10.orig/include/linux/preempt.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/preempt.h 2015-10-12 22:33:32.276676754 +0200 +@@ -33,6 +33,20 @@ + #define preempt_count_inc() preempt_count_add(1) + #define preempt_count_dec() preempt_count_sub(1) + ++#ifdef CONFIG_PREEMPT_LAZY ++#define add_preempt_lazy_count(val) do { preempt_lazy_count() += (val); } while (0) ++#define sub_preempt_lazy_count(val) do { preempt_lazy_count() -= (val); } while (0) ++#define inc_preempt_lazy_count() add_preempt_lazy_count(1) ++#define dec_preempt_lazy_count() sub_preempt_lazy_count(1) ++#define preempt_lazy_count() (current_thread_info()->preempt_lazy_count) ++#else ++#define add_preempt_lazy_count(val) do { } while (0) ++#define sub_preempt_lazy_count(val) do { } while (0) ++#define inc_preempt_lazy_count() do { } while (0) ++#define dec_preempt_lazy_count() do { } while (0) ++#define preempt_lazy_count() (0) ++#endif + -+ if (discard_supported && -+ mddev->queue->limits.max_discard_sectors >= stripe && -+ mddev->queue->limits.discard_granularity >= stripe) -+ queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, -+ mddev->queue); -+ else -+ queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD, -+ mddev->queue); -+ } + #ifdef CONFIG_PREEMPT_COUNT + + #define preempt_disable() \ +@@ -41,13 +55,25 @@ + barrier(); \ + } while (0) + ++#define preempt_lazy_disable() \ ++do { \ ++ inc_preempt_lazy_count(); \ ++ barrier(); \ ++} while (0) + -+ return 0; -+abort: -+ md_unregister_thread(&mddev->thread); -+ print_raid5_conf(conf); -+ free_conf(conf); -+ mddev->private = NULL; -+ printk(KERN_ALERT "md/raid:%s: failed to run raid set.\n", mdname(mddev)); -+ return -EIO; -+} + #define sched_preempt_enable_no_resched() \ + do { \ + barrier(); \ + preempt_count_dec(); \ + } while (0) + +-#define preempt_enable_no_resched() sched_preempt_enable_no_resched() ++#ifdef CONFIG_PREEMPT_RT_BASE ++# define preempt_enable_no_resched() sched_preempt_enable_no_resched() ++# define preempt_check_resched_rt() preempt_check_resched() ++#else ++# define preempt_enable_no_resched() preempt_enable() ++# define preempt_check_resched_rt() barrier(); ++#endif + + #ifdef CONFIG_PREEMPT + #define preempt_enable() \ +@@ -63,6 +89,13 @@ + __preempt_schedule(); \ + } while (0) + ++#define preempt_lazy_enable() \ ++do { \ ++ dec_preempt_lazy_count(); \ ++ barrier(); \ ++ preempt_check_resched(); \ ++} while (0) + -+static void raid5_free(struct mddev *mddev, void *priv) -+{ -+ struct r5conf *conf = priv; + #else + #define preempt_enable() \ + do { \ +@@ -121,6 +154,7 @@ + #define preempt_disable_notrace() barrier() + #define preempt_enable_no_resched_notrace() barrier() + #define preempt_enable_notrace() barrier() ++#define preempt_check_resched_rt() barrier() + + #endif /* CONFIG_PREEMPT_COUNT */ + +@@ -140,10 +174,31 @@ + } while (0) + #define preempt_fold_need_resched() \ + do { \ +- if (tif_need_resched()) \ ++ if (tif_need_resched_now()) \ + set_preempt_need_resched(); \ + } while (0) + ++#ifdef CONFIG_PREEMPT_RT_FULL ++# define preempt_disable_rt() preempt_disable() ++# define preempt_enable_rt() preempt_enable() ++# define preempt_disable_nort() barrier() ++# define preempt_enable_nort() barrier() ++# ifdef CONFIG_SMP ++ extern void migrate_disable(void); ++ extern void migrate_enable(void); ++# else /* CONFIG_SMP */ ++# define migrate_disable() barrier() ++# define migrate_enable() barrier() ++# endif /* CONFIG_SMP */ ++#else ++# define preempt_disable_rt() barrier() ++# define preempt_enable_rt() barrier() ++# define preempt_disable_nort() preempt_disable() ++# define preempt_enable_nort() preempt_enable() ++# define migrate_disable() preempt_disable() ++# define migrate_enable() preempt_enable() ++#endif + -+ free_conf(conf); -+ mddev->to_remove = &raid5_attrs_group; -+} + #ifdef CONFIG_PREEMPT_NOTIFIERS + + struct preempt_notifier; +diff -Nur linux-4.1.10.orig/include/linux/preempt_mask.h linux-4.1.10/include/linux/preempt_mask.h +--- linux-4.1.10.orig/include/linux/preempt_mask.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/preempt_mask.h 2015-10-12 22:33:32.276676754 +0200 +@@ -44,16 +44,26 @@ + #define HARDIRQ_OFFSET (1UL << HARDIRQ_SHIFT) + #define NMI_OFFSET (1UL << NMI_SHIFT) + +-#define SOFTIRQ_DISABLE_OFFSET (2 * SOFTIRQ_OFFSET) ++#ifndef CONFIG_PREEMPT_RT_FULL ++# define SOFTIRQ_DISABLE_OFFSET (2 * SOFTIRQ_OFFSET) ++#else ++# define SOFTIRQ_DISABLE_OFFSET (0) ++#endif + + #define PREEMPT_ACTIVE_BITS 1 + #define PREEMPT_ACTIVE_SHIFT (NMI_SHIFT + NMI_BITS) + #define PREEMPT_ACTIVE (__IRQ_MASK(PREEMPT_ACTIVE_BITS) << PREEMPT_ACTIVE_SHIFT) + + #define hardirq_count() (preempt_count() & HARDIRQ_MASK) +-#define softirq_count() (preempt_count() & SOFTIRQ_MASK) + #define irq_count() (preempt_count() & (HARDIRQ_MASK | SOFTIRQ_MASK \ + | NMI_MASK)) ++#ifndef CONFIG_PREEMPT_RT_FULL ++# define softirq_count() (preempt_count() & SOFTIRQ_MASK) ++# define in_serving_softirq() (softirq_count() & SOFTIRQ_OFFSET) ++#else ++# define softirq_count() (0UL) ++extern int in_serving_softirq(void); ++#endif + + /* + * Are we doing bottom half or hardware interrupt processing? +@@ -64,7 +74,6 @@ + #define in_irq() (hardirq_count()) + #define in_softirq() (softirq_count()) + #define in_interrupt() (irq_count()) +-#define in_serving_softirq() (softirq_count() & SOFTIRQ_OFFSET) + + /* + * Are we in NMI context? +diff -Nur linux-4.1.10.orig/include/linux/printk.h linux-4.1.10/include/linux/printk.h +--- linux-4.1.10.orig/include/linux/printk.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/printk.h 2015-10-12 22:33:32.276676754 +0200 +@@ -115,9 +115,11 @@ + #ifdef CONFIG_EARLY_PRINTK + extern asmlinkage __printf(1, 2) + void early_printk(const char *fmt, ...); ++extern void printk_kill(void); + #else + static inline __printf(1, 2) __cold + void early_printk(const char *s, ...) { } ++static inline void printk_kill(void) { } + #endif + + typedef int(*printk_func_t)(const char *fmt, va_list args); +diff -Nur linux-4.1.10.orig/include/linux/radix-tree.h linux-4.1.10/include/linux/radix-tree.h +--- linux-4.1.10.orig/include/linux/radix-tree.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/radix-tree.h 2015-10-12 22:33:32.280676490 +0200 +@@ -277,8 +277,13 @@ + unsigned int radix_tree_gang_lookup_slot(struct radix_tree_root *root, + void ***results, unsigned long *indices, + unsigned long first_index, unsigned int max_items); ++#ifndef CONFIG_PREEMPT_RT_FULL + int radix_tree_preload(gfp_t gfp_mask); + int radix_tree_maybe_preload(gfp_t gfp_mask); ++#else ++static inline int radix_tree_preload(gfp_t gm) { return 0; } ++static inline int radix_tree_maybe_preload(gfp_t gfp_mask) { return 0; } ++#endif + void radix_tree_init(void); + void *radix_tree_tag_set(struct radix_tree_root *root, + unsigned long index, unsigned int tag); +@@ -303,7 +308,7 @@ + + static inline void radix_tree_preload_end(void) + { +- preempt_enable(); ++ preempt_enable_nort(); + } + + /** +diff -Nur linux-4.1.10.orig/include/linux/random.h linux-4.1.10/include/linux/random.h +--- linux-4.1.10.orig/include/linux/random.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/random.h 2015-10-12 22:33:32.280676490 +0200 +@@ -11,7 +11,7 @@ + extern void add_device_randomness(const void *, unsigned int); + extern void add_input_randomness(unsigned int type, unsigned int code, + unsigned int value); +-extern void add_interrupt_randomness(int irq, int irq_flags); ++extern void add_interrupt_randomness(int irq, int irq_flags, __u64 ip); + + extern void get_random_bytes(void *buf, int nbytes); + extern void get_random_bytes_arch(void *buf, int nbytes); +diff -Nur linux-4.1.10.orig/include/linux/rcupdate.h linux-4.1.10/include/linux/rcupdate.h +--- linux-4.1.10.orig/include/linux/rcupdate.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/rcupdate.h 2015-10-12 22:33:32.280676490 +0200 +@@ -167,6 +167,9 @@ + + #endif /* #else #ifdef CONFIG_PREEMPT_RCU */ + ++#ifdef CONFIG_PREEMPT_RT_FULL ++#define call_rcu_bh call_rcu ++#else + /** + * call_rcu_bh() - Queue an RCU for invocation after a quicker grace period. + * @head: structure to be used for queueing the RCU updates. +@@ -190,6 +193,7 @@ + */ + void call_rcu_bh(struct rcu_head *head, + void (*func)(struct rcu_head *head)); ++#endif + + /** + * call_rcu_sched() - Queue an RCU for invocation after sched grace period. +@@ -260,6 +264,11 @@ + * types of kernel builds, the rcu_read_lock() nesting depth is unknowable. + */ + #define rcu_preempt_depth() (current->rcu_read_lock_nesting) ++#ifndef CONFIG_PREEMPT_RT_FULL ++#define sched_rcu_preempt_depth() rcu_preempt_depth() ++#else ++static inline int sched_rcu_preempt_depth(void) { return 0; } ++#endif + + #else /* #ifdef CONFIG_PREEMPT_RCU */ + +@@ -283,6 +292,8 @@ + return 0; + } + ++#define sched_rcu_preempt_depth() rcu_preempt_depth() + -+static void status(struct seq_file *seq, struct mddev *mddev) + #endif /* #else #ifdef CONFIG_PREEMPT_RCU */ + + /* Internal to kernel */ +@@ -463,7 +474,14 @@ + int debug_lockdep_rcu_enabled(void); + + int rcu_read_lock_held(void); ++#ifdef CONFIG_PREEMPT_RT_FULL ++static inline int rcu_read_lock_bh_held(void) +{ -+ struct r5conf *conf = mddev->private; -+ int i; -+ -+ seq_printf(seq, " level %d, %dk chunk, algorithm %d", mddev->level, -+ mddev->chunk_sectors / 2, mddev->layout); -+ seq_printf (seq, " [%d/%d] [", conf->raid_disks, conf->raid_disks - mddev->degraded); -+ for (i = 0; i < conf->raid_disks; i++) -+ seq_printf (seq, "%s", -+ conf->disks[i].rdev && -+ test_bit(In_sync, &conf->disks[i].rdev->flags) ? "U" : "_"); -+ seq_printf (seq, "]"); ++ return rcu_read_lock_held(); +} ++#else + int rcu_read_lock_bh_held(void); ++#endif + + /** + * rcu_read_lock_sched_held() - might we be in RCU-sched read-side critical section? +@@ -990,10 +1008,14 @@ + static inline void rcu_read_lock_bh(void) + { + local_bh_disable(); ++#ifdef CONFIG_PREEMPT_RT_FULL ++ rcu_read_lock(); ++#else + __acquire(RCU_BH); + rcu_lock_acquire(&rcu_bh_lock_map); + rcu_lockdep_assert(rcu_is_watching(), + "rcu_read_lock_bh() used illegally while idle"); ++#endif + } + + /* +@@ -1003,10 +1025,14 @@ + */ + static inline void rcu_read_unlock_bh(void) + { ++#ifdef CONFIG_PREEMPT_RT_FULL ++ rcu_read_unlock(); ++#else + rcu_lockdep_assert(rcu_is_watching(), + "rcu_read_unlock_bh() used illegally while idle"); + rcu_lock_release(&rcu_bh_lock_map); + __release(RCU_BH); ++#endif + local_bh_enable(); + } + +diff -Nur linux-4.1.10.orig/include/linux/rcutree.h linux-4.1.10/include/linux/rcutree.h +--- linux-4.1.10.orig/include/linux/rcutree.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/rcutree.h 2015-10-12 22:33:32.280676490 +0200 +@@ -46,7 +46,11 @@ + rcu_note_context_switch(); + } + ++#ifdef CONFIG_PREEMPT_RT_FULL ++# define synchronize_rcu_bh synchronize_rcu ++#else + void synchronize_rcu_bh(void); ++#endif + void synchronize_sched_expedited(void); + void synchronize_rcu_expedited(void); + +@@ -74,7 +78,11 @@ + } + + void rcu_barrier(void); ++#ifdef CONFIG_PREEMPT_RT_FULL ++# define rcu_barrier_bh rcu_barrier ++#else + void rcu_barrier_bh(void); ++#endif + void rcu_barrier_sched(void); + unsigned long get_state_synchronize_rcu(void); + void cond_synchronize_rcu(unsigned long oldstate); +@@ -85,12 +93,10 @@ + unsigned long rcu_batches_started_bh(void); + unsigned long rcu_batches_started_sched(void); + unsigned long rcu_batches_completed(void); +-unsigned long rcu_batches_completed_bh(void); + unsigned long rcu_batches_completed_sched(void); + void show_rcu_gp_kthreads(void); + + void rcu_force_quiescent_state(void); +-void rcu_bh_force_quiescent_state(void); + void rcu_sched_force_quiescent_state(void); + + void exit_rcu(void); +@@ -100,6 +106,14 @@ + + bool rcu_is_watching(void); + ++#ifndef CONFIG_PREEMPT_RT_FULL ++void rcu_bh_force_quiescent_state(void); ++unsigned long rcu_batches_completed_bh(void); ++#else ++# define rcu_bh_force_quiescent_state rcu_force_quiescent_state ++# define rcu_batches_completed_bh rcu_batches_completed ++#endif + -+static void print_raid5_conf (struct r5conf *conf) -+{ -+ int i; -+ struct disk_info *tmp; + void rcu_all_qs(void); + + #endif /* __LINUX_RCUTREE_H */ +diff -Nur linux-4.1.10.orig/include/linux/rtmutex.h linux-4.1.10/include/linux/rtmutex.h +--- linux-4.1.10.orig/include/linux/rtmutex.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/rtmutex.h 2015-10-12 22:33:32.280676490 +0200 +@@ -14,10 +14,14 @@ + + #include + #include +-#include ++#include + + extern int max_lock_depth; /* for sysctl */ + ++#ifdef CONFIG_DEBUG_MUTEXES ++#include ++#endif + -+ printk(KERN_DEBUG "RAID conf printout:\n"); -+ if (!conf) { -+ printk("(conf==NULL)\n"); -+ return; -+ } -+ printk(KERN_DEBUG " --- level:%d rd:%d wd:%d\n", conf->level, -+ conf->raid_disks, -+ conf->raid_disks - conf->mddev->degraded); -+ -+ for (i = 0; i < conf->raid_disks; i++) { -+ char b[BDEVNAME_SIZE]; -+ tmp = conf->disks + i; -+ if (tmp->rdev) -+ printk(KERN_DEBUG " disk %d, o:%d, dev:%s\n", -+ i, !test_bit(Faulty, &tmp->rdev->flags), -+ bdevname(tmp->rdev->bdev, b)); -+ } -+} + /** + * The rt_mutex structure + * +@@ -31,8 +35,8 @@ + struct rb_root waiters; + struct rb_node *waiters_leftmost; + struct task_struct *owner; +-#ifdef CONFIG_DEBUG_RT_MUTEXES + int save_state; ++#ifdef CONFIG_DEBUG_RT_MUTEXES + const char *name, *file; + int line; + void *magic; +@@ -55,22 +59,33 @@ + # define rt_mutex_debug_check_no_locks_held(task) do { } while (0) + #endif + ++# define rt_mutex_init(mutex) \ ++ do { \ ++ raw_spin_lock_init(&(mutex)->wait_lock); \ ++ __rt_mutex_init(mutex, #mutex); \ ++ } while (0) + -+static int raid5_spare_active(struct mddev *mddev) -+{ -+ int i; -+ struct r5conf *conf = mddev->private; -+ struct disk_info *tmp; -+ int count = 0; -+ unsigned long flags; + #ifdef CONFIG_DEBUG_RT_MUTEXES + # define __DEBUG_RT_MUTEX_INITIALIZER(mutexname) \ + , .name = #mutexname, .file = __FILE__, .line = __LINE__ +-# define rt_mutex_init(mutex) __rt_mutex_init(mutex, __func__) + extern void rt_mutex_debug_task_free(struct task_struct *tsk); + #else + # define __DEBUG_RT_MUTEX_INITIALIZER(mutexname) +-# define rt_mutex_init(mutex) __rt_mutex_init(mutex, NULL) + # define rt_mutex_debug_task_free(t) do { } while (0) + #endif + +-#define __RT_MUTEX_INITIALIZER(mutexname) \ +- { .wait_lock = __RAW_SPIN_LOCK_UNLOCKED(mutexname.wait_lock) \ ++#define __RT_MUTEX_INITIALIZER_PLAIN(mutexname) \ ++ .wait_lock = __RAW_SPIN_LOCK_UNLOCKED(mutexname.wait_lock) \ + , .waiters = RB_ROOT \ + , .owner = NULL \ +- __DEBUG_RT_MUTEX_INITIALIZER(mutexname)} ++ __DEBUG_RT_MUTEX_INITIALIZER(mutexname) + -+ for (i = 0; i < conf->raid_disks; i++) { -+ tmp = conf->disks + i; -+ if (tmp->replacement -+ && tmp->replacement->recovery_offset == MaxSector -+ && !test_bit(Faulty, &tmp->replacement->flags) -+ && !test_and_set_bit(In_sync, &tmp->replacement->flags)) { -+ /* Replacement has just become active. */ -+ if (!tmp->rdev -+ || !test_and_clear_bit(In_sync, &tmp->rdev->flags)) -+ count++; -+ if (tmp->rdev) { -+ /* Replaced device not technically faulty, -+ * but we need to be sure it gets removed -+ * and never re-added. -+ */ -+ set_bit(Faulty, &tmp->rdev->flags); -+ sysfs_notify_dirent_safe( -+ tmp->rdev->sysfs_state); -+ } -+ sysfs_notify_dirent_safe(tmp->replacement->sysfs_state); -+ } else if (tmp->rdev -+ && tmp->rdev->recovery_offset == MaxSector -+ && !test_bit(Faulty, &tmp->rdev->flags) -+ && !test_and_set_bit(In_sync, &tmp->rdev->flags)) { -+ count++; -+ sysfs_notify_dirent_safe(tmp->rdev->sysfs_state); -+ } -+ } -+ spin_lock_irqsave(&conf->device_lock, flags); -+ mddev->degraded = calc_degraded(conf); -+ spin_unlock_irqrestore(&conf->device_lock, flags); -+ print_raid5_conf(conf); -+ return count; -+} ++#define __RT_MUTEX_INITIALIZER(mutexname) \ ++ { __RT_MUTEX_INITIALIZER_PLAIN(mutexname) } + -+static int raid5_remove_disk(struct mddev *mddev, struct md_rdev *rdev) -+{ -+ struct r5conf *conf = mddev->private; -+ int err = 0; -+ int number = rdev->raid_disk; -+ struct md_rdev **rdevp; -+ struct disk_info *p = conf->disks + number; -+ -+ print_raid5_conf(conf); -+ if (rdev == p->rdev) -+ rdevp = &p->rdev; -+ else if (rdev == p->replacement) -+ rdevp = &p->replacement; -+ else -+ return 0; ++#define __RT_MUTEX_INITIALIZER_SAVE_STATE(mutexname) \ ++ { __RT_MUTEX_INITIALIZER_PLAIN(mutexname) \ ++ , .save_state = 1 } + + #define DEFINE_RT_MUTEX(mutexname) \ + struct rt_mutex mutexname = __RT_MUTEX_INITIALIZER(mutexname) +@@ -91,6 +106,7 @@ + + extern void rt_mutex_lock(struct rt_mutex *lock); + extern int rt_mutex_lock_interruptible(struct rt_mutex *lock); ++extern int rt_mutex_lock_killable(struct rt_mutex *lock); + extern int rt_mutex_timed_lock(struct rt_mutex *lock, + struct hrtimer_sleeper *timeout); + +diff -Nur linux-4.1.10.orig/include/linux/rwlock_rt.h linux-4.1.10/include/linux/rwlock_rt.h +--- linux-4.1.10.orig/include/linux/rwlock_rt.h 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/include/linux/rwlock_rt.h 2015-10-12 22:33:32.280676490 +0200 +@@ -0,0 +1,99 @@ ++#ifndef __LINUX_RWLOCK_RT_H ++#define __LINUX_RWLOCK_RT_H + -+ if (number >= conf->raid_disks && -+ conf->reshape_progress == MaxSector) -+ clear_bit(In_sync, &rdev->flags); ++#ifndef __LINUX_SPINLOCK_H ++#error Do not include directly. Use spinlock.h ++#endif + -+ if (test_bit(In_sync, &rdev->flags) || -+ atomic_read(&rdev->nr_pending)) { -+ err = -EBUSY; -+ goto abort; -+ } -+ /* Only remove non-faulty devices if recovery -+ * isn't possible. -+ */ -+ if (!test_bit(Faulty, &rdev->flags) && -+ mddev->recovery_disabled != conf->recovery_disabled && -+ !has_failed(conf) && -+ (!p->replacement || p->replacement == rdev) && -+ number < conf->raid_disks) { -+ err = -EBUSY; -+ goto abort; -+ } -+ *rdevp = NULL; -+ synchronize_rcu(); -+ if (atomic_read(&rdev->nr_pending)) { -+ /* lost the race, try later */ -+ err = -EBUSY; -+ *rdevp = rdev; -+ } else if (p->replacement) { -+ /* We must have just cleared 'rdev' */ -+ p->rdev = p->replacement; -+ clear_bit(Replacement, &p->replacement->flags); -+ smp_mb(); /* Make sure other CPUs may see both as identical -+ * but will never see neither - if they are careful -+ */ -+ p->replacement = NULL; -+ clear_bit(WantReplacement, &rdev->flags); -+ } else -+ /* We might have just removed the Replacement as faulty- -+ * clear the bit just in case -+ */ -+ clear_bit(WantReplacement, &rdev->flags); -+abort: ++#define rwlock_init(rwl) \ ++do { \ ++ static struct lock_class_key __key; \ ++ \ ++ rt_mutex_init(&(rwl)->lock); \ ++ __rt_rwlock_init(rwl, #rwl, &__key); \ ++} while (0) + -+ print_raid5_conf(conf); -+ return err; -+} ++extern void __lockfunc rt_write_lock(rwlock_t *rwlock); ++extern void __lockfunc rt_read_lock(rwlock_t *rwlock); ++extern int __lockfunc rt_write_trylock(rwlock_t *rwlock); ++extern int __lockfunc rt_write_trylock_irqsave(rwlock_t *trylock, unsigned long *flags); ++extern int __lockfunc rt_read_trylock(rwlock_t *rwlock); ++extern void __lockfunc rt_write_unlock(rwlock_t *rwlock); ++extern void __lockfunc rt_read_unlock(rwlock_t *rwlock); ++extern unsigned long __lockfunc rt_write_lock_irqsave(rwlock_t *rwlock); ++extern unsigned long __lockfunc rt_read_lock_irqsave(rwlock_t *rwlock); ++extern void __rt_rwlock_init(rwlock_t *rwlock, char *name, struct lock_class_key *key); + -+static int raid5_add_disk(struct mddev *mddev, struct md_rdev *rdev) -+{ -+ struct r5conf *conf = mddev->private; -+ int err = -EEXIST; -+ int disk; -+ struct disk_info *p; -+ int first = 0; -+ int last = conf->raid_disks - 1; -+ -+ if (mddev->recovery_disabled == conf->recovery_disabled) -+ return -EBUSY; ++#define read_trylock(lock) __cond_lock(lock, rt_read_trylock(lock)) ++#define write_trylock(lock) __cond_lock(lock, rt_write_trylock(lock)) + -+ if (rdev->saved_raid_disk < 0 && has_failed(conf)) -+ /* no point adding a device */ -+ return -EINVAL; ++#define write_trylock_irqsave(lock, flags) \ ++ __cond_lock(lock, rt_write_trylock_irqsave(lock, &flags)) + -+ if (rdev->raid_disk >= 0) -+ first = last = rdev->raid_disk; ++#define read_lock_irqsave(lock, flags) \ ++ do { \ ++ typecheck(unsigned long, flags); \ ++ flags = rt_read_lock_irqsave(lock); \ ++ } while (0) + -+ /* -+ * find the disk ... but prefer rdev->saved_raid_disk -+ * if possible. -+ */ -+ if (rdev->saved_raid_disk >= 0 && -+ rdev->saved_raid_disk >= first && -+ conf->disks[rdev->saved_raid_disk].rdev == NULL) -+ first = rdev->saved_raid_disk; -+ -+ for (disk = first; disk <= last; disk++) { -+ p = conf->disks + disk; -+ if (p->rdev == NULL) { -+ clear_bit(In_sync, &rdev->flags); -+ rdev->raid_disk = disk; -+ err = 0; -+ if (rdev->saved_raid_disk != disk) -+ conf->fullsync = 1; -+ rcu_assign_pointer(p->rdev, rdev); -+ goto out; -+ } -+ } -+ for (disk = first; disk <= last; disk++) { -+ p = conf->disks + disk; -+ if (test_bit(WantReplacement, &p->rdev->flags) && -+ p->replacement == NULL) { -+ clear_bit(In_sync, &rdev->flags); -+ set_bit(Replacement, &rdev->flags); -+ rdev->raid_disk = disk; -+ err = 0; -+ conf->fullsync = 1; -+ rcu_assign_pointer(p->replacement, rdev); -+ break; -+ } -+ } -+out: -+ print_raid5_conf(conf); -+ return err; -+} ++#define write_lock_irqsave(lock, flags) \ ++ do { \ ++ typecheck(unsigned long, flags); \ ++ flags = rt_write_lock_irqsave(lock); \ ++ } while (0) + -+static int raid5_resize(struct mddev *mddev, sector_t sectors) -+{ -+ /* no resync is happening, and there is enough space -+ * on all devices, so we can resize. -+ * We need to make sure resync covers any new space. -+ * If the array is shrinking we should possibly wait until -+ * any io in the removed space completes, but it hardly seems -+ * worth it. -+ */ -+ sector_t newsize; -+ sectors &= ~((sector_t)mddev->chunk_sectors - 1); -+ newsize = raid5_size(mddev, sectors, mddev->raid_disks); -+ if (mddev->external_size && -+ mddev->array_sectors > newsize) -+ return -EINVAL; -+ if (mddev->bitmap) { -+ int ret = bitmap_resize(mddev->bitmap, sectors, 0, 0); -+ if (ret) -+ return ret; -+ } -+ md_set_array_sectors(mddev, newsize); -+ set_capacity(mddev->gendisk, mddev->array_sectors); -+ revalidate_disk(mddev->gendisk); -+ if (sectors > mddev->dev_sectors && -+ mddev->recovery_cp > mddev->dev_sectors) { -+ mddev->recovery_cp = mddev->dev_sectors; -+ set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); -+ } -+ mddev->dev_sectors = sectors; -+ mddev->resync_max_sectors = sectors; -+ return 0; -+} ++#define read_lock(lock) rt_read_lock(lock) + -+static int check_stripe_cache(struct mddev *mddev) -+{ -+ /* Can only proceed if there are plenty of stripe_heads. -+ * We need a minimum of one full stripe,, and for sensible progress -+ * it is best to have about 4 times that. -+ * If we require 4 times, then the default 256 4K stripe_heads will -+ * allow for chunk sizes up to 256K, which is probably OK. -+ * If the chunk size is greater, user-space should request more -+ * stripe_heads first. -+ */ -+ struct r5conf *conf = mddev->private; -+ if (((mddev->chunk_sectors << 9) / STRIPE_SIZE) * 4 -+ > conf->min_nr_stripes || -+ ((mddev->new_chunk_sectors << 9) / STRIPE_SIZE) * 4 -+ > conf->min_nr_stripes) { -+ printk(KERN_WARNING "md/raid:%s: reshape: not enough stripes. Needed %lu\n", -+ mdname(mddev), -+ ((max(mddev->chunk_sectors, mddev->new_chunk_sectors) << 9) -+ / STRIPE_SIZE)*4); -+ return 0; -+ } -+ return 1; -+} -+ -+static int check_reshape(struct mddev *mddev) -+{ -+ struct r5conf *conf = mddev->private; ++#define read_lock_bh(lock) \ ++ do { \ ++ local_bh_disable(); \ ++ rt_read_lock(lock); \ ++ } while (0) + -+ if (mddev->delta_disks == 0 && -+ mddev->new_layout == mddev->layout && -+ mddev->new_chunk_sectors == mddev->chunk_sectors) -+ return 0; /* nothing to do */ -+ if (has_failed(conf)) -+ return -EINVAL; -+ if (mddev->delta_disks < 0 && mddev->reshape_position == MaxSector) { -+ /* We might be able to shrink, but the devices must -+ * be made bigger first. -+ * For raid6, 4 is the minimum size. -+ * Otherwise 2 is the minimum -+ */ -+ int min = 2; -+ if (mddev->level == 6) -+ min = 4; -+ if (mddev->raid_disks + mddev->delta_disks < min) -+ return -EINVAL; -+ } ++#define read_lock_irq(lock) read_lock(lock) + -+ if (!check_stripe_cache(mddev)) -+ return -ENOSPC; -+ -+ if (mddev->new_chunk_sectors > mddev->chunk_sectors || -+ mddev->delta_disks > 0) -+ if (resize_chunks(conf, -+ conf->previous_raid_disks -+ + max(0, mddev->delta_disks), -+ max(mddev->new_chunk_sectors, -+ mddev->chunk_sectors) -+ ) < 0) -+ return -ENOMEM; -+ return resize_stripes(conf, (conf->previous_raid_disks -+ + mddev->delta_disks)); -+} ++#define write_lock(lock) rt_write_lock(lock) + -+static int raid5_start_reshape(struct mddev *mddev) -+{ -+ struct r5conf *conf = mddev->private; -+ struct md_rdev *rdev; -+ int spares = 0; -+ unsigned long flags; ++#define write_lock_bh(lock) \ ++ do { \ ++ local_bh_disable(); \ ++ rt_write_lock(lock); \ ++ } while (0) + -+ if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery)) -+ return -EBUSY; ++#define write_lock_irq(lock) write_lock(lock) + -+ if (!check_stripe_cache(mddev)) -+ return -ENOSPC; ++#define read_unlock(lock) rt_read_unlock(lock) + -+ if (has_failed(conf)) -+ return -EINVAL; ++#define read_unlock_bh(lock) \ ++ do { \ ++ rt_read_unlock(lock); \ ++ local_bh_enable(); \ ++ } while (0) + -+ rdev_for_each(rdev, mddev) { -+ if (!test_bit(In_sync, &rdev->flags) -+ && !test_bit(Faulty, &rdev->flags)) -+ spares++; -+ } ++#define read_unlock_irq(lock) read_unlock(lock) + -+ if (spares - mddev->degraded < mddev->delta_disks - conf->max_degraded) -+ /* Not enough devices even to make a degraded array -+ * of that size -+ */ -+ return -EINVAL; ++#define write_unlock(lock) rt_write_unlock(lock) + -+ /* Refuse to reduce size of the array. Any reductions in -+ * array size must be through explicit setting of array_size -+ * attribute. -+ */ -+ if (raid5_size(mddev, 0, conf->raid_disks + mddev->delta_disks) -+ < mddev->array_sectors) { -+ printk(KERN_ERR "md/raid:%s: array size must be reduced " -+ "before number of disks\n", mdname(mddev)); -+ return -EINVAL; -+ } ++#define write_unlock_bh(lock) \ ++ do { \ ++ rt_write_unlock(lock); \ ++ local_bh_enable(); \ ++ } while (0) + -+ atomic_set(&conf->reshape_stripes, 0); -+ spin_lock_irq(&conf->device_lock); -+ write_seqcount_begin(&conf->gen_lock); -+ conf->previous_raid_disks = conf->raid_disks; -+ conf->raid_disks += mddev->delta_disks; -+ conf->prev_chunk_sectors = conf->chunk_sectors; -+ conf->chunk_sectors = mddev->new_chunk_sectors; -+ conf->prev_algo = conf->algorithm; -+ conf->algorithm = mddev->new_layout; -+ conf->generation++; -+ /* Code that selects data_offset needs to see the generation update -+ * if reshape_progress has been set - so a memory barrier needed. -+ */ -+ smp_mb(); -+ if (mddev->reshape_backwards) -+ conf->reshape_progress = raid5_size(mddev, 0, 0); -+ else -+ conf->reshape_progress = 0; -+ conf->reshape_safe = conf->reshape_progress; -+ write_seqcount_end(&conf->gen_lock); -+ spin_unlock_irq(&conf->device_lock); -+ -+ /* Now make sure any requests that proceeded on the assumption -+ * the reshape wasn't running - like Discard or Read - have -+ * completed. -+ */ -+ mddev_suspend(mddev); -+ mddev_resume(mddev); -+ -+ /* Add some new drives, as many as will fit. -+ * We know there are enough to make the newly sized array work. -+ * Don't add devices if we are reducing the number of -+ * devices in the array. This is because it is not possible -+ * to correctly record the "partially reconstructed" state of -+ * such devices during the reshape and confusion could result. -+ */ -+ if (mddev->delta_disks >= 0) { -+ rdev_for_each(rdev, mddev) -+ if (rdev->raid_disk < 0 && -+ !test_bit(Faulty, &rdev->flags)) { -+ if (raid5_add_disk(mddev, rdev) == 0) { -+ if (rdev->raid_disk -+ >= conf->previous_raid_disks) -+ set_bit(In_sync, &rdev->flags); -+ else -+ rdev->recovery_offset = 0; -+ -+ if (sysfs_link_rdev(mddev, rdev)) -+ /* Failure here is OK */; -+ } -+ } else if (rdev->raid_disk >= conf->previous_raid_disks -+ && !test_bit(Faulty, &rdev->flags)) { -+ /* This is a spare that was manually added */ -+ set_bit(In_sync, &rdev->flags); -+ } ++#define write_unlock_irq(lock) write_unlock(lock) + -+ /* When a reshape changes the number of devices, -+ * ->degraded is measured against the larger of the -+ * pre and post number of devices. -+ */ -+ spin_lock_irqsave(&conf->device_lock, flags); -+ mddev->degraded = calc_degraded(conf); -+ spin_unlock_irqrestore(&conf->device_lock, flags); -+ } -+ mddev->raid_disks = conf->raid_disks; -+ mddev->reshape_position = conf->reshape_progress; -+ set_bit(MD_CHANGE_DEVS, &mddev->flags); -+ -+ clear_bit(MD_RECOVERY_SYNC, &mddev->recovery); -+ clear_bit(MD_RECOVERY_CHECK, &mddev->recovery); -+ clear_bit(MD_RECOVERY_DONE, &mddev->recovery); -+ set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery); -+ set_bit(MD_RECOVERY_RUNNING, &mddev->recovery); -+ mddev->sync_thread = md_register_thread(md_do_sync, mddev, -+ "reshape"); -+ if (!mddev->sync_thread) { -+ mddev->recovery = 0; -+ spin_lock_irq(&conf->device_lock); -+ write_seqcount_begin(&conf->gen_lock); -+ mddev->raid_disks = conf->raid_disks = conf->previous_raid_disks; -+ mddev->new_chunk_sectors = -+ conf->chunk_sectors = conf->prev_chunk_sectors; -+ mddev->new_layout = conf->algorithm = conf->prev_algo; -+ rdev_for_each(rdev, mddev) -+ rdev->new_data_offset = rdev->data_offset; -+ smp_wmb(); -+ conf->generation --; -+ conf->reshape_progress = MaxSector; -+ mddev->reshape_position = MaxSector; -+ write_seqcount_end(&conf->gen_lock); -+ spin_unlock_irq(&conf->device_lock); -+ return -EAGAIN; -+ } -+ conf->reshape_checkpoint = jiffies; -+ md_wakeup_thread(mddev->sync_thread); -+ md_new_event(mddev); -+ return 0; -+} ++#define read_unlock_irqrestore(lock, flags) \ ++ do { \ ++ typecheck(unsigned long, flags); \ ++ (void) flags; \ ++ rt_read_unlock(lock); \ ++ } while (0) + -+/* This is called from the reshape thread and should make any -+ * changes needed in 'conf' -+ */ -+static void end_reshape(struct r5conf *conf) -+{ ++#define write_unlock_irqrestore(lock, flags) \ ++ do { \ ++ typecheck(unsigned long, flags); \ ++ (void) flags; \ ++ rt_write_unlock(lock); \ ++ } while (0) + -+ if (!test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery)) { -+ struct md_rdev *rdev; ++#endif +diff -Nur linux-4.1.10.orig/include/linux/rwlock_types.h linux-4.1.10/include/linux/rwlock_types.h +--- linux-4.1.10.orig/include/linux/rwlock_types.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/rwlock_types.h 2015-10-12 22:33:32.280676490 +0200 +@@ -1,6 +1,10 @@ + #ifndef __LINUX_RWLOCK_TYPES_H + #define __LINUX_RWLOCK_TYPES_H + ++#if !defined(__LINUX_SPINLOCK_TYPES_H) ++# error "Do not include directly, include spinlock_types.h" ++#endif + -+ spin_lock_irq(&conf->device_lock); -+ conf->previous_raid_disks = conf->raid_disks; -+ rdev_for_each(rdev, conf->mddev) -+ rdev->data_offset = rdev->new_data_offset; -+ smp_wmb(); -+ conf->reshape_progress = MaxSector; -+ spin_unlock_irq(&conf->device_lock); -+ wake_up(&conf->wait_for_overlap); + /* + * include/linux/rwlock_types.h - generic rwlock type definitions + * and initializers +@@ -43,6 +47,7 @@ + RW_DEP_MAP_INIT(lockname) } + #endif + +-#define DEFINE_RWLOCK(x) rwlock_t x = __RW_LOCK_UNLOCKED(x) ++#define DEFINE_RWLOCK(name) \ ++ rwlock_t name __cacheline_aligned_in_smp = __RW_LOCK_UNLOCKED(name) + + #endif /* __LINUX_RWLOCK_TYPES_H */ +diff -Nur linux-4.1.10.orig/include/linux/rwlock_types_rt.h linux-4.1.10/include/linux/rwlock_types_rt.h +--- linux-4.1.10.orig/include/linux/rwlock_types_rt.h 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/include/linux/rwlock_types_rt.h 2015-10-12 22:33:32.280676490 +0200 +@@ -0,0 +1,33 @@ ++#ifndef __LINUX_RWLOCK_TYPES_RT_H ++#define __LINUX_RWLOCK_TYPES_RT_H + -+ /* read-ahead size must cover two whole stripes, which is -+ * 2 * (datadisks) * chunksize where 'n' is the number of raid devices -+ */ -+ if (conf->mddev->queue) { -+ int data_disks = conf->raid_disks - conf->max_degraded; -+ int stripe = data_disks * ((conf->chunk_sectors << 9) -+ / PAGE_SIZE); -+ if (conf->mddev->queue->backing_dev_info.ra_pages < 2 * stripe) -+ conf->mddev->queue->backing_dev_info.ra_pages = 2 * stripe; -+ } -+ } -+} ++#ifndef __LINUX_SPINLOCK_TYPES_H ++#error "Do not include directly. Include spinlock_types.h instead" ++#endif + -+/* This is called from the raid5d thread with mddev_lock held. -+ * It makes config changes to the device. ++/* ++ * rwlocks - rtmutex which allows single reader recursion + */ -+static void raid5_finish_reshape(struct mddev *mddev) -+{ -+ struct r5conf *conf = mddev->private; ++typedef struct { ++ struct rt_mutex lock; ++ int read_depth; ++ unsigned int break_lock; ++#ifdef CONFIG_DEBUG_LOCK_ALLOC ++ struct lockdep_map dep_map; ++#endif ++} rwlock_t; + -+ if (!test_bit(MD_RECOVERY_INTR, &mddev->recovery)) { ++#ifdef CONFIG_DEBUG_LOCK_ALLOC ++# define RW_DEP_MAP_INIT(lockname) .dep_map = { .name = #lockname } ++#else ++# define RW_DEP_MAP_INIT(lockname) ++#endif + -+ if (mddev->delta_disks > 0) { -+ md_set_array_sectors(mddev, raid5_size(mddev, 0, 0)); -+ set_capacity(mddev->gendisk, mddev->array_sectors); -+ revalidate_disk(mddev->gendisk); -+ } else { -+ int d; -+ spin_lock_irq(&conf->device_lock); -+ mddev->degraded = calc_degraded(conf); -+ spin_unlock_irq(&conf->device_lock); -+ for (d = conf->raid_disks ; -+ d < conf->raid_disks - mddev->delta_disks; -+ d++) { -+ struct md_rdev *rdev = conf->disks[d].rdev; -+ if (rdev) -+ clear_bit(In_sync, &rdev->flags); -+ rdev = conf->disks[d].replacement; -+ if (rdev) -+ clear_bit(In_sync, &rdev->flags); -+ } -+ } -+ mddev->layout = conf->algorithm; -+ mddev->chunk_sectors = conf->chunk_sectors; -+ mddev->reshape_position = MaxSector; -+ mddev->delta_disks = 0; -+ mddev->reshape_backwards = 0; -+ } -+} ++#define __RW_LOCK_UNLOCKED(name) \ ++ { .lock = __RT_MUTEX_INITIALIZER_SAVE_STATE(name.lock), \ ++ RW_DEP_MAP_INIT(name) } + -+static void raid5_quiesce(struct mddev *mddev, int state) -+{ -+ struct r5conf *conf = mddev->private; ++#define DEFINE_RWLOCK(name) \ ++ rwlock_t name __cacheline_aligned_in_smp = __RW_LOCK_UNLOCKED(name) + -+ switch(state) { -+ case 2: /* resume for a suspend */ -+ wake_up(&conf->wait_for_overlap); -+ break; ++#endif +diff -Nur linux-4.1.10.orig/include/linux/rwsem.h linux-4.1.10/include/linux/rwsem.h +--- linux-4.1.10.orig/include/linux/rwsem.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/rwsem.h 2015-10-12 22:33:32.280676490 +0200 +@@ -18,6 +18,10 @@ + #include + #endif + ++#ifdef CONFIG_PREEMPT_RT_FULL ++#include ++#else /* PREEMPT_RT_FULL */ + -+ case 1: /* stop all writes */ -+ lock_all_device_hash_locks_irq(conf); -+ /* '2' tells resync/reshape to pause so that all -+ * active stripes can drain -+ */ -+ conf->quiesce = 2; -+ wait_event_cmd(conf->wait_for_stripe, -+ atomic_read(&conf->active_stripes) == 0 && -+ atomic_read(&conf->active_aligned_reads) == 0, -+ unlock_all_device_hash_locks_irq(conf), -+ lock_all_device_hash_locks_irq(conf)); -+ conf->quiesce = 1; -+ unlock_all_device_hash_locks_irq(conf); -+ /* allow reshape to continue */ -+ wake_up(&conf->wait_for_overlap); -+ break; + struct rw_semaphore; + + #ifdef CONFIG_RWSEM_GENERIC_SPINLOCK +@@ -177,4 +181,6 @@ + # define up_read_non_owner(sem) up_read(sem) + #endif + ++#endif /* !PREEMPT_RT_FULL */ + -+ case 0: /* re-enable writes */ -+ lock_all_device_hash_locks_irq(conf); -+ conf->quiesce = 0; -+ wake_up(&conf->wait_for_stripe); -+ wake_up(&conf->wait_for_overlap); -+ unlock_all_device_hash_locks_irq(conf); -+ break; -+ } -+} + #endif /* _LINUX_RWSEM_H */ +diff -Nur linux-4.1.10.orig/include/linux/rwsem_rt.h linux-4.1.10/include/linux/rwsem_rt.h +--- linux-4.1.10.orig/include/linux/rwsem_rt.h 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/include/linux/rwsem_rt.h 2015-10-12 22:33:32.280676490 +0200 +@@ -0,0 +1,140 @@ ++#ifndef _LINUX_RWSEM_RT_H ++#define _LINUX_RWSEM_RT_H + -+static void *raid45_takeover_raid0(struct mddev *mddev, int level) -+{ -+ struct r0conf *raid0_conf = mddev->private; -+ sector_t sectors; -+ -+ /* for raid0 takeover only one zone is supported */ -+ if (raid0_conf->nr_strip_zones > 1) { -+ printk(KERN_ERR "md/raid:%s: cannot takeover raid0 with more than one zone.\n", -+ mdname(mddev)); -+ return ERR_PTR(-EINVAL); -+ } ++#ifndef _LINUX_RWSEM_H ++#error "Include rwsem.h" ++#endif + -+ sectors = raid0_conf->strip_zone[0].zone_end; -+ sector_div(sectors, raid0_conf->strip_zone[0].nb_dev); -+ mddev->dev_sectors = sectors; -+ mddev->new_level = level; -+ mddev->new_layout = ALGORITHM_PARITY_N; -+ mddev->new_chunk_sectors = mddev->chunk_sectors; -+ mddev->raid_disks += 1; -+ mddev->delta_disks = 1; -+ /* make sure it will be not marked as dirty */ -+ mddev->recovery_cp = MaxSector; -+ -+ return setup_conf(mddev); -+} ++/* ++ * RW-semaphores are a spinlock plus a reader-depth count. ++ * ++ * Note that the semantics are different from the usual ++ * Linux rw-sems, in PREEMPT_RT mode we do not allow ++ * multiple readers to hold the lock at once, we only allow ++ * a read-lock owner to read-lock recursively. This is ++ * better for latency, makes the implementation inherently ++ * fair and makes it simpler as well. ++ */ + -+static void *raid5_takeover_raid1(struct mddev *mddev) -+{ -+ int chunksect; ++#include + -+ if (mddev->raid_disks != 2 || -+ mddev->degraded > 1) -+ return ERR_PTR(-EINVAL); ++struct rw_semaphore { ++ struct rt_mutex lock; ++ int read_depth; ++#ifdef CONFIG_DEBUG_LOCK_ALLOC ++ struct lockdep_map dep_map; ++#endif ++}; + -+ /* Should check if there are write-behind devices? */ ++#define __RWSEM_INITIALIZER(name) \ ++ { .lock = __RT_MUTEX_INITIALIZER(name.lock), \ ++ RW_DEP_MAP_INIT(name) } + -+ chunksect = 64*2; /* 64K by default */ ++#define DECLARE_RWSEM(lockname) \ ++ struct rw_semaphore lockname = __RWSEM_INITIALIZER(lockname) + -+ /* The array must be an exact multiple of chunksize */ -+ while (chunksect && (mddev->array_sectors & (chunksect-1))) -+ chunksect >>= 1; ++extern void __rt_rwsem_init(struct rw_semaphore *rwsem, const char *name, ++ struct lock_class_key *key); + -+ if ((chunksect<<9) < STRIPE_SIZE) -+ /* array size does not allow a suitable chunk size */ -+ return ERR_PTR(-EINVAL); ++#define __rt_init_rwsem(sem, name, key) \ ++ do { \ ++ rt_mutex_init(&(sem)->lock); \ ++ __rt_rwsem_init((sem), (name), (key));\ ++ } while (0) + -+ mddev->new_level = 5; -+ mddev->new_layout = ALGORITHM_LEFT_SYMMETRIC; -+ mddev->new_chunk_sectors = chunksect; ++#define __init_rwsem(sem, name, key) __rt_init_rwsem(sem, name, key) + -+ return setup_conf(mddev); -+} ++# define rt_init_rwsem(sem) \ ++do { \ ++ static struct lock_class_key __key; \ ++ \ ++ __rt_init_rwsem((sem), #sem, &__key); \ ++} while (0) + -+static void *raid5_takeover_raid6(struct mddev *mddev) -+{ -+ int new_layout; ++extern void rt_down_write(struct rw_semaphore *rwsem); ++extern void rt_down_read_nested(struct rw_semaphore *rwsem, int subclass); ++extern void rt_down_write_nested(struct rw_semaphore *rwsem, int subclass); ++extern void rt_down_write_nested_lock(struct rw_semaphore *rwsem, ++ struct lockdep_map *nest); ++extern void rt_down_read(struct rw_semaphore *rwsem); ++extern int rt_down_write_trylock(struct rw_semaphore *rwsem); ++extern int rt_down_read_trylock(struct rw_semaphore *rwsem); ++extern void __rt_up_read(struct rw_semaphore *rwsem); ++extern void rt_up_read(struct rw_semaphore *rwsem); ++extern void rt_up_write(struct rw_semaphore *rwsem); ++extern void rt_downgrade_write(struct rw_semaphore *rwsem); + -+ switch (mddev->layout) { -+ case ALGORITHM_LEFT_ASYMMETRIC_6: -+ new_layout = ALGORITHM_LEFT_ASYMMETRIC; -+ break; -+ case ALGORITHM_RIGHT_ASYMMETRIC_6: -+ new_layout = ALGORITHM_RIGHT_ASYMMETRIC; -+ break; -+ case ALGORITHM_LEFT_SYMMETRIC_6: -+ new_layout = ALGORITHM_LEFT_SYMMETRIC; -+ break; -+ case ALGORITHM_RIGHT_SYMMETRIC_6: -+ new_layout = ALGORITHM_RIGHT_SYMMETRIC; -+ break; -+ case ALGORITHM_PARITY_0_6: -+ new_layout = ALGORITHM_PARITY_0; -+ break; -+ case ALGORITHM_PARITY_N: -+ new_layout = ALGORITHM_PARITY_N; -+ break; -+ default: -+ return ERR_PTR(-EINVAL); -+ } -+ mddev->new_level = 5; -+ mddev->new_layout = new_layout; -+ mddev->delta_disks = -1; -+ mddev->raid_disks -= 1; -+ return setup_conf(mddev); -+} ++#define init_rwsem(sem) rt_init_rwsem(sem) ++#define rwsem_is_locked(s) rt_mutex_is_locked(&(s)->lock) + -+static int raid5_check_reshape(struct mddev *mddev) ++static inline int rwsem_is_contended(struct rw_semaphore *sem) +{ -+ /* For a 2-drive array, the layout and chunk size can be changed -+ * immediately as not restriping is needed. -+ * For larger arrays we record the new value - after validation -+ * to be used by a reshape pass. -+ */ -+ struct r5conf *conf = mddev->private; -+ int new_chunk = mddev->new_chunk_sectors; -+ -+ if (mddev->new_layout >= 0 && !algorithm_valid_raid5(mddev->new_layout)) -+ return -EINVAL; -+ if (new_chunk > 0) { -+ if (!is_power_of_2(new_chunk)) -+ return -EINVAL; -+ if (new_chunk < (PAGE_SIZE>>9)) -+ return -EINVAL; -+ if (mddev->array_sectors & (new_chunk-1)) -+ /* not factor of array size */ -+ return -EINVAL; -+ } -+ -+ /* They look valid */ -+ -+ if (mddev->raid_disks == 2) { -+ /* can make the change immediately */ -+ if (mddev->new_layout >= 0) { -+ conf->algorithm = mddev->new_layout; -+ mddev->layout = mddev->new_layout; -+ } -+ if (new_chunk > 0) { -+ conf->chunk_sectors = new_chunk ; -+ mddev->chunk_sectors = new_chunk; -+ } -+ set_bit(MD_CHANGE_DEVS, &mddev->flags); -+ md_wakeup_thread(mddev->thread); -+ } -+ return check_reshape(mddev); ++ /* rt_mutex_has_waiters() */ ++ return !RB_EMPTY_ROOT(&sem->lock.waiters); +} + -+static int raid6_check_reshape(struct mddev *mddev) ++static inline void down_read(struct rw_semaphore *sem) +{ -+ int new_chunk = mddev->new_chunk_sectors; -+ -+ if (mddev->new_layout >= 0 && !algorithm_valid_raid6(mddev->new_layout)) -+ return -EINVAL; -+ if (new_chunk > 0) { -+ if (!is_power_of_2(new_chunk)) -+ return -EINVAL; -+ if (new_chunk < (PAGE_SIZE >> 9)) -+ return -EINVAL; -+ if (mddev->array_sectors & (new_chunk-1)) -+ /* not factor of array size */ -+ return -EINVAL; -+ } -+ -+ /* They look valid */ -+ return check_reshape(mddev); ++ rt_down_read(sem); +} + -+static void *raid5_takeover(struct mddev *mddev) ++static inline int down_read_trylock(struct rw_semaphore *sem) +{ -+ /* raid5 can take over: -+ * raid0 - if there is only one strip zone - make it a raid4 layout -+ * raid1 - if there are two drives. We need to know the chunk size -+ * raid4 - trivial - just use a raid4 layout. -+ * raid6 - Providing it is a *_6 layout -+ */ -+ if (mddev->level == 0) -+ return raid45_takeover_raid0(mddev, 5); -+ if (mddev->level == 1) -+ return raid5_takeover_raid1(mddev); -+ if (mddev->level == 4) { -+ mddev->new_layout = ALGORITHM_PARITY_N; -+ mddev->new_level = 5; -+ return setup_conf(mddev); -+ } -+ if (mddev->level == 6) -+ return raid5_takeover_raid6(mddev); -+ -+ return ERR_PTR(-EINVAL); ++ return rt_down_read_trylock(sem); +} + -+static void *raid4_takeover(struct mddev *mddev) ++static inline void down_write(struct rw_semaphore *sem) +{ -+ /* raid4 can take over: -+ * raid0 - if there is only one strip zone -+ * raid5 - if layout is right -+ */ -+ if (mddev->level == 0) -+ return raid45_takeover_raid0(mddev, 4); -+ if (mddev->level == 5 && -+ mddev->layout == ALGORITHM_PARITY_N) { -+ mddev->new_layout = 0; -+ mddev->new_level = 4; -+ return setup_conf(mddev); -+ } -+ return ERR_PTR(-EINVAL); ++ rt_down_write(sem); +} + -+static struct md_personality raid5_personality; ++static inline int down_write_trylock(struct rw_semaphore *sem) ++{ ++ return rt_down_write_trylock(sem); ++} + -+static void *raid6_takeover(struct mddev *mddev) ++static inline void __up_read(struct rw_semaphore *sem) +{ -+ /* Currently can only take over a raid5. We map the -+ * personality to an equivalent raid6 personality -+ * with the Q block at the end. -+ */ -+ int new_layout; -+ -+ if (mddev->pers != &raid5_personality) -+ return ERR_PTR(-EINVAL); -+ if (mddev->degraded > 1) -+ return ERR_PTR(-EINVAL); -+ if (mddev->raid_disks > 253) -+ return ERR_PTR(-EINVAL); -+ if (mddev->raid_disks < 3) -+ return ERR_PTR(-EINVAL); -+ -+ switch (mddev->layout) { -+ case ALGORITHM_LEFT_ASYMMETRIC: -+ new_layout = ALGORITHM_LEFT_ASYMMETRIC_6; -+ break; -+ case ALGORITHM_RIGHT_ASYMMETRIC: -+ new_layout = ALGORITHM_RIGHT_ASYMMETRIC_6; -+ break; -+ case ALGORITHM_LEFT_SYMMETRIC: -+ new_layout = ALGORITHM_LEFT_SYMMETRIC_6; -+ break; -+ case ALGORITHM_RIGHT_SYMMETRIC: -+ new_layout = ALGORITHM_RIGHT_SYMMETRIC_6; -+ break; -+ case ALGORITHM_PARITY_0: -+ new_layout = ALGORITHM_PARITY_0_6; -+ break; -+ case ALGORITHM_PARITY_N: -+ new_layout = ALGORITHM_PARITY_N; -+ break; -+ default: -+ return ERR_PTR(-EINVAL); -+ } -+ mddev->new_level = 6; -+ mddev->new_layout = new_layout; -+ mddev->delta_disks = 1; -+ mddev->raid_disks += 1; -+ return setup_conf(mddev); ++ __rt_up_read(sem); +} + -+static struct md_personality raid6_personality = ++static inline void up_read(struct rw_semaphore *sem) +{ -+ .name = "raid6", -+ .level = 6, -+ .owner = THIS_MODULE, -+ .make_request = make_request, -+ .run = run, -+ .free = raid5_free, -+ .status = status, -+ .error_handler = error, -+ .hot_add_disk = raid5_add_disk, -+ .hot_remove_disk= raid5_remove_disk, -+ .spare_active = raid5_spare_active, -+ .sync_request = sync_request, -+ .resize = raid5_resize, -+ .size = raid5_size, -+ .check_reshape = raid6_check_reshape, -+ .start_reshape = raid5_start_reshape, -+ .finish_reshape = raid5_finish_reshape, -+ .quiesce = raid5_quiesce, -+ .takeover = raid6_takeover, -+ .congested = raid5_congested, -+ .mergeable_bvec = raid5_mergeable_bvec, -+}; -+static struct md_personality raid5_personality = ++ rt_up_read(sem); ++} ++ ++static inline void up_write(struct rw_semaphore *sem) +{ -+ .name = "raid5", -+ .level = 5, -+ .owner = THIS_MODULE, -+ .make_request = make_request, -+ .run = run, -+ .free = raid5_free, -+ .status = status, -+ .error_handler = error, -+ .hot_add_disk = raid5_add_disk, -+ .hot_remove_disk= raid5_remove_disk, -+ .spare_active = raid5_spare_active, -+ .sync_request = sync_request, -+ .resize = raid5_resize, -+ .size = raid5_size, -+ .check_reshape = raid5_check_reshape, -+ .start_reshape = raid5_start_reshape, -+ .finish_reshape = raid5_finish_reshape, -+ .quiesce = raid5_quiesce, -+ .takeover = raid5_takeover, -+ .congested = raid5_congested, -+ .mergeable_bvec = raid5_mergeable_bvec, -+}; ++ rt_up_write(sem); ++} + -+static struct md_personality raid4_personality = ++static inline void downgrade_write(struct rw_semaphore *sem) +{ -+ .name = "raid4", -+ .level = 4, -+ .owner = THIS_MODULE, -+ .make_request = make_request, -+ .run = run, -+ .free = raid5_free, -+ .status = status, -+ .error_handler = error, -+ .hot_add_disk = raid5_add_disk, -+ .hot_remove_disk= raid5_remove_disk, -+ .spare_active = raid5_spare_active, -+ .sync_request = sync_request, -+ .resize = raid5_resize, -+ .size = raid5_size, -+ .check_reshape = raid5_check_reshape, -+ .start_reshape = raid5_start_reshape, -+ .finish_reshape = raid5_finish_reshape, -+ .quiesce = raid5_quiesce, -+ .takeover = raid4_takeover, -+ .congested = raid5_congested, -+ .mergeable_bvec = raid5_mergeable_bvec, -+}; ++ rt_downgrade_write(sem); ++} + -+static int __init raid5_init(void) ++static inline void down_read_nested(struct rw_semaphore *sem, int subclass) +{ -+ raid5_wq = alloc_workqueue("raid5wq", -+ WQ_UNBOUND|WQ_MEM_RECLAIM|WQ_CPU_INTENSIVE|WQ_SYSFS, 0); -+ if (!raid5_wq) -+ return -ENOMEM; -+ register_md_personality(&raid6_personality); -+ register_md_personality(&raid5_personality); -+ register_md_personality(&raid4_personality); -+ return 0; ++ return rt_down_read_nested(sem, subclass); +} + -+static void raid5_exit(void) ++static inline void down_write_nested(struct rw_semaphore *sem, int subclass) ++{ ++ rt_down_write_nested(sem, subclass); ++} ++#ifdef CONFIG_DEBUG_LOCK_ALLOC ++static inline void down_write_nest_lock(struct rw_semaphore *sem, ++ struct rw_semaphore *nest_lock) +{ -+ unregister_md_personality(&raid6_personality); -+ unregister_md_personality(&raid5_personality); -+ unregister_md_personality(&raid4_personality); -+ destroy_workqueue(raid5_wq); ++ rt_down_write_nested_lock(sem, &nest_lock->dep_map); +} + -+module_init(raid5_init); -+module_exit(raid5_exit); -+MODULE_LICENSE("GPL"); -+MODULE_DESCRIPTION("RAID4/5/6 (striping with parity) personality for MD"); -+MODULE_ALIAS("md-personality-4"); /* RAID5 */ -+MODULE_ALIAS("md-raid5"); -+MODULE_ALIAS("md-raid4"); -+MODULE_ALIAS("md-level-5"); -+MODULE_ALIAS("md-level-4"); -+MODULE_ALIAS("md-personality-8"); /* RAID6 */ -+MODULE_ALIAS("md-raid6"); -+MODULE_ALIAS("md-level-6"); -+ -+/* This used to be two separate modules, they were: */ -+MODULE_ALIAS("raid5"); -+MODULE_ALIAS("raid6"); -diff -Nur linux-4.1.10.orig/drivers/md/raid5.h linux-4.1.10/drivers/md/raid5.h ---- linux-4.1.10.orig/drivers/md/raid5.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/md/raid5.h 2015-10-07 18:00:08.000000000 +0200 -@@ -495,6 +495,7 @@ - int recovery_disabled; - /* per cpu variables */ - struct raid5_percpu { -+ spinlock_t lock; /* Protection for -RT */ - struct page *spare_page; /* Used when checking P/Q in raid6 */ - struct flex_array *scribble; /* space for constructing buffer - * lists and performing address -diff -Nur linux-4.1.10.orig/drivers/md/raid5.h.orig linux-4.1.10/drivers/md/raid5.h.orig ---- linux-4.1.10.orig/drivers/md/raid5.h.orig 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.10/drivers/md/raid5.h.orig 2015-10-03 13:49:38.000000000 +0200 -@@ -0,0 +1,608 @@ -+#ifndef _RAID5_H -+#define _RAID5_H ++#else + -+#include -+#include ++static inline void down_write_nest_lock(struct rw_semaphore *sem, ++ struct rw_semaphore *nest_lock) ++{ ++ rt_down_write_nested_lock(sem, NULL); ++} ++#endif ++#endif +diff -Nur linux-4.1.10.orig/include/linux/sched.h linux-4.1.10/include/linux/sched.h +--- linux-4.1.10.orig/include/linux/sched.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/sched.h 2015-10-12 22:33:32.280676490 +0200 +@@ -26,6 +26,7 @@ + #include + #include + #include ++#include + + #include + #include +@@ -234,10 +235,7 @@ + TASK_UNINTERRUPTIBLE | __TASK_STOPPED | \ + __TASK_TRACED | EXIT_ZOMBIE | EXIT_DEAD) + +-#define task_is_traced(task) ((task->state & __TASK_TRACED) != 0) + #define task_is_stopped(task) ((task->state & __TASK_STOPPED) != 0) +-#define task_is_stopped_or_traced(task) \ +- ((task->state & (__TASK_STOPPED | __TASK_TRACED)) != 0) + #define task_contributes_to_load(task) \ + ((task->state & TASK_UNINTERRUPTIBLE) != 0 && \ + (task->flags & PF_FROZEN) == 0) +@@ -302,6 +300,11 @@ + + #endif + ++#define __set_current_state_no_track(state_value) \ ++ do { current->state = (state_value); } while (0) ++#define set_current_state_no_track(state_value) \ ++ set_mb(current->state, (state_value)) + -+/* -+ * -+ * Each stripe contains one buffer per device. Each buffer can be in -+ * one of a number of states stored in "flags". Changes between -+ * these states happen *almost* exclusively under the protection of the -+ * STRIPE_ACTIVE flag. Some very specific changes can happen in bi_end_io, and -+ * these are not protected by STRIPE_ACTIVE. -+ * -+ * The flag bits that are used to represent these states are: -+ * R5_UPTODATE and R5_LOCKED -+ * -+ * State Empty == !UPTODATE, !LOCK -+ * We have no data, and there is no active request -+ * State Want == !UPTODATE, LOCK -+ * A read request is being submitted for this block -+ * State Dirty == UPTODATE, LOCK -+ * Some new data is in this buffer, and it is being written out -+ * State Clean == UPTODATE, !LOCK -+ * We have valid data which is the same as on disc -+ * -+ * The possible state transitions are: -+ * -+ * Empty -> Want - on read or write to get old data for parity calc -+ * Empty -> Dirty - on compute_parity to satisfy write/sync request. -+ * Empty -> Clean - on compute_block when computing a block for failed drive -+ * Want -> Empty - on failed read -+ * Want -> Clean - on successful completion of read request -+ * Dirty -> Clean - on successful completion of write request -+ * Dirty -> Clean - on failed write -+ * Clean -> Dirty - on compute_parity to satisfy write/sync (RECONSTRUCT or RMW) -+ * -+ * The Want->Empty, Want->Clean, Dirty->Clean, transitions -+ * all happen in b_end_io at interrupt time. -+ * Each sets the Uptodate bit before releasing the Lock bit. -+ * This leaves one multi-stage transition: -+ * Want->Dirty->Clean -+ * This is safe because thinking that a Clean buffer is actually dirty -+ * will at worst delay some action, and the stripe will be scheduled -+ * for attention after the transition is complete. -+ * -+ * There is one possibility that is not covered by these states. That -+ * is if one drive has failed and there is a spare being rebuilt. We -+ * can't distinguish between a clean block that has been generated -+ * from parity calculations, and a clean block that has been -+ * successfully written to the spare ( or to parity when resyncing). -+ * To distinguish these states we have a stripe bit STRIPE_INSYNC that -+ * is set whenever a write is scheduled to the spare, or to the parity -+ * disc if there is no spare. A sync request clears this bit, and -+ * when we find it set with no buffers locked, we know the sync is -+ * complete. -+ * -+ * Buffers for the md device that arrive via make_request are attached -+ * to the appropriate stripe in one of two lists linked on b_reqnext. -+ * One list (bh_read) for read requests, one (bh_write) for write. -+ * There should never be more than one buffer on the two lists -+ * together, but we are not guaranteed of that so we allow for more. -+ * -+ * If a buffer is on the read list when the associated cache buffer is -+ * Uptodate, the data is copied into the read buffer and it's b_end_io -+ * routine is called. This may happen in the end_request routine only -+ * if the buffer has just successfully been read. end_request should -+ * remove the buffers from the list and then set the Uptodate bit on -+ * the buffer. Other threads may do this only if they first check -+ * that the Uptodate bit is set. Once they have checked that they may -+ * take buffers off the read queue. -+ * -+ * When a buffer on the write list is committed for write it is copied -+ * into the cache buffer, which is then marked dirty, and moved onto a -+ * third list, the written list (bh_written). Once both the parity -+ * block and the cached buffer are successfully written, any buffer on -+ * a written list can be returned with b_end_io. -+ * -+ * The write list and read list both act as fifos. The read list, -+ * write list and written list are protected by the device_lock. -+ * The device_lock is only for list manipulations and will only be -+ * held for a very short time. It can be claimed from interrupts. -+ * -+ * -+ * Stripes in the stripe cache can be on one of two lists (or on -+ * neither). The "inactive_list" contains stripes which are not -+ * currently being used for any request. They can freely be reused -+ * for another stripe. The "handle_list" contains stripes that need -+ * to be handled in some way. Both of these are fifo queues. Each -+ * stripe is also (potentially) linked to a hash bucket in the hash -+ * table so that it can be found by sector number. Stripes that are -+ * not hashed must be on the inactive_list, and will normally be at -+ * the front. All stripes start life this way. -+ * -+ * The inactive_list, handle_list and hash bucket lists are all protected by the -+ * device_lock. -+ * - stripes have a reference counter. If count==0, they are on a list. -+ * - If a stripe might need handling, STRIPE_HANDLE is set. -+ * - When refcount reaches zero, then if STRIPE_HANDLE it is put on -+ * handle_list else inactive_list -+ * -+ * This, combined with the fact that STRIPE_HANDLE is only ever -+ * cleared while a stripe has a non-zero count means that if the -+ * refcount is 0 and STRIPE_HANDLE is set, then it is on the -+ * handle_list and if recount is 0 and STRIPE_HANDLE is not set, then -+ * the stripe is on inactive_list. -+ * -+ * The possible transitions are: -+ * activate an unhashed/inactive stripe (get_active_stripe()) -+ * lockdev check-hash unlink-stripe cnt++ clean-stripe hash-stripe unlockdev -+ * activate a hashed, possibly active stripe (get_active_stripe()) -+ * lockdev check-hash if(!cnt++)unlink-stripe unlockdev -+ * attach a request to an active stripe (add_stripe_bh()) -+ * lockdev attach-buffer unlockdev -+ * handle a stripe (handle_stripe()) -+ * setSTRIPE_ACTIVE, clrSTRIPE_HANDLE ... -+ * (lockdev check-buffers unlockdev) .. -+ * change-state .. -+ * record io/ops needed clearSTRIPE_ACTIVE schedule io/ops -+ * release an active stripe (release_stripe()) -+ * lockdev if (!--cnt) { if STRIPE_HANDLE, add to handle_list else add to inactive-list } unlockdev + /* Task command name length */ + #define TASK_COMM_LEN 16 + +@@ -900,6 +903,50 @@ + #define SCHED_CAPACITY_SCALE (1L << SCHED_CAPACITY_SHIFT) + + /* ++ * Wake-queues are lists of tasks with a pending wakeup, whose ++ * callers have already marked the task as woken internally, ++ * and can thus carry on. A common use case is being able to ++ * do the wakeups once the corresponding user lock as been ++ * released. + * -+ * The refcount counts each thread that have activated the stripe, -+ * plus raid5d if it is handling it, plus one for each active request -+ * on a cached buffer, and plus one if the stripe is undergoing stripe -+ * operations. ++ * We hold reference to each task in the list across the wakeup, ++ * thus guaranteeing that the memory is still valid by the time ++ * the actual wakeups are performed in wake_up_q(). + * -+ * The stripe operations are: -+ * -copying data between the stripe cache and user application buffers -+ * -computing blocks to save a disk access, or to recover a missing block -+ * -updating the parity on a write operation (reconstruct write and -+ * read-modify-write) -+ * -checking parity correctness -+ * -running i/o to disk -+ * These operations are carried out by raid5_run_ops which uses the async_tx -+ * api to (optionally) offload operations to dedicated hardware engines. -+ * When requesting an operation handle_stripe sets the pending bit for the -+ * operation and increments the count. raid5_run_ops is then run whenever -+ * the count is non-zero. -+ * There are some critical dependencies between the operations that prevent some -+ * from being requested while another is in flight. -+ * 1/ Parity check operations destroy the in cache version of the parity block, -+ * so we prevent parity dependent operations like writes and compute_blocks -+ * from starting while a check is in progress. Some dma engines can perform -+ * the check without damaging the parity block, in these cases the parity -+ * block is re-marked up to date (assuming the check was successful) and is -+ * not re-read from disk. -+ * 2/ When a write operation is requested we immediately lock the affected -+ * blocks, and mark them as not up to date. This causes new read requests -+ * to be held off, as well as parity checks and compute block operations. -+ * 3/ Once a compute block operation has been requested handle_stripe treats -+ * that block as if it is up to date. raid5_run_ops guaruntees that any -+ * operation that is dependent on the compute block result is initiated after -+ * the compute block completes. -+ */ -+ -+/* -+ * Operations state - intermediate states that are visible outside of -+ * STRIPE_ACTIVE. -+ * In general _idle indicates nothing is running, _run indicates a data -+ * processing operation is active, and _result means the data processing result -+ * is stable and can be acted upon. For simple operations like biofill and -+ * compute that only have an _idle and _run state they are indicated with -+ * sh->state flags (STRIPE_BIOFILL_RUN and STRIPE_COMPUTE_RUN) -+ */ -+/** -+ * enum check_states - handles syncing / repairing a stripe -+ * @check_state_idle - check operations are quiesced -+ * @check_state_run - check operation is running -+ * @check_state_result - set outside lock when check result is valid -+ * @check_state_compute_run - check failed and we are repairing -+ * @check_state_compute_result - set outside lock when compute result is valid -+ */ -+enum check_states { -+ check_state_idle = 0, -+ check_state_run, /* xor parity check */ -+ check_state_run_q, /* q-parity check */ -+ check_state_run_pq, /* pq dual parity check */ -+ check_state_check_result, -+ check_state_compute_run, /* parity repair */ -+ check_state_compute_result, -+}; -+ -+/** -+ * enum reconstruct_states - handles writing or expanding a stripe -+ */ -+enum reconstruct_states { -+ reconstruct_state_idle = 0, -+ reconstruct_state_prexor_drain_run, /* prexor-write */ -+ reconstruct_state_drain_run, /* write */ -+ reconstruct_state_run, /* expand */ -+ reconstruct_state_prexor_drain_result, -+ reconstruct_state_drain_result, -+ reconstruct_state_result, -+}; -+ -+struct stripe_head { -+ struct hlist_node hash; -+ struct list_head lru; /* inactive_list or handle_list */ -+ struct llist_node release_list; -+ struct r5conf *raid_conf; -+ short generation; /* increments with every -+ * reshape */ -+ sector_t sector; /* sector of this row */ -+ short pd_idx; /* parity disk index */ -+ short qd_idx; /* 'Q' disk index for raid6 */ -+ short ddf_layout;/* use DDF ordering to calculate Q */ -+ short hash_lock_index; -+ unsigned long state; /* state flags */ -+ atomic_t count; /* nr of active thread/requests */ -+ int bm_seq; /* sequence number for bitmap flushes */ -+ int disks; /* disks in stripe */ -+ int overwrite_disks; /* total overwrite disks in stripe, -+ * this is only checked when stripe -+ * has STRIPE_BATCH_READY -+ */ -+ enum check_states check_state; -+ enum reconstruct_states reconstruct_state; -+ spinlock_t stripe_lock; -+ int cpu; -+ struct r5worker_group *group; -+ -+ struct stripe_head *batch_head; /* protected by stripe lock */ -+ spinlock_t batch_lock; /* only header's lock is useful */ -+ struct list_head batch_list; /* protected by head's batch lock*/ -+ /** -+ * struct stripe_operations -+ * @target - STRIPE_OP_COMPUTE_BLK target -+ * @target2 - 2nd compute target in the raid6 case -+ * @zero_sum_result - P and Q verification flags -+ * @request - async service request flags for raid_run_ops -+ */ -+ struct stripe_operations { -+ int target, target2; -+ enum sum_check_flags zero_sum_result; -+ } ops; -+ struct r5dev { -+ /* rreq and rvec are used for the replacement device when -+ * writing data to both devices. -+ */ -+ struct bio req, rreq; -+ struct bio_vec vec, rvec; -+ struct page *page, *orig_page; -+ struct bio *toread, *read, *towrite, *written; -+ sector_t sector; /* sector of this page */ -+ unsigned long flags; -+ } dev[1]; /* allocated with extra space depending of RAID geometry */ -+}; -+ -+/* stripe_head_state - collects and tracks the dynamic state of a stripe_head -+ * for handle_stripe. -+ */ -+struct stripe_head_state { -+ /* 'syncing' means that we need to read all devices, either -+ * to check/correct parity, or to reconstruct a missing device. -+ * 'replacing' means we are replacing one or more drives and -+ * the source is valid at this point so we don't need to -+ * read all devices, just the replacement targets. -+ */ -+ int syncing, expanding, expanded, replacing; -+ int locked, uptodate, to_read, to_write, failed, written; -+ int to_fill, compute, req_compute, non_overwrite; -+ int failed_num[2]; -+ int p_failed, q_failed; -+ int dec_preread_active; -+ unsigned long ops_request; -+ -+ struct bio *return_bi; -+ struct md_rdev *blocked_rdev; -+ int handle_bad_blocks; -+}; -+ -+/* Flags for struct r5dev.flags */ -+enum r5dev_flags { -+ R5_UPTODATE, /* page contains current data */ -+ R5_LOCKED, /* IO has been submitted on "req" */ -+ R5_DOUBLE_LOCKED,/* Cannot clear R5_LOCKED until 2 writes complete */ -+ R5_OVERWRITE, /* towrite covers whole page */ -+/* and some that are internal to handle_stripe */ -+ R5_Insync, /* rdev && rdev->in_sync at start */ -+ R5_Wantread, /* want to schedule a read */ -+ R5_Wantwrite, -+ R5_Overlap, /* There is a pending overlapping request -+ * on this block */ -+ R5_ReadNoMerge, /* prevent bio from merging in block-layer */ -+ R5_ReadError, /* seen a read error here recently */ -+ R5_ReWrite, /* have tried to over-write the readerror */ -+ -+ R5_Expanded, /* This block now has post-expand data */ -+ R5_Wantcompute, /* compute_block in progress treat as -+ * uptodate -+ */ -+ R5_Wantfill, /* dev->toread contains a bio that needs -+ * filling -+ */ -+ R5_Wantdrain, /* dev->towrite needs to be drained */ -+ R5_WantFUA, /* Write should be FUA */ -+ R5_SyncIO, /* The IO is sync */ -+ R5_WriteError, /* got a write error - need to record it */ -+ R5_MadeGood, /* A bad block has been fixed by writing to it */ -+ R5_ReadRepl, /* Will/did read from replacement rather than orig */ -+ R5_MadeGoodRepl,/* A bad block on the replacement device has been -+ * fixed by writing to it */ -+ R5_NeedReplace, /* This device has a replacement which is not -+ * up-to-date at this stripe. */ -+ R5_WantReplace, /* We need to update the replacement, we have read -+ * data in, and now is a good time to write it out. -+ */ -+ R5_Discard, /* Discard the stripe */ -+ R5_SkipCopy, /* Don't copy data from bio to stripe cache */ -+}; -+ -+/* -+ * Stripe state -+ */ -+enum { -+ STRIPE_ACTIVE, -+ STRIPE_HANDLE, -+ STRIPE_SYNC_REQUESTED, -+ STRIPE_SYNCING, -+ STRIPE_INSYNC, -+ STRIPE_REPLACED, -+ STRIPE_PREREAD_ACTIVE, -+ STRIPE_DELAYED, -+ STRIPE_DEGRADED, -+ STRIPE_BIT_DELAY, -+ STRIPE_EXPANDING, -+ STRIPE_EXPAND_SOURCE, -+ STRIPE_EXPAND_READY, -+ STRIPE_IO_STARTED, /* do not count towards 'bypass_count' */ -+ STRIPE_FULL_WRITE, /* all blocks are set to be overwritten */ -+ STRIPE_BIOFILL_RUN, -+ STRIPE_COMPUTE_RUN, -+ STRIPE_OPS_REQ_PENDING, -+ STRIPE_ON_UNPLUG_LIST, -+ STRIPE_DISCARD, -+ STRIPE_ON_RELEASE_LIST, -+ STRIPE_BATCH_READY, -+ STRIPE_BATCH_ERR, -+ STRIPE_BITMAP_PENDING, /* Being added to bitmap, don't add -+ * to batch yet. -+ */ -+}; -+ -+#define STRIPE_EXPAND_SYNC_FLAGS \ -+ ((1 << STRIPE_EXPAND_SOURCE) |\ -+ (1 << STRIPE_EXPAND_READY) |\ -+ (1 << STRIPE_EXPANDING) |\ -+ (1 << STRIPE_SYNC_REQUESTED)) -+/* -+ * Operation request flags -+ */ -+enum { -+ STRIPE_OP_BIOFILL, -+ STRIPE_OP_COMPUTE_BLK, -+ STRIPE_OP_PREXOR, -+ STRIPE_OP_BIODRAIN, -+ STRIPE_OP_RECONSTRUCT, -+ STRIPE_OP_CHECK, -+}; -+ -+/* -+ * RAID parity calculation preferences -+ */ -+enum { -+ PARITY_DISABLE_RMW = 0, -+ PARITY_ENABLE_RMW, -+ PARITY_PREFER_RMW, -+}; -+ -+/* -+ * Pages requested from set_syndrome_sources() -+ */ -+enum { -+ SYNDROME_SRC_ALL, -+ SYNDROME_SRC_WANT_DRAIN, -+ SYNDROME_SRC_WRITTEN, -+}; -+/* -+ * Plugging: ++ * One per task suffices, because there's never a need for a task to be ++ * in two wake queues simultaneously; it is forbidden to abandon a task ++ * in a wake queue (a call to wake_up_q() _must_ follow), so if a task is ++ * already in a wake queue, the wakeup will happen soon and the second ++ * waker can just skip it. + * -+ * To improve write throughput, we need to delay the handling of some -+ * stripes until there has been a chance that several write requests -+ * for the one stripe have all been collected. -+ * In particular, any write request that would require pre-reading -+ * is put on a "delayed" queue until there are no stripes currently -+ * in a pre-read phase. Further, if the "delayed" queue is empty when -+ * a stripe is put on it then we "plug" the queue and do not process it -+ * until an unplug call is made. (the unplug_io_fn() is called). ++ * The WAKE_Q macro declares and initializes the list head. ++ * wake_up_q() does NOT reinitialize the list; it's expected to be ++ * called near the end of a function, where the fact that the queue is ++ * not used again will be easy to see by inspection. + * -+ * When preread is initiated on a stripe, we set PREREAD_ACTIVE and add -+ * it to the count of prereading stripes. -+ * When write is initiated, or the stripe refcnt == 0 (just in case) we -+ * clear the PREREAD_ACTIVE flag and decrement the count -+ * Whenever the 'handle' queue is empty and the device is not plugged, we -+ * move any strips from delayed to handle and clear the DELAYED flag and set -+ * PREREAD_ACTIVE. -+ * In stripe_handle, if we find pre-reading is necessary, we do it if -+ * PREREAD_ACTIVE is set, else we set DELAYED which will send it to the delayed queue. -+ * HANDLE gets cleared if stripe_handle leaves nothing locked. -+ */ -+ -+struct disk_info { -+ struct md_rdev *rdev, *replacement; -+}; -+ -+/* NOTE NR_STRIPE_HASH_LOCKS must remain below 64. -+ * This is because we sometimes take all the spinlocks -+ * and creating that much locking depth can cause -+ * problems. ++ * Note that this can cause spurious wakeups. schedule() callers ++ * must ensure the call is done inside a loop, confirming that the ++ * wakeup condition has in fact occurred. + */ -+#define NR_STRIPE_HASH_LOCKS 8 -+#define STRIPE_HASH_LOCKS_MASK (NR_STRIPE_HASH_LOCKS - 1) -+ -+struct r5worker { -+ struct work_struct work; -+ struct r5worker_group *group; -+ struct list_head temp_inactive_list[NR_STRIPE_HASH_LOCKS]; -+ bool working; ++struct wake_q_node { ++ struct wake_q_node *next; +}; + -+struct r5worker_group { -+ struct list_head handle_list; -+ struct r5conf *conf; -+ struct r5worker *workers; -+ int stripes_cnt; ++struct wake_q_head { ++ struct wake_q_node *first; ++ struct wake_q_node **lastp; +}; + -+struct r5conf { -+ struct hlist_head *stripe_hashtbl; -+ /* only protect corresponding hash list and inactive_list */ -+ spinlock_t hash_locks[NR_STRIPE_HASH_LOCKS]; -+ struct mddev *mddev; -+ int chunk_sectors; -+ int level, algorithm, rmw_level; -+ int max_degraded; -+ int raid_disks; -+ int max_nr_stripes; -+ int min_nr_stripes; -+ -+ /* reshape_progress is the leading edge of a 'reshape' -+ * It has value MaxSector when no reshape is happening -+ * If delta_disks < 0, it is the last sector we started work on, -+ * else is it the next sector to work on. -+ */ -+ sector_t reshape_progress; -+ /* reshape_safe is the trailing edge of a reshape. We know that -+ * before (or after) this address, all reshape has completed. -+ */ -+ sector_t reshape_safe; -+ int previous_raid_disks; -+ int prev_chunk_sectors; -+ int prev_algo; -+ short generation; /* increments with every reshape */ -+ seqcount_t gen_lock; /* lock against generation changes */ -+ unsigned long reshape_checkpoint; /* Time we last updated -+ * metadata */ -+ long long min_offset_diff; /* minimum difference between -+ * data_offset and -+ * new_data_offset across all -+ * devices. May be negative, -+ * but is closest to zero. -+ */ -+ -+ struct list_head handle_list; /* stripes needing handling */ -+ struct list_head hold_list; /* preread ready stripes */ -+ struct list_head delayed_list; /* stripes that have plugged requests */ -+ struct list_head bitmap_list; /* stripes delaying awaiting bitmap update */ -+ struct bio *retry_read_aligned; /* currently retrying aligned bios */ -+ struct bio *retry_read_aligned_list; /* aligned bios retry list */ -+ atomic_t preread_active_stripes; /* stripes with scheduled io */ -+ atomic_t active_aligned_reads; -+ atomic_t pending_full_writes; /* full write backlog */ -+ int bypass_count; /* bypassed prereads */ -+ int bypass_threshold; /* preread nice */ -+ int skip_copy; /* Don't copy data from bio to stripe cache */ -+ struct list_head *last_hold; /* detect hold_list promotions */ -+ -+ atomic_t reshape_stripes; /* stripes with pending writes for reshape */ -+ /* unfortunately we need two cache names as we temporarily have -+ * two caches. -+ */ -+ int active_name; -+ char cache_name[2][32]; -+ struct kmem_cache *slab_cache; /* for allocating stripes */ -+ struct mutex cache_size_mutex; /* Protect changes to cache size */ -+ -+ int seq_flush, seq_write; -+ int quiesce; -+ -+ int fullsync; /* set to 1 if a full sync is needed, -+ * (fresh device added). -+ * Cleared when a sync completes. -+ */ -+ int recovery_disabled; -+ /* per cpu variables */ -+ struct raid5_percpu { -+ struct page *spare_page; /* Used when checking P/Q in raid6 */ -+ struct flex_array *scribble; /* space for constructing buffer -+ * lists and performing address -+ * conversions -+ */ -+ } __percpu *percpu; -+#ifdef CONFIG_HOTPLUG_CPU -+ struct notifier_block cpu_notify; -+#endif ++#define WAKE_Q_TAIL ((struct wake_q_node *) 0x01) + -+ /* -+ * Free stripes pool -+ */ -+ atomic_t active_stripes; -+ struct list_head inactive_list[NR_STRIPE_HASH_LOCKS]; -+ atomic_t empty_inactive_list_nr; -+ struct llist_head released_stripes; -+ wait_queue_head_t wait_for_stripe; -+ wait_queue_head_t wait_for_overlap; -+ unsigned long cache_state; -+#define R5_INACTIVE_BLOCKED 1 /* release of inactive stripes blocked, -+ * waiting for 25% to be free -+ */ -+#define R5_ALLOC_MORE 2 /* It might help to allocate another -+ * stripe. -+ */ -+#define R5_DID_ALLOC 4 /* A stripe was allocated, don't allocate -+ * more until at least one has been -+ * released. This avoids flooding -+ * the cache. -+ */ -+ struct shrinker shrinker; -+ int pool_size; /* number of disks in stripeheads in pool */ -+ spinlock_t device_lock; -+ struct disk_info *disks; -+ -+ /* When taking over an array from a different personality, we store -+ * the new thread here until we fully activate the array. -+ */ -+ struct md_thread *thread; -+ struct list_head temp_inactive_list[NR_STRIPE_HASH_LOCKS]; -+ struct r5worker_group *worker_groups; -+ int group_cnt; -+ int worker_cnt_per_group; -+}; ++#define WAKE_Q(name) \ ++ struct wake_q_head name = { WAKE_Q_TAIL, &name.first } + ++extern void wake_q_add(struct wake_q_head *head, ++ struct task_struct *task); ++extern void wake_up_q(struct wake_q_head *head); + +/* -+ * Our supported algorithms -+ */ -+#define ALGORITHM_LEFT_ASYMMETRIC 0 /* Rotating Parity N with Data Restart */ -+#define ALGORITHM_RIGHT_ASYMMETRIC 1 /* Rotating Parity 0 with Data Restart */ -+#define ALGORITHM_LEFT_SYMMETRIC 2 /* Rotating Parity N with Data Continuation */ -+#define ALGORITHM_RIGHT_SYMMETRIC 3 /* Rotating Parity 0 with Data Continuation */ -+ -+/* Define non-rotating (raid4) algorithms. These allow -+ * conversion of raid4 to raid5. -+ */ -+#define ALGORITHM_PARITY_0 4 /* P or P,Q are initial devices */ -+#define ALGORITHM_PARITY_N 5 /* P or P,Q are final devices. */ -+ -+/* DDF RAID6 layouts differ from md/raid6 layouts in two ways. -+ * Firstly, the exact positioning of the parity block is slightly -+ * different between the 'LEFT_*' modes of md and the "_N_*" modes -+ * of DDF. -+ * Secondly, or order of datablocks over which the Q syndrome is computed -+ * is different. -+ * Consequently we have different layouts for DDF/raid6 than md/raid6. -+ * These layouts are from the DDFv1.2 spec. -+ * Interestingly DDFv1.2-Errata-A does not specify N_CONTINUE but -+ * leaves RLQ=3 as 'Vendor Specific' -+ */ -+ -+#define ALGORITHM_ROTATING_ZERO_RESTART 8 /* DDF PRL=6 RLQ=1 */ -+#define ALGORITHM_ROTATING_N_RESTART 9 /* DDF PRL=6 RLQ=2 */ -+#define ALGORITHM_ROTATING_N_CONTINUE 10 /*DDF PRL=6 RLQ=3 */ -+ -+/* For every RAID5 algorithm we define a RAID6 algorithm -+ * with exactly the same layout for data and parity, and -+ * with the Q block always on the last device (N-1). -+ * This allows trivial conversion from RAID5 to RAID6 -+ */ -+#define ALGORITHM_LEFT_ASYMMETRIC_6 16 -+#define ALGORITHM_RIGHT_ASYMMETRIC_6 17 -+#define ALGORITHM_LEFT_SYMMETRIC_6 18 -+#define ALGORITHM_RIGHT_SYMMETRIC_6 19 -+#define ALGORITHM_PARITY_0_6 20 -+#define ALGORITHM_PARITY_N_6 ALGORITHM_PARITY_N -+ -+static inline int algorithm_valid_raid5(int layout) -+{ -+ return (layout >= 0) && -+ (layout <= 5); -+} -+static inline int algorithm_valid_raid6(int layout) -+{ -+ return (layout >= 0 && layout <= 5) -+ || -+ (layout >= 8 && layout <= 10) -+ || -+ (layout >= 16 && layout <= 20); -+} -+ -+static inline int algorithm_is_DDF(int layout) -+{ -+ return layout >= 8 && layout <= 10; -+} -+ -+extern void md_raid5_kick_device(struct r5conf *conf); -+extern int raid5_set_cache_size(struct mddev *mddev, int size); + * sched-domains (multiprocessor balancing) declarations: + */ + #ifdef CONFIG_SMP +@@ -1291,6 +1338,7 @@ + + struct task_struct { + volatile long state; /* -1 unrunnable, 0 runnable, >0 stopped */ ++ volatile long saved_state; /* saved state for "spinlock sleepers" */ + void *stack; + atomic_t usage; + unsigned int flags; /* per process flags, defined below */ +@@ -1327,6 +1375,12 @@ + #endif + + unsigned int policy; ++#ifdef CONFIG_PREEMPT_RT_FULL ++ int migrate_disable; ++# ifdef CONFIG_SCHED_DEBUG ++ int migrate_disable_atomic; ++# endif +#endif -diff -Nur linux-4.1.10.orig/drivers/misc/hwlat_detector.c linux-4.1.10/drivers/misc/hwlat_detector.c ---- linux-4.1.10.orig/drivers/misc/hwlat_detector.c 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.10/drivers/misc/hwlat_detector.c 2015-10-07 18:00:08.000000000 +0200 -@@ -0,0 +1,1240 @@ -+/* -+ * hwlat_detector.c - A simple Hardware Latency detector. -+ * -+ * Use this module to detect large system latencies induced by the behavior of -+ * certain underlying system hardware or firmware, independent of Linux itself. -+ * The code was developed originally to detect the presence of SMIs on Intel -+ * and AMD systems, although there is no dependency upon x86 herein. -+ * -+ * The classical example usage of this module is in detecting the presence of -+ * SMIs or System Management Interrupts on Intel and AMD systems. An SMI is a -+ * somewhat special form of hardware interrupt spawned from earlier CPU debug -+ * modes in which the (BIOS/EFI/etc.) firmware arranges for the South Bridge -+ * LPC (or other device) to generate a special interrupt under certain -+ * circumstances, for example, upon expiration of a special SMI timer device, -+ * due to certain external thermal readings, on certain I/O address accesses, -+ * and other situations. An SMI hits a special CPU pin, triggers a special -+ * SMI mode (complete with special memory map), and the OS is unaware. -+ * -+ * Although certain hardware-inducing latencies are necessary (for example, -+ * a modern system often requires an SMI handler for correct thermal control -+ * and remote management) they can wreak havoc upon any OS-level performance -+ * guarantees toward low-latency, especially when the OS is not even made -+ * aware of the presence of these interrupts. For this reason, we need a -+ * somewhat brute force mechanism to detect these interrupts. In this case, -+ * we do it by hogging all of the CPU(s) for configurable timer intervals, -+ * sampling the built-in CPU timer, looking for discontiguous readings. -+ * -+ * WARNING: This implementation necessarily introduces latencies. Therefore, -+ * you should NEVER use this module in a production environment -+ * requiring any kind of low-latency performance guarantee(s). -+ * -+ * Copyright (C) 2008-2009 Jon Masters, Red Hat, Inc. -+ * -+ * Includes useful feedback from Clark Williams -+ * -+ * This file is licensed under the terms of the GNU General Public -+ * License version 2. This program is licensed "as is" without any -+ * warranty of any kind, whether express or implied. -+ */ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#define BUF_SIZE_DEFAULT 262144UL /* 8K*(sizeof(entry)) */ -+#define BUF_FLAGS (RB_FL_OVERWRITE) /* no block on full */ -+#define U64STR_SIZE 22 /* 20 digits max */ -+ -+#define VERSION "1.0.0" -+#define BANNER "hwlat_detector: " -+#define DRVNAME "hwlat_detector" -+#define DEFAULT_SAMPLE_WINDOW 1000000 /* 1s */ -+#define DEFAULT_SAMPLE_WIDTH 500000 /* 0.5s */ -+#define DEFAULT_LAT_THRESHOLD 10 /* 10us */ -+ -+/* Module metadata */ -+ -+MODULE_LICENSE("GPL"); -+MODULE_AUTHOR("Jon Masters "); -+MODULE_DESCRIPTION("A simple hardware latency detector"); -+MODULE_VERSION(VERSION); -+ -+/* Module parameters */ -+ -+static int debug; -+static int enabled; -+static int threshold; -+ -+module_param(debug, int, 0); /* enable debug */ -+module_param(enabled, int, 0); /* enable detector */ -+module_param(threshold, int, 0); /* latency threshold */ -+ -+/* Buffering and sampling */ -+ -+static struct ring_buffer *ring_buffer; /* sample buffer */ -+static DEFINE_MUTEX(ring_buffer_mutex); /* lock changes */ -+static unsigned long buf_size = BUF_SIZE_DEFAULT; -+static struct task_struct *kthread; /* sampling thread */ -+ -+/* DebugFS filesystem entries */ -+ -+static struct dentry *debug_dir; /* debugfs directory */ -+static struct dentry *debug_max; /* maximum TSC delta */ -+static struct dentry *debug_count; /* total detect count */ -+static struct dentry *debug_sample_width; /* sample width us */ -+static struct dentry *debug_sample_window; /* sample window us */ -+static struct dentry *debug_sample; /* raw samples us */ -+static struct dentry *debug_threshold; /* threshold us */ -+static struct dentry *debug_enable; /* enable/disable */ -+ -+/* Individual samples and global state */ -+ -+struct sample; /* latency sample */ -+struct data; /* Global state */ + int nr_cpus_allowed; + cpumask_t cpus_allowed; + +@@ -1434,7 +1488,8 @@ + struct cputime prev_cputime; + #endif + #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN +- seqlock_t vtime_seqlock; ++ raw_spinlock_t vtime_lock; ++ seqcount_t vtime_seq; + unsigned long long vtime_snap; + enum { + VTIME_SLEEPING = 0, +@@ -1450,6 +1505,9 @@ + + struct task_cputime cputime_expires; + struct list_head cpu_timers[3]; ++#ifdef CONFIG_PREEMPT_RT_BASE ++ struct task_struct *posix_timer_list; ++#endif + + /* process credentials */ + const struct cred __rcu *real_cred; /* objective and real subjective task +@@ -1482,10 +1540,15 @@ + /* signal handlers */ + struct signal_struct *signal; + struct sighand_struct *sighand; ++ struct sigqueue *sigqueue_cache; + + sigset_t blocked, real_blocked; + sigset_t saved_sigmask; /* restored if set_restore_sigmask() was used */ + struct sigpending pending; ++#ifdef CONFIG_PREEMPT_RT_FULL ++ /* TODO: move me into ->restart_block ? */ ++ struct siginfo forced_info; ++#endif + + unsigned long sas_ss_sp; + size_t sas_ss_size; +@@ -1511,6 +1574,8 @@ + /* Protection of the PI data structures: */ + raw_spinlock_t pi_lock; + ++ struct wake_q_node wake_q; + -+/* Sampling functions */ -+static int __buffer_add_sample(struct sample *sample); -+static struct sample *buffer_get_sample(struct sample *sample); + #ifdef CONFIG_RT_MUTEXES + /* PI waiters blocked on a rt_mutex held by this task */ + struct rb_root pi_waiters; +@@ -1705,6 +1770,12 @@ + unsigned long trace; + /* bitmask and counter of trace recursion */ + unsigned long trace_recursion; ++#ifdef CONFIG_WAKEUP_LATENCY_HIST ++ u64 preempt_timestamp_hist; ++#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST ++ long timer_offset; ++#endif ++#endif + #endif /* CONFIG_TRACING */ + #ifdef CONFIG_MEMCG + struct memcg_oom_info { +@@ -1721,14 +1792,23 @@ + unsigned int sequential_io; + unsigned int sequential_io_avg; + #endif ++#ifdef CONFIG_PREEMPT_RT_BASE ++ struct rcu_head put_rcu; ++ int softirq_nestcnt; ++ unsigned int softirqs_raised; ++#endif ++#ifdef CONFIG_PREEMPT_RT_FULL ++# if defined CONFIG_HIGHMEM || defined CONFIG_X86_32 ++ int kmap_idx; ++ pte_t kmap_pte[KM_TYPE_NR]; ++# endif ++#endif + #ifdef CONFIG_DEBUG_ATOMIC_SLEEP + unsigned long task_state_change; + #endif ++ int pagefault_disabled; + }; + +-/* Future-safe accessor for struct task_struct's cpus_allowed. */ +-#define tsk_cpus_allowed(tsk) (&(tsk)->cpus_allowed) +- + #define TNF_MIGRATED 0x01 + #define TNF_NO_GROUP 0x02 + #define TNF_SHARED 0x04 +@@ -1917,6 +1997,15 @@ + extern void free_task(struct task_struct *tsk); + #define get_task_struct(tsk) do { atomic_inc(&(tsk)->usage); } while(0) + ++#ifdef CONFIG_PREEMPT_RT_BASE ++extern void __put_task_struct_cb(struct rcu_head *rhp); + -+/* Threading and state */ -+static int kthread_fn(void *unused); -+static int start_kthread(void); -+static int stop_kthread(void); -+static void __reset_stats(void); -+static int init_stats(void); ++static inline void put_task_struct(struct task_struct *t) ++{ ++ if (atomic_dec_and_test(&t->usage)) ++ call_rcu(&t->put_rcu, __put_task_struct_cb); ++} ++#else + extern void __put_task_struct(struct task_struct *t); + + static inline void put_task_struct(struct task_struct *t) +@@ -1924,6 +2013,7 @@ + if (atomic_dec_and_test(&t->usage)) + __put_task_struct(t); + } ++#endif + + #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN + extern void task_cputime(struct task_struct *t, +@@ -1962,6 +2052,7 @@ + /* + * Per process flags + */ ++#define PF_IN_SOFTIRQ 0x00000001 /* Task is serving softirq */ + #define PF_EXITING 0x00000004 /* getting shut down */ + #define PF_EXITPIDONE 0x00000008 /* pi exit done on shut down */ + #define PF_VCPU 0x00000010 /* I'm a virtual CPU */ +@@ -2126,6 +2217,10 @@ + + extern int set_cpus_allowed_ptr(struct task_struct *p, + const struct cpumask *new_mask); ++int migrate_me(void); ++void tell_sched_cpu_down_begin(int cpu); ++void tell_sched_cpu_down_done(int cpu); + -+/* Debugfs interface */ -+static ssize_t simple_data_read(struct file *filp, char __user *ubuf, -+ size_t cnt, loff_t *ppos, const u64 *entry); -+static ssize_t simple_data_write(struct file *filp, const char __user *ubuf, -+ size_t cnt, loff_t *ppos, u64 *entry); -+static int debug_sample_fopen(struct inode *inode, struct file *filp); -+static ssize_t debug_sample_fread(struct file *filp, char __user *ubuf, -+ size_t cnt, loff_t *ppos); -+static int debug_sample_release(struct inode *inode, struct file *filp); -+static int debug_enable_fopen(struct inode *inode, struct file *filp); -+static ssize_t debug_enable_fread(struct file *filp, char __user *ubuf, -+ size_t cnt, loff_t *ppos); -+static ssize_t debug_enable_fwrite(struct file *file, -+ const char __user *user_buffer, -+ size_t user_size, loff_t *offset); + #else + static inline void do_set_cpus_allowed(struct task_struct *p, + const struct cpumask *new_mask) +@@ -2138,6 +2233,9 @@ + return -EINVAL; + return 0; + } ++static inline int migrate_me(void) { return 0; } ++static inline void tell_sched_cpu_down_begin(int cpu) { } ++static inline void tell_sched_cpu_down_done(int cpu) { } + #endif + + #ifdef CONFIG_NO_HZ_COMMON +@@ -2354,6 +2452,7 @@ + + extern int wake_up_state(struct task_struct *tsk, unsigned int state); + extern int wake_up_process(struct task_struct *tsk); ++extern int wake_up_lock_sleeper(struct task_struct * tsk); + extern void wake_up_new_task(struct task_struct *tsk); + #ifdef CONFIG_SMP + extern void kick_process(struct task_struct *tsk); +@@ -2470,12 +2569,24 @@ + + /* mmdrop drops the mm and the page tables */ + extern void __mmdrop(struct mm_struct *); + -+/* Initialization functions */ -+static int init_debugfs(void); -+static void free_debugfs(void); -+static int detector_init(void); -+static void detector_exit(void); + static inline void mmdrop(struct mm_struct * mm) + { + if (unlikely(atomic_dec_and_test(&mm->mm_count))) + __mmdrop(mm); + } + ++#ifdef CONFIG_PREEMPT_RT_BASE ++extern void __mmdrop_delayed(struct rcu_head *rhp); ++static inline void mmdrop_delayed(struct mm_struct *mm) ++{ ++ if (atomic_dec_and_test(&mm->mm_count)) ++ call_rcu(&mm->delayed_drop, __mmdrop_delayed); ++} ++#else ++# define mmdrop_delayed(mm) mmdrop(mm) ++#endif + -+/* Individual latency samples are stored here when detected and packed into -+ * the ring_buffer circular buffer, where they are overwritten when -+ * more than buf_size/sizeof(sample) samples are received. */ -+struct sample { -+ u64 seqnum; /* unique sequence */ -+ u64 duration; /* ktime delta */ -+ u64 outer_duration; /* ktime delta (outer loop) */ -+ struct timespec timestamp; /* wall time */ -+ unsigned long lost; -+}; + /* mmput gets rid of the mappings and all user-space */ + extern void mmput(struct mm_struct *); + /* Grab a reference to a task's mm, if it is not already going away */ +@@ -2787,6 +2898,43 @@ + return unlikely(test_tsk_thread_flag(tsk,TIF_NEED_RESCHED)); + } + ++#ifdef CONFIG_PREEMPT_LAZY ++static inline void set_tsk_need_resched_lazy(struct task_struct *tsk) ++{ ++ set_tsk_thread_flag(tsk,TIF_NEED_RESCHED_LAZY); ++} + -+/* keep the global state somewhere. */ -+static struct data { ++static inline void clear_tsk_need_resched_lazy(struct task_struct *tsk) ++{ ++ clear_tsk_thread_flag(tsk,TIF_NEED_RESCHED_LAZY); ++} + -+ struct mutex lock; /* protect changes */ ++static inline int test_tsk_need_resched_lazy(struct task_struct *tsk) ++{ ++ return unlikely(test_tsk_thread_flag(tsk,TIF_NEED_RESCHED_LAZY)); ++} + -+ u64 count; /* total since reset */ -+ u64 max_sample; /* max hardware latency */ -+ u64 threshold; /* sample threshold level */ ++static inline int need_resched_lazy(void) ++{ ++ return test_thread_flag(TIF_NEED_RESCHED_LAZY); ++} + -+ u64 sample_window; /* total sampling window (on+off) */ -+ u64 sample_width; /* active sampling portion of window */ ++static inline int need_resched_now(void) ++{ ++ return test_thread_flag(TIF_NEED_RESCHED); ++} + -+ atomic_t sample_open; /* whether the sample file is open */ ++#else ++static inline void clear_tsk_need_resched_lazy(struct task_struct *tsk) { } ++static inline int need_resched_lazy(void) { return 0; } + -+ wait_queue_head_t wq; /* waitqeue for new sample values */ ++static inline int need_resched_now(void) ++{ ++ return test_thread_flag(TIF_NEED_RESCHED); ++} + -+} data; ++#endif + -+/** -+ * __buffer_add_sample - add a new latency sample recording to the ring buffer -+ * @sample: The new latency sample value -+ * -+ * This receives a new latency sample and records it in a global ring buffer. -+ * No additional locking is used in this case. -+ */ -+static int __buffer_add_sample(struct sample *sample) + static inline int restart_syscall(void) + { + set_tsk_thread_flag(current, TIF_SIGPENDING); +@@ -2818,6 +2966,51 @@ + return (state & TASK_INTERRUPTIBLE) || __fatal_signal_pending(p); + } + ++static inline bool __task_is_stopped_or_traced(struct task_struct *task) +{ -+ return ring_buffer_write(ring_buffer, -+ sizeof(struct sample), sample); ++ if (task->state & (__TASK_STOPPED | __TASK_TRACED)) ++ return true; ++#ifdef CONFIG_PREEMPT_RT_FULL ++ if (task->saved_state & (__TASK_STOPPED | __TASK_TRACED)) ++ return true; ++#endif ++ return false; +} + -+/** -+ * buffer_get_sample - remove a hardware latency sample from the ring buffer -+ * @sample: Pre-allocated storage for the sample -+ * -+ * This retrieves a hardware latency sample from the global circular buffer -+ */ -+static struct sample *buffer_get_sample(struct sample *sample) ++static inline bool task_is_stopped_or_traced(struct task_struct *task) +{ -+ struct ring_buffer_event *e = NULL; -+ struct sample *s = NULL; -+ unsigned int cpu = 0; ++ bool traced_stopped; + -+ if (!sample) -+ return NULL; ++#ifdef CONFIG_PREEMPT_RT_FULL ++ unsigned long flags; + -+ mutex_lock(&ring_buffer_mutex); -+ for_each_online_cpu(cpu) { -+ e = ring_buffer_consume(ring_buffer, cpu, NULL, &sample->lost); -+ if (e) -+ break; -+ } ++ raw_spin_lock_irqsave(&task->pi_lock, flags); ++ traced_stopped = __task_is_stopped_or_traced(task); ++ raw_spin_unlock_irqrestore(&task->pi_lock, flags); ++#else ++ traced_stopped = __task_is_stopped_or_traced(task); ++#endif ++ return traced_stopped; ++} + -+ if (e) { -+ s = ring_buffer_event_data(e); -+ memcpy(sample, s, sizeof(struct sample)); -+ } else -+ sample = NULL; -+ mutex_unlock(&ring_buffer_mutex); ++static inline bool task_is_traced(struct task_struct *task) ++{ ++ bool traced = false; + -+ return sample; ++ if (task->state & __TASK_TRACED) ++ return true; ++#ifdef CONFIG_PREEMPT_RT_FULL ++ /* in case the task is sleeping on tasklist_lock */ ++ raw_spin_lock_irq(&task->pi_lock); ++ if (task->state & __TASK_TRACED) ++ traced = true; ++ else if (task->saved_state & __TASK_TRACED) ++ traced = true; ++ raw_spin_unlock_irq(&task->pi_lock); ++#endif ++ return traced; +} + -+#ifndef CONFIG_TRACING -+#define time_type ktime_t -+#define time_get() ktime_get() -+#define time_to_us(x) ktime_to_us(x) -+#define time_sub(a, b) ktime_sub(a, b) -+#define init_time(a, b) (a).tv64 = b -+#define time_u64(a) ((a).tv64) + /* + * cond_resched() and cond_resched_lock(): latency reduction via + * explicit rescheduling in places that are safe. The return +@@ -2834,7 +3027,7 @@ + + extern int __cond_resched_lock(spinlock_t *lock); + +-#ifdef CONFIG_PREEMPT_COUNT ++#if defined(CONFIG_PREEMPT_COUNT) && !defined(CONFIG_PREEMPT_RT_FULL) + #define PREEMPT_LOCK_OFFSET PREEMPT_OFFSET + #else + #define PREEMPT_LOCK_OFFSET 0 +@@ -2845,12 +3038,16 @@ + __cond_resched_lock(lock); \ + }) + ++#ifndef CONFIG_PREEMPT_RT_FULL + extern int __cond_resched_softirq(void); + + #define cond_resched_softirq() ({ \ + ___might_sleep(__FILE__, __LINE__, SOFTIRQ_DISABLE_OFFSET); \ + __cond_resched_softirq(); \ + }) +#else -+#define time_type u64 -+#define time_get() trace_clock_local() -+#define time_to_us(x) div_u64(x, 1000) -+#define time_sub(a, b) ((a) - (b)) -+#define init_time(a, b) (a = b) -+#define time_u64(a) a ++# define cond_resched_softirq() cond_resched() +#endif -+/** -+ * get_sample - sample the CPU TSC and look for likely hardware latencies -+ * -+ * Used to repeatedly capture the CPU TSC (or similar), looking for potential -+ * hardware-induced latency. Called with interrupts disabled and with -+ * data.lock held. -+ */ -+static int get_sample(void) + + static inline void cond_resched_rcu(void) + { +@@ -3017,6 +3214,26 @@ + + #endif /* CONFIG_SMP */ + ++static inline int __migrate_disabled(struct task_struct *p) +{ -+ time_type start, t1, t2, last_t2; -+ s64 diff, total = 0; -+ u64 sample = 0; -+ u64 outer_sample = 0; -+ int ret = -1; -+ -+ init_time(last_t2, 0); -+ start = time_get(); /* start timestamp */ ++#ifdef CONFIG_PREEMPT_RT_FULL ++ return p->migrate_disable; ++#else ++ return 0; ++#endif ++} + -+ do { ++/* Future-safe accessor for struct task_struct's cpus_allowed. */ ++static inline const struct cpumask *tsk_cpus_allowed(struct task_struct *p) ++{ ++#ifdef CONFIG_PREEMPT_RT_FULL ++ if (p->migrate_disable) ++ return cpumask_of(task_cpu(p)); ++#endif + -+ t1 = time_get(); /* we'll look for a discontinuity */ -+ t2 = time_get(); ++ return &p->cpus_allowed; ++} + -+ if (time_u64(last_t2)) { -+ /* Check the delta from outer loop (t2 to next t1) */ -+ diff = time_to_us(time_sub(t1, last_t2)); -+ /* This shouldn't happen */ -+ if (diff < 0) { -+ pr_err(BANNER "time running backwards\n"); -+ goto out; -+ } -+ if (diff > outer_sample) -+ outer_sample = diff; -+ } -+ last_t2 = t2; + extern long sched_setaffinity(pid_t pid, const struct cpumask *new_mask); + extern long sched_getaffinity(pid_t pid, struct cpumask *mask); + +diff -Nur linux-4.1.10.orig/include/linux/seqlock.h linux-4.1.10/include/linux/seqlock.h +--- linux-4.1.10.orig/include/linux/seqlock.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/seqlock.h 2015-10-12 22:33:32.280676490 +0200 +@@ -219,20 +219,30 @@ + return __read_seqcount_retry(s, start); + } + +- +- +-static inline void raw_write_seqcount_begin(seqcount_t *s) ++static inline void __raw_write_seqcount_begin(seqcount_t *s) + { + s->sequence++; + smp_wmb(); + } + +-static inline void raw_write_seqcount_end(seqcount_t *s) ++static inline void raw_write_seqcount_begin(seqcount_t *s) ++{ ++ preempt_disable_rt(); ++ __raw_write_seqcount_begin(s); ++} + -+ total = time_to_us(time_sub(t2, start)); /* sample width */ ++static inline void __raw_write_seqcount_end(seqcount_t *s) + { + smp_wmb(); + s->sequence++; + } + ++static inline void raw_write_seqcount_end(seqcount_t *s) ++{ ++ __raw_write_seqcount_end(s); ++ preempt_enable_rt(); ++} + -+ /* This checks the inner loop (t1 to t2) */ -+ diff = time_to_us(time_sub(t2, t1)); /* current diff */ + /* + * raw_write_seqcount_latch - redirect readers to even/odd copy + * @s: pointer to seqcount_t +@@ -305,10 +315,32 @@ + /* + * Read side functions for starting and finalizing a read side section. + */ ++#ifndef CONFIG_PREEMPT_RT_FULL + static inline unsigned read_seqbegin(const seqlock_t *sl) + { + return read_seqcount_begin(&sl->seqcount); + } ++#else ++/* ++ * Starvation safe read side for RT ++ */ ++static inline unsigned read_seqbegin(seqlock_t *sl) ++{ ++ unsigned ret; + -+ /* This shouldn't happen */ -+ if (diff < 0) { -+ pr_err(BANNER "time running backwards\n"); -+ goto out; -+ } ++repeat: ++ ret = ACCESS_ONCE(sl->seqcount.sequence); ++ if (unlikely(ret & 1)) { ++ /* ++ * Take the lock and let the writer proceed (i.e. evtl ++ * boost it), otherwise we could loop here forever. ++ */ ++ spin_unlock_wait(&sl->lock); ++ goto repeat; ++ } ++ return ret; ++} ++#endif + + static inline unsigned read_seqretry(const seqlock_t *sl, unsigned start) + { +@@ -323,36 +355,36 @@ + static inline void write_seqlock(seqlock_t *sl) + { + spin_lock(&sl->lock); +- write_seqcount_begin(&sl->seqcount); ++ __raw_write_seqcount_begin(&sl->seqcount); + } + + static inline void write_sequnlock(seqlock_t *sl) + { +- write_seqcount_end(&sl->seqcount); ++ __raw_write_seqcount_end(&sl->seqcount); + spin_unlock(&sl->lock); + } + + static inline void write_seqlock_bh(seqlock_t *sl) + { + spin_lock_bh(&sl->lock); +- write_seqcount_begin(&sl->seqcount); ++ __raw_write_seqcount_begin(&sl->seqcount); + } + + static inline void write_sequnlock_bh(seqlock_t *sl) + { +- write_seqcount_end(&sl->seqcount); ++ __raw_write_seqcount_end(&sl->seqcount); + spin_unlock_bh(&sl->lock); + } + + static inline void write_seqlock_irq(seqlock_t *sl) + { + spin_lock_irq(&sl->lock); +- write_seqcount_begin(&sl->seqcount); ++ __raw_write_seqcount_begin(&sl->seqcount); + } + + static inline void write_sequnlock_irq(seqlock_t *sl) + { +- write_seqcount_end(&sl->seqcount); ++ __raw_write_seqcount_end(&sl->seqcount); + spin_unlock_irq(&sl->lock); + } + +@@ -361,7 +393,7 @@ + unsigned long flags; + + spin_lock_irqsave(&sl->lock, flags); +- write_seqcount_begin(&sl->seqcount); ++ __raw_write_seqcount_begin(&sl->seqcount); + return flags; + } + +@@ -371,7 +403,7 @@ + static inline void + write_sequnlock_irqrestore(seqlock_t *sl, unsigned long flags) + { +- write_seqcount_end(&sl->seqcount); ++ __raw_write_seqcount_end(&sl->seqcount); + spin_unlock_irqrestore(&sl->lock, flags); + } + +diff -Nur linux-4.1.10.orig/include/linux/signal.h linux-4.1.10/include/linux/signal.h +--- linux-4.1.10.orig/include/linux/signal.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/signal.h 2015-10-12 22:33:32.280676490 +0200 +@@ -218,6 +218,7 @@ + } + + extern void flush_sigqueue(struct sigpending *queue); ++extern void flush_task_sigqueue(struct task_struct *tsk); + + /* Test if 'sig' is valid signal. Use this instead of testing _NSIG directly */ + static inline int valid_signal(unsigned long sig) +diff -Nur linux-4.1.10.orig/include/linux/skbuff.h linux-4.1.10/include/linux/skbuff.h +--- linux-4.1.10.orig/include/linux/skbuff.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/skbuff.h 2015-10-12 22:33:32.280676490 +0200 +@@ -187,6 +187,7 @@ + + __u32 qlen; + spinlock_t lock; ++ raw_spinlock_t raw_lock; + }; + + struct sk_buff; +@@ -1336,6 +1337,12 @@ + __skb_queue_head_init(list); + } + ++static inline void skb_queue_head_init_raw(struct sk_buff_head *list) ++{ ++ raw_spin_lock_init(&list->raw_lock); ++ __skb_queue_head_init(list); ++} + -+ if (diff > sample) -+ sample = diff; /* only want highest value */ + static inline void skb_queue_head_init_class(struct sk_buff_head *list, + struct lock_class_key *class) + { +diff -Nur linux-4.1.10.orig/include/linux/smp.h linux-4.1.10/include/linux/smp.h +--- linux-4.1.10.orig/include/linux/smp.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/smp.h 2015-10-12 22:33:32.280676490 +0200 +@@ -185,6 +185,9 @@ + #define get_cpu() ({ preempt_disable(); smp_processor_id(); }) + #define put_cpu() preempt_enable() + ++#define get_cpu_light() ({ migrate_disable(); smp_processor_id(); }) ++#define put_cpu_light() migrate_enable() + -+ } while (total <= data.sample_width); + /* + * Callback to arch code if there's nosmp or maxcpus=0 on the + * boot command line: +diff -Nur linux-4.1.10.orig/include/linux/spinlock_api_smp.h linux-4.1.10/include/linux/spinlock_api_smp.h +--- linux-4.1.10.orig/include/linux/spinlock_api_smp.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/spinlock_api_smp.h 2015-10-12 22:33:32.280676490 +0200 +@@ -189,6 +189,8 @@ + return 0; + } + +-#include ++#ifndef CONFIG_PREEMPT_RT_FULL ++# include ++#endif + + #endif /* __LINUX_SPINLOCK_API_SMP_H */ +diff -Nur linux-4.1.10.orig/include/linux/spinlock.h linux-4.1.10/include/linux/spinlock.h +--- linux-4.1.10.orig/include/linux/spinlock.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/spinlock.h 2015-10-12 22:33:32.280676490 +0200 +@@ -281,7 +281,11 @@ + #define raw_spin_can_lock(lock) (!raw_spin_is_locked(lock)) + + /* Include rwlock functions */ +-#include ++#ifdef CONFIG_PREEMPT_RT_FULL ++# include ++#else ++# include ++#endif + + /* + * Pull the _spin_*()/_read_*()/_write_*() functions/declarations: +@@ -292,6 +296,10 @@ + # include + #endif + ++#ifdef CONFIG_PREEMPT_RT_FULL ++# include ++#else /* PREEMPT_RT_FULL */ + -+ ret = 0; + /* + * Map the spin_lock functions to the raw variants for PREEMPT_RT=n + */ +@@ -426,4 +434,6 @@ + #define atomic_dec_and_lock(atomic, lock) \ + __cond_lock(lock, _atomic_dec_and_lock(atomic, lock)) + ++#endif /* !PREEMPT_RT_FULL */ + -+ /* If we exceed the threshold value, we have found a hardware latency */ -+ if (sample > data.threshold || outer_sample > data.threshold) { -+ struct sample s; + #endif /* __LINUX_SPINLOCK_H */ +diff -Nur linux-4.1.10.orig/include/linux/spinlock_rt.h linux-4.1.10/include/linux/spinlock_rt.h +--- linux-4.1.10.orig/include/linux/spinlock_rt.h 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/include/linux/spinlock_rt.h 2015-10-12 22:33:32.280676490 +0200 +@@ -0,0 +1,174 @@ ++#ifndef __LINUX_SPINLOCK_RT_H ++#define __LINUX_SPINLOCK_RT_H + -+ ret = 1; ++#ifndef __LINUX_SPINLOCK_H ++#error Do not include directly. Use spinlock.h ++#endif + -+ data.count++; -+ s.seqnum = data.count; -+ s.duration = sample; -+ s.outer_duration = outer_sample; -+ s.timestamp = CURRENT_TIME; -+ __buffer_add_sample(&s); ++#include + -+ /* Keep a running maximum ever recorded hardware latency */ -+ if (sample > data.max_sample) -+ data.max_sample = sample; -+ } ++extern void ++__rt_spin_lock_init(spinlock_t *lock, char *name, struct lock_class_key *key); + -+out: -+ return ret; -+} ++#define spin_lock_init(slock) \ ++do { \ ++ static struct lock_class_key __key; \ ++ \ ++ rt_mutex_init(&(slock)->lock); \ ++ __rt_spin_lock_init(slock, #slock, &__key); \ ++} while (0) ++ ++extern void __lockfunc rt_spin_lock(spinlock_t *lock); ++extern unsigned long __lockfunc rt_spin_lock_trace_flags(spinlock_t *lock); ++extern void __lockfunc rt_spin_lock_nested(spinlock_t *lock, int subclass); ++extern void __lockfunc rt_spin_unlock(spinlock_t *lock); ++extern void __lockfunc rt_spin_unlock_wait(spinlock_t *lock); ++extern int __lockfunc rt_spin_trylock_irqsave(spinlock_t *lock, unsigned long *flags); ++extern int __lockfunc rt_spin_trylock_bh(spinlock_t *lock); ++extern int __lockfunc rt_spin_trylock(spinlock_t *lock); ++extern int atomic_dec_and_spin_lock(atomic_t *atomic, spinlock_t *lock); + +/* -+ * kthread_fn - The CPU time sampling/hardware latency detection kernel thread -+ * @unused: A required part of the kthread API. -+ * -+ * Used to periodically sample the CPU TSC via a call to get_sample. We -+ * disable interrupts, which does (intentionally) introduce latency since we -+ * need to ensure nothing else might be running (and thus pre-empting). -+ * Obviously this should never be used in production environments. -+ * -+ * Currently this runs on which ever CPU it was scheduled on, but most -+ * real-worald hardware latency situations occur across several CPUs, -+ * but we might later generalize this if we find there are any actualy -+ * systems with alternate SMI delivery or other hardware latencies. ++ * lockdep-less calls, for derived types like rwlock: ++ * (for trylock they can use rt_mutex_trylock() directly. + */ -+static int kthread_fn(void *unused) -+{ -+ int ret; -+ u64 interval; ++extern void __lockfunc __rt_spin_lock(struct rt_mutex *lock); ++extern void __lockfunc __rt_spin_unlock(struct rt_mutex *lock); ++extern int __lockfunc __rt_spin_trylock(struct rt_mutex *lock); + -+ while (!kthread_should_stop()) { ++#define spin_lock(lock) \ ++ do { \ ++ migrate_disable(); \ ++ rt_spin_lock(lock); \ ++ } while (0) + -+ mutex_lock(&data.lock); ++#define spin_lock_bh(lock) \ ++ do { \ ++ local_bh_disable(); \ ++ migrate_disable(); \ ++ rt_spin_lock(lock); \ ++ } while (0) + -+ local_irq_disable(); -+ ret = get_sample(); -+ local_irq_enable(); ++#define spin_lock_irq(lock) spin_lock(lock) + -+ if (ret > 0) -+ wake_up(&data.wq); /* wake up reader(s) */ ++#define spin_do_trylock(lock) __cond_lock(lock, rt_spin_trylock(lock)) + -+ interval = data.sample_window - data.sample_width; -+ do_div(interval, USEC_PER_MSEC); /* modifies interval value */ ++#define spin_trylock(lock) \ ++({ \ ++ int __locked; \ ++ migrate_disable(); \ ++ __locked = spin_do_trylock(lock); \ ++ if (!__locked) \ ++ migrate_enable(); \ ++ __locked; \ ++}) + -+ mutex_unlock(&data.lock); ++#ifdef CONFIG_LOCKDEP ++# define spin_lock_nested(lock, subclass) \ ++ do { \ ++ migrate_disable(); \ ++ rt_spin_lock_nested(lock, subclass); \ ++ } while (0) + -+ if (msleep_interruptible(interval)) -+ break; -+ } ++#define spin_lock_bh_nested(lock, subclass) \ ++ do { \ ++ local_bh_disable(); \ ++ migrate_disable(); \ ++ rt_spin_lock_nested(lock, subclass); \ ++ } while (0) + -+ return 0; -+} ++# define spin_lock_irqsave_nested(lock, flags, subclass) \ ++ do { \ ++ typecheck(unsigned long, flags); \ ++ flags = 0; \ ++ migrate_disable(); \ ++ rt_spin_lock_nested(lock, subclass); \ ++ } while (0) ++#else ++# define spin_lock_nested(lock, subclass) spin_lock(lock) ++# define spin_lock_bh_nested(lock, subclass) spin_lock_bh(lock) + -+/** -+ * start_kthread - Kick off the hardware latency sampling/detector kthread -+ * -+ * This starts a kernel thread that will sit and sample the CPU timestamp -+ * counter (TSC or similar) and look for potential hardware latencies. -+ */ -+static int start_kthread(void) -+{ -+ kthread = kthread_run(kthread_fn, NULL, -+ DRVNAME); -+ if (IS_ERR(kthread)) { -+ pr_err(BANNER "could not start sampling thread\n"); -+ enabled = 0; -+ return -ENOMEM; -+ } ++# define spin_lock_irqsave_nested(lock, flags, subclass) \ ++ do { \ ++ typecheck(unsigned long, flags); \ ++ flags = 0; \ ++ spin_lock(lock); \ ++ } while (0) ++#endif + -+ return 0; -+} ++#define spin_lock_irqsave(lock, flags) \ ++ do { \ ++ typecheck(unsigned long, flags); \ ++ flags = 0; \ ++ spin_lock(lock); \ ++ } while (0) + -+/** -+ * stop_kthread - Inform the hardware latency samping/detector kthread to stop -+ * -+ * This kicks the running hardware latency sampling/detector kernel thread and -+ * tells it to stop sampling now. Use this on unload and at system shutdown. -+ */ -+static int stop_kthread(void) ++static inline unsigned long spin_lock_trace_flags(spinlock_t *lock) +{ -+ int ret; -+ -+ ret = kthread_stop(kthread); -+ -+ return ret; ++ unsigned long flags = 0; ++#ifdef CONFIG_TRACE_IRQFLAGS ++ flags = rt_spin_lock_trace_flags(lock); ++#else ++ spin_lock(lock); /* lock_local */ ++#endif ++ return flags; +} + -+/** -+ * __reset_stats - Reset statistics for the hardware latency detector -+ * -+ * We use data to store various statistics and global state. We call this -+ * function in order to reset those when "enable" is toggled on or off, and -+ * also at initialization. Should be called with data.lock held. -+ */ -+static void __reset_stats(void) -+{ -+ data.count = 0; -+ data.max_sample = 0; -+ ring_buffer_reset(ring_buffer); /* flush out old sample entries */ -+} ++/* FIXME: we need rt_spin_lock_nest_lock */ ++#define spin_lock_nest_lock(lock, nest_lock) spin_lock_nested(lock, 0) + -+/** -+ * init_stats - Setup global state statistics for the hardware latency detector -+ * -+ * We use data to store various statistics and global state. We also use -+ * a global ring buffer (ring_buffer) to keep raw samples of detected hardware -+ * induced system latencies. This function initializes these structures and -+ * allocates the global ring buffer also. -+ */ -+static int init_stats(void) -+{ -+ int ret = -ENOMEM; ++#define spin_unlock(lock) \ ++ do { \ ++ rt_spin_unlock(lock); \ ++ migrate_enable(); \ ++ } while (0) + -+ mutex_init(&data.lock); -+ init_waitqueue_head(&data.wq); -+ atomic_set(&data.sample_open, 0); ++#define spin_unlock_bh(lock) \ ++ do { \ ++ rt_spin_unlock(lock); \ ++ migrate_enable(); \ ++ local_bh_enable(); \ ++ } while (0) + -+ ring_buffer = ring_buffer_alloc(buf_size, BUF_FLAGS); ++#define spin_unlock_irq(lock) spin_unlock(lock) + -+ if (WARN(!ring_buffer, KERN_ERR BANNER -+ "failed to allocate ring buffer!\n")) -+ goto out; ++#define spin_unlock_irqrestore(lock, flags) \ ++ do { \ ++ typecheck(unsigned long, flags); \ ++ (void) flags; \ ++ spin_unlock(lock); \ ++ } while (0) + -+ __reset_stats(); -+ data.threshold = threshold ?: DEFAULT_LAT_THRESHOLD; /* threshold us */ -+ data.sample_window = DEFAULT_SAMPLE_WINDOW; /* window us */ -+ data.sample_width = DEFAULT_SAMPLE_WIDTH; /* width us */ ++#define spin_trylock_bh(lock) __cond_lock(lock, rt_spin_trylock_bh(lock)) ++#define spin_trylock_irq(lock) spin_trylock(lock) + -+ ret = 0; ++#define spin_trylock_irqsave(lock, flags) \ ++ rt_spin_trylock_irqsave(lock, &(flags)) + -+out: -+ return ret; ++#define spin_unlock_wait(lock) rt_spin_unlock_wait(lock) + ++#ifdef CONFIG_GENERIC_LOCKBREAK ++# define spin_is_contended(lock) ((lock)->break_lock) ++#else ++# define spin_is_contended(lock) (((void)(lock), 0)) ++#endif ++ ++static inline int spin_can_lock(spinlock_t *lock) ++{ ++ return !rt_mutex_is_locked(&lock->lock); +} + -+/* -+ * simple_data_read - Wrapper read function for global state debugfs entries -+ * @filp: The active open file structure for the debugfs "file" -+ * @ubuf: The userspace provided buffer to read value into -+ * @cnt: The maximum number of bytes to read -+ * @ppos: The current "file" position -+ * @entry: The entry to read from -+ * -+ * This function provides a generic read implementation for the global state -+ * "data" structure debugfs filesystem entries. It would be nice to use -+ * simple_attr_read directly, but we need to make sure that the data.lock -+ * is held during the actual read. -+ */ -+static ssize_t simple_data_read(struct file *filp, char __user *ubuf, -+ size_t cnt, loff_t *ppos, const u64 *entry) ++static inline int spin_is_locked(spinlock_t *lock) +{ -+ char buf[U64STR_SIZE]; -+ u64 val = 0; -+ int len = 0; ++ return rt_mutex_is_locked(&lock->lock); ++} + -+ memset(buf, 0, sizeof(buf)); ++static inline void assert_spin_locked(spinlock_t *lock) ++{ ++ BUG_ON(!spin_is_locked(lock)); ++} + -+ if (!entry) -+ return -EFAULT; ++#define atomic_dec_and_lock(atomic, lock) \ ++ atomic_dec_and_spin_lock(atomic, lock) + -+ mutex_lock(&data.lock); -+ val = *entry; -+ mutex_unlock(&data.lock); -+ -+ len = snprintf(buf, sizeof(buf), "%llu\n", (unsigned long long)val); -+ -+ return simple_read_from_buffer(ubuf, cnt, ppos, buf, len); ++#endif +diff -Nur linux-4.1.10.orig/include/linux/spinlock_types.h linux-4.1.10/include/linux/spinlock_types.h +--- linux-4.1.10.orig/include/linux/spinlock_types.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/spinlock_types.h 2015-10-12 22:33:32.280676490 +0200 +@@ -9,80 +9,15 @@ + * Released under the General Public License (GPL). + */ + +-#if defined(CONFIG_SMP) +-# include +-#else +-# include +-#endif +- +-#include +- +-typedef struct raw_spinlock { +- arch_spinlock_t raw_lock; +-#ifdef CONFIG_GENERIC_LOCKBREAK +- unsigned int break_lock; +-#endif +-#ifdef CONFIG_DEBUG_SPINLOCK +- unsigned int magic, owner_cpu; +- void *owner; +-#endif +-#ifdef CONFIG_DEBUG_LOCK_ALLOC +- struct lockdep_map dep_map; +-#endif +-} raw_spinlock_t; +- +-#define SPINLOCK_MAGIC 0xdead4ead +- +-#define SPINLOCK_OWNER_INIT ((void *)-1L) +- +-#ifdef CONFIG_DEBUG_LOCK_ALLOC +-# define SPIN_DEP_MAP_INIT(lockname) .dep_map = { .name = #lockname } +-#else +-# define SPIN_DEP_MAP_INIT(lockname) +-#endif ++#include + +-#ifdef CONFIG_DEBUG_SPINLOCK +-# define SPIN_DEBUG_INIT(lockname) \ +- .magic = SPINLOCK_MAGIC, \ +- .owner_cpu = -1, \ +- .owner = SPINLOCK_OWNER_INIT, ++#ifndef CONFIG_PREEMPT_RT_FULL ++# include ++# include + #else +-# define SPIN_DEBUG_INIT(lockname) ++# include ++# include ++# include + #endif + +-#define __RAW_SPIN_LOCK_INITIALIZER(lockname) \ +- { \ +- .raw_lock = __ARCH_SPIN_LOCK_UNLOCKED, \ +- SPIN_DEBUG_INIT(lockname) \ +- SPIN_DEP_MAP_INIT(lockname) } +- +-#define __RAW_SPIN_LOCK_UNLOCKED(lockname) \ +- (raw_spinlock_t) __RAW_SPIN_LOCK_INITIALIZER(lockname) +- +-#define DEFINE_RAW_SPINLOCK(x) raw_spinlock_t x = __RAW_SPIN_LOCK_UNLOCKED(x) +- +-typedef struct spinlock { +- union { +- struct raw_spinlock rlock; +- +-#ifdef CONFIG_DEBUG_LOCK_ALLOC +-# define LOCK_PADSIZE (offsetof(struct raw_spinlock, dep_map)) +- struct { +- u8 __padding[LOCK_PADSIZE]; +- struct lockdep_map dep_map; +- }; +-#endif +- }; +-} spinlock_t; +- +-#define __SPIN_LOCK_INITIALIZER(lockname) \ +- { { .rlock = __RAW_SPIN_LOCK_INITIALIZER(lockname) } } +- +-#define __SPIN_LOCK_UNLOCKED(lockname) \ +- (spinlock_t ) __SPIN_LOCK_INITIALIZER(lockname) +- +-#define DEFINE_SPINLOCK(x) spinlock_t x = __SPIN_LOCK_UNLOCKED(x) +- +-#include +- + #endif /* __LINUX_SPINLOCK_TYPES_H */ +diff -Nur linux-4.1.10.orig/include/linux/spinlock_types_nort.h linux-4.1.10/include/linux/spinlock_types_nort.h +--- linux-4.1.10.orig/include/linux/spinlock_types_nort.h 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/include/linux/spinlock_types_nort.h 2015-10-12 22:33:32.280676490 +0200 +@@ -0,0 +1,33 @@ ++#ifndef __LINUX_SPINLOCK_TYPES_NORT_H ++#define __LINUX_SPINLOCK_TYPES_NORT_H + -+} ++#ifndef __LINUX_SPINLOCK_TYPES_H ++#error "Do not include directly. Include spinlock_types.h instead" ++#endif + +/* -+ * simple_data_write - Wrapper write function for global state debugfs entries -+ * @filp: The active open file structure for the debugfs "file" -+ * @ubuf: The userspace provided buffer to write value from -+ * @cnt: The maximum number of bytes to write -+ * @ppos: The current "file" position -+ * @entry: The entry to write to -+ * -+ * This function provides a generic write implementation for the global state -+ * "data" structure debugfs filesystem entries. It would be nice to use -+ * simple_attr_write directly, but we need to make sure that the data.lock -+ * is held during the actual write. ++ * The non RT version maps spinlocks to raw_spinlocks + */ -+static ssize_t simple_data_write(struct file *filp, const char __user *ubuf, -+ size_t cnt, loff_t *ppos, u64 *entry) -+{ -+ char buf[U64STR_SIZE]; -+ int csize = min(cnt, sizeof(buf)); -+ u64 val = 0; -+ int err = 0; -+ -+ memset(buf, '\0', sizeof(buf)); -+ if (copy_from_user(buf, ubuf, csize)) -+ return -EFAULT; -+ -+ buf[U64STR_SIZE-1] = '\0'; /* just in case */ -+ err = kstrtoull(buf, 10, &val); -+ if (err) -+ return -EINVAL; ++typedef struct spinlock { ++ union { ++ struct raw_spinlock rlock; + -+ mutex_lock(&data.lock); -+ *entry = val; -+ mutex_unlock(&data.lock); ++#ifdef CONFIG_DEBUG_LOCK_ALLOC ++# define LOCK_PADSIZE (offsetof(struct raw_spinlock, dep_map)) ++ struct { ++ u8 __padding[LOCK_PADSIZE]; ++ struct lockdep_map dep_map; ++ }; ++#endif ++ }; ++} spinlock_t; + -+ return csize; -+} ++#define __SPIN_LOCK_INITIALIZER(lockname) \ ++ { { .rlock = __RAW_SPIN_LOCK_INITIALIZER(lockname) } } + -+/** -+ * debug_count_fopen - Open function for "count" debugfs entry -+ * @inode: The in-kernel inode representation of the debugfs "file" -+ * @filp: The active open file structure for the debugfs "file" -+ * -+ * This function provides an open implementation for the "count" debugfs -+ * interface to the hardware latency detector. -+ */ -+static int debug_count_fopen(struct inode *inode, struct file *filp) -+{ -+ return 0; -+} ++#define __SPIN_LOCK_UNLOCKED(lockname) \ ++ (spinlock_t ) __SPIN_LOCK_INITIALIZER(lockname) + -+/** -+ * debug_count_fread - Read function for "count" debugfs entry -+ * @filp: The active open file structure for the debugfs "file" -+ * @ubuf: The userspace provided buffer to read value into -+ * @cnt: The maximum number of bytes to read -+ * @ppos: The current "file" position -+ * -+ * This function provides a read implementation for the "count" debugfs -+ * interface to the hardware latency detector. Can be used to read the -+ * number of latency readings exceeding the configured threshold since -+ * the detector was last reset (e.g. by writing a zero into "count"). -+ */ -+static ssize_t debug_count_fread(struct file *filp, char __user *ubuf, -+ size_t cnt, loff_t *ppos) -+{ -+ return simple_data_read(filp, ubuf, cnt, ppos, &data.count); -+} ++#define DEFINE_SPINLOCK(x) spinlock_t x = __SPIN_LOCK_UNLOCKED(x) + -+/** -+ * debug_count_fwrite - Write function for "count" debugfs entry -+ * @filp: The active open file structure for the debugfs "file" -+ * @ubuf: The user buffer that contains the value to write -+ * @cnt: The maximum number of bytes to write to "file" -+ * @ppos: The current position in the debugfs "file" -+ * -+ * This function provides a write implementation for the "count" debugfs -+ * interface to the hardware latency detector. Can be used to write a -+ * desired value, especially to zero the total count. -+ */ -+static ssize_t debug_count_fwrite(struct file *filp, -+ const char __user *ubuf, -+ size_t cnt, -+ loff_t *ppos) -+{ -+ return simple_data_write(filp, ubuf, cnt, ppos, &data.count); -+} ++#endif +diff -Nur linux-4.1.10.orig/include/linux/spinlock_types_raw.h linux-4.1.10/include/linux/spinlock_types_raw.h +--- linux-4.1.10.orig/include/linux/spinlock_types_raw.h 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/include/linux/spinlock_types_raw.h 2015-10-12 22:33:32.284676225 +0200 +@@ -0,0 +1,56 @@ ++#ifndef __LINUX_SPINLOCK_TYPES_RAW_H ++#define __LINUX_SPINLOCK_TYPES_RAW_H + -+/** -+ * debug_enable_fopen - Dummy open function for "enable" debugfs interface -+ * @inode: The in-kernel inode representation of the debugfs "file" -+ * @filp: The active open file structure for the debugfs "file" -+ * -+ * This function provides an open implementation for the "enable" debugfs -+ * interface to the hardware latency detector. -+ */ -+static int debug_enable_fopen(struct inode *inode, struct file *filp) -+{ -+ return 0; -+} ++#if defined(CONFIG_SMP) ++# include ++#else ++# include ++#endif + -+/** -+ * debug_enable_fread - Read function for "enable" debugfs interface -+ * @filp: The active open file structure for the debugfs "file" -+ * @ubuf: The userspace provided buffer to read value into -+ * @cnt: The maximum number of bytes to read -+ * @ppos: The current "file" position -+ * -+ * This function provides a read implementation for the "enable" debugfs -+ * interface to the hardware latency detector. Can be used to determine -+ * whether the detector is currently enabled ("0\n" or "1\n" returned). -+ */ -+static ssize_t debug_enable_fread(struct file *filp, char __user *ubuf, -+ size_t cnt, loff_t *ppos) -+{ -+ char buf[4]; ++#include + -+ if ((cnt < sizeof(buf)) || (*ppos)) -+ return 0; ++typedef struct raw_spinlock { ++ arch_spinlock_t raw_lock; ++#ifdef CONFIG_GENERIC_LOCKBREAK ++ unsigned int break_lock; ++#endif ++#ifdef CONFIG_DEBUG_SPINLOCK ++ unsigned int magic, owner_cpu; ++ void *owner; ++#endif ++#ifdef CONFIG_DEBUG_LOCK_ALLOC ++ struct lockdep_map dep_map; ++#endif ++} raw_spinlock_t; + -+ buf[0] = enabled ? '1' : '0'; -+ buf[1] = '\n'; -+ buf[2] = '\0'; -+ if (copy_to_user(ubuf, buf, strlen(buf))) -+ return -EFAULT; -+ return *ppos = strlen(buf); -+} ++#define SPINLOCK_MAGIC 0xdead4ead + -+/** -+ * debug_enable_fwrite - Write function for "enable" debugfs interface -+ * @filp: The active open file structure for the debugfs "file" -+ * @ubuf: The user buffer that contains the value to write -+ * @cnt: The maximum number of bytes to write to "file" -+ * @ppos: The current position in the debugfs "file" -+ * -+ * This function provides a write implementation for the "enable" debugfs -+ * interface to the hardware latency detector. Can be used to enable or -+ * disable the detector, which will have the side-effect of possibly -+ * also resetting the global stats and kicking off the measuring -+ * kthread (on an enable) or the converse (upon a disable). -+ */ -+static ssize_t debug_enable_fwrite(struct file *filp, -+ const char __user *ubuf, -+ size_t cnt, -+ loff_t *ppos) -+{ -+ char buf[4]; -+ int csize = min(cnt, sizeof(buf)); -+ long val = 0; -+ int err = 0; ++#define SPINLOCK_OWNER_INIT ((void *)-1L) + -+ memset(buf, '\0', sizeof(buf)); -+ if (copy_from_user(buf, ubuf, csize)) -+ return -EFAULT; ++#ifdef CONFIG_DEBUG_LOCK_ALLOC ++# define SPIN_DEP_MAP_INIT(lockname) .dep_map = { .name = #lockname } ++#else ++# define SPIN_DEP_MAP_INIT(lockname) ++#endif + -+ buf[sizeof(buf)-1] = '\0'; /* just in case */ -+ err = kstrtoul(buf, 10, &val); -+ if (0 != err) -+ return -EINVAL; ++#ifdef CONFIG_DEBUG_SPINLOCK ++# define SPIN_DEBUG_INIT(lockname) \ ++ .magic = SPINLOCK_MAGIC, \ ++ .owner_cpu = -1, \ ++ .owner = SPINLOCK_OWNER_INIT, ++#else ++# define SPIN_DEBUG_INIT(lockname) ++#endif + -+ if (val) { -+ if (enabled) -+ goto unlock; -+ enabled = 1; -+ __reset_stats(); -+ if (start_kthread()) -+ return -EFAULT; -+ } else { -+ if (!enabled) -+ goto unlock; -+ enabled = 0; -+ err = stop_kthread(); -+ if (err) { -+ pr_err(BANNER "cannot stop kthread\n"); -+ return -EFAULT; -+ } -+ wake_up(&data.wq); /* reader(s) should return */ -+ } -+unlock: -+ return csize; -+} ++#define __RAW_SPIN_LOCK_INITIALIZER(lockname) \ ++ { \ ++ .raw_lock = __ARCH_SPIN_LOCK_UNLOCKED, \ ++ SPIN_DEBUG_INIT(lockname) \ ++ SPIN_DEP_MAP_INIT(lockname) } + -+/** -+ * debug_max_fopen - Open function for "max" debugfs entry -+ * @inode: The in-kernel inode representation of the debugfs "file" -+ * @filp: The active open file structure for the debugfs "file" -+ * -+ * This function provides an open implementation for the "max" debugfs -+ * interface to the hardware latency detector. -+ */ -+static int debug_max_fopen(struct inode *inode, struct file *filp) -+{ -+ return 0; -+} ++#define __RAW_SPIN_LOCK_UNLOCKED(lockname) \ ++ (raw_spinlock_t) __RAW_SPIN_LOCK_INITIALIZER(lockname) + -+/** -+ * debug_max_fread - Read function for "max" debugfs entry -+ * @filp: The active open file structure for the debugfs "file" -+ * @ubuf: The userspace provided buffer to read value into -+ * @cnt: The maximum number of bytes to read -+ * @ppos: The current "file" position -+ * -+ * This function provides a read implementation for the "max" debugfs -+ * interface to the hardware latency detector. Can be used to determine -+ * the maximum latency value observed since it was last reset. -+ */ -+static ssize_t debug_max_fread(struct file *filp, char __user *ubuf, -+ size_t cnt, loff_t *ppos) -+{ -+ return simple_data_read(filp, ubuf, cnt, ppos, &data.max_sample); -+} ++#define DEFINE_RAW_SPINLOCK(x) raw_spinlock_t x = __RAW_SPIN_LOCK_UNLOCKED(x) + -+/** -+ * debug_max_fwrite - Write function for "max" debugfs entry -+ * @filp: The active open file structure for the debugfs "file" -+ * @ubuf: The user buffer that contains the value to write -+ * @cnt: The maximum number of bytes to write to "file" -+ * @ppos: The current position in the debugfs "file" -+ * -+ * This function provides a write implementation for the "max" debugfs -+ * interface to the hardware latency detector. Can be used to reset the -+ * maximum or set it to some other desired value - if, then, subsequent -+ * measurements exceed this value, the maximum will be updated. -+ */ -+static ssize_t debug_max_fwrite(struct file *filp, -+ const char __user *ubuf, -+ size_t cnt, -+ loff_t *ppos) -+{ -+ return simple_data_write(filp, ubuf, cnt, ppos, &data.max_sample); -+} ++#endif +diff -Nur linux-4.1.10.orig/include/linux/spinlock_types_rt.h linux-4.1.10/include/linux/spinlock_types_rt.h +--- linux-4.1.10.orig/include/linux/spinlock_types_rt.h 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/include/linux/spinlock_types_rt.h 2015-10-12 22:33:32.284676225 +0200 +@@ -0,0 +1,51 @@ ++#ifndef __LINUX_SPINLOCK_TYPES_RT_H ++#define __LINUX_SPINLOCK_TYPES_RT_H + ++#ifndef __LINUX_SPINLOCK_TYPES_H ++#error "Do not include directly. Include spinlock_types.h instead" ++#endif + -+/** -+ * debug_sample_fopen - An open function for "sample" debugfs interface -+ * @inode: The in-kernel inode representation of this debugfs "file" -+ * @filp: The active open file structure for the debugfs "file" -+ * -+ * This function handles opening the "sample" file within the hardware -+ * latency detector debugfs directory interface. This file is used to read -+ * raw samples from the global ring_buffer and allows the user to see a -+ * running latency history. Can be opened blocking or non-blocking, -+ * affecting whether it behaves as a buffer read pipe, or does not. -+ * Implements simple locking to prevent multiple simultaneous use. -+ */ -+static int debug_sample_fopen(struct inode *inode, struct file *filp) -+{ -+ if (!atomic_add_unless(&data.sample_open, 1, 1)) -+ return -EBUSY; -+ else -+ return 0; -+} ++#include + -+/** -+ * debug_sample_fread - A read function for "sample" debugfs interface -+ * @filp: The active open file structure for the debugfs "file" -+ * @ubuf: The user buffer that will contain the samples read -+ * @cnt: The maximum bytes to read from the debugfs "file" -+ * @ppos: The current position in the debugfs "file" -+ * -+ * This function handles reading from the "sample" file within the hardware -+ * latency detector debugfs directory interface. This file is used to read -+ * raw samples from the global ring_buffer and allows the user to see a -+ * running latency history. By default this will block pending a new -+ * value written into the sample buffer, unless there are already a -+ * number of value(s) waiting in the buffer, or the sample file was -+ * previously opened in a non-blocking mode of operation. ++/* ++ * PREEMPT_RT: spinlocks - an RT mutex plus lock-break field: + */ -+static ssize_t debug_sample_fread(struct file *filp, char __user *ubuf, -+ size_t cnt, loff_t *ppos) -+{ -+ int len = 0; -+ char buf[64]; -+ struct sample *sample = NULL; -+ -+ if (!enabled) -+ return 0; -+ -+ sample = kzalloc(sizeof(struct sample), GFP_KERNEL); -+ if (!sample) -+ return -ENOMEM; -+ -+ while (!buffer_get_sample(sample)) { -+ -+ DEFINE_WAIT(wait); -+ -+ if (filp->f_flags & O_NONBLOCK) { -+ len = -EAGAIN; -+ goto out; -+ } -+ -+ prepare_to_wait(&data.wq, &wait, TASK_INTERRUPTIBLE); -+ schedule(); -+ finish_wait(&data.wq, &wait); -+ -+ if (signal_pending(current)) { -+ len = -EINTR; -+ goto out; -+ } ++typedef struct spinlock { ++ struct rt_mutex lock; ++ unsigned int break_lock; ++#ifdef CONFIG_DEBUG_LOCK_ALLOC ++ struct lockdep_map dep_map; ++#endif ++} spinlock_t; + -+ if (!enabled) { /* enable was toggled */ -+ len = 0; -+ goto out; -+ } ++#ifdef CONFIG_DEBUG_RT_MUTEXES ++# define __RT_SPIN_INITIALIZER(name) \ ++ { \ ++ .wait_lock = __RAW_SPIN_LOCK_UNLOCKED(name.wait_lock), \ ++ .save_state = 1, \ ++ .file = __FILE__, \ ++ .line = __LINE__ , \ + } ++#else ++# define __RT_SPIN_INITIALIZER(name) \ ++ { \ ++ .wait_lock = __RAW_SPIN_LOCK_UNLOCKED(name.wait_lock), \ ++ .save_state = 1, \ ++ } ++#endif + -+ len = snprintf(buf, sizeof(buf), "%010lu.%010lu\t%llu\t%llu\n", -+ sample->timestamp.tv_sec, -+ sample->timestamp.tv_nsec, -+ sample->duration, -+ sample->outer_duration); -+ ++/* ++.wait_list = PLIST_HEAD_INIT_RAW((name).lock.wait_list, (name).lock.wait_lock) ++*/ + -+ /* handling partial reads is more trouble than it's worth */ -+ if (len > cnt) -+ goto out; ++#define __SPIN_LOCK_UNLOCKED(name) \ ++ { .lock = __RT_SPIN_INITIALIZER(name.lock), \ ++ SPIN_DEP_MAP_INIT(name) } + -+ if (copy_to_user(ubuf, buf, len)) -+ len = -EFAULT; ++#define __DEFINE_SPINLOCK(name) \ ++ spinlock_t name = __SPIN_LOCK_UNLOCKED(name) + -+out: -+ kfree(sample); -+ return len; -+} ++#define DEFINE_SPINLOCK(name) \ ++ spinlock_t name __cacheline_aligned_in_smp = __SPIN_LOCK_UNLOCKED(name) + -+/** -+ * debug_sample_release - Release function for "sample" debugfs interface -+ * @inode: The in-kernel inode represenation of the debugfs "file" -+ * @filp: The active open file structure for the debugfs "file" -+ * -+ * This function completes the close of the debugfs interface "sample" file. -+ * Frees the sample_open "lock" so that other users may open the interface. -+ */ -+static int debug_sample_release(struct inode *inode, struct file *filp) -+{ -+ atomic_dec(&data.sample_open); ++#endif +diff -Nur linux-4.1.10.orig/include/linux/srcu.h linux-4.1.10/include/linux/srcu.h +--- linux-4.1.10.orig/include/linux/srcu.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/srcu.h 2015-10-12 22:33:32.284676225 +0200 +@@ -84,10 +84,10 @@ + + void process_srcu(struct work_struct *work); + +-#define __SRCU_STRUCT_INIT(name) \ ++#define __SRCU_STRUCT_INIT(name, pcpu_name) \ + { \ + .completed = -300, \ +- .per_cpu_ref = &name##_srcu_array, \ ++ .per_cpu_ref = &pcpu_name, \ + .queue_lock = __SPIN_LOCK_UNLOCKED(name.queue_lock), \ + .running = false, \ + .batch_queue = RCU_BATCH_INIT(name.batch_queue), \ +@@ -104,7 +104,7 @@ + */ + #define __DEFINE_SRCU(name, is_static) \ + static DEFINE_PER_CPU(struct srcu_struct_array, name##_srcu_array);\ +- is_static struct srcu_struct name = __SRCU_STRUCT_INIT(name) ++ is_static struct srcu_struct name = __SRCU_STRUCT_INIT(name, name##_srcu_array) + #define DEFINE_SRCU(name) __DEFINE_SRCU(name, /* not static */) + #define DEFINE_STATIC_SRCU(name) __DEFINE_SRCU(name, static) + +diff -Nur linux-4.1.10.orig/include/linux/swap.h linux-4.1.10/include/linux/swap.h +--- linux-4.1.10.orig/include/linux/swap.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/swap.h 2015-10-12 22:33:32.284676225 +0200 +@@ -11,6 +11,7 @@ + #include + #include + #include ++#include + #include + + struct notifier_block; +@@ -252,7 +253,8 @@ + void *workingset_eviction(struct address_space *mapping, struct page *page); + bool workingset_refault(void *shadow); + void workingset_activation(struct page *page); +-extern struct list_lru workingset_shadow_nodes; ++extern struct list_lru __workingset_shadow_nodes; ++DECLARE_LOCAL_IRQ_LOCK(workingset_shadow_lock); + + static inline unsigned int workingset_node_pages(struct radix_tree_node *node) + { +@@ -296,6 +298,7 @@ + + + /* linux/mm/swap.c */ ++DECLARE_LOCAL_IRQ_LOCK(swapvec_lock); + extern void lru_cache_add(struct page *); + extern void lru_cache_add_anon(struct page *page); + extern void lru_cache_add_file(struct page *page); +diff -Nur linux-4.1.10.orig/include/linux/thread_info.h linux-4.1.10/include/linux/thread_info.h +--- linux-4.1.10.orig/include/linux/thread_info.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/thread_info.h 2015-10-12 22:33:32.284676225 +0200 +@@ -102,7 +102,17 @@ + #define test_thread_flag(flag) \ + test_ti_thread_flag(current_thread_info(), flag) + +-#define tif_need_resched() test_thread_flag(TIF_NEED_RESCHED) ++#ifdef CONFIG_PREEMPT_LAZY ++#define tif_need_resched() (test_thread_flag(TIF_NEED_RESCHED) || \ ++ test_thread_flag(TIF_NEED_RESCHED_LAZY)) ++#define tif_need_resched_now() (test_thread_flag(TIF_NEED_RESCHED)) ++#define tif_need_resched_lazy() test_thread_flag(TIF_NEED_RESCHED_LAZY)) + -+ return 0; ++#else ++#define tif_need_resched() test_thread_flag(TIF_NEED_RESCHED) ++#define tif_need_resched_now() test_thread_flag(TIF_NEED_RESCHED) ++#define tif_need_resched_lazy() 0 ++#endif + + #if defined TIF_RESTORE_SIGMASK && !defined HAVE_SET_RESTORE_SIGMASK + /* +diff -Nur linux-4.1.10.orig/include/linux/timer.h linux-4.1.10/include/linux/timer.h +--- linux-4.1.10.orig/include/linux/timer.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/timer.h 2015-10-12 22:33:32.284676225 +0200 +@@ -241,7 +241,7 @@ + + extern int try_to_del_timer_sync(struct timer_list *timer); + +-#ifdef CONFIG_SMP ++#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT_FULL) + extern int del_timer_sync(struct timer_list *timer); + #else + # define del_timer_sync(t) del_timer(t) +diff -Nur linux-4.1.10.orig/include/linux/uaccess.h linux-4.1.10/include/linux/uaccess.h +--- linux-4.1.10.orig/include/linux/uaccess.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/uaccess.h 2015-10-12 22:33:32.284676225 +0200 +@@ -1,21 +1,31 @@ + #ifndef __LINUX_UACCESS_H__ + #define __LINUX_UACCESS_H__ + +-#include ++#include + #include + ++static __always_inline void pagefault_disabled_inc(void) ++{ ++ current->pagefault_disabled++; +} + -+/** -+ * debug_threshold_fopen - Open function for "threshold" debugfs entry -+ * @inode: The in-kernel inode representation of the debugfs "file" -+ * @filp: The active open file structure for the debugfs "file" -+ * -+ * This function provides an open implementation for the "threshold" debugfs -+ * interface to the hardware latency detector. -+ */ -+static int debug_threshold_fopen(struct inode *inode, struct file *filp) ++static __always_inline void pagefault_disabled_dec(void) +{ -+ return 0; ++ current->pagefault_disabled--; ++ WARN_ON(current->pagefault_disabled < 0); +} + -+/** -+ * debug_threshold_fread - Read function for "threshold" debugfs entry -+ * @filp: The active open file structure for the debugfs "file" -+ * @ubuf: The userspace provided buffer to read value into -+ * @cnt: The maximum number of bytes to read -+ * @ppos: The current "file" position -+ * -+ * This function provides a read implementation for the "threshold" debugfs -+ * interface to the hardware latency detector. It can be used to determine -+ * the current threshold level at which a latency will be recorded in the -+ * global ring buffer, typically on the order of 10us. + /* +- * These routines enable/disable the pagefault handler in that +- * it will not take any locks and go straight to the fixup table. ++ * These routines enable/disable the pagefault handler. If disabled, it will ++ * not take any locks and go straight to the fixup table. + * +- * They have great resemblance to the preempt_disable/enable calls +- * and in fact they are identical; this is because currently there is +- * no other way to make the pagefault handlers do this. So we do +- * disable preemption but we don't necessarily care about that. ++ * User access methods will not sleep when called from a pagefault_disabled() ++ * environment. + */ + static inline void pagefault_disable(void) + { +- preempt_count_inc(); ++ migrate_disable(); ++ pagefault_disabled_inc(); + /* + * make sure to have issued the store before a pagefault + * can hit. +@@ -25,18 +35,32 @@ + + static inline void pagefault_enable(void) + { +-#ifndef CONFIG_PREEMPT + /* + * make sure to issue those last loads/stores before enabling + * the pagefault handler again. + */ + barrier(); +- preempt_count_dec(); +-#else +- preempt_enable(); +-#endif ++ pagefault_disabled_dec(); ++ migrate_enable(); + } + ++/* ++ * Is the pagefault handler disabled? If so, user access methods will not sleep. + */ -+static ssize_t debug_threshold_fread(struct file *filp, char __user *ubuf, -+ size_t cnt, loff_t *ppos) -+{ -+ return simple_data_read(filp, ubuf, cnt, ppos, &data.threshold); -+} ++#define pagefault_disabled() (current->pagefault_disabled != 0) + -+/** -+ * debug_threshold_fwrite - Write function for "threshold" debugfs entry -+ * @filp: The active open file structure for the debugfs "file" -+ * @ubuf: The user buffer that contains the value to write -+ * @cnt: The maximum number of bytes to write to "file" -+ * @ppos: The current position in the debugfs "file" ++/* ++ * The pagefault handler is in general disabled by pagefault_disable() or ++ * when in irq context (via in_atomic()). + * -+ * This function provides a write implementation for the "threshold" debugfs -+ * interface to the hardware latency detector. It can be used to configure -+ * the threshold level at which any subsequently detected latencies will -+ * be recorded into the global ring buffer. ++ * This function should only be used by the fault handlers. Other users should ++ * stick to pagefault_disabled(). ++ * Please NEVER use preempt_disable() to disable the fault handler. With ++ * !CONFIG_PREEMPT_COUNT, this is like a NOP. So the handler won't be disabled. ++ * in_atomic() will report different values based on !CONFIG_PREEMPT_COUNT. + */ -+static ssize_t debug_threshold_fwrite(struct file *filp, -+ const char __user *ubuf, -+ size_t cnt, -+ loff_t *ppos) -+{ -+ int ret; -+ -+ ret = simple_data_write(filp, ubuf, cnt, ppos, &data.threshold); -+ -+ if (enabled) -+ wake_up_process(kthread); ++#define faulthandler_disabled() (pagefault_disabled() || in_atomic()) + -+ return ret; -+} + #ifndef ARCH_HAS_NOCACHE_UACCESS + + static inline unsigned long __copy_from_user_inatomic_nocache(void *to, +diff -Nur linux-4.1.10.orig/include/linux/uprobes.h linux-4.1.10/include/linux/uprobes.h +--- linux-4.1.10.orig/include/linux/uprobes.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/uprobes.h 2015-10-12 22:33:32.284676225 +0200 +@@ -27,6 +27,7 @@ + #include + #include + #include ++#include + + struct vm_area_struct; + struct mm_struct; +diff -Nur linux-4.1.10.orig/include/linux/vmstat.h linux-4.1.10/include/linux/vmstat.h +--- linux-4.1.10.orig/include/linux/vmstat.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/vmstat.h 2015-10-12 22:33:32.284676225 +0200 +@@ -33,7 +33,9 @@ + */ + static inline void __count_vm_event(enum vm_event_item item) + { ++ preempt_disable_rt(); + raw_cpu_inc(vm_event_states.event[item]); ++ preempt_enable_rt(); + } + + static inline void count_vm_event(enum vm_event_item item) +@@ -43,7 +45,9 @@ + + static inline void __count_vm_events(enum vm_event_item item, long delta) + { ++ preempt_disable_rt(); + raw_cpu_add(vm_event_states.event[item], delta); ++ preempt_enable_rt(); + } + + static inline void count_vm_events(enum vm_event_item item, long delta) +diff -Nur linux-4.1.10.orig/include/linux/wait.h linux-4.1.10/include/linux/wait.h +--- linux-4.1.10.orig/include/linux/wait.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/wait.h 2015-10-12 22:33:32.284676225 +0200 +@@ -8,6 +8,7 @@ + #include + #include + #include ++#include + + typedef struct __wait_queue wait_queue_t; + typedef int (*wait_queue_func_t)(wait_queue_t *wait, unsigned mode, int flags, void *key); +diff -Nur linux-4.1.10.orig/include/linux/wait-simple.h linux-4.1.10/include/linux/wait-simple.h +--- linux-4.1.10.orig/include/linux/wait-simple.h 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/include/linux/wait-simple.h 2015-10-12 22:33:32.284676225 +0200 +@@ -0,0 +1,207 @@ ++#ifndef _LINUX_WAIT_SIMPLE_H ++#define _LINUX_WAIT_SIMPLE_H + -+/** -+ * debug_width_fopen - Open function for "width" debugfs entry -+ * @inode: The in-kernel inode representation of the debugfs "file" -+ * @filp: The active open file structure for the debugfs "file" -+ * -+ * This function provides an open implementation for the "width" debugfs -+ * interface to the hardware latency detector. -+ */ -+static int debug_width_fopen(struct inode *inode, struct file *filp) -+{ -+ return 0; -+} ++#include ++#include + -+/** -+ * debug_width_fread - Read function for "width" debugfs entry -+ * @filp: The active open file structure for the debugfs "file" -+ * @ubuf: The userspace provided buffer to read value into -+ * @cnt: The maximum number of bytes to read -+ * @ppos: The current "file" position -+ * -+ * This function provides a read implementation for the "width" debugfs -+ * interface to the hardware latency detector. It can be used to determine -+ * for how many us of the total window us we will actively sample for any -+ * hardware-induced latecy periods. Obviously, it is not possible to -+ * sample constantly and have the system respond to a sample reader, or, -+ * worse, without having the system appear to have gone out to lunch. -+ */ -+static ssize_t debug_width_fread(struct file *filp, char __user *ubuf, -+ size_t cnt, loff_t *ppos) -+{ -+ return simple_data_read(filp, ubuf, cnt, ppos, &data.sample_width); -+} ++#include + -+/** -+ * debug_width_fwrite - Write function for "width" debugfs entry -+ * @filp: The active open file structure for the debugfs "file" -+ * @ubuf: The user buffer that contains the value to write -+ * @cnt: The maximum number of bytes to write to "file" -+ * @ppos: The current position in the debugfs "file" -+ * -+ * This function provides a write implementation for the "width" debugfs -+ * interface to the hardware latency detector. It can be used to configure -+ * for how many us of the total window us we will actively sample for any -+ * hardware-induced latency periods. Obviously, it is not possible to -+ * sample constantly and have the system respond to a sample reader, or, -+ * worse, without having the system appear to have gone out to lunch. It -+ * is enforced that width is less that the total window size. -+ */ -+static ssize_t debug_width_fwrite(struct file *filp, -+ const char __user *ubuf, -+ size_t cnt, -+ loff_t *ppos) -+{ -+ char buf[U64STR_SIZE]; -+ int csize = min(cnt, sizeof(buf)); -+ u64 val = 0; -+ int err = 0; ++struct swaiter { ++ struct task_struct *task; ++ struct list_head node; ++}; + -+ memset(buf, '\0', sizeof(buf)); -+ if (copy_from_user(buf, ubuf, csize)) -+ return -EFAULT; ++#define DEFINE_SWAITER(name) \ ++ struct swaiter name = { \ ++ .task = current, \ ++ .node = LIST_HEAD_INIT((name).node), \ ++ } + -+ buf[U64STR_SIZE-1] = '\0'; /* just in case */ -+ err = kstrtoull(buf, 10, &val); -+ if (0 != err) -+ return -EINVAL; ++struct swait_head { ++ raw_spinlock_t lock; ++ struct list_head list; ++}; + -+ mutex_lock(&data.lock); -+ if (val < data.sample_window) -+ data.sample_width = val; -+ else { -+ mutex_unlock(&data.lock); -+ return -EINVAL; ++#define SWAIT_HEAD_INITIALIZER(name) { \ ++ .lock = __RAW_SPIN_LOCK_UNLOCKED(name.lock), \ ++ .list = LIST_HEAD_INIT((name).list), \ + } -+ mutex_unlock(&data.lock); + -+ if (enabled) -+ wake_up_process(kthread); ++#define DEFINE_SWAIT_HEAD(name) \ ++ struct swait_head name = SWAIT_HEAD_INITIALIZER(name) + -+ return csize; -+} ++extern void __init_swait_head(struct swait_head *h, struct lock_class_key *key); + -+/** -+ * debug_window_fopen - Open function for "window" debugfs entry -+ * @inode: The in-kernel inode representation of the debugfs "file" -+ * @filp: The active open file structure for the debugfs "file" -+ * -+ * This function provides an open implementation for the "window" debugfs -+ * interface to the hardware latency detector. The window is the total time -+ * in us that will be considered one sample period. Conceptually, windows -+ * occur back-to-back and contain a sample width period during which -+ * actual sampling occurs. -+ */ -+static int debug_window_fopen(struct inode *inode, struct file *filp) -+{ -+ return 0; -+} ++#define init_swait_head(swh) \ ++ do { \ ++ static struct lock_class_key __key; \ ++ \ ++ __init_swait_head((swh), &__key); \ ++ } while (0) + -+/** -+ * debug_window_fread - Read function for "window" debugfs entry -+ * @filp: The active open file structure for the debugfs "file" -+ * @ubuf: The userspace provided buffer to read value into -+ * @cnt: The maximum number of bytes to read -+ * @ppos: The current "file" position -+ * -+ * This function provides a read implementation for the "window" debugfs -+ * interface to the hardware latency detector. The window is the total time -+ * in us that will be considered one sample period. Conceptually, windows -+ * occur back-to-back and contain a sample width period during which -+ * actual sampling occurs. Can be used to read the total window size. ++/* ++ * Waiter functions + */ -+static ssize_t debug_window_fread(struct file *filp, char __user *ubuf, -+ size_t cnt, loff_t *ppos) -+{ -+ return simple_data_read(filp, ubuf, cnt, ppos, &data.sample_window); -+} ++extern void swait_prepare_locked(struct swait_head *head, struct swaiter *w); ++extern void swait_prepare(struct swait_head *head, struct swaiter *w, int state); ++extern void swait_finish_locked(struct swait_head *head, struct swaiter *w); ++extern void swait_finish(struct swait_head *head, struct swaiter *w); + -+/** -+ * debug_window_fwrite - Write function for "window" debugfs entry -+ * @filp: The active open file structure for the debugfs "file" -+ * @ubuf: The user buffer that contains the value to write -+ * @cnt: The maximum number of bytes to write to "file" -+ * @ppos: The current position in the debugfs "file" -+ * -+ * This function provides a write implementation for the "window" debufds -+ * interface to the hardware latency detetector. The window is the total time -+ * in us that will be considered one sample period. Conceptually, windows -+ * occur back-to-back and contain a sample width period during which -+ * actual sampling occurs. Can be used to write a new total window size. It -+ * is enfoced that any value written must be greater than the sample width -+ * size, or an error results. -+ */ -+static ssize_t debug_window_fwrite(struct file *filp, -+ const char __user *ubuf, -+ size_t cnt, -+ loff_t *ppos) ++/* Check whether a head has waiters enqueued */ ++static inline bool swaitqueue_active(struct swait_head *h) +{ -+ char buf[U64STR_SIZE]; -+ int csize = min(cnt, sizeof(buf)); -+ u64 val = 0; -+ int err = 0; -+ -+ memset(buf, '\0', sizeof(buf)); -+ if (copy_from_user(buf, ubuf, csize)) -+ return -EFAULT; -+ -+ buf[U64STR_SIZE-1] = '\0'; /* just in case */ -+ err = kstrtoull(buf, 10, &val); -+ if (0 != err) -+ return -EINVAL; -+ -+ mutex_lock(&data.lock); -+ if (data.sample_width < val) -+ data.sample_window = val; -+ else { -+ mutex_unlock(&data.lock); -+ return -EINVAL; -+ } -+ mutex_unlock(&data.lock); -+ -+ return csize; ++ /* Make sure the condition is visible before checking list_empty() */ ++ smp_mb(); ++ return !list_empty(&h->list); +} + +/* -+ * Function pointers for the "count" debugfs file operations ++ * Wakeup functions + */ -+static const struct file_operations count_fops = { -+ .open = debug_count_fopen, -+ .read = debug_count_fread, -+ .write = debug_count_fwrite, -+ .owner = THIS_MODULE, -+}; ++extern unsigned int __swait_wake(struct swait_head *head, unsigned int state, unsigned int num); ++extern unsigned int __swait_wake_locked(struct swait_head *head, unsigned int state, unsigned int num); + -+/* -+ * Function pointers for the "enable" debugfs file operations -+ */ -+static const struct file_operations enable_fops = { -+ .open = debug_enable_fopen, -+ .read = debug_enable_fread, -+ .write = debug_enable_fwrite, -+ .owner = THIS_MODULE, -+}; ++#define swait_wake(head) __swait_wake(head, TASK_NORMAL, 1) ++#define swait_wake_interruptible(head) __swait_wake(head, TASK_INTERRUPTIBLE, 1) ++#define swait_wake_all(head) __swait_wake(head, TASK_NORMAL, 0) ++#define swait_wake_all_interruptible(head) __swait_wake(head, TASK_INTERRUPTIBLE, 0) + +/* -+ * Function pointers for the "max" debugfs file operations ++ * Event API + */ -+static const struct file_operations max_fops = { -+ .open = debug_max_fopen, -+ .read = debug_max_fread, -+ .write = debug_max_fwrite, -+ .owner = THIS_MODULE, -+}; ++#define __swait_event(wq, condition) \ ++do { \ ++ DEFINE_SWAITER(__wait); \ ++ \ ++ for (;;) { \ ++ swait_prepare(&wq, &__wait, TASK_UNINTERRUPTIBLE); \ ++ if (condition) \ ++ break; \ ++ schedule(); \ ++ } \ ++ swait_finish(&wq, &__wait); \ ++} while (0) + -+/* -+ * Function pointers for the "sample" debugfs file operations ++/** ++ * swait_event - sleep until a condition gets true ++ * @wq: the waitqueue to wait on ++ * @condition: a C expression for the event to wait for ++ * ++ * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the ++ * @condition evaluates to true. The @condition is checked each time ++ * the waitqueue @wq is woken up. ++ * ++ * wake_up() has to be called after changing any variable that could ++ * change the result of the wait condition. + */ -+static const struct file_operations sample_fops = { -+ .open = debug_sample_fopen, -+ .read = debug_sample_fread, -+ .release = debug_sample_release, -+ .owner = THIS_MODULE, -+}; -+ -+/* -+ * Function pointers for the "threshold" debugfs file operations -+ */ -+static const struct file_operations threshold_fops = { -+ .open = debug_threshold_fopen, -+ .read = debug_threshold_fread, -+ .write = debug_threshold_fwrite, -+ .owner = THIS_MODULE, -+}; ++#define swait_event(wq, condition) \ ++do { \ ++ if (condition) \ ++ break; \ ++ __swait_event(wq, condition); \ ++} while (0) + -+/* -+ * Function pointers for the "width" debugfs file operations -+ */ -+static const struct file_operations width_fops = { -+ .open = debug_width_fopen, -+ .read = debug_width_fread, -+ .write = debug_width_fwrite, -+ .owner = THIS_MODULE, -+}; ++#define __swait_event_interruptible(wq, condition, ret) \ ++do { \ ++ DEFINE_SWAITER(__wait); \ ++ \ ++ for (;;) { \ ++ swait_prepare(&wq, &__wait, TASK_INTERRUPTIBLE); \ ++ if (condition) \ ++ break; \ ++ if (signal_pending(current)) { \ ++ ret = -ERESTARTSYS; \ ++ break; \ ++ } \ ++ schedule(); \ ++ } \ ++ swait_finish(&wq, &__wait); \ ++} while (0) + -+/* -+ * Function pointers for the "window" debugfs file operations -+ */ -+static const struct file_operations window_fops = { -+ .open = debug_window_fopen, -+ .read = debug_window_fread, -+ .write = debug_window_fwrite, -+ .owner = THIS_MODULE, -+}; ++#define __swait_event_interruptible_timeout(wq, condition, ret) \ ++do { \ ++ DEFINE_SWAITER(__wait); \ ++ \ ++ for (;;) { \ ++ swait_prepare(&wq, &__wait, TASK_INTERRUPTIBLE); \ ++ if (condition) \ ++ break; \ ++ if (signal_pending(current)) { \ ++ ret = -ERESTARTSYS; \ ++ break; \ ++ } \ ++ ret = schedule_timeout(ret); \ ++ if (!ret) \ ++ break; \ ++ } \ ++ swait_finish(&wq, &__wait); \ ++} while (0) + +/** -+ * init_debugfs - A function to initialize the debugfs interface files ++ * swait_event_interruptible - sleep until a condition gets true ++ * @wq: the waitqueue to wait on ++ * @condition: a C expression for the event to wait for + * -+ * This function creates entries in debugfs for "hwlat_detector", including -+ * files to read values from the detector, current samples, and the -+ * maximum sample that has been captured since the hardware latency -+ * dectector was started. ++ * The process is put to sleep (TASK_INTERRUPTIBLE) until the ++ * @condition evaluates to true. The @condition is checked each time ++ * the waitqueue @wq is woken up. ++ * ++ * wake_up() has to be called after changing any variable that could ++ * change the result of the wait condition. + */ -+static int init_debugfs(void) -+{ -+ int ret = -ENOMEM; ++#define swait_event_interruptible(wq, condition) \ ++({ \ ++ int __ret = 0; \ ++ if (!(condition)) \ ++ __swait_event_interruptible(wq, condition, __ret); \ ++ __ret; \ ++}) + -+ debug_dir = debugfs_create_dir(DRVNAME, NULL); -+ if (!debug_dir) -+ goto err_debug_dir; ++#define swait_event_interruptible_timeout(wq, condition, timeout) \ ++({ \ ++ int __ret = timeout; \ ++ if (!(condition)) \ ++ __swait_event_interruptible_timeout(wq, condition, __ret); \ ++ __ret; \ ++}) + -+ debug_sample = debugfs_create_file("sample", 0444, -+ debug_dir, NULL, -+ &sample_fops); -+ if (!debug_sample) -+ goto err_sample; ++#define __swait_event_timeout(wq, condition, ret) \ ++do { \ ++ DEFINE_SWAITER(__wait); \ ++ \ ++ for (;;) { \ ++ swait_prepare(&wq, &__wait, TASK_UNINTERRUPTIBLE); \ ++ if (condition) \ ++ break; \ ++ ret = schedule_timeout(ret); \ ++ if (!ret) \ ++ break; \ ++ } \ ++ swait_finish(&wq, &__wait); \ ++} while (0) + -+ debug_count = debugfs_create_file("count", 0444, -+ debug_dir, NULL, -+ &count_fops); -+ if (!debug_count) -+ goto err_count; ++/** ++ * swait_event_timeout - sleep until a condition gets true or a timeout elapses ++ * @wq: the waitqueue to wait on ++ * @condition: a C expression for the event to wait for ++ * @timeout: timeout, in jiffies ++ * ++ * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the ++ * @condition evaluates to true. The @condition is checked each time ++ * the waitqueue @wq is woken up. ++ * ++ * wake_up() has to be called after changing any variable that could ++ * change the result of the wait condition. ++ * ++ * The function returns 0 if the @timeout elapsed, and the remaining ++ * jiffies if the condition evaluated to true before the timeout elapsed. ++ */ ++#define swait_event_timeout(wq, condition, timeout) \ ++({ \ ++ long __ret = timeout; \ ++ if (!(condition)) \ ++ __swait_event_timeout(wq, condition, __ret); \ ++ __ret; \ ++}) + -+ debug_max = debugfs_create_file("max", 0444, -+ debug_dir, NULL, -+ &max_fops); -+ if (!debug_max) -+ goto err_max; ++#endif +diff -Nur linux-4.1.10.orig/include/linux/work-simple.h linux-4.1.10/include/linux/work-simple.h +--- linux-4.1.10.orig/include/linux/work-simple.h 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/include/linux/work-simple.h 2015-10-12 22:33:32.284676225 +0200 +@@ -0,0 +1,24 @@ ++#ifndef _LINUX_SWORK_H ++#define _LINUX_SWORK_H + -+ debug_sample_window = debugfs_create_file("window", 0644, -+ debug_dir, NULL, -+ &window_fops); -+ if (!debug_sample_window) -+ goto err_window; ++#include + -+ debug_sample_width = debugfs_create_file("width", 0644, -+ debug_dir, NULL, -+ &width_fops); -+ if (!debug_sample_width) -+ goto err_width; ++struct swork_event { ++ struct list_head item; ++ unsigned long flags; ++ void (*func)(struct swork_event *); ++}; + -+ debug_threshold = debugfs_create_file("threshold", 0644, -+ debug_dir, NULL, -+ &threshold_fops); -+ if (!debug_threshold) -+ goto err_threshold; ++static inline void INIT_SWORK(struct swork_event *event, ++ void (*func)(struct swork_event *)) ++{ ++ event->flags = 0; ++ event->func = func; ++} + -+ debug_enable = debugfs_create_file("enable", 0644, -+ debug_dir, &enabled, -+ &enable_fops); -+ if (!debug_enable) -+ goto err_enable; ++bool swork_queue(struct swork_event *sev); + -+ else { -+ ret = 0; -+ goto out; -+ } ++int swork_get(void); ++void swork_put(void); + -+err_enable: -+ debugfs_remove(debug_threshold); -+err_threshold: -+ debugfs_remove(debug_sample_width); -+err_width: -+ debugfs_remove(debug_sample_window); -+err_window: -+ debugfs_remove(debug_max); -+err_max: -+ debugfs_remove(debug_count); -+err_count: -+ debugfs_remove(debug_sample); -+err_sample: -+ debugfs_remove(debug_dir); -+err_debug_dir: -+out: -+ return ret; -+} ++#endif /* _LINUX_SWORK_H */ +diff -Nur linux-4.1.10.orig/include/net/dst.h linux-4.1.10/include/net/dst.h +--- linux-4.1.10.orig/include/net/dst.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/net/dst.h 2015-10-12 22:33:32.284676225 +0200 +@@ -403,7 +403,7 @@ + static inline int dst_neigh_output(struct dst_entry *dst, struct neighbour *n, + struct sk_buff *skb) + { +- const struct hh_cache *hh; ++ struct hh_cache *hh; + + if (dst->pending_confirm) { + unsigned long now = jiffies; +diff -Nur linux-4.1.10.orig/include/net/neighbour.h linux-4.1.10/include/net/neighbour.h +--- linux-4.1.10.orig/include/net/neighbour.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/net/neighbour.h 2015-10-12 22:33:32.284676225 +0200 +@@ -445,7 +445,7 @@ + } + #endif + +-static inline int neigh_hh_output(const struct hh_cache *hh, struct sk_buff *skb) ++static inline int neigh_hh_output(struct hh_cache *hh, struct sk_buff *skb) + { + unsigned int seq; + int hh_len; +@@ -500,7 +500,7 @@ + + #define NEIGH_CB(skb) ((struct neighbour_cb *)(skb)->cb) + +-static inline void neigh_ha_snapshot(char *dst, const struct neighbour *n, ++static inline void neigh_ha_snapshot(char *dst, struct neighbour *n, + const struct net_device *dev) + { + unsigned int seq; +diff -Nur linux-4.1.10.orig/include/net/netns/ipv4.h linux-4.1.10/include/net/netns/ipv4.h +--- linux-4.1.10.orig/include/net/netns/ipv4.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/net/netns/ipv4.h 2015-10-12 22:33:32.284676225 +0200 +@@ -69,6 +69,7 @@ + + int sysctl_icmp_echo_ignore_all; + int sysctl_icmp_echo_ignore_broadcasts; ++ int sysctl_icmp_echo_sysrq; + int sysctl_icmp_ignore_bogus_error_responses; + int sysctl_icmp_ratelimit; + int sysctl_icmp_ratemask; +diff -Nur linux-4.1.10.orig/include/trace/events/hist.h linux-4.1.10/include/trace/events/hist.h +--- linux-4.1.10.orig/include/trace/events/hist.h 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/include/trace/events/hist.h 2015-10-12 22:33:32.284676225 +0200 +@@ -0,0 +1,72 @@ ++#undef TRACE_SYSTEM ++#define TRACE_SYSTEM hist + -+/** -+ * free_debugfs - A function to cleanup the debugfs file interface -+ */ -+static void free_debugfs(void) -+{ -+ /* could also use a debugfs_remove_recursive */ -+ debugfs_remove(debug_enable); -+ debugfs_remove(debug_threshold); -+ debugfs_remove(debug_sample_width); -+ debugfs_remove(debug_sample_window); -+ debugfs_remove(debug_max); -+ debugfs_remove(debug_count); -+ debugfs_remove(debug_sample); -+ debugfs_remove(debug_dir); -+} ++#if !defined(_TRACE_HIST_H) || defined(TRACE_HEADER_MULTI_READ) ++#define _TRACE_HIST_H + -+/** -+ * detector_init - Standard module initialization code -+ */ -+static int detector_init(void) -+{ -+ int ret = -ENOMEM; ++#include "latency_hist.h" ++#include + -+ pr_info(BANNER "version %s\n", VERSION); ++#if !defined(CONFIG_PREEMPT_OFF_HIST) && !defined(CONFIG_INTERRUPT_OFF_HIST) ++#define trace_preemptirqsoff_hist(a, b) ++#else ++TRACE_EVENT(preemptirqsoff_hist, + -+ ret = init_stats(); -+ if (0 != ret) -+ goto out; ++ TP_PROTO(int reason, int starthist), + -+ ret = init_debugfs(); -+ if (0 != ret) -+ goto err_stats; ++ TP_ARGS(reason, starthist), + -+ if (enabled) -+ ret = start_kthread(); ++ TP_STRUCT__entry( ++ __field(int, reason) ++ __field(int, starthist) ++ ), + -+ goto out; ++ TP_fast_assign( ++ __entry->reason = reason; ++ __entry->starthist = starthist; ++ ), + -+err_stats: -+ ring_buffer_free(ring_buffer); -+out: -+ return ret; ++ TP_printk("reason=%s starthist=%s", getaction(__entry->reason), ++ __entry->starthist ? "start" : "stop") ++); ++#endif + -+} ++#ifndef CONFIG_MISSED_TIMER_OFFSETS_HIST ++#define trace_hrtimer_interrupt(a, b, c, d) ++#else ++TRACE_EVENT(hrtimer_interrupt, + -+/** -+ * detector_exit - Standard module cleanup code -+ */ -+static void detector_exit(void) -+{ -+ int err; ++ TP_PROTO(int cpu, long long offset, struct task_struct *curr, ++ struct task_struct *task), + -+ if (enabled) { -+ enabled = 0; -+ err = stop_kthread(); -+ if (err) -+ pr_err(BANNER "cannot stop kthread\n"); -+ } ++ TP_ARGS(cpu, offset, curr, task), + -+ free_debugfs(); -+ ring_buffer_free(ring_buffer); /* free up the ring buffer */ ++ TP_STRUCT__entry( ++ __field(int, cpu) ++ __field(long long, offset) ++ __array(char, ccomm, TASK_COMM_LEN) ++ __field(int, cprio) ++ __array(char, tcomm, TASK_COMM_LEN) ++ __field(int, tprio) ++ ), + -+} ++ TP_fast_assign( ++ __entry->cpu = cpu; ++ __entry->offset = offset; ++ memcpy(__entry->ccomm, curr->comm, TASK_COMM_LEN); ++ __entry->cprio = curr->prio; ++ memcpy(__entry->tcomm, task != NULL ? task->comm : "", ++ task != NULL ? TASK_COMM_LEN : 7); ++ __entry->tprio = task != NULL ? task->prio : -1; ++ ), + -+module_init(detector_init); -+module_exit(detector_exit); -diff -Nur linux-4.1.10.orig/drivers/misc/Kconfig linux-4.1.10/drivers/misc/Kconfig ---- linux-4.1.10.orig/drivers/misc/Kconfig 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/misc/Kconfig 2015-10-07 18:00:08.000000000 +0200 -@@ -54,6 +54,7 @@ - config ATMEL_TCLIB - bool "Atmel AT32/AT91 Timer/Counter Library" - depends on (AVR32 || ARCH_AT91) -+ default y if PREEMPT_RT_FULL - help - Select this if you want a library to allocate the Timer/Counter - blocks found on many Atmel processors. This facilitates using -@@ -69,8 +70,7 @@ - are combined to make a single 32-bit timer. - - When GENERIC_CLOCKEVENTS is defined, the third timer channel -- may be used as a clock event device supporting oneshot mode -- (delays of up to two seconds) based on the 32 KiHz clock. -+ may be used as a clock event device supporting oneshot mode. - - config ATMEL_TCB_CLKSRC_BLOCK - int -@@ -84,6 +84,15 @@ - TC can be used for other purposes, such as PWM generation and - interval timing. - -+config ATMEL_TCB_CLKSRC_USE_SLOW_CLOCK -+ bool "TC Block use 32 KiHz clock" -+ depends on ATMEL_TCB_CLKSRC -+ default y if !PREEMPT_RT_FULL -+ help -+ Select this to use 32 KiHz base clock rate as TC block clock -+ source for clock events. ++ TP_printk("cpu=%d offset=%lld curr=%s[%d] thread=%s[%d]", ++ __entry->cpu, __entry->offset, __entry->ccomm, ++ __entry->cprio, __entry->tcomm, __entry->tprio) ++); ++#endif + ++#endif /* _TRACE_HIST_H */ + - config DUMMY_IRQ - tristate "Dummy IRQ handler" - default n -@@ -113,6 +122,35 @@ - for information on the specific driver level and support statement - for your IBM server. - -+config HWLAT_DETECTOR -+ tristate "Testing module to detect hardware-induced latencies" -+ depends on DEBUG_FS -+ depends on RING_BUFFER -+ default m -+ ---help--- -+ A simple hardware latency detector. Use this module to detect -+ large latencies introduced by the behavior of the underlying -+ system firmware external to Linux. We do this using periodic -+ use of stop_machine to grab all available CPUs and measure -+ for unexplainable gaps in the CPU timestamp counter(s). By -+ default, the module is not enabled until the "enable" file -+ within the "hwlat_detector" debugfs directory is toggled. ++/* This part must be outside protection */ ++#include +diff -Nur linux-4.1.10.orig/include/trace/events/latency_hist.h linux-4.1.10/include/trace/events/latency_hist.h +--- linux-4.1.10.orig/include/trace/events/latency_hist.h 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/include/trace/events/latency_hist.h 2015-10-12 22:33:32.284676225 +0200 +@@ -0,0 +1,29 @@ ++#ifndef _LATENCY_HIST_H ++#define _LATENCY_HIST_H + -+ This module is often used to detect SMI (System Management -+ Interrupts) on x86 systems, though is not x86 specific. To -+ this end, we default to using a sample window of 1 second, -+ during which we will sample for 0.5 seconds. If an SMI or -+ similar event occurs during that time, it is recorded -+ into an 8K samples global ring buffer until retreived. ++enum hist_action { ++ IRQS_ON, ++ PREEMPT_ON, ++ TRACE_STOP, ++ IRQS_OFF, ++ PREEMPT_OFF, ++ TRACE_START, ++}; + -+ WARNING: This software should never be enabled (it can be built -+ but should not be turned on after it is loaded) in a production -+ environment where high latencies are a concern since the -+ sampling mechanism actually introduces latencies for -+ regular tasks while the CPU(s) are being held. ++static char *actions[] = { ++ "IRQS_ON", ++ "PREEMPT_ON", ++ "TRACE_STOP", ++ "IRQS_OFF", ++ "PREEMPT_OFF", ++ "TRACE_START", ++}; + -+ If unsure, say N ++static inline char *getaction(int action) ++{ ++ if (action >= 0 && action <= sizeof(actions)/sizeof(actions[0])) ++ return actions[action]; ++ return "unknown"; ++} + - config PHANTOM - tristate "Sensable PHANToM (PCI)" - depends on PCI -diff -Nur linux-4.1.10.orig/drivers/misc/Makefile linux-4.1.10/drivers/misc/Makefile ---- linux-4.1.10.orig/drivers/misc/Makefile 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/misc/Makefile 2015-10-07 18:00:08.000000000 +0200 -@@ -38,6 +38,7 @@ - obj-$(CONFIG_HMC6352) += hmc6352.o - obj-y += eeprom/ - obj-y += cb710/ -+obj-$(CONFIG_HWLAT_DETECTOR) += hwlat_detector.o - obj-$(CONFIG_SPEAR13XX_PCIE_GADGET) += spear13xx_pcie_gadget.o - obj-$(CONFIG_VMWARE_BALLOON) += vmw_balloon.o - obj-$(CONFIG_ARM_CHARLCD) += arm-charlcd.o -diff -Nur linux-4.1.10.orig/drivers/mmc/host/mmci.c linux-4.1.10/drivers/mmc/host/mmci.c ---- linux-4.1.10.orig/drivers/mmc/host/mmci.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/mmc/host/mmci.c 2015-10-07 18:00:08.000000000 +0200 -@@ -1155,15 +1155,12 @@ - struct sg_mapping_iter *sg_miter = &host->sg_miter; - struct variant_data *variant = host->variant; - void __iomem *base = host->base; -- unsigned long flags; - u32 status; - - status = readl(base + MMCISTATUS); ++#endif /* _LATENCY_HIST_H */ +diff -Nur linux-4.1.10.orig/init/Kconfig linux-4.1.10/init/Kconfig +--- linux-4.1.10.orig/init/Kconfig 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/init/Kconfig 2015-10-12 22:33:32.284676225 +0200 +@@ -637,7 +637,7 @@ - dev_dbg(mmc_dev(host->mmc), "irq1 (pio) %08x\n", status); + config RCU_FAST_NO_HZ + bool "Accelerate last non-dyntick-idle CPU's grace periods" +- depends on NO_HZ_COMMON && SMP ++ depends on NO_HZ_COMMON && SMP && !PREEMPT_RT_FULL + default n + help + This option permits CPUs to enter dynticks-idle state even if +@@ -664,7 +664,7 @@ + config RCU_BOOST + bool "Enable RCU priority boosting" + depends on RT_MUTEXES && PREEMPT_RCU +- default n ++ default y if PREEMPT_RT_FULL + help + This option boosts the priority of preempted RCU readers that + block the current preemptible RCU grace period for too long. +@@ -1101,6 +1101,7 @@ + config RT_GROUP_SCHED + bool "Group scheduling for SCHED_RR/FIFO" + depends on CGROUP_SCHED ++ depends on !PREEMPT_RT_FULL + default n + help + This feature lets you explicitly allocate real CPU bandwidth +@@ -1688,6 +1689,7 @@ -- local_irq_save(flags); -- - do { - unsigned int remain, len; - char *buffer; -@@ -1203,8 +1200,6 @@ - - sg_miter_stop(sg_miter); - -- local_irq_restore(flags); -- - /* - * If we have less than the fifo 'half-full' threshold to transfer, - * trigger a PIO interrupt as soon as any data is available. -diff -Nur linux-4.1.10.orig/drivers/net/ethernet/3com/3c59x.c linux-4.1.10/drivers/net/ethernet/3com/3c59x.c ---- linux-4.1.10.orig/drivers/net/ethernet/3com/3c59x.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/net/ethernet/3com/3c59x.c 2015-10-07 18:00:08.000000000 +0200 -@@ -842,9 +842,9 @@ - { - struct vortex_private *vp = netdev_priv(dev); - unsigned long flags; -- local_irq_save(flags); -+ local_irq_save_nort(flags); - (vp->full_bus_master_rx ? boomerang_interrupt:vortex_interrupt)(dev->irq,dev); -- local_irq_restore(flags); -+ local_irq_restore_nort(flags); - } - #endif + config SLAB + bool "SLAB" ++ depends on !PREEMPT_RT_FULL + help + The regular slab allocator that is established and known to work + well in all environments. It organizes cache hot objects in +@@ -1706,6 +1708,7 @@ + config SLOB + depends on EXPERT + bool "SLOB (Simple Allocator)" ++ depends on !PREEMPT_RT_FULL + help + SLOB replaces the stock allocator with a drastically simpler + allocator. SLOB is generally more space efficient but +@@ -1715,7 +1718,7 @@ -@@ -1916,12 +1916,12 @@ - * Block interrupts because vortex_interrupt does a bare spin_lock() - */ - unsigned long flags; -- local_irq_save(flags); -+ local_irq_save_nort(flags); - if (vp->full_bus_master_tx) - boomerang_interrupt(dev->irq, dev); - else - vortex_interrupt(dev->irq, dev); -- local_irq_restore(flags); -+ local_irq_restore_nort(flags); - } - } + config SLUB_CPU_PARTIAL + default y +- depends on SLUB && SMP ++ depends on SLUB && SMP && !PREEMPT_RT_FULL + bool "SLUB per cpu partial cache" + help + Per cpu partial caches accellerate objects allocation and freeing +diff -Nur linux-4.1.10.orig/init/main.c linux-4.1.10/init/main.c +--- linux-4.1.10.orig/init/main.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/init/main.c 2015-10-12 22:33:32.284676225 +0200 +@@ -525,6 +525,7 @@ + setup_command_line(command_line); + setup_nr_cpu_ids(); + setup_per_cpu_areas(); ++ softirq_early_init(); + smp_prepare_boot_cpu(); /* arch-specific boot-cpu hooks */ -diff -Nur linux-4.1.10.orig/drivers/net/ethernet/atheros/atl1c/atl1c_main.c linux-4.1.10/drivers/net/ethernet/atheros/atl1c/atl1c_main.c ---- linux-4.1.10.orig/drivers/net/ethernet/atheros/atl1c/atl1c_main.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/net/ethernet/atheros/atl1c/atl1c_main.c 2015-10-07 18:00:08.000000000 +0200 -@@ -2213,11 +2213,7 @@ - } + build_all_zonelists(NULL, NULL); +diff -Nur linux-4.1.10.orig/init/Makefile linux-4.1.10/init/Makefile +--- linux-4.1.10.orig/init/Makefile 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/init/Makefile 2015-10-12 22:33:32.284676225 +0200 +@@ -33,4 +33,4 @@ + include/generated/compile.h: FORCE + @$($(quiet)chk_compile.h) + $(Q)$(CONFIG_SHELL) $(srctree)/scripts/mkcompile_h $@ \ +- "$(UTS_MACHINE)" "$(CONFIG_SMP)" "$(CONFIG_PREEMPT)" "$(CC) $(KBUILD_CFLAGS)" ++ "$(UTS_MACHINE)" "$(CONFIG_SMP)" "$(CONFIG_PREEMPT)" "$(CONFIG_PREEMPT_RT_FULL)" "$(CC) $(KBUILD_CFLAGS)" +diff -Nur linux-4.1.10.orig/ipc/mqueue.c linux-4.1.10/ipc/mqueue.c +--- linux-4.1.10.orig/ipc/mqueue.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/ipc/mqueue.c 2015-10-12 22:33:32.284676225 +0200 +@@ -47,8 +47,7 @@ + #define RECV 1 - tpd_req = atl1c_cal_tpd_req(skb); -- if (!spin_trylock_irqsave(&adapter->tx_lock, flags)) { -- if (netif_msg_pktdata(adapter)) -- dev_info(&adapter->pdev->dev, "tx locked\n"); -- return NETDEV_TX_LOCKED; -- } -+ spin_lock_irqsave(&adapter->tx_lock, flags); + #define STATE_NONE 0 +-#define STATE_PENDING 1 +-#define STATE_READY 2 ++#define STATE_READY 1 - if (atl1c_tpd_avail(adapter, type) < tpd_req) { - /* no enough descriptor, just stop queue */ -diff -Nur linux-4.1.10.orig/drivers/net/ethernet/atheros/atl1e/atl1e_main.c linux-4.1.10/drivers/net/ethernet/atheros/atl1e/atl1e_main.c ---- linux-4.1.10.orig/drivers/net/ethernet/atheros/atl1e/atl1e_main.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/net/ethernet/atheros/atl1e/atl1e_main.c 2015-10-07 18:00:08.000000000 +0200 -@@ -1880,8 +1880,7 @@ - return NETDEV_TX_OK; - } - tpd_req = atl1e_cal_tdp_req(skb); -- if (!spin_trylock_irqsave(&adapter->tx_lock, flags)) -- return NETDEV_TX_LOCKED; -+ spin_lock_irqsave(&adapter->tx_lock, flags); + struct posix_msg_tree_node { + struct rb_node rb_node; +@@ -568,15 +567,12 @@ + wq_add(info, sr, ewp); - if (atl1e_tpd_avail(adapter) < tpd_req) { - /* no enough descriptor, just stop queue */ -diff -Nur linux-4.1.10.orig/drivers/net/ethernet/chelsio/cxgb/sge.c linux-4.1.10/drivers/net/ethernet/chelsio/cxgb/sge.c ---- linux-4.1.10.orig/drivers/net/ethernet/chelsio/cxgb/sge.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/net/ethernet/chelsio/cxgb/sge.c 2015-10-07 18:00:08.000000000 +0200 -@@ -1664,8 +1664,7 @@ - struct cmdQ *q = &sge->cmdQ[qid]; - unsigned int credits, pidx, genbit, count, use_sched_skb = 0; + for (;;) { +- set_current_state(TASK_INTERRUPTIBLE); ++ __set_current_state(TASK_INTERRUPTIBLE); -- if (!spin_trylock(&q->lock)) -- return NETDEV_TX_LOCKED; -+ spin_lock(&q->lock); + spin_unlock(&info->lock); + time = schedule_hrtimeout_range_clock(timeout, 0, + HRTIMER_MODE_ABS, CLOCK_REALTIME); - reclaim_completed_tx(sge, q); +- while (ewp->state == STATE_PENDING) +- cpu_relax(); +- + if (ewp->state == STATE_READY) { + retval = 0; + goto out; +@@ -904,11 +900,15 @@ + * list of waiting receivers. A sender checks that list before adding the new + * message into the message array. If there is a waiting receiver, then it + * bypasses the message array and directly hands the message over to the +- * receiver. +- * The receiver accepts the message and returns without grabbing the queue +- * spinlock. Therefore an intermediate STATE_PENDING state and memory barriers +- * are necessary. The same algorithm is used for sysv semaphores, see +- * ipc/sem.c for more details. ++ * receiver. The receiver accepts the message and returns without grabbing the ++ * queue spinlock: ++ * ++ * - Set pointer to message. ++ * - Queue the receiver task for later wakeup (without the info->lock). ++ * - Update its state to STATE_READY. Now the receiver can continue. ++ * - Wake up the process after the lock is dropped. Should the process wake up ++ * before this wakeup (due to a timeout or a signal) it will either see ++ * STATE_READY and continue or acquire the lock to check the state again. + * + * The same algorithm is used for senders. + */ +@@ -916,21 +916,29 @@ + /* pipelined_send() - send a message directly to the task waiting in + * sys_mq_timedreceive() (without inserting message into a queue). + */ +-static inline void pipelined_send(struct mqueue_inode_info *info, ++static inline void pipelined_send(struct wake_q_head *wake_q, ++ struct mqueue_inode_info *info, + struct msg_msg *message, + struct ext_wait_queue *receiver) + { + receiver->msg = message; + list_del(&receiver->list); +- receiver->state = STATE_PENDING; +- wake_up_process(receiver->task); +- smp_wmb(); ++ wake_q_add(wake_q, receiver->task); ++ /* ++ * Rely on the implicit cmpxchg barrier from wake_q_add such ++ * that we can ensure that updating receiver->state is the last ++ * write operation: As once set, the receiver can continue, ++ * and if we don't have the reference count from the wake_q, ++ * yet, at that point we can later have a use-after-free ++ * condition and bogus wakeup. ++ */ + receiver->state = STATE_READY; + } -diff -Nur linux-4.1.10.orig/drivers/net/ethernet/freescale/gianfar.c linux-4.1.10/drivers/net/ethernet/freescale/gianfar.c ---- linux-4.1.10.orig/drivers/net/ethernet/freescale/gianfar.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/net/ethernet/freescale/gianfar.c 2015-10-07 18:00:08.000000000 +0200 -@@ -1540,7 +1540,7 @@ + /* pipelined_receive() - if there is task waiting in sys_mq_timedsend() + * gets its message and put to the queue (we have one free place for sure). */ +-static inline void pipelined_receive(struct mqueue_inode_info *info) ++static inline void pipelined_receive(struct wake_q_head *wake_q, ++ struct mqueue_inode_info *info) + { + struct ext_wait_queue *sender = wq_get_first_waiter(info, SEND); - if (netif_running(ndev)) { +@@ -941,10 +949,9 @@ + } + if (msg_insert(sender->msg, info)) + return; ++ + list_del(&sender->list); +- sender->state = STATE_PENDING; +- wake_up_process(sender->task); +- smp_wmb(); ++ wake_q_add(wake_q, sender->task); + sender->state = STATE_READY; + } -- local_irq_save(flags); -+ local_irq_save_nort(flags); - lock_tx_qs(priv); +@@ -962,6 +969,7 @@ + struct timespec ts; + struct posix_msg_tree_node *new_leaf = NULL; + int ret = 0; ++ WAKE_Q(wake_q); - gfar_halt_nodisable(priv); -@@ -1556,7 +1556,7 @@ - gfar_write(®s->maccfg1, tempval); + if (u_abs_timeout) { + int res = prepare_timeout(u_abs_timeout, &expires, &ts); +@@ -1045,7 +1053,7 @@ + } else { + receiver = wq_get_first_waiter(info, RECV); + if (receiver) { +- pipelined_send(info, msg_ptr, receiver); ++ pipelined_send(&wake_q, info, msg_ptr, receiver); + } else { + /* adds message to the queue */ + ret = msg_insert(msg_ptr, info); +@@ -1058,6 +1066,7 @@ + } + out_unlock: + spin_unlock(&info->lock); ++ wake_up_q(&wake_q); + out_free: + if (ret) + free_msg(msg_ptr); +@@ -1144,14 +1153,17 @@ + msg_ptr = wait.msg; + } + } else { ++ WAKE_Q(wake_q); ++ + msg_ptr = msg_get(info); - unlock_tx_qs(priv); -- local_irq_restore(flags); -+ local_irq_restore_nort(flags); + inode->i_atime = inode->i_mtime = inode->i_ctime = + CURRENT_TIME; - disable_napi(priv); + /* There is now free space in queue. */ +- pipelined_receive(info); ++ pipelined_receive(&wake_q, info); + spin_unlock(&info->lock); ++ wake_up_q(&wake_q); + ret = 0; + } + if (ret == 0) { +diff -Nur linux-4.1.10.orig/ipc/msg.c linux-4.1.10/ipc/msg.c +--- linux-4.1.10.orig/ipc/msg.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/ipc/msg.c 2015-10-12 22:33:32.288675961 +0200 +@@ -188,6 +188,12 @@ + struct msg_receiver *msr, *t; -@@ -1598,7 +1598,7 @@ - /* Disable Magic Packet mode, in case something - * else woke us up. - */ -- local_irq_save(flags); -+ local_irq_save_nort(flags); - lock_tx_qs(priv); + list_for_each_entry_safe(msr, t, &msq->q_receivers, r_list) { ++ /* ++ * Make sure that the wakeup doesnt preempt ++ * this CPU prematurely. (on PREEMPT_RT) ++ */ ++ preempt_disable_rt(); ++ + msr->r_msg = NULL; /* initialize expunge ordering */ + wake_up_process(msr->r_tsk); + /* +@@ -198,6 +204,8 @@ + */ + smp_mb(); + msr->r_msg = ERR_PTR(res); ++ ++ preempt_enable_rt(); + } + } - tempval = gfar_read(®s->maccfg2); -@@ -1608,7 +1608,7 @@ - gfar_start(priv); +@@ -574,6 +582,11 @@ + if (testmsg(msg, msr->r_msgtype, msr->r_mode) && + !security_msg_queue_msgrcv(msq, msg, msr->r_tsk, + msr->r_msgtype, msr->r_mode)) { ++ /* ++ * Make sure that the wakeup doesnt preempt ++ * this CPU prematurely. (on PREEMPT_RT) ++ */ ++ preempt_disable_rt(); - unlock_tx_qs(priv); -- local_irq_restore(flags); -+ local_irq_restore_nort(flags); + list_del(&msr->r_list); + if (msr->r_maxsize < msg->m_ts) { +@@ -595,12 +608,13 @@ + */ + smp_mb(); + msr->r_msg = msg; ++ preempt_enable_rt(); - netif_device_attach(ndev); + return 1; + } ++ preempt_enable_rt(); + } + } +- + return 0; + } -@@ -3418,14 +3418,14 @@ - dev->stats.tx_dropped++; - atomic64_inc(&priv->extra_stats.tx_underrun); +diff -Nur linux-4.1.10.orig/ipc/sem.c linux-4.1.10/ipc/sem.c +--- linux-4.1.10.orig/ipc/sem.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/ipc/sem.c 2015-10-12 22:33:32.288675961 +0200 +@@ -690,6 +690,13 @@ + static void wake_up_sem_queue_prepare(struct list_head *pt, + struct sem_queue *q, int error) + { ++#ifdef CONFIG_PREEMPT_RT_BASE ++ struct task_struct *p = q->sleeper; ++ get_task_struct(p); ++ q->status = error; ++ wake_up_process(p); ++ put_task_struct(p); ++#else + if (list_empty(pt)) { + /* + * Hold preempt off so that we don't get preempted and have the +@@ -701,6 +708,7 @@ + q->pid = error; -- local_irq_save(flags); -+ local_irq_save_nort(flags); - lock_tx_qs(priv); + list_add_tail(&q->list, pt); ++#endif + } - /* Reactivate the Tx Queues */ - gfar_write(®s->tstat, gfargrp->tstat); + /** +@@ -714,6 +722,7 @@ + */ + static void wake_up_sem_queue_do(struct list_head *pt) + { ++#ifndef CONFIG_PREEMPT_RT_BASE + struct sem_queue *q, *t; + int did_something; - unlock_tx_qs(priv); -- local_irq_restore(flags); -+ local_irq_restore_nort(flags); - } - netif_dbg(priv, tx_err, dev, "Transmit Error\n"); +@@ -726,6 +735,7 @@ } -diff -Nur linux-4.1.10.orig/drivers/net/ethernet/neterion/s2io.c linux-4.1.10/drivers/net/ethernet/neterion/s2io.c ---- linux-4.1.10.orig/drivers/net/ethernet/neterion/s2io.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/net/ethernet/neterion/s2io.c 2015-10-07 18:00:08.000000000 +0200 -@@ -4084,12 +4084,7 @@ - [skb->priority & (MAX_TX_FIFOS - 1)]; - fifo = &mac_control->fifos[queue]; + if (did_something) + preempt_enable(); ++#endif + } -- if (do_spin_lock) -- spin_lock_irqsave(&fifo->tx_lock, flags); -- else { -- if (unlikely(!spin_trylock_irqsave(&fifo->tx_lock, flags))) -- return NETDEV_TX_LOCKED; -- } -+ spin_lock_irqsave(&fifo->tx_lock, flags); + static void unlink_queue(struct sem_array *sma, struct sem_queue *q) +diff -Nur linux-4.1.10.orig/kernel/cgroup.c linux-4.1.10/kernel/cgroup.c +--- linux-4.1.10.orig/kernel/cgroup.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/cgroup.c 2015-10-12 22:33:32.288675961 +0200 +@@ -4422,10 +4422,10 @@ + queue_work(cgroup_destroy_wq, &css->destroy_work); + } - if (sp->config.multiq) { - if (__netif_subqueue_stopped(dev, fifo->fifo_no)) { -diff -Nur linux-4.1.10.orig/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c linux-4.1.10/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c ---- linux-4.1.10.orig/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c 2015-10-07 18:00:08.000000000 +0200 -@@ -2137,10 +2137,8 @@ - struct pch_gbe_tx_ring *tx_ring = adapter->tx_ring; - unsigned long flags; +-static void css_release_work_fn(struct work_struct *work) ++static void css_release_work_fn(struct swork_event *sev) + { + struct cgroup_subsys_state *css = +- container_of(work, struct cgroup_subsys_state, destroy_work); ++ container_of(sev, struct cgroup_subsys_state, destroy_swork); + struct cgroup_subsys *ss = css->ss; + struct cgroup *cgrp = css->cgroup; -- if (!spin_trylock_irqsave(&tx_ring->tx_lock, flags)) { -- /* Collision - tell upper layer to requeue */ -- return NETDEV_TX_LOCKED; -- } -+ spin_lock_irqsave(&tx_ring->tx_lock, flags); -+ - if (unlikely(!PCH_GBE_DESC_UNUSED(tx_ring))) { - netif_stop_queue(netdev); - spin_unlock_irqrestore(&tx_ring->tx_lock, flags); -diff -Nur linux-4.1.10.orig/drivers/net/ethernet/realtek/8139too.c linux-4.1.10/drivers/net/ethernet/realtek/8139too.c ---- linux-4.1.10.orig/drivers/net/ethernet/realtek/8139too.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/net/ethernet/realtek/8139too.c 2015-10-07 18:00:08.000000000 +0200 -@@ -2229,7 +2229,7 @@ - struct rtl8139_private *tp = netdev_priv(dev); - const int irq = tp->pci_dev->irq; +@@ -4464,8 +4464,8 @@ + struct cgroup_subsys_state *css = + container_of(ref, struct cgroup_subsys_state, refcnt); -- disable_irq(irq); -+ disable_irq_nosync(irq); - rtl8139_interrupt(irq, dev); - enable_irq(irq); +- INIT_WORK(&css->destroy_work, css_release_work_fn); +- queue_work(cgroup_destroy_wq, &css->destroy_work); ++ INIT_SWORK(&css->destroy_swork, css_release_work_fn); ++ swork_queue(&css->destroy_swork); } -diff -Nur linux-4.1.10.orig/drivers/net/ethernet/tehuti/tehuti.c linux-4.1.10/drivers/net/ethernet/tehuti/tehuti.c ---- linux-4.1.10.orig/drivers/net/ethernet/tehuti/tehuti.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/net/ethernet/tehuti/tehuti.c 2015-10-07 18:00:08.000000000 +0200 -@@ -1629,13 +1629,8 @@ - unsigned long flags; - - ENTER; -- local_irq_save(flags); -- if (!spin_trylock(&priv->tx_lock)) { -- local_irq_restore(flags); -- DBG("%s[%s]: TX locked, returning NETDEV_TX_LOCKED\n", -- BDX_DRV_NAME, ndev->name); -- return NETDEV_TX_LOCKED; -- } -+ -+ spin_lock_irqsave(&priv->tx_lock, flags); - - /* build tx descriptor */ - BDX_ASSERT(f->m.wptr >= f->m.memsz); /* started with valid wptr */ -diff -Nur linux-4.1.10.orig/drivers/net/rionet.c linux-4.1.10/drivers/net/rionet.c ---- linux-4.1.10.orig/drivers/net/rionet.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/net/rionet.c 2015-10-07 18:00:08.000000000 +0200 -@@ -174,11 +174,7 @@ - unsigned long flags; - int add_num = 1; -- local_irq_save(flags); -- if (!spin_trylock(&rnet->tx_lock)) { -- local_irq_restore(flags); -- return NETDEV_TX_LOCKED; -- } -+ spin_lock_irqsave(&rnet->tx_lock, flags); + static void init_and_link_css(struct cgroup_subsys_state *css, +@@ -5069,6 +5069,7 @@ + */ + cgroup_destroy_wq = alloc_workqueue("cgroup_destroy", 0, 1); + BUG_ON(!cgroup_destroy_wq); ++ BUG_ON(swork_get()); - if (is_multicast_ether_addr(eth->h_dest)) - add_num = nets[rnet->mport->id].nact; -diff -Nur linux-4.1.10.orig/drivers/net/wireless/orinoco/orinoco_usb.c linux-4.1.10/drivers/net/wireless/orinoco/orinoco_usb.c ---- linux-4.1.10.orig/drivers/net/wireless/orinoco/orinoco_usb.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/net/wireless/orinoco/orinoco_usb.c 2015-10-07 18:00:08.000000000 +0200 -@@ -697,7 +697,7 @@ - while (!ctx->done.done && msecs--) - udelay(1000); - } else { -- wait_event_interruptible(ctx->done.wait, -+ swait_event_interruptible(ctx->done.wait, - ctx->done.done); - } - break; -diff -Nur linux-4.1.10.orig/drivers/pci/access.c linux-4.1.10/drivers/pci/access.c ---- linux-4.1.10.orig/drivers/pci/access.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/pci/access.c 2015-10-07 18:00:08.000000000 +0200 -@@ -580,7 +580,7 @@ - WARN_ON(!dev->block_cfg_access); + /* + * Used to destroy pidlists and separate to serve as flush domain. +diff -Nur linux-4.1.10.orig/kernel/cpu.c linux-4.1.10/kernel/cpu.c +--- linux-4.1.10.orig/kernel/cpu.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/cpu.c 2015-10-12 22:33:32.292675697 +0200 +@@ -74,8 +74,8 @@ + #endif + } cpu_hotplug = { + .active_writer = NULL, +- .wq = __WAIT_QUEUE_HEAD_INITIALIZER(cpu_hotplug.wq), + .lock = __MUTEX_INITIALIZER(cpu_hotplug.lock), ++ .wq = __WAIT_QUEUE_HEAD_INITIALIZER(cpu_hotplug.wq), + #ifdef CONFIG_DEBUG_LOCK_ALLOC + .dep_map = {.name = "cpu_hotplug.lock" }, + #endif +@@ -88,6 +88,289 @@ + #define cpuhp_lock_acquire() lock_map_acquire(&cpu_hotplug.dep_map) + #define cpuhp_lock_release() lock_map_release(&cpu_hotplug.dep_map) - dev->block_cfg_access = 0; -- wake_up_all(&pci_cfg_wait); -+ wake_up_all_locked(&pci_cfg_wait); - raw_spin_unlock_irqrestore(&pci_lock, flags); - } - EXPORT_SYMBOL_GPL(pci_cfg_access_unlock); -diff -Nur linux-4.1.10.orig/drivers/pci/access.c.orig linux-4.1.10/drivers/pci/access.c.orig ---- linux-4.1.10.orig/drivers/pci/access.c.orig 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.10/drivers/pci/access.c.orig 2015-10-03 13:49:38.000000000 +0200 -@@ -0,0 +1,784 @@ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include "pci.h" -+ -+/* -+ * This interrupt-safe spinlock protects all accesses to PCI -+ * configuration space. -+ */ -+ -+DEFINE_RAW_SPINLOCK(pci_lock); -+ -+/* -+ * Wrappers for all PCI configuration access functions. They just check -+ * alignment, do locking and call the low-level functions pointed to -+ * by pci_dev->ops. -+ */ -+ -+#define PCI_byte_BAD 0 -+#define PCI_word_BAD (pos & 1) -+#define PCI_dword_BAD (pos & 3) -+ -+#define PCI_OP_READ(size,type,len) \ -+int pci_bus_read_config_##size \ -+ (struct pci_bus *bus, unsigned int devfn, int pos, type *value) \ -+{ \ -+ int res; \ -+ unsigned long flags; \ -+ u32 data = 0; \ -+ if (PCI_##size##_BAD) return PCIBIOS_BAD_REGISTER_NUMBER; \ -+ raw_spin_lock_irqsave(&pci_lock, flags); \ -+ res = bus->ops->read(bus, devfn, pos, len, &data); \ -+ *value = (type)data; \ -+ raw_spin_unlock_irqrestore(&pci_lock, flags); \ -+ return res; \ -+} -+ -+#define PCI_OP_WRITE(size,type,len) \ -+int pci_bus_write_config_##size \ -+ (struct pci_bus *bus, unsigned int devfn, int pos, type value) \ -+{ \ -+ int res; \ -+ unsigned long flags; \ -+ if (PCI_##size##_BAD) return PCIBIOS_BAD_REGISTER_NUMBER; \ -+ raw_spin_lock_irqsave(&pci_lock, flags); \ -+ res = bus->ops->write(bus, devfn, pos, len, value); \ -+ raw_spin_unlock_irqrestore(&pci_lock, flags); \ -+ return res; \ -+} -+ -+PCI_OP_READ(byte, u8, 1) -+PCI_OP_READ(word, u16, 2) -+PCI_OP_READ(dword, u32, 4) -+PCI_OP_WRITE(byte, u8, 1) -+PCI_OP_WRITE(word, u16, 2) -+PCI_OP_WRITE(dword, u32, 4) -+ -+EXPORT_SYMBOL(pci_bus_read_config_byte); -+EXPORT_SYMBOL(pci_bus_read_config_word); -+EXPORT_SYMBOL(pci_bus_read_config_dword); -+EXPORT_SYMBOL(pci_bus_write_config_byte); -+EXPORT_SYMBOL(pci_bus_write_config_word); -+EXPORT_SYMBOL(pci_bus_write_config_dword); -+ -+int pci_generic_config_read(struct pci_bus *bus, unsigned int devfn, -+ int where, int size, u32 *val) -+{ -+ void __iomem *addr; -+ -+ addr = bus->ops->map_bus(bus, devfn, where); -+ if (!addr) { -+ *val = ~0; -+ return PCIBIOS_DEVICE_NOT_FOUND; -+ } -+ -+ if (size == 1) -+ *val = readb(addr); -+ else if (size == 2) -+ *val = readw(addr); -+ else -+ *val = readl(addr); -+ -+ return PCIBIOS_SUCCESSFUL; -+} -+EXPORT_SYMBOL_GPL(pci_generic_config_read); -+ -+int pci_generic_config_write(struct pci_bus *bus, unsigned int devfn, -+ int where, int size, u32 val) -+{ -+ void __iomem *addr; -+ -+ addr = bus->ops->map_bus(bus, devfn, where); -+ if (!addr) -+ return PCIBIOS_DEVICE_NOT_FOUND; -+ -+ if (size == 1) -+ writeb(val, addr); -+ else if (size == 2) -+ writew(val, addr); -+ else -+ writel(val, addr); -+ -+ return PCIBIOS_SUCCESSFUL; -+} -+EXPORT_SYMBOL_GPL(pci_generic_config_write); -+ -+int pci_generic_config_read32(struct pci_bus *bus, unsigned int devfn, -+ int where, int size, u32 *val) -+{ -+ void __iomem *addr; -+ -+ addr = bus->ops->map_bus(bus, devfn, where & ~0x3); -+ if (!addr) { -+ *val = ~0; -+ return PCIBIOS_DEVICE_NOT_FOUND; -+ } -+ -+ *val = readl(addr); -+ -+ if (size <= 2) -+ *val = (*val >> (8 * (where & 3))) & ((1 << (size * 8)) - 1); -+ -+ return PCIBIOS_SUCCESSFUL; -+} -+EXPORT_SYMBOL_GPL(pci_generic_config_read32); -+ -+int pci_generic_config_write32(struct pci_bus *bus, unsigned int devfn, -+ int where, int size, u32 val) -+{ -+ void __iomem *addr; -+ u32 mask, tmp; -+ -+ addr = bus->ops->map_bus(bus, devfn, where & ~0x3); -+ if (!addr) -+ return PCIBIOS_DEVICE_NOT_FOUND; -+ -+ if (size == 4) { -+ writel(val, addr); -+ return PCIBIOS_SUCCESSFUL; -+ } else { -+ mask = ~(((1 << (size * 8)) - 1) << ((where & 0x3) * 8)); -+ } -+ -+ tmp = readl(addr) & mask; -+ tmp |= val << ((where & 0x3) * 8); -+ writel(tmp, addr); -+ -+ return PCIBIOS_SUCCESSFUL; -+} -+EXPORT_SYMBOL_GPL(pci_generic_config_write32); -+ +/** -+ * pci_bus_set_ops - Set raw operations of pci bus -+ * @bus: pci bus struct -+ * @ops: new raw operations ++ * hotplug_pcp - per cpu hotplug descriptor ++ * @unplug: set when pin_current_cpu() needs to sync tasks ++ * @sync_tsk: the task that waits for tasks to finish pinned sections ++ * @refcount: counter of tasks in pinned sections ++ * @grab_lock: set when the tasks entering pinned sections should wait ++ * @synced: notifier for @sync_tsk to tell cpu_down it's finished ++ * @mutex: the mutex to make tasks wait (used when @grab_lock is true) ++ * @mutex_init: zero if the mutex hasn't been initialized yet. + * -+ * Return previous raw operations ++ * Although @unplug and @sync_tsk may point to the same task, the @unplug ++ * is used as a flag and still exists after @sync_tsk has exited and ++ * @sync_tsk set to NULL. + */ -+struct pci_ops *pci_bus_set_ops(struct pci_bus *bus, struct pci_ops *ops) -+{ -+ struct pci_ops *old_ops; -+ unsigned long flags; ++struct hotplug_pcp { ++ struct task_struct *unplug; ++ struct task_struct *sync_tsk; ++ int refcount; ++ int grab_lock; ++ struct completion synced; ++ struct completion unplug_wait; ++#ifdef CONFIG_PREEMPT_RT_FULL ++ /* ++ * Note, on PREEMPT_RT, the hotplug lock must save the state of ++ * the task, otherwise the mutex will cause the task to fail ++ * to sleep when required. (Because it's called from migrate_disable()) ++ * ++ * The spinlock_t on PREEMPT_RT is a mutex that saves the task's ++ * state. ++ */ ++ spinlock_t lock; ++#else ++ struct mutex mutex; ++#endif ++ int mutex_init; ++}; + -+ raw_spin_lock_irqsave(&pci_lock, flags); -+ old_ops = bus->ops; -+ bus->ops = ops; -+ raw_spin_unlock_irqrestore(&pci_lock, flags); -+ return old_ops; -+} -+EXPORT_SYMBOL(pci_bus_set_ops); ++#ifdef CONFIG_PREEMPT_RT_FULL ++# define hotplug_lock(hp) rt_spin_lock(&(hp)->lock) ++# define hotplug_unlock(hp) rt_spin_unlock(&(hp)->lock) ++#else ++# define hotplug_lock(hp) mutex_lock(&(hp)->mutex) ++# define hotplug_unlock(hp) mutex_unlock(&(hp)->mutex) ++#endif + -+/** -+ * pci_read_vpd - Read one entry from Vital Product Data -+ * @dev: pci device struct -+ * @pos: offset in vpd space -+ * @count: number of bytes to read -+ * @buf: pointer to where to store result -+ * -+ */ -+ssize_t pci_read_vpd(struct pci_dev *dev, loff_t pos, size_t count, void *buf) -+{ -+ if (!dev->vpd || !dev->vpd->ops) -+ return -ENODEV; -+ return dev->vpd->ops->read(dev, pos, count, buf); -+} -+EXPORT_SYMBOL(pci_read_vpd); ++static DEFINE_PER_CPU(struct hotplug_pcp, hotplug_pcp); + +/** -+ * pci_write_vpd - Write entry to Vital Product Data -+ * @dev: pci device struct -+ * @pos: offset in vpd space -+ * @count: number of bytes to write -+ * @buf: buffer containing write data -+ * -+ */ -+ssize_t pci_write_vpd(struct pci_dev *dev, loff_t pos, size_t count, const void *buf) -+{ -+ if (!dev->vpd || !dev->vpd->ops) -+ return -ENODEV; -+ return dev->vpd->ops->write(dev, pos, count, buf); -+} -+EXPORT_SYMBOL(pci_write_vpd); -+ -+/* -+ * The following routines are to prevent the user from accessing PCI config -+ * space when it's unsafe to do so. Some devices require this during BIST and -+ * we're required to prevent it during D-state transitions. ++ * pin_current_cpu - Prevent the current cpu from being unplugged + * -+ * We have a bit per device to indicate it's blocked and a global wait queue -+ * for callers to sleep on until devices are unblocked. -+ */ -+static DECLARE_WAIT_QUEUE_HEAD(pci_cfg_wait); -+ -+static noinline void pci_wait_cfg(struct pci_dev *dev) -+{ -+ DECLARE_WAITQUEUE(wait, current); -+ -+ __add_wait_queue(&pci_cfg_wait, &wait); -+ do { -+ set_current_state(TASK_UNINTERRUPTIBLE); -+ raw_spin_unlock_irq(&pci_lock); -+ schedule(); -+ raw_spin_lock_irq(&pci_lock); -+ } while (dev->block_cfg_access); -+ __remove_wait_queue(&pci_cfg_wait, &wait); -+} -+ -+/* Returns 0 on success, negative values indicate error. */ -+#define PCI_USER_READ_CONFIG(size,type) \ -+int pci_user_read_config_##size \ -+ (struct pci_dev *dev, int pos, type *val) \ -+{ \ -+ int ret = PCIBIOS_SUCCESSFUL; \ -+ u32 data = -1; \ -+ if (PCI_##size##_BAD) \ -+ return -EINVAL; \ -+ raw_spin_lock_irq(&pci_lock); \ -+ if (unlikely(dev->block_cfg_access)) \ -+ pci_wait_cfg(dev); \ -+ ret = dev->bus->ops->read(dev->bus, dev->devfn, \ -+ pos, sizeof(type), &data); \ -+ raw_spin_unlock_irq(&pci_lock); \ -+ *val = (type)data; \ -+ return pcibios_err_to_errno(ret); \ -+} \ -+EXPORT_SYMBOL_GPL(pci_user_read_config_##size); -+ -+/* Returns 0 on success, negative values indicate error. */ -+#define PCI_USER_WRITE_CONFIG(size,type) \ -+int pci_user_write_config_##size \ -+ (struct pci_dev *dev, int pos, type val) \ -+{ \ -+ int ret = PCIBIOS_SUCCESSFUL; \ -+ if (PCI_##size##_BAD) \ -+ return -EINVAL; \ -+ raw_spin_lock_irq(&pci_lock); \ -+ if (unlikely(dev->block_cfg_access)) \ -+ pci_wait_cfg(dev); \ -+ ret = dev->bus->ops->write(dev->bus, dev->devfn, \ -+ pos, sizeof(type), val); \ -+ raw_spin_unlock_irq(&pci_lock); \ -+ return pcibios_err_to_errno(ret); \ -+} \ -+EXPORT_SYMBOL_GPL(pci_user_write_config_##size); -+ -+PCI_USER_READ_CONFIG(byte, u8) -+PCI_USER_READ_CONFIG(word, u16) -+PCI_USER_READ_CONFIG(dword, u32) -+PCI_USER_WRITE_CONFIG(byte, u8) -+PCI_USER_WRITE_CONFIG(word, u16) -+PCI_USER_WRITE_CONFIG(dword, u32) -+ -+/* VPD access through PCI 2.2+ VPD capability */ -+ -+#define PCI_VPD_PCI22_SIZE (PCI_VPD_ADDR_MASK + 1) -+ -+struct pci_vpd_pci22 { -+ struct pci_vpd base; -+ struct mutex lock; -+ u16 flag; -+ bool busy; -+ u8 cap; -+}; -+ -+/* -+ * Wait for last operation to complete. -+ * This code has to spin since there is no other notification from the PCI -+ * hardware. Since the VPD is often implemented by serial attachment to an -+ * EEPROM, it may take many milliseconds to complete. ++ * Lightweight version of get_online_cpus() to prevent cpu from being ++ * unplugged when code runs in a migration disabled region. + * -+ * Returns 0 on success, negative values indicate error. ++ * Must be called with preemption disabled (preempt_count = 1)! + */ -+static int pci_vpd_pci22_wait(struct pci_dev *dev) ++void pin_current_cpu(void) +{ -+ struct pci_vpd_pci22 *vpd = -+ container_of(dev->vpd, struct pci_vpd_pci22, base); -+ unsigned long timeout = jiffies + HZ/20 + 2; -+ u16 status; -+ int ret; -+ -+ if (!vpd->busy) -+ return 0; -+ -+ for (;;) { -+ ret = pci_user_read_config_word(dev, vpd->cap + PCI_VPD_ADDR, -+ &status); -+ if (ret < 0) -+ return ret; ++ struct hotplug_pcp *hp; ++ int force = 0; + -+ if ((status & PCI_VPD_ADDR_F) == vpd->flag) { -+ vpd->busy = false; -+ return 0; -+ } ++retry: ++ hp = this_cpu_ptr(&hotplug_pcp); + -+ if (time_after(jiffies, timeout)) { -+ dev_printk(KERN_DEBUG, &dev->dev, "vpd r/w failed. This is likely a firmware bug on this device. Contact the card vendor for a firmware update\n"); -+ return -ETIMEDOUT; -+ } -+ if (fatal_signal_pending(current)) -+ return -EINTR; -+ if (!cond_resched()) -+ udelay(10); ++ if (!hp->unplug || hp->refcount || force || preempt_count() > 1 || ++ hp->unplug == current) { ++ hp->refcount++; ++ return; + } -+} -+ -+static ssize_t pci_vpd_pci22_read(struct pci_dev *dev, loff_t pos, size_t count, -+ void *arg) -+{ -+ struct pci_vpd_pci22 *vpd = -+ container_of(dev->vpd, struct pci_vpd_pci22, base); -+ int ret; -+ loff_t end = pos + count; -+ u8 *buf = arg; -+ -+ if (pos < 0 || pos > vpd->base.len || end > vpd->base.len) -+ return -EINVAL; -+ -+ if (mutex_lock_killable(&vpd->lock)) -+ return -EINTR; -+ -+ ret = pci_vpd_pci22_wait(dev); -+ if (ret < 0) -+ goto out; -+ -+ while (pos < end) { -+ u32 val; -+ unsigned int i, skip; -+ -+ ret = pci_user_write_config_word(dev, vpd->cap + PCI_VPD_ADDR, -+ pos & ~3); -+ if (ret < 0) -+ break; -+ vpd->busy = true; -+ vpd->flag = PCI_VPD_ADDR_F; -+ ret = pci_vpd_pci22_wait(dev); -+ if (ret < 0) -+ break; -+ -+ ret = pci_user_read_config_dword(dev, vpd->cap + PCI_VPD_DATA, &val); -+ if (ret < 0) -+ break; -+ -+ skip = pos & 3; -+ for (i = 0; i < sizeof(u32); i++) { -+ if (i >= skip) { -+ *buf++ = val; -+ if (++pos == end) -+ break; ++ if (hp->grab_lock) { ++ preempt_enable(); ++ hotplug_lock(hp); ++ hotplug_unlock(hp); ++ } else { ++ preempt_enable(); ++ /* ++ * Try to push this task off of this CPU. ++ */ ++ if (!migrate_me()) { ++ preempt_disable(); ++ hp = this_cpu_ptr(&hotplug_pcp); ++ if (!hp->grab_lock) { ++ /* ++ * Just let it continue it's already pinned ++ * or about to sleep. ++ */ ++ force = 1; ++ goto retry; + } -+ val >>= 8; ++ preempt_enable(); + } + } -+out: -+ mutex_unlock(&vpd->lock); -+ return ret ? ret : count; ++ preempt_disable(); ++ goto retry; +} + -+static ssize_t pci_vpd_pci22_write(struct pci_dev *dev, loff_t pos, size_t count, -+ const void *arg) ++/** ++ * unpin_current_cpu - Allow unplug of current cpu ++ * ++ * Must be called with preemption or interrupts disabled! ++ */ ++void unpin_current_cpu(void) +{ -+ struct pci_vpd_pci22 *vpd = -+ container_of(dev->vpd, struct pci_vpd_pci22, base); -+ const u8 *buf = arg; -+ loff_t end = pos + count; -+ int ret = 0; -+ -+ if (pos < 0 || (pos & 3) || (count & 3) || end > vpd->base.len) -+ return -EINVAL; -+ -+ if (mutex_lock_killable(&vpd->lock)) -+ return -EINTR; -+ -+ ret = pci_vpd_pci22_wait(dev); -+ if (ret < 0) -+ goto out; -+ -+ while (pos < end) { -+ u32 val; -+ -+ val = *buf++; -+ val |= *buf++ << 8; -+ val |= *buf++ << 16; -+ val |= *buf++ << 24; -+ -+ ret = pci_user_write_config_dword(dev, vpd->cap + PCI_VPD_DATA, val); -+ if (ret < 0) -+ break; -+ ret = pci_user_write_config_word(dev, vpd->cap + PCI_VPD_ADDR, -+ pos | PCI_VPD_ADDR_F); -+ if (ret < 0) -+ break; -+ -+ vpd->busy = true; -+ vpd->flag = 0; -+ ret = pci_vpd_pci22_wait(dev); -+ if (ret < 0) -+ break; ++ struct hotplug_pcp *hp = this_cpu_ptr(&hotplug_pcp); + -+ pos += sizeof(u32); -+ } -+out: -+ mutex_unlock(&vpd->lock); -+ return ret ? ret : count; -+} ++ WARN_ON(hp->refcount <= 0); + -+static void pci_vpd_pci22_release(struct pci_dev *dev) -+{ -+ kfree(container_of(dev->vpd, struct pci_vpd_pci22, base)); ++ /* This is safe. sync_unplug_thread is pinned to this cpu */ ++ if (!--hp->refcount && hp->unplug && hp->unplug != current) ++ wake_up_process(hp->unplug); +} + -+static const struct pci_vpd_ops pci_vpd_pci22_ops = { -+ .read = pci_vpd_pci22_read, -+ .write = pci_vpd_pci22_write, -+ .release = pci_vpd_pci22_release, -+}; -+ -+static ssize_t pci_vpd_f0_read(struct pci_dev *dev, loff_t pos, size_t count, -+ void *arg) ++static void wait_for_pinned_cpus(struct hotplug_pcp *hp) +{ -+ struct pci_dev *tdev = pci_get_slot(dev->bus, PCI_SLOT(dev->devfn)); -+ ssize_t ret; -+ -+ if (!tdev) -+ return -ENODEV; -+ -+ ret = pci_read_vpd(tdev, pos, count, arg); -+ pci_dev_put(tdev); -+ return ret; ++ set_current_state(TASK_UNINTERRUPTIBLE); ++ while (hp->refcount) { ++ schedule_preempt_disabled(); ++ set_current_state(TASK_UNINTERRUPTIBLE); ++ } +} + -+static ssize_t pci_vpd_f0_write(struct pci_dev *dev, loff_t pos, size_t count, -+ const void *arg) ++static int sync_unplug_thread(void *data) +{ -+ struct pci_dev *tdev = pci_get_slot(dev->bus, PCI_SLOT(dev->devfn)); -+ ssize_t ret; -+ -+ if (!tdev) -+ return -ENODEV; -+ -+ ret = pci_write_vpd(tdev, pos, count, arg); -+ pci_dev_put(tdev); -+ return ret; -+} -+ -+static const struct pci_vpd_ops pci_vpd_f0_ops = { -+ .read = pci_vpd_f0_read, -+ .write = pci_vpd_f0_write, -+ .release = pci_vpd_pci22_release, -+}; ++ struct hotplug_pcp *hp = data; + -+static int pci_vpd_f0_dev_check(struct pci_dev *dev) -+{ -+ struct pci_dev *tdev = pci_get_slot(dev->bus, PCI_SLOT(dev->devfn)); -+ int ret = 0; ++ wait_for_completion(&hp->unplug_wait); ++ preempt_disable(); ++ hp->unplug = current; ++ wait_for_pinned_cpus(hp); + -+ if (!tdev) -+ return -ENODEV; -+ if (!tdev->vpd || !tdev->multifunction || -+ dev->class != tdev->class || dev->vendor != tdev->vendor || -+ dev->device != tdev->device) -+ ret = -ENODEV; ++ /* ++ * This thread will synchronize the cpu_down() with threads ++ * that have pinned the CPU. When the pinned CPU count reaches ++ * zero, we inform the cpu_down code to continue to the next step. ++ */ ++ set_current_state(TASK_UNINTERRUPTIBLE); ++ preempt_enable(); ++ complete(&hp->synced); + -+ pci_dev_put(tdev); -+ return ret; -+} ++ /* ++ * If all succeeds, the next step will need tasks to wait till ++ * the CPU is offline before continuing. To do this, the grab_lock ++ * is set and tasks going into pin_current_cpu() will block on the ++ * mutex. But we still need to wait for those that are already in ++ * pinned CPU sections. If the cpu_down() failed, the kthread_should_stop() ++ * will kick this thread out. ++ */ ++ while (!hp->grab_lock && !kthread_should_stop()) { ++ schedule(); ++ set_current_state(TASK_UNINTERRUPTIBLE); ++ } + -+int pci_vpd_pci22_init(struct pci_dev *dev) -+{ -+ struct pci_vpd_pci22 *vpd; -+ u8 cap; ++ /* Make sure grab_lock is seen before we see a stale completion */ ++ smp_mb(); + -+ cap = pci_find_capability(dev, PCI_CAP_ID_VPD); -+ if (!cap) -+ return -ENODEV; -+ if (dev->dev_flags & PCI_DEV_FLAGS_VPD_REF_F0) { -+ int ret = pci_vpd_f0_dev_check(dev); ++ /* ++ * Now just before cpu_down() enters stop machine, we need to make ++ * sure all tasks that are in pinned CPU sections are out, and new ++ * tasks will now grab the lock, keeping them from entering pinned ++ * CPU sections. ++ */ ++ if (!kthread_should_stop()) { ++ preempt_disable(); ++ wait_for_pinned_cpus(hp); ++ preempt_enable(); ++ complete(&hp->synced); ++ } + -+ if (ret) -+ return ret; ++ set_current_state(TASK_UNINTERRUPTIBLE); ++ while (!kthread_should_stop()) { ++ schedule(); ++ set_current_state(TASK_UNINTERRUPTIBLE); + } -+ vpd = kzalloc(sizeof(*vpd), GFP_ATOMIC); -+ if (!vpd) -+ return -ENOMEM; ++ set_current_state(TASK_RUNNING); + -+ vpd->base.len = PCI_VPD_PCI22_SIZE; -+ if (dev->dev_flags & PCI_DEV_FLAGS_VPD_REF_F0) -+ vpd->base.ops = &pci_vpd_f0_ops; -+ else -+ vpd->base.ops = &pci_vpd_pci22_ops; -+ mutex_init(&vpd->lock); -+ vpd->cap = cap; -+ vpd->busy = false; -+ dev->vpd = &vpd->base; ++ /* ++ * Force this thread off this CPU as it's going down and ++ * we don't want any more work on this CPU. ++ */ ++ current->flags &= ~PF_NO_SETAFFINITY; ++ set_cpus_allowed_ptr(current, cpu_present_mask); ++ migrate_me(); + return 0; +} + -+/** -+ * pci_cfg_access_lock - Lock PCI config reads/writes -+ * @dev: pci device struct -+ * -+ * When access is locked, any userspace reads or writes to config -+ * space and concurrent lock requests will sleep until access is -+ * allowed via pci_cfg_access_unlocked again. -+ */ -+void pci_cfg_access_lock(struct pci_dev *dev) ++static void __cpu_unplug_sync(struct hotplug_pcp *hp) +{ -+ might_sleep(); -+ -+ raw_spin_lock_irq(&pci_lock); -+ if (dev->block_cfg_access) -+ pci_wait_cfg(dev); -+ dev->block_cfg_access = 1; -+ raw_spin_unlock_irq(&pci_lock); ++ wake_up_process(hp->sync_tsk); ++ wait_for_completion(&hp->synced); +} -+EXPORT_SYMBOL_GPL(pci_cfg_access_lock); + -+/** -+ * pci_cfg_access_trylock - try to lock PCI config reads/writes -+ * @dev: pci device struct -+ * -+ * Same as pci_cfg_access_lock, but will return 0 if access is -+ * already locked, 1 otherwise. This function can be used from -+ * atomic contexts. -+ */ -+bool pci_cfg_access_trylock(struct pci_dev *dev) ++static void __cpu_unplug_wait(unsigned int cpu) +{ -+ unsigned long flags; -+ bool locked = true; -+ -+ raw_spin_lock_irqsave(&pci_lock, flags); -+ if (dev->block_cfg_access) -+ locked = false; -+ else -+ dev->block_cfg_access = 1; -+ raw_spin_unlock_irqrestore(&pci_lock, flags); ++ struct hotplug_pcp *hp = &per_cpu(hotplug_pcp, cpu); + -+ return locked; ++ complete(&hp->unplug_wait); ++ wait_for_completion(&hp->synced); +} -+EXPORT_SYMBOL_GPL(pci_cfg_access_trylock); + -+/** -+ * pci_cfg_access_unlock - Unlock PCI config reads/writes -+ * @dev: pci device struct -+ * -+ * This function allows PCI config accesses to resume. ++/* ++ * Start the sync_unplug_thread on the target cpu and wait for it to ++ * complete. + */ -+void pci_cfg_access_unlock(struct pci_dev *dev) -+{ -+ unsigned long flags; -+ -+ raw_spin_lock_irqsave(&pci_lock, flags); -+ -+ /* This indicates a problem in the caller, but we don't need -+ * to kill them, unlike a double-block above. */ -+ WARN_ON(!dev->block_cfg_access); -+ -+ dev->block_cfg_access = 0; -+ wake_up_all(&pci_cfg_wait); -+ raw_spin_unlock_irqrestore(&pci_lock, flags); -+} -+EXPORT_SYMBOL_GPL(pci_cfg_access_unlock); -+ -+static inline int pcie_cap_version(const struct pci_dev *dev) -+{ -+ return pcie_caps_reg(dev) & PCI_EXP_FLAGS_VERS; -+} -+ -+bool pcie_cap_has_lnkctl(const struct pci_dev *dev) -+{ -+ int type = pci_pcie_type(dev); -+ -+ return type == PCI_EXP_TYPE_ENDPOINT || -+ type == PCI_EXP_TYPE_LEG_END || -+ type == PCI_EXP_TYPE_ROOT_PORT || -+ type == PCI_EXP_TYPE_UPSTREAM || -+ type == PCI_EXP_TYPE_DOWNSTREAM || -+ type == PCI_EXP_TYPE_PCI_BRIDGE || -+ type == PCI_EXP_TYPE_PCIE_BRIDGE; -+} -+ -+static inline bool pcie_cap_has_sltctl(const struct pci_dev *dev) -+{ -+ int type = pci_pcie_type(dev); -+ -+ return (type == PCI_EXP_TYPE_ROOT_PORT || -+ type == PCI_EXP_TYPE_DOWNSTREAM) && -+ pcie_caps_reg(dev) & PCI_EXP_FLAGS_SLOT; -+} -+ -+static inline bool pcie_cap_has_rtctl(const struct pci_dev *dev) -+{ -+ int type = pci_pcie_type(dev); -+ -+ return type == PCI_EXP_TYPE_ROOT_PORT || -+ type == PCI_EXP_TYPE_RC_EC; -+} -+ -+static bool pcie_capability_reg_implemented(struct pci_dev *dev, int pos) ++static int cpu_unplug_begin(unsigned int cpu) +{ -+ if (!pci_is_pcie(dev)) -+ return false; ++ struct hotplug_pcp *hp = &per_cpu(hotplug_pcp, cpu); ++ int err; + -+ switch (pos) { -+ case PCI_EXP_FLAGS: -+ return true; -+ case PCI_EXP_DEVCAP: -+ case PCI_EXP_DEVCTL: -+ case PCI_EXP_DEVSTA: -+ return true; -+ case PCI_EXP_LNKCAP: -+ case PCI_EXP_LNKCTL: -+ case PCI_EXP_LNKSTA: -+ return pcie_cap_has_lnkctl(dev); -+ case PCI_EXP_SLTCAP: -+ case PCI_EXP_SLTCTL: -+ case PCI_EXP_SLTSTA: -+ return pcie_cap_has_sltctl(dev); -+ case PCI_EXP_RTCTL: -+ case PCI_EXP_RTCAP: -+ case PCI_EXP_RTSTA: -+ return pcie_cap_has_rtctl(dev); -+ case PCI_EXP_DEVCAP2: -+ case PCI_EXP_DEVCTL2: -+ case PCI_EXP_LNKCAP2: -+ case PCI_EXP_LNKCTL2: -+ case PCI_EXP_LNKSTA2: -+ return pcie_cap_version(dev) > 1; -+ default: -+ return false; ++ /* Protected by cpu_hotplug.lock */ ++ if (!hp->mutex_init) { ++#ifdef CONFIG_PREEMPT_RT_FULL ++ spin_lock_init(&hp->lock); ++#else ++ mutex_init(&hp->mutex); ++#endif ++ hp->mutex_init = 1; + } -+} + -+/* -+ * Note that these accessor functions are only for the "PCI Express -+ * Capability" (see PCIe spec r3.0, sec 7.8). They do not apply to the -+ * other "PCI Express Extended Capabilities" (AER, VC, ACS, MFVC, etc.) -+ */ -+int pcie_capability_read_word(struct pci_dev *dev, int pos, u16 *val) -+{ -+ int ret; ++ /* Inform the scheduler to migrate tasks off this CPU */ ++ tell_sched_cpu_down_begin(cpu); + -+ *val = 0; -+ if (pos & 1) -+ return -EINVAL; ++ init_completion(&hp->synced); ++ init_completion(&hp->unplug_wait); + -+ if (pcie_capability_reg_implemented(dev, pos)) { -+ ret = pci_read_config_word(dev, pci_pcie_cap(dev) + pos, val); -+ /* -+ * Reset *val to 0 if pci_read_config_word() fails, it may -+ * have been written as 0xFFFF if hardware error happens -+ * during pci_read_config_word(). -+ */ -+ if (ret) -+ *val = 0; -+ return ret; ++ hp->sync_tsk = kthread_create(sync_unplug_thread, hp, "sync_unplug/%d", cpu); ++ if (IS_ERR(hp->sync_tsk)) { ++ err = PTR_ERR(hp->sync_tsk); ++ hp->sync_tsk = NULL; ++ return err; + } ++ kthread_bind(hp->sync_tsk, cpu); + + /* -+ * For Functions that do not implement the Slot Capabilities, -+ * Slot Status, and Slot Control registers, these spaces must -+ * be hardwired to 0b, with the exception of the Presence Detect -+ * State bit in the Slot Status register of Downstream Ports, -+ * which must be hardwired to 1b. (PCIe Base Spec 3.0, sec 7.8) ++ * Wait for tasks to get out of the pinned sections, ++ * it's still OK if new tasks enter. Some CPU notifiers will ++ * wait for tasks that are going to enter these sections and ++ * we must not have them block. + */ -+ if (pci_is_pcie(dev) && pos == PCI_EXP_SLTSTA && -+ pci_pcie_type(dev) == PCI_EXP_TYPE_DOWNSTREAM) { -+ *val = PCI_EXP_SLTSTA_PDS; -+ } -+ ++ wake_up_process(hp->sync_tsk); + return 0; +} -+EXPORT_SYMBOL(pcie_capability_read_word); + -+int pcie_capability_read_dword(struct pci_dev *dev, int pos, u32 *val) ++static void cpu_unplug_sync(unsigned int cpu) +{ -+ int ret; -+ -+ *val = 0; -+ if (pos & 3) -+ return -EINVAL; -+ -+ if (pcie_capability_reg_implemented(dev, pos)) { -+ ret = pci_read_config_dword(dev, pci_pcie_cap(dev) + pos, val); -+ /* -+ * Reset *val to 0 if pci_read_config_dword() fails, it may -+ * have been written as 0xFFFFFFFF if hardware error happens -+ * during pci_read_config_dword(). -+ */ -+ if (ret) -+ *val = 0; -+ return ret; -+ } -+ -+ if (pci_is_pcie(dev) && pos == PCI_EXP_SLTCTL && -+ pci_pcie_type(dev) == PCI_EXP_TYPE_DOWNSTREAM) { -+ *val = PCI_EXP_SLTSTA_PDS; -+ } ++ struct hotplug_pcp *hp = &per_cpu(hotplug_pcp, cpu); + -+ return 0; -+} -+EXPORT_SYMBOL(pcie_capability_read_dword); ++ init_completion(&hp->synced); ++ /* The completion needs to be initialzied before setting grab_lock */ ++ smp_wmb(); + -+int pcie_capability_write_word(struct pci_dev *dev, int pos, u16 val) -+{ -+ if (pos & 1) -+ return -EINVAL; ++ /* Grab the mutex before setting grab_lock */ ++ hotplug_lock(hp); ++ hp->grab_lock = 1; + -+ if (!pcie_capability_reg_implemented(dev, pos)) -+ return 0; ++ /* ++ * The CPU notifiers have been completed. ++ * Wait for tasks to get out of pinned CPU sections and have new ++ * tasks block until the CPU is completely down. ++ */ ++ __cpu_unplug_sync(hp); + -+ return pci_write_config_word(dev, pci_pcie_cap(dev) + pos, val); ++ /* All done with the sync thread */ ++ kthread_stop(hp->sync_tsk); ++ hp->sync_tsk = NULL; +} -+EXPORT_SYMBOL(pcie_capability_write_word); + -+int pcie_capability_write_dword(struct pci_dev *dev, int pos, u32 val) ++static void cpu_unplug_done(unsigned int cpu) +{ -+ if (pos & 3) -+ return -EINVAL; -+ -+ if (!pcie_capability_reg_implemented(dev, pos)) -+ return 0; ++ struct hotplug_pcp *hp = &per_cpu(hotplug_pcp, cpu); + -+ return pci_write_config_dword(dev, pci_pcie_cap(dev) + pos, val); -+} -+EXPORT_SYMBOL(pcie_capability_write_dword); ++ hp->unplug = NULL; ++ /* Let all tasks know cpu unplug is finished before cleaning up */ ++ smp_wmb(); + -+int pcie_capability_clear_and_set_word(struct pci_dev *dev, int pos, -+ u16 clear, u16 set) -+{ -+ int ret; -+ u16 val; ++ if (hp->sync_tsk) ++ kthread_stop(hp->sync_tsk); + -+ ret = pcie_capability_read_word(dev, pos, &val); -+ if (!ret) { -+ val &= ~clear; -+ val |= set; -+ ret = pcie_capability_write_word(dev, pos, val); ++ if (hp->grab_lock) { ++ hotplug_unlock(hp); ++ /* protected by cpu_hotplug.lock */ ++ hp->grab_lock = 0; + } -+ -+ return ret; ++ tell_sched_cpu_down_done(cpu); +} -+EXPORT_SYMBOL(pcie_capability_clear_and_set_word); -+ -+int pcie_capability_clear_and_set_dword(struct pci_dev *dev, int pos, -+ u32 clear, u32 set) -+{ -+ int ret; -+ u32 val; + + void get_online_cpus(void) + { +@@ -349,13 +632,15 @@ + /* Requires cpu_add_remove_lock to be held */ + static int __ref _cpu_down(unsigned int cpu, int tasks_frozen) + { +- int err, nr_calls = 0; ++ int mycpu, err, nr_calls = 0; + void *hcpu = (void *)(long)cpu; + unsigned long mod = tasks_frozen ? CPU_TASKS_FROZEN : 0; + struct take_cpu_down_param tcd_param = { + .mod = mod, + .hcpu = hcpu, + }; ++ cpumask_var_t cpumask; ++ cpumask_var_t cpumask_org; + + if (num_online_cpus() == 1) + return -EBUSY; +@@ -363,7 +648,34 @@ + if (!cpu_online(cpu)) + return -EINVAL; + ++ /* Move the downtaker off the unplug cpu */ ++ if (!alloc_cpumask_var(&cpumask, GFP_KERNEL)) ++ return -ENOMEM; ++ if (!alloc_cpumask_var(&cpumask_org, GFP_KERNEL)) { ++ free_cpumask_var(cpumask); ++ return -ENOMEM; ++ } + -+ ret = pcie_capability_read_dword(dev, pos, &val); -+ if (!ret) { -+ val &= ~clear; -+ val |= set; -+ ret = pcie_capability_write_dword(dev, pos, val); ++ cpumask_copy(cpumask_org, tsk_cpus_allowed(current)); ++ cpumask_andnot(cpumask, cpu_online_mask, cpumask_of(cpu)); ++ set_cpus_allowed_ptr(current, cpumask); ++ free_cpumask_var(cpumask); ++ migrate_disable(); ++ mycpu = smp_processor_id(); ++ if (mycpu == cpu) { ++ printk(KERN_ERR "Yuck! Still on unplug CPU\n!"); ++ migrate_enable(); ++ err = -EBUSY; ++ goto restore_cpus; + } ++ migrate_enable(); + -+ return ret; -+} -+EXPORT_SYMBOL(pcie_capability_clear_and_set_dword); -diff -Nur linux-4.1.10.orig/drivers/scsi/fcoe/fcoe.c linux-4.1.10/drivers/scsi/fcoe/fcoe.c ---- linux-4.1.10.orig/drivers/scsi/fcoe/fcoe.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/scsi/fcoe/fcoe.c 2015-10-07 18:00:08.000000000 +0200 -@@ -1287,7 +1287,7 @@ - struct sk_buff *skb; - #ifdef CONFIG_SMP - struct fcoe_percpu_s *p0; -- unsigned targ_cpu = get_cpu(); -+ unsigned targ_cpu = get_cpu_light(); - #endif /* CONFIG_SMP */ - - FCOE_DBG("Destroying receive thread for CPU %d\n", cpu); -@@ -1343,7 +1343,7 @@ - kfree_skb(skb); - spin_unlock_bh(&p->fcoe_rx_list.lock); - } -- put_cpu(); -+ put_cpu_light(); - #else - /* - * This a non-SMP scenario where the singular Rx thread is -@@ -1567,11 +1567,11 @@ - static int fcoe_alloc_paged_crc_eof(struct sk_buff *skb, int tlen) - { - struct fcoe_percpu_s *fps; -- int rc; -+ int rc, cpu = get_cpu_light(); + cpu_hotplug_begin(); ++ err = cpu_unplug_begin(cpu); ++ if (err) { ++ printk("cpu_unplug_begin(%d) failed\n", cpu); ++ goto out_cancel; ++ } -- fps = &get_cpu_var(fcoe_percpu); -+ fps = &per_cpu(fcoe_percpu, cpu); - rc = fcoe_get_paged_crc_eof(skb, tlen, fps); -- put_cpu_var(fcoe_percpu); -+ put_cpu_light(); + err = __cpu_notify(CPU_DOWN_PREPARE | mod, hcpu, -1, &nr_calls); + if (err) { +@@ -389,8 +701,12 @@ + #endif + synchronize_rcu(); - return rc; - } -@@ -1767,11 +1767,11 @@ - return 0; - } ++ __cpu_unplug_wait(cpu); + smpboot_park_threads(cpu); -- stats = per_cpu_ptr(lport->stats, get_cpu()); -+ stats = per_cpu_ptr(lport->stats, get_cpu_light()); - stats->InvalidCRCCount++; - if (stats->InvalidCRCCount < 5) - printk(KERN_WARNING "fcoe: dropping frame with CRC error\n"); -- put_cpu(); -+ put_cpu_light(); - return -EINVAL; - } - -@@ -1847,13 +1847,13 @@ - goto drop; ++ /* Notifiers are done. Don't let any more tasks pin this CPU. */ ++ cpu_unplug_sync(cpu); ++ + /* + * So now all preempt/rcu users must observe !cpu_active(). + */ +@@ -427,9 +743,14 @@ + check_for_tasks(cpu); - if (!fcoe_filter_frames(lport, fp)) { -- put_cpu(); -+ put_cpu_light(); - fc_exch_recv(lport, fp); - return; - } - drop: - stats->ErrorFrames++; -- put_cpu(); -+ put_cpu_light(); - kfree_skb(skb); + out_release: ++ cpu_unplug_done(cpu); ++out_cancel: + cpu_hotplug_done(); + if (!err) + cpu_notify_nofail(CPU_POST_DEAD | mod, hcpu); ++restore_cpus: ++ set_cpus_allowed_ptr(current, cpumask_org); ++ free_cpumask_var(cpumask_org); + return err; } -diff -Nur linux-4.1.10.orig/drivers/scsi/fcoe/fcoe_ctlr.c linux-4.1.10/drivers/scsi/fcoe/fcoe_ctlr.c ---- linux-4.1.10.orig/drivers/scsi/fcoe/fcoe_ctlr.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/scsi/fcoe/fcoe_ctlr.c 2015-10-07 18:00:08.000000000 +0200 -@@ -831,7 +831,7 @@ - - INIT_LIST_HEAD(&del_list); +diff -Nur linux-4.1.10.orig/kernel/debug/kdb/kdb_io.c linux-4.1.10/kernel/debug/kdb/kdb_io.c +--- linux-4.1.10.orig/kernel/debug/kdb/kdb_io.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/debug/kdb/kdb_io.c 2015-10-12 22:33:32.292675697 +0200 +@@ -554,7 +554,6 @@ + int linecount; + int colcount; + int logging, saved_loglevel = 0; +- int saved_trap_printk; + int got_printf_lock = 0; + int retlen = 0; + int fnd, len; +@@ -565,8 +564,6 @@ + unsigned long uninitialized_var(flags); -- stats = per_cpu_ptr(fip->lp->stats, get_cpu()); -+ stats = per_cpu_ptr(fip->lp->stats, get_cpu_light()); + preempt_disable(); +- saved_trap_printk = kdb_trap_printk; +- kdb_trap_printk = 0; - list_for_each_entry_safe(fcf, next, &fip->fcfs, list) { - deadline = fcf->time + fcf->fka_period + fcf->fka_period / 2; -@@ -867,7 +867,7 @@ - sel_time = fcf->time; - } + /* Serialize kdb_printf if multiple cpus try to write at once. + * But if any cpu goes recursive in kdb, just print the output, +@@ -855,7 +852,6 @@ + } else { + __release(kdb_printf_lock); } -- put_cpu(); -+ put_cpu_light(); +- kdb_trap_printk = saved_trap_printk; + preempt_enable(); + return retlen; + } +@@ -865,9 +861,11 @@ + va_list ap; + int r; - list_for_each_entry_safe(fcf, next, &del_list, list) { - /* Removes fcf from current list */ -diff -Nur linux-4.1.10.orig/drivers/scsi/libfc/fc_exch.c linux-4.1.10/drivers/scsi/libfc/fc_exch.c ---- linux-4.1.10.orig/drivers/scsi/libfc/fc_exch.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/scsi/libfc/fc_exch.c 2015-10-07 18:00:08.000000000 +0200 -@@ -814,10 +814,10 @@ - } - memset(ep, 0, sizeof(*ep)); ++ kdb_trap_printk++; + va_start(ap, fmt); + r = vkdb_printf(KDB_MSGSRC_INTERNAL, fmt, ap); + va_end(ap); ++ kdb_trap_printk--; -- cpu = get_cpu(); -+ cpu = get_cpu_light(); - pool = per_cpu_ptr(mp->pool, cpu); - spin_lock_bh(&pool->lock); -- put_cpu(); -+ put_cpu_light(); + return r; + } +diff -Nur linux-4.1.10.orig/kernel/events/core.c linux-4.1.10/kernel/events/core.c +--- linux-4.1.10.orig/kernel/events/core.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/events/core.c 2015-10-12 22:33:32.292675697 +0200 +@@ -6933,6 +6933,7 @@ - /* peek cache of free slot */ - if (pool->left != FC_XID_UNKNOWN) { -diff -Nur linux-4.1.10.orig/drivers/scsi/libsas/sas_ata.c linux-4.1.10/drivers/scsi/libsas/sas_ata.c ---- linux-4.1.10.orig/drivers/scsi/libsas/sas_ata.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/scsi/libsas/sas_ata.c 2015-10-07 18:00:08.000000000 +0200 -@@ -190,7 +190,7 @@ - /* TODO: audit callers to ensure they are ready for qc_issue to - * unconditionally re-enable interrupts - */ -- local_irq_save(flags); -+ local_irq_save_nort(flags); - spin_unlock(ap->lock); + hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + hwc->hrtimer.function = perf_swevent_hrtimer; ++ hwc->hrtimer.irqsafe = 1; - /* If the device fell off, no sense in issuing commands */ -@@ -255,7 +255,7 @@ + /* + * Since hrtimers have a fixed rate, we can do a static freq->period +diff -Nur linux-4.1.10.orig/kernel/exit.c linux-4.1.10/kernel/exit.c +--- linux-4.1.10.orig/kernel/exit.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/exit.c 2015-10-12 22:33:32.292675697 +0200 +@@ -144,7 +144,7 @@ + * Do this under ->siglock, we can race with another thread + * doing sigqueue_free() if we have SIGQUEUE_PREALLOC signals. + */ +- flush_sigqueue(&tsk->pending); ++ flush_task_sigqueue(tsk); + tsk->sighand = NULL; + spin_unlock(&sighand->siglock); - out: - spin_lock(ap->lock); -- local_irq_restore(flags); -+ local_irq_restore_nort(flags); - return ret; - } +diff -Nur linux-4.1.10.orig/kernel/fork.c linux-4.1.10/kernel/fork.c +--- linux-4.1.10.orig/kernel/fork.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/fork.c 2015-10-12 22:33:32.292675697 +0200 +@@ -108,7 +108,7 @@ -diff -Nur linux-4.1.10.orig/drivers/scsi/qla2xxx/qla_inline.h linux-4.1.10/drivers/scsi/qla2xxx/qla_inline.h ---- linux-4.1.10.orig/drivers/scsi/qla2xxx/qla_inline.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/scsi/qla2xxx/qla_inline.h 2015-10-07 18:00:08.000000000 +0200 -@@ -59,12 +59,12 @@ - { - unsigned long flags; - struct qla_hw_data *ha = rsp->hw; -- local_irq_save(flags); -+ local_irq_save_nort(flags); - if (IS_P3P_TYPE(ha)) - qla82xx_poll(0, rsp); - else - ha->isp_ops->intr_handler(0, rsp); -- local_irq_restore(flags); -+ local_irq_restore_nort(flags); - } + DEFINE_PER_CPU(unsigned long, process_counts) = 0; - static inline uint8_t * -diff -Nur linux-4.1.10.orig/drivers/thermal/x86_pkg_temp_thermal.c linux-4.1.10/drivers/thermal/x86_pkg_temp_thermal.c ---- linux-4.1.10.orig/drivers/thermal/x86_pkg_temp_thermal.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/thermal/x86_pkg_temp_thermal.c 2015-10-07 18:00:08.000000000 +0200 -@@ -29,6 +29,7 @@ - #include - #include - #include -+#include - #include - #include +-__cacheline_aligned DEFINE_RWLOCK(tasklist_lock); /* outer */ ++DEFINE_RWLOCK(tasklist_lock); /* outer */ -@@ -352,7 +353,7 @@ - } + #ifdef CONFIG_PROVE_RCU + int lockdep_tasklist_lock_is_held(void) +@@ -244,7 +244,9 @@ + if (atomic_dec_and_test(&sig->sigcnt)) + free_signal_struct(sig); } - --static int pkg_temp_thermal_platform_thermal_notify(__u64 msr_val) -+static void platform_thermal_notify_work(struct swork_event *event) +- ++#ifdef CONFIG_PREEMPT_RT_BASE ++static ++#endif + void __put_task_struct(struct task_struct *tsk) { - unsigned long flags; - int cpu = smp_processor_id(); -@@ -369,7 +370,7 @@ - pkg_work_scheduled[phy_id]) { - disable_pkg_thres_interrupt(); - spin_unlock_irqrestore(&pkg_work_lock, flags); -- return -EINVAL; -+ return; - } - pkg_work_scheduled[phy_id] = 1; - spin_unlock_irqrestore(&pkg_work_lock, flags); -@@ -378,9 +379,48 @@ - schedule_delayed_work_on(cpu, - &per_cpu(pkg_temp_thermal_threshold_work, cpu), - msecs_to_jiffies(notify_delay_ms)); -+} -+ -+#ifdef CONFIG_PREEMPT_RT_FULL -+static struct swork_event notify_work; -+ -+static int thermal_notify_work_init(void) + WARN_ON(!tsk->exit_state); +@@ -260,7 +262,18 @@ + if (!profile_handoff_task(tsk)) + free_task(tsk); + } ++#ifndef CONFIG_PREEMPT_RT_BASE + EXPORT_SYMBOL_GPL(__put_task_struct); ++#else ++void __put_task_struct_cb(struct rcu_head *rhp) +{ -+ int err; ++ struct task_struct *tsk = container_of(rhp, struct task_struct, put_rcu); + -+ err = swork_get(); -+ if (err) -+ return err; ++ __put_task_struct(tsk); + -+ INIT_SWORK(¬ify_work, platform_thermal_notify_work); - return 0; ++} ++EXPORT_SYMBOL_GPL(__put_task_struct_cb); ++#endif + + void __init __weak arch_task_cache_init(void) { } + +@@ -680,6 +693,19 @@ } + EXPORT_SYMBOL_GPL(__mmdrop); -+static void thermal_notify_work_cleanup(void) -+{ -+ swork_put(); -+} -+ -+static int pkg_temp_thermal_platform_thermal_notify(__u64 msr_val) -+{ -+ swork_queue(¬ify_work); -+ return 0; -+} -+ -+#else /* !CONFIG_PREEMPT_RT_FULL */ -+ -+static int thermal_notify_work_init(void) { return 0; } -+ -+static void thermal_notify_work_cleanup(void) { } -+ -+static int pkg_temp_thermal_platform_thermal_notify(__u64 msr_val) ++#ifdef CONFIG_PREEMPT_RT_BASE ++/* ++ * RCU callback for delayed mm drop. Not strictly rcu, but we don't ++ * want another facility to make this work. ++ */ ++void __mmdrop_delayed(struct rcu_head *rhp) +{ -+ platform_thermal_notify_work(NULL); ++ struct mm_struct *mm = container_of(rhp, struct mm_struct, delayed_drop); + -+ return 0; ++ __mmdrop(mm); +} -+#endif /* CONFIG_PREEMPT_RT_FULL */ ++#endif + - static int find_siblings_cpu(int cpu) + /* + * Decrement the use count and release all resources for an mm. + */ +@@ -1214,6 +1240,9 @@ + */ + static void posix_cpu_timers_init(struct task_struct *tsk) { - int i; -@@ -584,6 +624,9 @@ - if (!x86_match_cpu(pkg_temp_thermal_ids)) - return -ENODEV; ++#ifdef CONFIG_PREEMPT_RT_BASE ++ tsk->posix_timer_list = NULL; ++#endif + tsk->cputime_expires.prof_exp = 0; + tsk->cputime_expires.virt_exp = 0; + tsk->cputime_expires.sched_exp = 0; +@@ -1338,6 +1367,7 @@ + spin_lock_init(&p->alloc_lock); -+ if (!thermal_notify_work_init()) -+ return -ENODEV; -+ - spin_lock_init(&pkg_work_lock); - platform_thermal_package_notify = - pkg_temp_thermal_platform_thermal_notify; -@@ -608,7 +651,7 @@ - kfree(pkg_work_scheduled); - platform_thermal_package_notify = NULL; - platform_thermal_package_rate_control = NULL; -- -+ thermal_notify_work_cleanup(); - return -ENODEV; - } + init_sigpending(&p->pending); ++ p->sigqueue_cache = NULL; -@@ -633,6 +676,7 @@ - mutex_unlock(&phy_dev_list_mutex); - platform_thermal_package_notify = NULL; - platform_thermal_package_rate_control = NULL; -+ thermal_notify_work_cleanup(); - for_each_online_cpu(i) - cancel_delayed_work_sync( - &per_cpu(pkg_temp_thermal_threshold_work, i)); -diff -Nur linux-4.1.10.orig/drivers/tty/serial/8250/8250_core.c linux-4.1.10/drivers/tty/serial/8250/8250_core.c ---- linux-4.1.10.orig/drivers/tty/serial/8250/8250_core.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/tty/serial/8250/8250_core.c 2015-10-07 18:00:08.000000000 +0200 -@@ -36,6 +36,7 @@ - #include - #include - #include -+#include - #include - #include - #ifdef CONFIG_SPARC -@@ -80,7 +81,16 @@ - #define DEBUG_INTR(fmt...) do { } while (0) + p->utime = p->stime = p->gtime = 0; + p->utimescaled = p->stimescaled = 0; +@@ -1345,7 +1375,8 @@ + p->prev_cputime.utime = p->prev_cputime.stime = 0; #endif + #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN +- seqlock_init(&p->vtime_seqlock); ++ raw_spin_lock_init(&p->vtime_lock); ++ seqcount_init(&p->vtime_seq); + p->vtime_snap = 0; + p->vtime_snap_whence = VTIME_SLEEPING; + #endif +@@ -1396,6 +1427,9 @@ + p->hardirq_context = 0; + p->softirq_context = 0; + #endif ++ ++ p->pagefault_disabled = 0; ++ + #ifdef CONFIG_LOCKDEP + p->lockdep_depth = 0; /* no locks held yet */ + p->curr_chain_key = 0; +diff -Nur linux-4.1.10.orig/kernel/futex.c linux-4.1.10/kernel/futex.c +--- linux-4.1.10.orig/kernel/futex.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/futex.c 2015-10-12 22:33:32.292675697 +0200 +@@ -738,7 +738,9 @@ + * task still owns the PI-state: + */ + if (head->next != next) { ++ raw_spin_unlock_irq(&curr->pi_lock); + spin_unlock(&hb->lock); ++ raw_spin_lock_irq(&curr->pi_lock); + continue; + } --#define PASS_LIMIT 512 -+/* -+ * On -rt we can have a more delays, and legitimately -+ * so - so don't drop work spuriously and spam the -+ * syslog: -+ */ -+#ifdef CONFIG_PREEMPT_RT_FULL -+# define PASS_LIMIT 1000000 -+#else -+# define PASS_LIMIT 512 -+#endif - - #define BOTH_EMPTY (UART_LSR_TEMT | UART_LSR_THRE) +@@ -1090,9 +1092,11 @@ -@@ -3364,7 +3374,7 @@ + /* + * The hash bucket lock must be held when this is called. +- * Afterwards, the futex_q must not be accessed. ++ * Afterwards, the futex_q must not be accessed. Callers ++ * must ensure to later call wake_up_q() for the actual ++ * wakeups to occur. + */ +-static void wake_futex(struct futex_q *q) ++static void mark_wake_futex(struct wake_q_head *wake_q, struct futex_q *q) + { + struct task_struct *p = q->task; - if (port->sysrq) - locked = 0; -- else if (oops_in_progress) -+ else if (oops_in_progress || in_kdb_printk()) - locked = spin_trylock_irqsave(&port->lock, flags); - else - spin_lock_irqsave(&port->lock, flags); -diff -Nur linux-4.1.10.orig/drivers/tty/serial/amba-pl011.c linux-4.1.10/drivers/tty/serial/amba-pl011.c ---- linux-4.1.10.orig/drivers/tty/serial/amba-pl011.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/tty/serial/amba-pl011.c 2015-10-07 18:00:08.000000000 +0200 -@@ -2000,13 +2000,19 @@ +@@ -1100,14 +1104,10 @@ + return; - clk_enable(uap->clk); + /* +- * We set q->lock_ptr = NULL _before_ we wake up the task. If +- * a non-futex wake up happens on another CPU then the task +- * might exit and p would dereference a non-existing task +- * struct. Prevent this by holding a reference on p across the +- * wake up. ++ * Queue the task for later wakeup for after we've released ++ * the hb->lock. wake_q_add() grabs reference to p. + */ +- get_task_struct(p); +- ++ wake_q_add(wake_q, p); + __unqueue_futex(q); + /* + * The waiting task can free the futex_q as soon as +@@ -1117,16 +1117,15 @@ + */ + smp_wmb(); + q->lock_ptr = NULL; +- +- wake_up_state(p, TASK_NORMAL); +- put_task_struct(p); + } -- local_irq_save(flags); -+ /* -+ * local_irq_save(flags); -+ * -+ * This local_irq_save() is nonsense. If we come in via sysrq -+ * handling then interrupts are already disabled. Aside of -+ * that the port.sysrq check is racy on SMP regardless. -+ */ - if (uap->port.sysrq) - locked = 0; - else if (oops_in_progress) -- locked = spin_trylock(&uap->port.lock); -+ locked = spin_trylock_irqsave(&uap->port.lock, flags); - else -- spin_lock(&uap->port.lock); -+ spin_lock_irqsave(&uap->port.lock, flags); +-static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_q *this) ++static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_q *this, ++ struct futex_hash_bucket *hb) + { + struct task_struct *new_owner; + struct futex_pi_state *pi_state = this->pi_state; + u32 uninitialized_var(curval), newval; ++ bool deboost; + int ret = 0; - /* - * First save the CR then disable the interrupts -@@ -2028,8 +2034,7 @@ - writew(old_cr, uap->port.membase + UART011_CR); + if (!pi_state) +@@ -1178,7 +1177,17 @@ + raw_spin_unlock_irq(&new_owner->pi_lock); - if (locked) -- spin_unlock(&uap->port.lock); -- local_irq_restore(flags); -+ spin_unlock_irqrestore(&uap->port.lock, flags); + raw_spin_unlock(&pi_state->pi_mutex.wait_lock); +- rt_mutex_unlock(&pi_state->pi_mutex); ++ ++ deboost = rt_mutex_futex_unlock(&pi_state->pi_mutex); ++ ++ /* ++ * We deboost after dropping hb->lock. That prevents a double ++ * wakeup on RT. ++ */ ++ spin_unlock(&hb->lock); ++ ++ if (deboost) ++ rt_mutex_adjust_prio(current); - clk_disable(uap->clk); + return 0; } -diff -Nur linux-4.1.10.orig/drivers/tty/serial/omap-serial.c linux-4.1.10/drivers/tty/serial/omap-serial.c ---- linux-4.1.10.orig/drivers/tty/serial/omap-serial.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/tty/serial/omap-serial.c 2015-10-07 18:00:08.000000000 +0200 -@@ -1282,13 +1282,10 @@ +@@ -1217,6 +1226,7 @@ + struct futex_q *this, *next; + union futex_key key = FUTEX_KEY_INIT; + int ret; ++ WAKE_Q(wake_q); - pm_runtime_get_sync(up->dev); + if (!bitset) + return -EINVAL; +@@ -1244,13 +1254,14 @@ + if (!(this->bitset & bitset)) + continue; -- local_irq_save(flags); -- if (up->port.sysrq) -- locked = 0; -- else if (oops_in_progress) -- locked = spin_trylock(&up->port.lock); -+ if (up->port.sysrq || oops_in_progress) -+ locked = spin_trylock_irqsave(&up->port.lock, flags); - else -- spin_lock(&up->port.lock); -+ spin_lock_irqsave(&up->port.lock, flags); +- wake_futex(this); ++ mark_wake_futex(&wake_q, this); + if (++ret >= nr_wake) + break; + } + } - /* - * First save the IER then disable the interrupts -@@ -1317,8 +1314,7 @@ - pm_runtime_mark_last_busy(up->dev); - pm_runtime_put_autosuspend(up->dev); - if (locked) -- spin_unlock(&up->port.lock); -- local_irq_restore(flags); -+ spin_unlock_irqrestore(&up->port.lock, flags); - } + spin_unlock(&hb->lock); ++ wake_up_q(&wake_q); + out_put_key: + put_futex_key(&key); + out: +@@ -1269,6 +1280,7 @@ + struct futex_hash_bucket *hb1, *hb2; + struct futex_q *this, *next; + int ret, op_ret; ++ WAKE_Q(wake_q); - static int __init -diff -Nur linux-4.1.10.orig/drivers/usb/core/hcd.c linux-4.1.10/drivers/usb/core/hcd.c ---- linux-4.1.10.orig/drivers/usb/core/hcd.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/usb/core/hcd.c 2015-10-07 18:00:08.000000000 +0200 -@@ -1684,9 +1684,9 @@ - * and no one may trigger the above deadlock situation when - * running complete() in tasklet. - */ -- local_irq_save(flags); -+ local_irq_save_nort(flags); - urb->complete(urb); -- local_irq_restore(flags); -+ local_irq_restore_nort(flags); + retry: + ret = get_futex_key(uaddr1, flags & FLAGS_SHARED, &key1, VERIFY_READ); +@@ -1320,7 +1332,7 @@ + ret = -EINVAL; + goto out_unlock; + } +- wake_futex(this); ++ mark_wake_futex(&wake_q, this); + if (++ret >= nr_wake) + break; + } +@@ -1334,7 +1346,7 @@ + ret = -EINVAL; + goto out_unlock; + } +- wake_futex(this); ++ mark_wake_futex(&wake_q, this); + if (++op_ret >= nr_wake2) + break; + } +@@ -1344,6 +1356,7 @@ - usb_anchor_resume_wakeups(anchor); - atomic_dec(&urb->use_count); -diff -Nur linux-4.1.10.orig/drivers/usb/gadget/function/f_fs.c linux-4.1.10/drivers/usb/gadget/function/f_fs.c ---- linux-4.1.10.orig/drivers/usb/gadget/function/f_fs.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/usb/gadget/function/f_fs.c 2015-10-07 18:00:08.000000000 +0200 -@@ -1405,7 +1405,7 @@ - pr_info("%s(): freeing\n", __func__); - ffs_data_clear(ffs); - BUG_ON(waitqueue_active(&ffs->ev.waitq) || -- waitqueue_active(&ffs->ep0req_completion.wait)); -+ swaitqueue_active(&ffs->ep0req_completion.wait)); - kfree(ffs->dev_name); - kfree(ffs); + out_unlock: + double_unlock_hb(hb1, hb2); ++ wake_up_q(&wake_q); + out_put_keys: + put_futex_key(&key2); + out_put_key1: +@@ -1503,6 +1516,7 @@ + struct futex_pi_state *pi_state = NULL; + struct futex_hash_bucket *hb1, *hb2; + struct futex_q *this, *next; ++ WAKE_Q(wake_q); + + if (requeue_pi) { + /* +@@ -1679,7 +1693,7 @@ + * woken by futex_unlock_pi(). + */ + if (++task_count <= nr_wake && !requeue_pi) { +- wake_futex(this); ++ mark_wake_futex(&wake_q, this); + continue; + } + +@@ -1705,6 +1719,16 @@ + requeue_pi_wake_futex(this, &key2, hb2); + drop_count++; + continue; ++ } else if (ret == -EAGAIN) { ++ /* ++ * Waiter was woken by timeout or ++ * signal and has set pi_blocked_on to ++ * PI_WAKEUP_INPROGRESS before we ++ * tried to enqueue it on the rtmutex. ++ */ ++ this->pi_state = NULL; ++ free_pi_state(pi_state); ++ continue; + } else if (ret) { + /* -EDEADLK */ + this->pi_state = NULL; +@@ -1719,6 +1743,7 @@ + out_unlock: + free_pi_state(pi_state); + double_unlock_hb(hb1, hb2); ++ wake_up_q(&wake_q); + hb_waiters_dec(hb2); + + /* +@@ -2412,13 +2437,26 @@ + */ + match = futex_top_waiter(hb, &key); + if (match) { +- ret = wake_futex_pi(uaddr, uval, match); ++ ret = wake_futex_pi(uaddr, uval, match, hb); ++ ++ /* ++ * In case of success wake_futex_pi dropped the hash ++ * bucket lock. ++ */ ++ if (!ret) ++ goto out_putkey; ++ + /* + * The atomic access to the futex value generated a + * pagefault, so retry the user-access and the wakeup: + */ + if (ret == -EFAULT) + goto pi_faulted; ++ ++ /* ++ * wake_futex_pi has detected invalid state. Tell user ++ * space. ++ */ + goto out_unlock; } -diff -Nur linux-4.1.10.orig/drivers/usb/gadget/legacy/inode.c linux-4.1.10/drivers/usb/gadget/legacy/inode.c ---- linux-4.1.10.orig/drivers/usb/gadget/legacy/inode.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/drivers/usb/gadget/legacy/inode.c 2015-10-07 18:00:08.000000000 +0200 -@@ -345,7 +345,7 @@ - spin_unlock_irq (&epdata->dev->lock); - if (likely (value == 0)) { -- value = wait_event_interruptible (done.wait, done.done); -+ value = swait_event_interruptible (done.wait, done.done); - if (value != 0) { - spin_lock_irq (&epdata->dev->lock); - if (likely (epdata->ep != NULL)) { -@@ -354,7 +354,7 @@ - usb_ep_dequeue (epdata->ep, epdata->req); - spin_unlock_irq (&epdata->dev->lock); +@@ -2439,6 +2477,7 @@ -- wait_event (done.wait, done.done); -+ swait_event (done.wait, done.done); - if (epdata->status == -ECONNRESET) - epdata->status = -EINTR; - } else { -diff -Nur linux-4.1.10.orig/fs/aio.c linux-4.1.10/fs/aio.c ---- linux-4.1.10.orig/fs/aio.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/fs/aio.c 2015-10-07 18:00:08.000000000 +0200 -@@ -40,6 +40,7 @@ - #include - #include - #include -+#include + out_unlock: + spin_unlock(&hb->lock); ++out_putkey: + put_futex_key(&key); + return ret; - #include - #include -@@ -115,7 +116,7 @@ - struct page **ring_pages; - long nr_pages; +@@ -2549,7 +2588,7 @@ + struct hrtimer_sleeper timeout, *to = NULL; + struct rt_mutex_waiter rt_waiter; + struct rt_mutex *pi_mutex = NULL; +- struct futex_hash_bucket *hb; ++ struct futex_hash_bucket *hb, *hb2; + union futex_key key2 = FUTEX_KEY_INIT; + struct futex_q q = futex_q_init; + int res, ret; +@@ -2574,10 +2613,7 @@ + * The waiter is allocated on our stack, manipulated by the requeue + * code while we sleep on uaddr. + */ +- debug_rt_mutex_init_waiter(&rt_waiter); +- RB_CLEAR_NODE(&rt_waiter.pi_tree_entry); +- RB_CLEAR_NODE(&rt_waiter.tree_entry); +- rt_waiter.task = NULL; ++ rt_mutex_init_waiter(&rt_waiter, false); -- struct work_struct free_work; -+ struct swork_event free_work; + ret = get_futex_key(uaddr2, flags & FLAGS_SHARED, &key2, VERIFY_WRITE); + if (unlikely(ret != 0)) +@@ -2608,20 +2644,55 @@ + /* Queue the futex_q, drop the hb lock, wait for wakeup. */ + futex_wait_queue_me(hb, &q, to); + +- spin_lock(&hb->lock); +- ret = handle_early_requeue_pi_wakeup(hb, &q, &key2, to); +- spin_unlock(&hb->lock); +- if (ret) +- goto out_put_keys; ++ /* ++ * On RT we must avoid races with requeue and trying to block ++ * on two mutexes (hb->lock and uaddr2's rtmutex) by ++ * serializing access to pi_blocked_on with pi_lock. ++ */ ++ raw_spin_lock_irq(¤t->pi_lock); ++ if (current->pi_blocked_on) { ++ /* ++ * We have been requeued or are in the process of ++ * being requeued. ++ */ ++ raw_spin_unlock_irq(¤t->pi_lock); ++ } else { ++ /* ++ * Setting pi_blocked_on to PI_WAKEUP_INPROGRESS ++ * prevents a concurrent requeue from moving us to the ++ * uaddr2 rtmutex. After that we can safely acquire ++ * (and possibly block on) hb->lock. ++ */ ++ current->pi_blocked_on = PI_WAKEUP_INPROGRESS; ++ raw_spin_unlock_irq(¤t->pi_lock); ++ ++ spin_lock(&hb->lock); ++ ++ /* ++ * Clean up pi_blocked_on. We might leak it otherwise ++ * when we succeeded with the hb->lock in the fast ++ * path. ++ */ ++ raw_spin_lock_irq(¤t->pi_lock); ++ current->pi_blocked_on = NULL; ++ raw_spin_unlock_irq(¤t->pi_lock); ++ ++ ret = handle_early_requeue_pi_wakeup(hb, &q, &key2, to); ++ spin_unlock(&hb->lock); ++ if (ret) ++ goto out_put_keys; ++ } /* - * signals when all in-flight requests are done -@@ -253,6 +254,7 @@ - .mount = aio_mount, - .kill_sb = kill_anon_super, - }; -+ BUG_ON(swork_get()); - aio_mnt = kern_mount(&aio_fs); - if (IS_ERR(aio_mnt)) - panic("Failed to create aio fs mount."); -@@ -559,9 +561,9 @@ - return cancel(&kiocb->common); - } +- * In order for us to be here, we know our q.key == key2, and since +- * we took the hb->lock above, we also know that futex_requeue() has +- * completed and we no longer have to concern ourselves with a wakeup +- * race with the atomic proxy lock acquisition by the requeue code. The +- * futex_requeue dropped our key1 reference and incremented our key2 +- * reference count. ++ * In order to be here, we have either been requeued, are in ++ * the process of being requeued, or requeue successfully ++ * acquired uaddr2 on our behalf. If pi_blocked_on was ++ * non-null above, we may be racing with a requeue. Do not ++ * rely on q->lock_ptr to be hb2->lock until after blocking on ++ * hb->lock or hb2->lock. The futex_requeue dropped our key1 ++ * reference and incremented our key2 reference count. + */ ++ hb2 = hash_futex(&key2); --static void free_ioctx(struct work_struct *work) -+static void free_ioctx(struct swork_event *sev) + /* Check if the requeue code acquired the second futex for us. */ + if (!q.rt_waiter) { +@@ -2630,9 +2701,10 @@ + * did a lock-steal - fix up the PI-state in that case. + */ + if (q.pi_state && (q.pi_state->owner != current)) { +- spin_lock(q.lock_ptr); ++ spin_lock(&hb2->lock); ++ BUG_ON(&hb2->lock != q.lock_ptr); + ret = fixup_pi_state_owner(uaddr2, &q, current); +- spin_unlock(q.lock_ptr); ++ spin_unlock(&hb2->lock); + } + } else { + /* +@@ -2645,7 +2717,8 @@ + ret = rt_mutex_finish_proxy_lock(pi_mutex, to, &rt_waiter); + debug_rt_mutex_free_waiter(&rt_waiter); + +- spin_lock(q.lock_ptr); ++ spin_lock(&hb2->lock); ++ BUG_ON(&hb2->lock != q.lock_ptr); + /* + * Fixup the pi_state owner and possibly acquire the lock if we + * haven't already. +diff -Nur linux-4.1.10.orig/kernel/irq/handle.c linux-4.1.10/kernel/irq/handle.c +--- linux-4.1.10.orig/kernel/irq/handle.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/irq/handle.c 2015-10-12 22:33:32.296675432 +0200 +@@ -133,6 +133,8 @@ + irqreturn_t + handle_irq_event_percpu(struct irq_desc *desc, struct irqaction *action) { -- struct kioctx *ctx = container_of(work, struct kioctx, free_work); -+ struct kioctx *ctx = container_of(sev, struct kioctx, free_work); ++ struct pt_regs *regs = get_irq_regs(); ++ u64 ip = regs ? instruction_pointer(regs) : 0; + irqreturn_t retval = IRQ_NONE; + unsigned int flags = 0, irq = desc->irq_data.irq; - pr_debug("freeing %p\n", ctx); +@@ -173,7 +175,11 @@ + action = action->next; + } while (action); -@@ -580,8 +582,8 @@ - if (ctx->rq_wait && atomic_dec_and_test(&ctx->rq_wait->count)) - complete(&ctx->rq_wait->comp); +- add_interrupt_randomness(irq, flags); ++#ifndef CONFIG_PREEMPT_RT_FULL ++ add_interrupt_randomness(irq, flags, ip); ++#else ++ desc->random_ip = ip; ++#endif -- INIT_WORK(&ctx->free_work, free_ioctx); -- schedule_work(&ctx->free_work); -+ INIT_SWORK(&ctx->free_work, free_ioctx); -+ swork_queue(&ctx->free_work); - } + if (!noirqdebug) + note_interrupt(irq, desc, retval); +diff -Nur linux-4.1.10.orig/kernel/irq/manage.c linux-4.1.10/kernel/irq/manage.c +--- linux-4.1.10.orig/kernel/irq/manage.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/irq/manage.c 2015-10-12 22:33:32.296675432 +0200 +@@ -22,6 +22,7 @@ + #include "internals.h" - /* -@@ -589,9 +591,9 @@ - * and ctx->users has dropped to 0, so we know no more kiocbs can be submitted - - * now it's safe to cancel any that need to be. - */ --static void free_ioctx_users(struct percpu_ref *ref) -+static void free_ioctx_users_work(struct swork_event *sev) - { -- struct kioctx *ctx = container_of(ref, struct kioctx, users); -+ struct kioctx *ctx = container_of(sev, struct kioctx, free_work); - struct aio_kiocb *req; + #ifdef CONFIG_IRQ_FORCED_THREADING ++# ifndef CONFIG_PREEMPT_RT_BASE + __read_mostly bool force_irqthreads; - spin_lock_irq(&ctx->ctx_lock); -@@ -610,6 +612,14 @@ - percpu_ref_put(&ctx->reqs); + static int __init setup_forced_irqthreads(char *arg) +@@ -30,6 +31,7 @@ + return 0; } + early_param("threadirqs", setup_forced_irqthreads); ++# endif + #endif -+static void free_ioctx_users(struct percpu_ref *ref) + static void __synchronize_hardirq(struct irq_desc *desc) +@@ -179,6 +181,62 @@ + irq_get_pending(struct cpumask *mask, struct irq_desc *desc) { } + #endif + ++#ifdef CONFIG_PREEMPT_RT_FULL ++static void _irq_affinity_notify(struct irq_affinity_notify *notify); ++static struct task_struct *set_affinity_helper; ++static LIST_HEAD(affinity_list); ++static DEFINE_RAW_SPINLOCK(affinity_list_lock); ++ ++static int set_affinity_thread(void *unused) +{ -+ struct kioctx *ctx = container_of(ref, struct kioctx, users); ++ while (1) { ++ struct irq_affinity_notify *notify; ++ int empty; + -+ INIT_SWORK(&ctx->free_work, free_ioctx_users_work); -+ swork_queue(&ctx->free_work); ++ set_current_state(TASK_INTERRUPTIBLE); ++ ++ raw_spin_lock_irq(&affinity_list_lock); ++ empty = list_empty(&affinity_list); ++ raw_spin_unlock_irq(&affinity_list_lock); ++ ++ if (empty) ++ schedule(); ++ if (kthread_should_stop()) ++ break; ++ set_current_state(TASK_RUNNING); ++try_next: ++ notify = NULL; ++ ++ raw_spin_lock_irq(&affinity_list_lock); ++ if (!list_empty(&affinity_list)) { ++ notify = list_first_entry(&affinity_list, ++ struct irq_affinity_notify, list); ++ list_del_init(¬ify->list); ++ } ++ raw_spin_unlock_irq(&affinity_list_lock); ++ ++ if (!notify) ++ continue; ++ _irq_affinity_notify(notify); ++ goto try_next; ++ } ++ return 0; +} + - static int ioctx_add_table(struct kioctx *ctx, struct mm_struct *mm) ++static void init_helper_thread(void) ++{ ++ if (set_affinity_helper) ++ return; ++ set_affinity_helper = kthread_run(set_affinity_thread, NULL, ++ "affinity-cb"); ++ WARN_ON(IS_ERR(set_affinity_helper)); ++} ++#else ++ ++static inline void init_helper_thread(void) { } ++ ++#endif ++ + int irq_do_set_affinity(struct irq_data *data, const struct cpumask *mask, + bool force) { - unsigned i, new_nr; -diff -Nur linux-4.1.10.orig/fs/autofs4/autofs_i.h linux-4.1.10/fs/autofs4/autofs_i.h ---- linux-4.1.10.orig/fs/autofs4/autofs_i.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/fs/autofs4/autofs_i.h 2015-10-07 18:00:08.000000000 +0200 -@@ -34,6 +34,7 @@ - #include - #include - #include -+#include - #include - #include +@@ -218,7 +276,17 @@ -diff -Nur linux-4.1.10.orig/fs/autofs4/expire.c linux-4.1.10/fs/autofs4/expire.c ---- linux-4.1.10.orig/fs/autofs4/expire.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/fs/autofs4/expire.c 2015-10-07 18:00:08.000000000 +0200 -@@ -150,7 +150,7 @@ - parent = p->d_parent; - if (!spin_trylock(&parent->d_lock)) { - spin_unlock(&p->d_lock); -- cpu_relax(); -+ cpu_chill(); - goto relock; - } - spin_unlock(&p->d_lock); -diff -Nur linux-4.1.10.orig/fs/buffer.c linux-4.1.10/fs/buffer.c ---- linux-4.1.10.orig/fs/buffer.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/fs/buffer.c 2015-10-07 18:00:08.000000000 +0200 -@@ -301,8 +301,7 @@ - * decide that the page is now completely done. - */ - first = page_buffers(page); -- local_irq_save(flags); -- bit_spin_lock(BH_Uptodate_Lock, &first->b_state); -+ flags = bh_uptodate_lock_irqsave(first); - clear_buffer_async_read(bh); - unlock_buffer(bh); - tmp = bh; -@@ -315,8 +314,7 @@ - } - tmp = tmp->b_this_page; - } while (tmp != bh); -- bit_spin_unlock(BH_Uptodate_Lock, &first->b_state); -- local_irq_restore(flags); -+ bh_uptodate_unlock_irqrestore(first, flags); + if (desc->affinity_notify) { + kref_get(&desc->affinity_notify->kref); ++ ++#ifdef CONFIG_PREEMPT_RT_FULL ++ raw_spin_lock(&affinity_list_lock); ++ if (list_empty(&desc->affinity_notify->list)) ++ list_add_tail(&affinity_list, ++ &desc->affinity_notify->list); ++ raw_spin_unlock(&affinity_list_lock); ++ wake_up_process(set_affinity_helper); ++#else + schedule_work(&desc->affinity_notify->work); ++#endif + } + irqd_set(data, IRQD_AFFINITY_SET); - /* - * If none of the buffers had errors and they are all -@@ -328,9 +326,7 @@ - return; +@@ -256,10 +324,8 @@ + } + EXPORT_SYMBOL_GPL(irq_set_affinity_hint); - still_busy: -- bit_spin_unlock(BH_Uptodate_Lock, &first->b_state); -- local_irq_restore(flags); -- return; -+ bh_uptodate_unlock_irqrestore(first, flags); +-static void irq_affinity_notify(struct work_struct *work) ++static void _irq_affinity_notify(struct irq_affinity_notify *notify) + { +- struct irq_affinity_notify *notify = +- container_of(work, struct irq_affinity_notify, work); + struct irq_desc *desc = irq_to_desc(notify->irq); + cpumask_var_t cpumask; + unsigned long flags; +@@ -281,6 +347,13 @@ + kref_put(¬ify->kref, notify->release); } - /* -@@ -358,8 +354,7 @@ ++static void irq_affinity_notify(struct work_struct *work) ++{ ++ struct irq_affinity_notify *notify = ++ container_of(work, struct irq_affinity_notify, work); ++ _irq_affinity_notify(notify); ++} ++ + /** + * irq_set_affinity_notifier - control notification of IRQ affinity changes + * @irq: Interrupt for which to enable/disable notification +@@ -310,6 +383,8 @@ + notify->irq = irq; + kref_init(¬ify->kref); + INIT_WORK(¬ify->work, irq_affinity_notify); ++ INIT_LIST_HEAD(¬ify->list); ++ init_helper_thread(); } - first = page_buffers(page); -- local_irq_save(flags); -- bit_spin_lock(BH_Uptodate_Lock, &first->b_state); -+ flags = bh_uptodate_lock_irqsave(first); - - clear_buffer_async_write(bh); - unlock_buffer(bh); -@@ -371,15 +366,12 @@ - } - tmp = tmp->b_this_page; - } -- bit_spin_unlock(BH_Uptodate_Lock, &first->b_state); -- local_irq_restore(flags); -+ bh_uptodate_unlock_irqrestore(first, flags); - end_page_writeback(page); - return; + raw_spin_lock_irqsave(&desc->lock, flags); +@@ -697,6 +772,12 @@ + return IRQ_NONE; + } - still_busy: -- bit_spin_unlock(BH_Uptodate_Lock, &first->b_state); -- local_irq_restore(flags); -- return; -+ bh_uptodate_unlock_irqrestore(first, flags); ++static irqreturn_t irq_forced_secondary_handler(int irq, void *dev_id) ++{ ++ WARN(1, "Secondary action handler called for irq %d\n", irq); ++ return IRQ_NONE; ++} ++ + static int irq_wait_for_interrupt(struct irqaction *action) + { + set_current_state(TASK_INTERRUPTIBLE); +@@ -723,7 +804,8 @@ + static void irq_finalize_oneshot(struct irq_desc *desc, + struct irqaction *action) + { +- if (!(desc->istate & IRQS_ONESHOT)) ++ if (!(desc->istate & IRQS_ONESHOT) || ++ action->handler == irq_forced_secondary_handler) + return; + again: + chip_bus_lock(desc); +@@ -825,7 +907,15 @@ + local_bh_disable(); + ret = action->thread_fn(action->irq, action->dev_id); + irq_finalize_oneshot(desc, action); +- local_bh_enable(); ++ /* ++ * Interrupts which have real time requirements can be set up ++ * to avoid softirq processing in the thread handler. This is ++ * safe as these interrupts do not raise soft interrupts. ++ */ ++ if (irq_settings_no_softirq_call(desc)) ++ _local_bh_enable(); ++ else ++ local_bh_enable(); + return ret; } - EXPORT_SYMBOL(end_buffer_async_write); - -@@ -3325,6 +3317,7 @@ - struct buffer_head *ret = kmem_cache_zalloc(bh_cachep, gfp_flags); - if (ret) { - INIT_LIST_HEAD(&ret->b_assoc_buffers); -+ buffer_head_init_locks(ret); - preempt_disable(); - __this_cpu_inc(bh_accounting.nr); - recalc_bh_state(); -diff -Nur linux-4.1.10.orig/fs/dcache.c linux-4.1.10/fs/dcache.c ---- linux-4.1.10.orig/fs/dcache.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/fs/dcache.c 2015-10-07 18:00:08.000000000 +0200 -@@ -19,6 +19,7 @@ - #include - #include - #include -+#include - #include - #include - #include -@@ -589,7 +590,7 @@ - failed: - spin_unlock(&dentry->d_lock); -- cpu_relax(); -+ cpu_chill(); - return dentry; /* try again with same dentry */ +@@ -877,6 +967,18 @@ + irq_finalize_oneshot(desc, action); } -@@ -2395,7 +2396,7 @@ - if (dentry->d_lockref.count == 1) { - if (!spin_trylock(&inode->i_lock)) { - spin_unlock(&dentry->d_lock); -- cpu_relax(); -+ cpu_chill(); - goto again; - } - dentry->d_flags &= ~DCACHE_CANT_MOUNT; -diff -Nur linux-4.1.10.orig/fs/eventpoll.c linux-4.1.10/fs/eventpoll.c ---- linux-4.1.10.orig/fs/eventpoll.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/fs/eventpoll.c 2015-10-07 18:00:08.000000000 +0200 -@@ -505,12 +505,12 @@ ++static void irq_wake_secondary(struct irq_desc *desc, struct irqaction *action) ++{ ++ struct irqaction *secondary = action->secondary; ++ ++ if (WARN_ON_ONCE(!secondary)) ++ return; ++ ++ raw_spin_lock_irq(&desc->lock); ++ __irq_wake_thread(desc, secondary); ++ raw_spin_unlock_irq(&desc->lock); ++} ++ + /* + * Interrupt handler thread */ - static void ep_poll_safewake(wait_queue_head_t *wq) - { -- int this_cpu = get_cpu(); -+ int this_cpu = get_cpu_light(); +@@ -907,7 +1009,15 @@ + action_ret = handler_fn(desc, action); + if (action_ret == IRQ_HANDLED) + atomic_inc(&desc->threads_handled); ++ if (action_ret == IRQ_WAKE_THREAD) ++ irq_wake_secondary(desc, action); - ep_call_nested(&poll_safewake_ncalls, EP_MAX_NESTS, - ep_poll_wakeup_proc, NULL, wq, (void *) (long) this_cpu); ++#ifdef CONFIG_PREEMPT_RT_FULL ++ migrate_disable(); ++ add_interrupt_randomness(action->irq, 0, ++ desc->random_ip ^ (unsigned long) action); ++ migrate_enable(); ++#endif + wake_threads_waitq(desc); + } -- put_cpu(); -+ put_cpu_light(); +@@ -951,20 +1061,36 @@ } + EXPORT_SYMBOL_GPL(irq_wake_thread); - static void ep_remove_wait_queue(struct eppoll_entry *pwq) -diff -Nur linux-4.1.10.orig/fs/exec.c linux-4.1.10/fs/exec.c ---- linux-4.1.10.orig/fs/exec.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/fs/exec.c 2015-10-07 18:00:08.000000000 +0200 -@@ -859,12 +859,14 @@ - } - } - task_lock(tsk); -+ preempt_disable_rt(); - active_mm = tsk->active_mm; - tsk->mm = mm; - tsk->active_mm = mm; - activate_mm(active_mm, mm); - tsk->mm->vmacache_seqnum = 0; - vmacache_flush(tsk); -+ preempt_enable_rt(); - task_unlock(tsk); - if (old_mm) { - up_read(&old_mm->mmap_sem); -diff -Nur linux-4.1.10.orig/fs/jbd/checkpoint.c linux-4.1.10/fs/jbd/checkpoint.c ---- linux-4.1.10.orig/fs/jbd/checkpoint.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/fs/jbd/checkpoint.c 2015-10-07 18:00:08.000000000 +0200 -@@ -129,6 +129,8 @@ - if (journal->j_flags & JFS_ABORT) - return; - spin_unlock(&journal->j_state_lock); -+ if (current->plug) -+ io_schedule(); - mutex_lock(&journal->j_checkpoint_mutex); +-static void irq_setup_forced_threading(struct irqaction *new) ++static int irq_setup_forced_threading(struct irqaction *new) + { + if (!force_irqthreads) +- return; ++ return 0; + if (new->flags & (IRQF_NO_THREAD | IRQF_PERCPU | IRQF_ONESHOT)) +- return; ++ return 0; - /* -diff -Nur linux-4.1.10.orig/fs/jbd2/checkpoint.c linux-4.1.10/fs/jbd2/checkpoint.c ---- linux-4.1.10.orig/fs/jbd2/checkpoint.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/fs/jbd2/checkpoint.c 2015-10-07 18:00:08.000000000 +0200 -@@ -116,6 +116,8 @@ - nblocks = jbd2_space_needed(journal); - while (jbd2_log_space_left(journal) < nblocks) { - write_unlock(&journal->j_state_lock); -+ if (current->plug) -+ io_schedule(); - mutex_lock(&journal->j_checkpoint_mutex); + new->flags |= IRQF_ONESHOT; - /* -diff -Nur linux-4.1.10.orig/fs/namespace.c linux-4.1.10/fs/namespace.c ---- linux-4.1.10.orig/fs/namespace.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/fs/namespace.c 2015-10-07 18:00:08.000000000 +0200 -@@ -14,6 +14,7 @@ - #include - #include - #include -+#include - #include - #include - #include /* init_rootfs */ -@@ -353,8 +354,11 @@ - * incremented count after it has set MNT_WRITE_HOLD. - */ - smp_mb(); -- while (ACCESS_ONCE(mnt->mnt.mnt_flags) & MNT_WRITE_HOLD) -- cpu_relax(); -+ while (ACCESS_ONCE(mnt->mnt.mnt_flags) & MNT_WRITE_HOLD) { -+ preempt_enable(); -+ cpu_chill(); -+ preempt_disable(); +- if (!new->thread_fn) { +- set_bit(IRQTF_FORCED_THREAD, &new->thread_flags); +- new->thread_fn = new->handler; +- new->handler = irq_default_primary_handler; +- } ++ /* ++ * Handle the case where we have a real primary handler and a ++ * thread handler. We force thread them as well by creating a ++ * secondary action. ++ */ ++ if (new->handler != irq_default_primary_handler && new->thread_fn) { ++ /* Allocate the secondary action */ ++ new->secondary = kzalloc(sizeof(struct irqaction), GFP_KERNEL); ++ if (!new->secondary) ++ return -ENOMEM; ++ new->secondary->handler = irq_forced_secondary_handler; ++ new->secondary->thread_fn = new->thread_fn; ++ new->secondary->dev_id = new->dev_id; ++ new->secondary->irq = new->irq; ++ new->secondary->name = new->name; ++ } ++ /* Deal with the primary handler */ ++ set_bit(IRQTF_FORCED_THREAD, &new->thread_flags); ++ new->thread_fn = new->handler; ++ new->handler = irq_default_primary_handler; ++ return 0; + } + + static int irq_request_resources(struct irq_desc *desc) +@@ -984,6 +1110,48 @@ + c->irq_release_resources(d); + } + ++static int ++setup_irq_thread(struct irqaction *new, unsigned int irq, bool secondary) ++{ ++ struct task_struct *t; ++ struct sched_param param = { ++ .sched_priority = MAX_USER_RT_PRIO/2, ++ }; ++ ++ if (!secondary) { ++ t = kthread_create(irq_thread, new, "irq/%d-%s", irq, ++ new->name); ++ } else { ++ t = kthread_create(irq_thread, new, "irq/%d-s-%s", irq, ++ new->name); ++ param.sched_priority += 1; + } ++ ++ if (IS_ERR(t)) ++ return PTR_ERR(t); ++ ++ sched_setscheduler_nocheck(t, SCHED_FIFO, ¶m); ++ ++ /* ++ * We keep the reference to the task struct even if ++ * the thread dies to avoid that the interrupt code ++ * references an already freed task_struct. ++ */ ++ get_task_struct(t); ++ new->thread = t; ++ /* ++ * Tell the thread to set its affinity. This is ++ * important for shared interrupt handlers as we do ++ * not invoke setup_affinity() for the secondary ++ * handlers as everything is already set up. Even for ++ * interrupts marked with IRQF_NO_BALANCE this is ++ * correct as we want the thread to move to the cpu(s) ++ * on which the requesting code placed the interrupt. ++ */ ++ set_bit(IRQTF_AFFINITY, &new->thread_flags); ++ return 0; ++} ++ + /* + * Internal function to register an irqaction - typically used to + * allocate special interrupts that are part of the architecture. +@@ -1004,6 +1172,8 @@ + if (!try_module_get(desc->owner)) + return -ENODEV; + ++ new->irq = irq; ++ /* - * After the slowpath clears MNT_WRITE_HOLD, mnt_is_readonly will - * be set to match its requirements. So we must not load that until -diff -Nur linux-4.1.10.orig/fs/ntfs/aops.c linux-4.1.10/fs/ntfs/aops.c ---- linux-4.1.10.orig/fs/ntfs/aops.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/fs/ntfs/aops.c 2015-10-07 18:00:08.000000000 +0200 -@@ -107,8 +107,7 @@ - "0x%llx.", (unsigned long long)bh->b_blocknr); + * Check whether the interrupt nests into another interrupt + * thread. +@@ -1021,8 +1191,11 @@ + */ + new->handler = irq_nested_primary_handler; + } else { +- if (irq_settings_can_thread(desc)) +- irq_setup_forced_threading(new); ++ if (irq_settings_can_thread(desc)) { ++ ret = irq_setup_forced_threading(new); ++ if (ret) ++ goto out_mput; ++ } } - first = page_buffers(page); -- local_irq_save(flags); -- bit_spin_lock(BH_Uptodate_Lock, &first->b_state); -+ flags = bh_uptodate_lock_irqsave(first); - clear_buffer_async_read(bh); - unlock_buffer(bh); - tmp = bh; -@@ -123,8 +122,7 @@ - } - tmp = tmp->b_this_page; - } while (tmp != bh); -- bit_spin_unlock(BH_Uptodate_Lock, &first->b_state); -- local_irq_restore(flags); -+ bh_uptodate_unlock_irqrestore(first, flags); - /* - * If none of the buffers had errors then we can set the page uptodate, - * but we first have to perform the post read mst fixups, if the -@@ -145,13 +143,13 @@ - recs = PAGE_CACHE_SIZE / rec_size; - /* Should have been verified before we got here... */ - BUG_ON(!recs); -- local_irq_save(flags); -+ local_irq_save_nort(flags); - kaddr = kmap_atomic(page); - for (i = 0; i < recs; i++) - post_read_mst_fixup((NTFS_RECORD*)(kaddr + - i * rec_size), rec_size); - kunmap_atomic(kaddr); -- local_irq_restore(flags); -+ local_irq_restore_nort(flags); - flush_dcache_page(page); - if (likely(page_uptodate && !PageError(page))) - SetPageUptodate(page); -@@ -159,9 +157,7 @@ - unlock_page(page); - return; - still_busy: -- bit_spin_unlock(BH_Uptodate_Lock, &first->b_state); -- local_irq_restore(flags); -- return; -+ bh_uptodate_unlock_irqrestore(first, flags); - } - /** -diff -Nur linux-4.1.10.orig/fs/timerfd.c linux-4.1.10/fs/timerfd.c ---- linux-4.1.10.orig/fs/timerfd.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/fs/timerfd.c 2015-10-07 18:00:08.000000000 +0200 -@@ -450,7 +450,10 @@ - break; + /* +@@ -1031,37 +1204,14 @@ + * thread. + */ + if (new->thread_fn && !nested) { +- struct task_struct *t; +- static const struct sched_param param = { +- .sched_priority = MAX_USER_RT_PRIO/2, +- }; +- +- t = kthread_create(irq_thread, new, "irq/%d-%s", irq, +- new->name); +- if (IS_ERR(t)) { +- ret = PTR_ERR(t); ++ ret = setup_irq_thread(new, irq, false); ++ if (ret) + goto out_mput; ++ if (new->secondary) { ++ ret = setup_irq_thread(new->secondary, irq, true); ++ if (ret) ++ goto out_thread; } - spin_unlock_irq(&ctx->wqh.lock); -- cpu_relax(); -+ if (isalarm(ctx)) -+ hrtimer_wait_for_timer(&ctx->t.alarm.timer); -+ else -+ hrtimer_wait_for_timer(&ctx->t.tmr); +- +- sched_setscheduler_nocheck(t, SCHED_FIFO, ¶m); +- +- /* +- * We keep the reference to the task struct even if +- * the thread dies to avoid that the interrupt code +- * references an already freed task_struct. +- */ +- get_task_struct(t); +- new->thread = t; +- /* +- * Tell the thread to set its affinity. This is +- * important for shared interrupt handlers as we do +- * not invoke setup_affinity() for the secondary +- * handlers as everything is already set up. Even for +- * interrupts marked with IRQF_NO_BALANCE this is +- * correct as we want the thread to move to the cpu(s) +- * on which the requesting code placed the interrupt. +- */ +- set_bit(IRQTF_AFFINITY, &new->thread_flags); } - /* -diff -Nur linux-4.1.10.orig/fs/xfs/xfs_inode.c linux-4.1.10/fs/xfs/xfs_inode.c ---- linux-4.1.10.orig/fs/xfs/xfs_inode.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/fs/xfs/xfs_inode.c 2015-10-07 18:00:08.000000000 +0200 -@@ -164,7 +164,7 @@ - (XFS_MMAPLOCK_SHARED | XFS_MMAPLOCK_EXCL)); - ASSERT((lock_flags & (XFS_ILOCK_SHARED | XFS_ILOCK_EXCL)) != - (XFS_ILOCK_SHARED | XFS_ILOCK_EXCL)); -- ASSERT((lock_flags & ~(XFS_LOCK_MASK | XFS_LOCK_DEP_MASK)) == 0); -+ ASSERT((lock_flags & ~(XFS_LOCK_MASK | XFS_LOCK_SUBCLASS_MASK)) == 0); + if (!alloc_cpumask_var(&mask, GFP_KERNEL)) { +@@ -1221,6 +1371,9 @@ + irqd_set(&desc->irq_data, IRQD_NO_BALANCING); + } - if (lock_flags & XFS_IOLOCK_EXCL) - mrupdate_nested(&ip->i_iolock, XFS_IOLOCK_DEP(lock_flags)); -@@ -212,7 +212,7 @@ - (XFS_MMAPLOCK_SHARED | XFS_MMAPLOCK_EXCL)); - ASSERT((lock_flags & (XFS_ILOCK_SHARED | XFS_ILOCK_EXCL)) != - (XFS_ILOCK_SHARED | XFS_ILOCK_EXCL)); -- ASSERT((lock_flags & ~(XFS_LOCK_MASK | XFS_LOCK_DEP_MASK)) == 0); -+ ASSERT((lock_flags & ~(XFS_LOCK_MASK | XFS_LOCK_SUBCLASS_MASK)) == 0); ++ if (new->flags & IRQF_NO_SOFTIRQ_CALL) ++ irq_settings_set_no_softirq_call(desc); ++ + /* Set default affinity mask once everything is setup */ + setup_affinity(irq, desc, mask); - if (lock_flags & XFS_IOLOCK_EXCL) { - if (!mrtryupdate(&ip->i_iolock)) -@@ -281,7 +281,7 @@ - (XFS_MMAPLOCK_SHARED | XFS_MMAPLOCK_EXCL)); - ASSERT((lock_flags & (XFS_ILOCK_SHARED | XFS_ILOCK_EXCL)) != - (XFS_ILOCK_SHARED | XFS_ILOCK_EXCL)); -- ASSERT((lock_flags & ~(XFS_LOCK_MASK | XFS_LOCK_DEP_MASK)) == 0); -+ ASSERT((lock_flags & ~(XFS_LOCK_MASK | XFS_LOCK_SUBCLASS_MASK)) == 0); - ASSERT(lock_flags != 0); +@@ -1234,7 +1387,6 @@ + irq, nmsk, omsk); + } - if (lock_flags & XFS_IOLOCK_EXCL) -@@ -364,30 +364,38 @@ +- new->irq = irq; + *old_ptr = new; - /* - * Bump the subclass so xfs_lock_inodes() acquires each lock with a different -- * value. This shouldn't be called for page fault locking, but we also need to -- * ensure we don't overrun the number of lockdep subclasses for the iolock or -- * mmaplock as that is limited to 12 by the mmap lock lockdep annotations. -+ * value. This can be called for any type of inode lock combination, including -+ * parent locking. Care must be taken to ensure we don't overrun the subclass -+ * storage fields in the class mask we build. - */ - static inline int - xfs_lock_inumorder(int lock_mode, int subclass) - { -+ int class = 0; -+ -+ ASSERT(!(lock_mode & (XFS_ILOCK_PARENT | XFS_ILOCK_RTBITMAP | -+ XFS_ILOCK_RTSUM))); -+ - if (lock_mode & (XFS_IOLOCK_SHARED|XFS_IOLOCK_EXCL)) { -- ASSERT(subclass + XFS_LOCK_INUMORDER < -- (1 << (XFS_MMAPLOCK_SHIFT - XFS_IOLOCK_SHIFT))); -- lock_mode |= (subclass + XFS_LOCK_INUMORDER) << XFS_IOLOCK_SHIFT; -+ ASSERT(subclass <= XFS_IOLOCK_MAX_SUBCLASS); -+ ASSERT(subclass + XFS_IOLOCK_PARENT_VAL < -+ MAX_LOCKDEP_SUBCLASSES); -+ class += subclass << XFS_IOLOCK_SHIFT; -+ if (lock_mode & XFS_IOLOCK_PARENT) -+ class += XFS_IOLOCK_PARENT_VAL << XFS_IOLOCK_SHIFT; - } + irq_pm_install_action(desc, new); +@@ -1260,6 +1412,8 @@ + */ + if (new->thread) + wake_up_process(new->thread); ++ if (new->secondary) ++ wake_up_process(new->secondary->thread); - if (lock_mode & (XFS_MMAPLOCK_SHARED|XFS_MMAPLOCK_EXCL)) { -- ASSERT(subclass + XFS_LOCK_INUMORDER < -- (1 << (XFS_ILOCK_SHIFT - XFS_MMAPLOCK_SHIFT))); -- lock_mode |= (subclass + XFS_LOCK_INUMORDER) << -- XFS_MMAPLOCK_SHIFT; -+ ASSERT(subclass <= XFS_MMAPLOCK_MAX_SUBCLASS); -+ class += subclass << XFS_MMAPLOCK_SHIFT; + register_irq_proc(irq, desc); + new->dir = NULL; +@@ -1290,6 +1444,13 @@ + kthread_stop(t); + put_task_struct(t); } - -- if (lock_mode & (XFS_ILOCK_SHARED|XFS_ILOCK_EXCL)) -- lock_mode |= (subclass + XFS_LOCK_INUMORDER) << XFS_ILOCK_SHIFT; -+ if (lock_mode & (XFS_ILOCK_SHARED|XFS_ILOCK_EXCL)) { -+ ASSERT(subclass <= XFS_ILOCK_MAX_SUBCLASS); -+ class += subclass << XFS_ILOCK_SHIFT; ++ if (new->secondary && new->secondary->thread) { ++ struct task_struct *t = new->secondary->thread; ++ ++ new->secondary->thread = NULL; ++ kthread_stop(t); ++ put_task_struct(t); + } + out_mput: + module_put(desc->owner); + return ret; +@@ -1397,9 +1558,14 @@ + if (action->thread) { + kthread_stop(action->thread); + put_task_struct(action->thread); ++ if (action->secondary && action->secondary->thread) { ++ kthread_stop(action->secondary->thread); ++ put_task_struct(action->secondary->thread); ++ } + } -- return lock_mode; -+ return (lock_mode & ~XFS_LOCK_SUBCLASS_MASK) | class; + module_put(desc->owner); ++ kfree(action->secondary); + return action; } - /* -@@ -399,6 +407,11 @@ - * transaction (such as truncate). This can result in deadlock since the long - * running trans might need to wait for the inode we just locked in order to - * push the tail and free space in the log. -+ * -+ * xfs_lock_inodes() can only be used to lock one type of lock at a time - -+ * the iolock, the mmaplock or the ilock, but not more than one at a time. If we -+ * lock more than one at a time, lockdep will report false positives saying we -+ * have violated locking orders. - */ - void - xfs_lock_inodes( -@@ -409,8 +422,29 @@ - int attempts = 0, i, j, try_lock; - xfs_log_item_t *lp; - -- /* currently supports between 2 and 5 inodes */ -+ /* -+ * Currently supports between 2 and 5 inodes with exclusive locking. We -+ * support an arbitrary depth of locking here, but absolute limits on -+ * inodes depend on the the type of locking and the limits placed by -+ * lockdep annotations in xfs_lock_inumorder. These are all checked by -+ * the asserts. -+ */ - ASSERT(ips && inodes >= 2 && inodes <= 5); -+ ASSERT(lock_mode & (XFS_IOLOCK_EXCL | XFS_MMAPLOCK_EXCL | -+ XFS_ILOCK_EXCL)); -+ ASSERT(!(lock_mode & (XFS_IOLOCK_SHARED | XFS_MMAPLOCK_SHARED | -+ XFS_ILOCK_SHARED))); -+ ASSERT(!(lock_mode & XFS_IOLOCK_EXCL) || -+ inodes <= XFS_IOLOCK_MAX_SUBCLASS + 1); -+ ASSERT(!(lock_mode & XFS_MMAPLOCK_EXCL) || -+ inodes <= XFS_MMAPLOCK_MAX_SUBCLASS + 1); -+ ASSERT(!(lock_mode & XFS_ILOCK_EXCL) || -+ inodes <= XFS_ILOCK_MAX_SUBCLASS + 1); -+ -+ if (lock_mode & XFS_IOLOCK_EXCL) { -+ ASSERT(!(lock_mode & (XFS_MMAPLOCK_EXCL | XFS_ILOCK_EXCL))); -+ } else if (lock_mode & XFS_MMAPLOCK_EXCL) -+ ASSERT(!(lock_mode & XFS_ILOCK_EXCL)); - - try_lock = 0; - i = 0; -diff -Nur linux-4.1.10.orig/fs/xfs/xfs_inode.h linux-4.1.10/fs/xfs/xfs_inode.h ---- linux-4.1.10.orig/fs/xfs/xfs_inode.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/fs/xfs/xfs_inode.h 2015-10-07 18:00:08.000000000 +0200 -@@ -284,9 +284,9 @@ - * Flags for lockdep annotations. - * - * XFS_LOCK_PARENT - for directory operations that require locking a -- * parent directory inode and a child entry inode. The parent gets locked -- * with this flag so it gets a lockdep subclass of 1 and the child entry -- * lock will have a lockdep subclass of 0. -+ * parent directory inode and a child entry inode. IOLOCK requires nesting, -+ * MMAPLOCK does not support this class, ILOCK requires a single subclass -+ * to differentiate parent from child. - * - * XFS_LOCK_RTBITMAP/XFS_LOCK_RTSUM - the realtime device bitmap and summary - * inodes do not participate in the normal lock order, and thus have their -@@ -295,30 +295,63 @@ - * XFS_LOCK_INUMORDER - for locking several inodes at the some time - * with xfs_lock_inodes(). This flag is used as the starting subclass - * and each subsequent lock acquired will increment the subclass by one. -- * So the first lock acquired will have a lockdep subclass of 4, the -- * second lock will have a lockdep subclass of 5, and so on. It is -- * the responsibility of the class builder to shift this to the correct -- * portion of the lock_mode lockdep mask. -+ * However, MAX_LOCKDEP_SUBCLASSES == 8, which means we are greatly -+ * limited to the subclasses we can represent via nesting. We need at least -+ * 5 inodes nest depth for the ILOCK through rename, and we also have to support -+ * XFS_ILOCK_PARENT, which gives 6 subclasses. Then we have XFS_ILOCK_RTBITMAP -+ * and XFS_ILOCK_RTSUM, which are another 2 unique subclasses, so that's all -+ * 8 subclasses supported by lockdep. -+ * -+ * This also means we have to number the sub-classes in the lowest bits of -+ * the mask we keep, and we have to ensure we never exceed 3 bits of lockdep -+ * mask and we can't use bit-masking to build the subclasses. What a mess. -+ * -+ * Bit layout: -+ * -+ * Bit Lock Region -+ * 16-19 XFS_IOLOCK_SHIFT dependencies -+ * 20-23 XFS_MMAPLOCK_SHIFT dependencies -+ * 24-31 XFS_ILOCK_SHIFT dependencies -+ * -+ * IOLOCK values -+ * -+ * 0-3 subclass value -+ * 4-7 PARENT subclass values -+ * -+ * MMAPLOCK values -+ * -+ * 0-3 subclass value -+ * 4-7 unused -+ * -+ * ILOCK values -+ * 0-4 subclass values -+ * 5 PARENT subclass (not nestable) -+ * 6 RTBITMAP subclass (not nestable) -+ * 7 RTSUM subclass (not nestable) -+ * - */ --#define XFS_LOCK_PARENT 1 --#define XFS_LOCK_RTBITMAP 2 --#define XFS_LOCK_RTSUM 3 --#define XFS_LOCK_INUMORDER 4 -- --#define XFS_IOLOCK_SHIFT 16 --#define XFS_IOLOCK_PARENT (XFS_LOCK_PARENT << XFS_IOLOCK_SHIFT) -+#define XFS_IOLOCK_SHIFT 16 -+#define XFS_IOLOCK_PARENT_VAL 4 -+#define XFS_IOLOCK_MAX_SUBCLASS (XFS_IOLOCK_PARENT_VAL - 1) -+#define XFS_IOLOCK_DEP_MASK 0x000f0000 -+#define XFS_IOLOCK_PARENT (XFS_IOLOCK_PARENT_VAL << XFS_IOLOCK_SHIFT) - --#define XFS_MMAPLOCK_SHIFT 20 -+#define XFS_MMAPLOCK_SHIFT 20 -+#define XFS_MMAPLOCK_NUMORDER 0 -+#define XFS_MMAPLOCK_MAX_SUBCLASS 3 -+#define XFS_MMAPLOCK_DEP_MASK 0x00f00000 - --#define XFS_ILOCK_SHIFT 24 --#define XFS_ILOCK_PARENT (XFS_LOCK_PARENT << XFS_ILOCK_SHIFT) --#define XFS_ILOCK_RTBITMAP (XFS_LOCK_RTBITMAP << XFS_ILOCK_SHIFT) --#define XFS_ILOCK_RTSUM (XFS_LOCK_RTSUM << XFS_ILOCK_SHIFT) -+#define XFS_ILOCK_SHIFT 24 -+#define XFS_ILOCK_PARENT_VAL 5 -+#define XFS_ILOCK_MAX_SUBCLASS (XFS_ILOCK_PARENT_VAL - 1) -+#define XFS_ILOCK_RTBITMAP_VAL 6 -+#define XFS_ILOCK_RTSUM_VAL 7 -+#define XFS_ILOCK_DEP_MASK 0xff000000 -+#define XFS_ILOCK_PARENT (XFS_ILOCK_PARENT_VAL << XFS_ILOCK_SHIFT) -+#define XFS_ILOCK_RTBITMAP (XFS_ILOCK_RTBITMAP_VAL << XFS_ILOCK_SHIFT) -+#define XFS_ILOCK_RTSUM (XFS_ILOCK_RTSUM_VAL << XFS_ILOCK_SHIFT) +@@ -1543,8 +1709,10 @@ + retval = __setup_irq(irq, desc, action); + chip_bus_sync_unlock(desc); --#define XFS_IOLOCK_DEP_MASK 0x000f0000 --#define XFS_MMAPLOCK_DEP_MASK 0x00f00000 --#define XFS_ILOCK_DEP_MASK 0xff000000 --#define XFS_LOCK_DEP_MASK (XFS_IOLOCK_DEP_MASK | \ -+#define XFS_LOCK_SUBCLASS_MASK (XFS_IOLOCK_DEP_MASK | \ - XFS_MMAPLOCK_DEP_MASK | \ - XFS_ILOCK_DEP_MASK) +- if (retval) ++ if (retval) { ++ kfree(action->secondary); + kfree(action); ++ } -diff -Nur linux-4.1.10.orig/include/acpi/platform/aclinux.h linux-4.1.10/include/acpi/platform/aclinux.h ---- linux-4.1.10.orig/include/acpi/platform/aclinux.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/acpi/platform/aclinux.h 2015-10-07 18:00:08.000000000 +0200 -@@ -123,6 +123,7 @@ + #ifdef CONFIG_DEBUG_SHIRQ_FIXME + if (!retval && (irqflags & IRQF_SHARED)) { +diff -Nur linux-4.1.10.orig/kernel/irq/settings.h linux-4.1.10/kernel/irq/settings.h +--- linux-4.1.10.orig/kernel/irq/settings.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/irq/settings.h 2015-10-12 22:33:32.296675432 +0200 +@@ -15,6 +15,7 @@ + _IRQ_NESTED_THREAD = IRQ_NESTED_THREAD, + _IRQ_PER_CPU_DEVID = IRQ_PER_CPU_DEVID, + _IRQ_IS_POLLED = IRQ_IS_POLLED, ++ _IRQ_NO_SOFTIRQ_CALL = IRQ_NO_SOFTIRQ_CALL, + _IRQF_MODIFY_MASK = IRQF_MODIFY_MASK, + }; - #define acpi_cache_t struct kmem_cache - #define acpi_spinlock spinlock_t * -+#define acpi_raw_spinlock raw_spinlock_t * - #define acpi_cpu_flags unsigned long +@@ -28,6 +29,7 @@ + #define IRQ_NESTED_THREAD GOT_YOU_MORON + #define IRQ_PER_CPU_DEVID GOT_YOU_MORON + #define IRQ_IS_POLLED GOT_YOU_MORON ++#define IRQ_NO_SOFTIRQ_CALL GOT_YOU_MORON + #undef IRQF_MODIFY_MASK + #define IRQF_MODIFY_MASK GOT_YOU_MORON - /* Use native linux version of acpi_os_allocate_zeroed */ -@@ -141,6 +142,20 @@ - #define ACPI_USE_ALTERNATE_PROTOTYPE_acpi_os_get_thread_id - #define ACPI_USE_ALTERNATE_PROTOTYPE_acpi_os_create_lock +@@ -38,6 +40,16 @@ + desc->status_use_accessors |= (set & _IRQF_MODIFY_MASK); + } -+#define acpi_os_create_raw_lock(__handle) \ -+({ \ -+ raw_spinlock_t *lock = ACPI_ALLOCATE(sizeof(*lock)); \ -+ \ -+ if (lock) { \ -+ *(__handle) = lock; \ -+ raw_spin_lock_init(*(__handle)); \ -+ } \ -+ lock ? AE_OK : AE_NO_MEMORY; \ -+ }) -+ -+#define acpi_os_delete_raw_lock(__handle) kfree(__handle) ++static inline bool irq_settings_no_softirq_call(struct irq_desc *desc) ++{ ++ return desc->status_use_accessors & _IRQ_NO_SOFTIRQ_CALL; ++} + ++static inline void irq_settings_set_no_softirq_call(struct irq_desc *desc) ++{ ++ desc->status_use_accessors |= _IRQ_NO_SOFTIRQ_CALL; ++} + - /* - * OSL interfaces used by debugger/disassembler - */ -diff -Nur linux-4.1.10.orig/include/asm-generic/bug.h linux-4.1.10/include/asm-generic/bug.h ---- linux-4.1.10.orig/include/asm-generic/bug.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/asm-generic/bug.h 2015-10-07 18:00:08.000000000 +0200 -@@ -206,6 +206,20 @@ - # define WARN_ON_SMP(x) ({0;}) - #endif + static inline bool irq_settings_is_per_cpu(struct irq_desc *desc) + { + return desc->status_use_accessors & _IRQ_PER_CPU; +diff -Nur linux-4.1.10.orig/kernel/irq/spurious.c linux-4.1.10/kernel/irq/spurious.c +--- linux-4.1.10.orig/kernel/irq/spurious.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/irq/spurious.c 2015-10-12 22:33:32.296675432 +0200 +@@ -444,6 +444,10 @@ + static int __init irqfixup_setup(char *str) + { +#ifdef CONFIG_PREEMPT_RT_BASE -+# define BUG_ON_RT(c) BUG_ON(c) -+# define BUG_ON_NONRT(c) do { } while (0) -+# define WARN_ON_RT(condition) WARN_ON(condition) -+# define WARN_ON_NONRT(condition) do { } while (0) -+# define WARN_ON_ONCE_NONRT(condition) do { } while (0) -+#else -+# define BUG_ON_RT(c) do { } while (0) -+# define BUG_ON_NONRT(c) BUG_ON(c) -+# define WARN_ON_RT(condition) do { } while (0) -+# define WARN_ON_NONRT(condition) WARN_ON(condition) -+# define WARN_ON_ONCE_NONRT(condition) WARN_ON_ONCE(condition) ++ pr_warn("irqfixup boot option not supported w/ CONFIG_PREEMPT_RT_BASE\n"); ++ return 1; +#endif -+ - #endif /* __ASSEMBLY__ */ + irqfixup = 1; + printk(KERN_WARNING "Misrouted IRQ fixup support enabled.\n"); + printk(KERN_WARNING "This may impact system performance.\n"); +@@ -456,6 +460,10 @@ - #endif -diff -Nur linux-4.1.10.orig/include/asm-generic/futex.h linux-4.1.10/include/asm-generic/futex.h ---- linux-4.1.10.orig/include/asm-generic/futex.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/asm-generic/futex.h 2015-10-07 18:00:08.000000000 +0200 -@@ -8,8 +8,7 @@ - #ifndef CONFIG_SMP - /* - * The following implementation only for uniprocessor machines. -- * For UP, it's relies on the fact that pagefault_disable() also disables -- * preemption to ensure mutual exclusion. -+ * It relies on preempt_disable() ensuring mutual exclusion. - * - */ + static int __init irqpoll_setup(char *str) + { ++#ifdef CONFIG_PREEMPT_RT_BASE ++ pr_warn("irqpoll boot option not supported w/ CONFIG_PREEMPT_RT_BASE\n"); ++ return 1; ++#endif + irqfixup = 2; + printk(KERN_WARNING "Misrouted IRQ fixup and polling support " + "enabled\n"); +diff -Nur linux-4.1.10.orig/kernel/irq_work.c linux-4.1.10/kernel/irq_work.c +--- linux-4.1.10.orig/kernel/irq_work.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/irq_work.c 2015-10-12 22:33:32.296675432 +0200 +@@ -17,6 +17,7 @@ + #include + #include + #include ++#include + #include -@@ -38,6 +37,7 @@ - if (encoded_op & (FUTEX_OP_OPARG_SHIFT << 28)) - oparg = 1 << oparg; -+ preempt_disable(); - pagefault_disable(); +@@ -65,6 +66,8 @@ + */ + bool irq_work_queue_on(struct irq_work *work, int cpu) + { ++ struct llist_head *list; ++ + /* All work should have been flushed before going offline */ + WARN_ON_ONCE(cpu_is_offline(cpu)); - ret = -EFAULT; -@@ -72,6 +72,7 @@ +@@ -75,7 +78,12 @@ + if (!irq_work_claim(work)) + return false; - out_pagefault_enable: - pagefault_enable(); -+ preempt_enable(); +- if (llist_add(&work->llnode, &per_cpu(raised_list, cpu))) ++ if (IS_ENABLED(CONFIG_PREEMPT_RT_FULL) && !(work->flags & IRQ_WORK_HARD_IRQ)) ++ list = &per_cpu(lazy_list, cpu); ++ else ++ list = &per_cpu(raised_list, cpu); ++ ++ if (llist_add(&work->llnode, list)) + arch_send_call_function_single_ipi(cpu); - if (ret == 0) { - switch (cmp) { -@@ -106,6 +107,7 @@ + return true; +@@ -86,6 +94,9 @@ + /* Enqueue the irq work @work on the current CPU */ + bool irq_work_queue(struct irq_work *work) { - u32 val; ++ struct llist_head *list; ++ bool lazy_work, realtime = IS_ENABLED(CONFIG_PREEMPT_RT_FULL); ++ + /* Only queue if not already pending */ + if (!irq_work_claim(work)) + return false; +@@ -93,13 +104,15 @@ + /* Queue the entry and raise the IPI if needed. */ + preempt_disable(); -+ preempt_disable(); - if (unlikely(get_user(val, uaddr) != 0)) - return -EFAULT; +- /* If the work is "lazy", handle it from next tick if any */ +- if (work->flags & IRQ_WORK_LAZY) { +- if (llist_add(&work->llnode, this_cpu_ptr(&lazy_list)) && +- tick_nohz_tick_stopped()) +- arch_irq_work_raise(); +- } else { +- if (llist_add(&work->llnode, this_cpu_ptr(&raised_list))) ++ lazy_work = work->flags & IRQ_WORK_LAZY; ++ ++ if (lazy_work || (realtime && !(work->flags & IRQ_WORK_HARD_IRQ))) ++ list = this_cpu_ptr(&lazy_list); ++ else ++ list = this_cpu_ptr(&raised_list); ++ ++ if (llist_add(&work->llnode, list)) { ++ if (!lazy_work || tick_nohz_tick_stopped()) + arch_irq_work_raise(); + } -@@ -113,6 +115,7 @@ - return -EFAULT; +@@ -116,9 +129,8 @@ + raised = this_cpu_ptr(&raised_list); + lazy = this_cpu_ptr(&lazy_list); - *uval = val; -+ preempt_enable(); +- if (llist_empty(raised) || arch_irq_work_has_interrupt()) +- if (llist_empty(lazy)) +- return false; ++ if (llist_empty(raised) && llist_empty(lazy)) ++ return false; - return 0; - } -diff -Nur linux-4.1.10.orig/include/linux/blkdev.h linux-4.1.10/include/linux/blkdev.h ---- linux-4.1.10.orig/include/linux/blkdev.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/blkdev.h 2015-10-07 18:00:08.000000000 +0200 -@@ -101,6 +101,7 @@ - struct list_head queuelist; - union { - struct call_single_data csd; -+ struct work_struct work; - unsigned long fifo_time; - }; + /* All work should have been flushed before going offline */ + WARN_ON_ONCE(cpu_is_offline(smp_processor_id())); +@@ -132,7 +144,7 @@ + struct irq_work *work; + struct llist_node *llnode; -@@ -482,7 +483,7 @@ - struct throtl_data *td; - #endif - struct rcu_head rcu_head; -- wait_queue_head_t mq_freeze_wq; -+ struct swait_head mq_freeze_wq; - struct percpu_ref mq_usage_counter; - struct list_head all_q_node; +- BUG_ON(!irqs_disabled()); ++ BUG_ON_NONRT(!irqs_disabled()); -diff -Nur linux-4.1.10.orig/include/linux/blk-mq.h linux-4.1.10/include/linux/blk-mq.h ---- linux-4.1.10.orig/include/linux/blk-mq.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/blk-mq.h 2015-10-07 18:00:08.000000000 +0200 -@@ -202,6 +202,7 @@ + if (llist_empty(list)) + return; +@@ -169,7 +181,16 @@ + void irq_work_run(void) + { + irq_work_run_list(this_cpu_ptr(&raised_list)); +- irq_work_run_list(this_cpu_ptr(&lazy_list)); ++ if (IS_ENABLED(CONFIG_PREEMPT_RT_FULL)) { ++ /* ++ * NOTE: we raise softirq via IPI for safety, ++ * and execute in irq_work_tick() to move the ++ * overhead from hard to soft irq context. ++ */ ++ if (!llist_empty(this_cpu_ptr(&lazy_list))) ++ raise_softirq(TIMER_SOFTIRQ); ++ } else ++ irq_work_run_list(this_cpu_ptr(&lazy_list)); + } + EXPORT_SYMBOL_GPL(irq_work_run); - struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *, const int ctx_index); - struct blk_mq_hw_ctx *blk_mq_alloc_single_hw_queue(struct blk_mq_tag_set *, unsigned int, int); -+void __blk_mq_complete_request_remote_work(struct work_struct *work); +diff -Nur linux-4.1.10.orig/kernel/Kconfig.locks linux-4.1.10/kernel/Kconfig.locks +--- linux-4.1.10.orig/kernel/Kconfig.locks 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/Kconfig.locks 2015-10-12 22:33:32.296675432 +0200 +@@ -225,11 +225,11 @@ - int blk_mq_request_started(struct request *rq); - void blk_mq_start_request(struct request *rq); -diff -Nur linux-4.1.10.orig/include/linux/bottom_half.h linux-4.1.10/include/linux/bottom_half.h ---- linux-4.1.10.orig/include/linux/bottom_half.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/bottom_half.h 2015-10-07 18:00:08.000000000 +0200 -@@ -4,6 +4,17 @@ - #include - #include + config MUTEX_SPIN_ON_OWNER + def_bool y +- depends on SMP && !DEBUG_MUTEXES && ARCH_SUPPORTS_ATOMIC_RMW ++ depends on SMP && !DEBUG_MUTEXES && ARCH_SUPPORTS_ATOMIC_RMW && !PREEMPT_RT_FULL -+#ifdef CONFIG_PREEMPT_RT_FULL + config RWSEM_SPIN_ON_OWNER + def_bool y +- depends on SMP && RWSEM_XCHGADD_ALGORITHM && ARCH_SUPPORTS_ATOMIC_RMW ++ depends on SMP && RWSEM_XCHGADD_ALGORITHM && ARCH_SUPPORTS_ATOMIC_RMW && !PREEMPT_RT_FULL + + config LOCK_SPIN_ON_OWNER + def_bool y +diff -Nur linux-4.1.10.orig/kernel/Kconfig.preempt linux-4.1.10/kernel/Kconfig.preempt +--- linux-4.1.10.orig/kernel/Kconfig.preempt 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/Kconfig.preempt 2015-10-12 22:33:32.296675432 +0200 +@@ -1,3 +1,16 @@ ++config PREEMPT ++ bool ++ select PREEMPT_COUNT + -+extern void local_bh_disable(void); -+extern void _local_bh_enable(void); -+extern void local_bh_enable(void); -+extern void local_bh_enable_ip(unsigned long ip); -+extern void __local_bh_disable_ip(unsigned long ip, unsigned int cnt); -+extern void __local_bh_enable_ip(unsigned long ip, unsigned int cnt); ++config PREEMPT_RT_BASE ++ bool ++ select PREEMPT + -+#else ++config HAVE_PREEMPT_LAZY ++ bool + - #ifdef CONFIG_TRACE_IRQFLAGS - extern void __local_bh_disable_ip(unsigned long ip, unsigned int cnt); - #else -@@ -31,5 +42,6 @@ - { - __local_bh_enable_ip(_THIS_IP_, SOFTIRQ_DISABLE_OFFSET); - } -+#endif ++config PREEMPT_LAZY ++ def_bool y if HAVE_PREEMPT_LAZY && PREEMPT_RT_FULL - #endif /* _LINUX_BH_H */ -diff -Nur linux-4.1.10.orig/include/linux/buffer_head.h linux-4.1.10/include/linux/buffer_head.h ---- linux-4.1.10.orig/include/linux/buffer_head.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/buffer_head.h 2015-10-07 18:00:08.000000000 +0200 -@@ -75,8 +75,52 @@ - struct address_space *b_assoc_map; /* mapping this buffer is - associated with */ - atomic_t b_count; /* users using this buffer_head */ -+#ifdef CONFIG_PREEMPT_RT_BASE -+ spinlock_t b_uptodate_lock; -+#if defined(CONFIG_JBD) || defined(CONFIG_JBD_MODULE) || \ -+ defined(CONFIG_JBD2) || defined(CONFIG_JBD2_MODULE) -+ spinlock_t b_state_lock; -+ spinlock_t b_journal_head_lock; -+#endif -+#endif - }; + choice + prompt "Preemption Model" +@@ -33,9 +46,9 @@ -+static inline unsigned long bh_uptodate_lock_irqsave(struct buffer_head *bh) -+{ -+ unsigned long flags; + Select this if you are building a kernel for a desktop system. + +-config PREEMPT ++config PREEMPT__LL + bool "Preemptible Kernel (Low-Latency Desktop)" +- select PREEMPT_COUNT ++ select PREEMPT + select UNINLINE_SPIN_UNLOCK if !ARCH_INLINE_SPIN_UNLOCK + help + This option reduces the latency of the kernel by making +@@ -52,6 +65,22 @@ + embedded system with latency requirements in the milliseconds + range. + ++config PREEMPT_RTB ++ bool "Preemptible Kernel (Basic RT)" ++ select PREEMPT_RT_BASE ++ help ++ This option is basically the same as (Low-Latency Desktop) but ++ enables changes which are preliminary for the full preemptible ++ RT kernel. + -+#ifndef CONFIG_PREEMPT_RT_BASE -+ local_irq_save(flags); -+ bit_spin_lock(BH_Uptodate_Lock, &bh->b_state); -+#else -+ spin_lock_irqsave(&bh->b_uptodate_lock, flags); -+#endif -+ return flags; -+} ++config PREEMPT_RT_FULL ++ bool "Fully Preemptible Kernel (RT)" ++ depends on IRQ_FORCED_THREADING ++ select PREEMPT_RT_BASE ++ select PREEMPT_RCU ++ help ++ All and everything + -+static inline void -+bh_uptodate_unlock_irqrestore(struct buffer_head *bh, unsigned long flags) + endchoice + + config PREEMPT_COUNT +diff -Nur linux-4.1.10.orig/kernel/ksysfs.c linux-4.1.10/kernel/ksysfs.c +--- linux-4.1.10.orig/kernel/ksysfs.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/ksysfs.c 2015-10-12 22:33:32.296675432 +0200 +@@ -136,6 +136,15 @@ + + #endif /* CONFIG_KEXEC */ + ++#if defined(CONFIG_PREEMPT_RT_FULL) ++static ssize_t realtime_show(struct kobject *kobj, ++ struct kobj_attribute *attr, char *buf) +{ -+#ifndef CONFIG_PREEMPT_RT_BASE -+ bit_spin_unlock(BH_Uptodate_Lock, &bh->b_state); -+ local_irq_restore(flags); -+#else -+ spin_unlock_irqrestore(&bh->b_uptodate_lock, flags); -+#endif ++ return sprintf(buf, "%d\n", 1); +} ++KERNEL_ATTR_RO(realtime); ++#endif + -+static inline void buffer_head_init_locks(struct buffer_head *bh) -+{ -+#ifdef CONFIG_PREEMPT_RT_BASE -+ spin_lock_init(&bh->b_uptodate_lock); -+#if defined(CONFIG_JBD) || defined(CONFIG_JBD_MODULE) || \ -+ defined(CONFIG_JBD2) || defined(CONFIG_JBD2_MODULE) -+ spin_lock_init(&bh->b_state_lock); -+ spin_lock_init(&bh->b_journal_head_lock); -+#endif + /* whether file capabilities are enabled */ + static ssize_t fscaps_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +@@ -203,6 +212,9 @@ + &vmcoreinfo_attr.attr, + #endif + &rcu_expedited_attr.attr, ++#ifdef CONFIG_PREEMPT_RT_FULL ++ &realtime_attr.attr, +#endif -+} -+ - /* - * macro tricks to expand the set_buffer_foo(), clear_buffer_foo() - * and buffer_foo() functions. -diff -Nur linux-4.1.10.orig/include/linux/cgroup.h linux-4.1.10/include/linux/cgroup.h ---- linux-4.1.10.orig/include/linux/cgroup.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/cgroup.h 2015-10-07 18:00:08.000000000 +0200 -@@ -22,6 +22,7 @@ - #include - #include - #include -+#include - - #ifdef CONFIG_CGROUPS - -@@ -91,6 +92,7 @@ - /* percpu_ref killing and RCU release */ - struct rcu_head rcu_head; - struct work_struct destroy_work; -+ struct swork_event destroy_swork; + NULL }; - /* bits in struct cgroup_subsys_state flags field */ -diff -Nur linux-4.1.10.orig/include/linux/completion.h linux-4.1.10/include/linux/completion.h ---- linux-4.1.10.orig/include/linux/completion.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/completion.h 2015-10-07 18:00:08.000000000 +0200 -@@ -7,8 +7,7 @@ - * Atomic wait-for-completion handler data structures. - * See kernel/sched/completion.c for details. - */ -- --#include -+#include +diff -Nur linux-4.1.10.orig/kernel/locking/lglock.c linux-4.1.10/kernel/locking/lglock.c +--- linux-4.1.10.orig/kernel/locking/lglock.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/locking/lglock.c 2015-10-12 22:33:32.296675432 +0200 +@@ -4,6 +4,15 @@ + #include + #include ++#ifndef CONFIG_PREEMPT_RT_FULL ++# define lg_lock_ptr arch_spinlock_t ++# define lg_do_lock(l) arch_spin_lock(l) ++# define lg_do_unlock(l) arch_spin_unlock(l) ++#else ++# define lg_lock_ptr struct rt_mutex ++# define lg_do_lock(l) __rt_spin_lock(l) ++# define lg_do_unlock(l) __rt_spin_unlock(l) ++#endif /* - * struct completion - structure used to maintain state for a "completion" -@@ -24,11 +23,11 @@ - */ - struct completion { - unsigned int done; -- wait_queue_head_t wait; -+ struct swait_head wait; - }; - - #define COMPLETION_INITIALIZER(work) \ -- { 0, __WAIT_QUEUE_HEAD_INITIALIZER((work).wait) } -+ { 0, SWAIT_HEAD_INITIALIZER((work).wait) } + * Note there is no uninit, so lglocks cannot be defined in + * modules (but it's fine to use them from there) +@@ -12,51 +21,60 @@ - #define COMPLETION_INITIALIZER_ONSTACK(work) \ - ({ init_completion(&work); work; }) -@@ -73,7 +72,7 @@ - static inline void init_completion(struct completion *x) + void lg_lock_init(struct lglock *lg, char *name) { - x->done = 0; -- init_waitqueue_head(&x->wait); -+ init_swait_head(&x->wait); - } - - /** -diff -Nur linux-4.1.10.orig/include/linux/cpu.h linux-4.1.10/include/linux/cpu.h ---- linux-4.1.10.orig/include/linux/cpu.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/cpu.h 2015-10-07 18:00:08.000000000 +0200 -@@ -231,6 +231,8 @@ - extern void put_online_cpus(void); - extern void cpu_hotplug_disable(void); - extern void cpu_hotplug_enable(void); -+extern void pin_current_cpu(void); -+extern void unpin_current_cpu(void); - #define hotcpu_notifier(fn, pri) cpu_notifier(fn, pri) - #define __hotcpu_notifier(fn, pri) __cpu_notifier(fn, pri) - #define register_hotcpu_notifier(nb) register_cpu_notifier(nb) -@@ -249,6 +251,8 @@ - #define put_online_cpus() do { } while (0) - #define cpu_hotplug_disable() do { } while (0) - #define cpu_hotplug_enable() do { } while (0) -+static inline void pin_current_cpu(void) { } -+static inline void unpin_current_cpu(void) { } - #define hotcpu_notifier(fn, pri) do { (void)(fn); } while (0) - #define __hotcpu_notifier(fn, pri) do { (void)(fn); } while (0) - /* These aren't inline functions due to a GCC bug. */ -diff -Nur linux-4.1.10.orig/include/linux/delay.h linux-4.1.10/include/linux/delay.h ---- linux-4.1.10.orig/include/linux/delay.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/delay.h 2015-10-07 18:00:08.000000000 +0200 -@@ -52,4 +52,10 @@ - msleep(seconds * 1000); - } - +#ifdef CONFIG_PREEMPT_RT_FULL -+extern void cpu_chill(void); -+#else -+# define cpu_chill() cpu_relax() -+#endif ++ int i; + - #endif /* defined(_LINUX_DELAY_H) */ -diff -Nur linux-4.1.10.orig/include/linux/ftrace_event.h linux-4.1.10/include/linux/ftrace_event.h ---- linux-4.1.10.orig/include/linux/ftrace_event.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/ftrace_event.h 2015-10-07 18:00:08.000000000 +0200 -@@ -66,6 +66,9 @@ - unsigned char flags; - unsigned char preempt_count; - int pid; -+ unsigned short migrate_disable; -+ unsigned short padding; -+ unsigned char preempt_lazy_count; - }; - - #define FTRACE_MAX_EVENT \ -diff -Nur linux-4.1.10.orig/include/linux/highmem.h linux-4.1.10/include/linux/highmem.h ---- linux-4.1.10.orig/include/linux/highmem.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/highmem.h 2015-10-07 18:00:08.000000000 +0200 -@@ -7,6 +7,7 @@ - #include - #include - #include -+#include - - #include - -@@ -65,6 +66,7 @@ - - static inline void *kmap_atomic(struct page *page) - { -+ preempt_disable_nort(); - pagefault_disable(); - return page_address(page); - } -@@ -73,6 +75,7 @@ - static inline void __kunmap_atomic(void *addr) - { - pagefault_enable(); -+ preempt_enable_nort(); ++ for_each_possible_cpu(i) { ++ struct rt_mutex *lock = per_cpu_ptr(lg->lock, i); ++ ++ rt_mutex_init(lock); ++ } ++#endif + LOCKDEP_INIT_MAP(&lg->lock_dep_map, name, &lg->lock_key, 0); } + EXPORT_SYMBOL(lg_lock_init); - #define kmap_atomic_pfn(pfn) kmap_atomic(pfn_to_page(pfn)) -@@ -85,32 +88,51 @@ - - #if defined(CONFIG_HIGHMEM) || defined(CONFIG_X86_32) + void lg_local_lock(struct lglock *lg) + { +- arch_spinlock_t *lock; ++ lg_lock_ptr *lock; -+#ifndef CONFIG_PREEMPT_RT_FULL - DECLARE_PER_CPU(int, __kmap_atomic_idx); -+#endif +- preempt_disable(); ++ migrate_disable(); + lock_acquire_shared(&lg->lock_dep_map, 0, 0, NULL, _RET_IP_); + lock = this_cpu_ptr(lg->lock); +- arch_spin_lock(lock); ++ lg_do_lock(lock); + } + EXPORT_SYMBOL(lg_local_lock); - static inline int kmap_atomic_idx_push(void) + void lg_local_unlock(struct lglock *lg) { -+#ifndef CONFIG_PREEMPT_RT_FULL - int idx = __this_cpu_inc_return(__kmap_atomic_idx) - 1; +- arch_spinlock_t *lock; ++ lg_lock_ptr *lock; --#ifdef CONFIG_DEBUG_HIGHMEM -+# ifdef CONFIG_DEBUG_HIGHMEM - WARN_ON_ONCE(in_irq() && !irqs_disabled()); - BUG_ON(idx >= KM_TYPE_NR); --#endif -+# endif - return idx; -+#else -+ current->kmap_idx++; -+ BUG_ON(current->kmap_idx > KM_TYPE_NR); -+ return current->kmap_idx - 1; -+#endif + lock_release(&lg->lock_dep_map, 1, _RET_IP_); + lock = this_cpu_ptr(lg->lock); +- arch_spin_unlock(lock); +- preempt_enable(); ++ lg_do_unlock(lock); ++ migrate_enable(); } + EXPORT_SYMBOL(lg_local_unlock); - static inline int kmap_atomic_idx(void) + void lg_local_lock_cpu(struct lglock *lg, int cpu) { -+#ifndef CONFIG_PREEMPT_RT_FULL - return __this_cpu_read(__kmap_atomic_idx) - 1; -+#else -+ return current->kmap_idx - 1; -+#endif +- arch_spinlock_t *lock; ++ lg_lock_ptr *lock; + +- preempt_disable(); ++ preempt_disable_nort(); + lock_acquire_shared(&lg->lock_dep_map, 0, 0, NULL, _RET_IP_); + lock = per_cpu_ptr(lg->lock, cpu); +- arch_spin_lock(lock); ++ lg_do_lock(lock); } + EXPORT_SYMBOL(lg_local_lock_cpu); - static inline void kmap_atomic_idx_pop(void) + void lg_local_unlock_cpu(struct lglock *lg, int cpu) { --#ifdef CONFIG_DEBUG_HIGHMEM -+#ifndef CONFIG_PREEMPT_RT_FULL -+# ifdef CONFIG_DEBUG_HIGHMEM - int idx = __this_cpu_dec_return(__kmap_atomic_idx); +- arch_spinlock_t *lock; ++ lg_lock_ptr *lock; - BUG_ON(idx < 0); --#else -+# else - __this_cpu_dec(__kmap_atomic_idx); -+# endif -+#else -+ current->kmap_idx--; -+# ifdef CONFIG_DEBUG_HIGHMEM -+ BUG_ON(current->kmap_idx < 0); -+# endif - #endif + lock_release(&lg->lock_dep_map, 1, _RET_IP_); + lock = per_cpu_ptr(lg->lock, cpu); +- arch_spin_unlock(lock); +- preempt_enable(); ++ lg_do_unlock(lock); ++ preempt_enable_nort(); } + EXPORT_SYMBOL(lg_local_unlock_cpu); -diff -Nur linux-4.1.10.orig/include/linux/hrtimer.h linux-4.1.10/include/linux/hrtimer.h ---- linux-4.1.10.orig/include/linux/hrtimer.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/hrtimer.h 2015-10-07 18:00:08.000000000 +0200 -@@ -111,6 +111,11 @@ - enum hrtimer_restart (*function)(struct hrtimer *); - struct hrtimer_clock_base *base; - unsigned long state; -+ struct list_head cb_entry; -+ int irqsafe; -+#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST -+ ktime_t praecox; -+#endif - #ifdef CONFIG_TIMER_STATS - int start_pid; - void *start_site; -@@ -147,6 +152,7 @@ - int index; - clockid_t clockid; - struct timerqueue_head active; -+ struct list_head expired; - ktime_t resolution; - ktime_t (*get_time)(void); - ktime_t softirq_time; -@@ -194,6 +200,9 @@ - unsigned long nr_hangs; - ktime_t max_hang_time; - #endif -+#ifdef CONFIG_PREEMPT_RT_BASE -+ wait_queue_head_t wait; -+#endif - struct hrtimer_clock_base clock_base[HRTIMER_MAX_CLOCK_BASES]; - }; +@@ -64,12 +82,12 @@ + { + int i; -@@ -381,6 +390,13 @@ - return hrtimer_start_expires(timer, HRTIMER_MODE_ABS); +- preempt_disable(); ++ preempt_disable_nort(); + lock_acquire_exclusive(&lg->lock_dep_map, 0, 0, NULL, _RET_IP_); + for_each_possible_cpu(i) { +- arch_spinlock_t *lock; ++ lg_lock_ptr *lock; + lock = per_cpu_ptr(lg->lock, i); +- arch_spin_lock(lock); ++ lg_do_lock(lock); + } } + EXPORT_SYMBOL(lg_global_lock); +@@ -80,10 +98,35 @@ -+/* Softirq preemption could deadlock timer removal */ -+#ifdef CONFIG_PREEMPT_RT_BASE -+ extern void hrtimer_wait_for_timer(const struct hrtimer *timer); -+#else -+# define hrtimer_wait_for_timer(timer) do { cpu_relax(); } while (0) -+#endif + lock_release(&lg->lock_dep_map, 1, _RET_IP_); + for_each_possible_cpu(i) { +- arch_spinlock_t *lock; ++ lg_lock_ptr *lock; + lock = per_cpu_ptr(lg->lock, i); +- arch_spin_unlock(lock); ++ lg_do_unlock(lock); + } +- preempt_enable(); ++ preempt_enable_nort(); + } + EXPORT_SYMBOL(lg_global_unlock); + - /* Query timers: */ - extern ktime_t hrtimer_get_remaining(const struct hrtimer *timer); - extern int hrtimer_get_res(const clockid_t which_clock, struct timespec *tp); -diff -Nur linux-4.1.10.orig/include/linux/idr.h linux-4.1.10/include/linux/idr.h ---- linux-4.1.10.orig/include/linux/idr.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/idr.h 2015-10-07 18:00:08.000000000 +0200 -@@ -95,10 +95,14 @@ - * Each idr_preload() should be matched with an invocation of this - * function. See idr_preload() for details. - */ +#ifdef CONFIG_PREEMPT_RT_FULL -+void idr_preload_end(void); -+#else - static inline void idr_preload_end(void) - { - preempt_enable(); - } -+#endif - - /** - * idr_find - return pointer for given id -diff -Nur linux-4.1.10.orig/include/linux/init_task.h linux-4.1.10/include/linux/init_task.h ---- linux-4.1.10.orig/include/linux/init_task.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/init_task.h 2015-10-07 18:00:08.000000000 +0200 -@@ -147,9 +147,16 @@ - # define INIT_PERF_EVENTS(tsk) - #endif - -+#ifdef CONFIG_PREEMPT_RT_BASE -+# define INIT_TIMER_LIST .posix_timer_list = NULL, -+#else -+# define INIT_TIMER_LIST -+#endif ++/* ++ * HACK: If you use this, you get to keep the pieces. ++ * Used in queue_stop_cpus_work() when stop machinery ++ * is called from inactive CPU, so we can't schedule. ++ */ ++# define lg_do_trylock_relax(l) \ ++ do { \ ++ while (!__rt_spin_trylock(l)) \ ++ cpu_relax(); \ ++ } while (0) + - #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN - # define INIT_VTIME(tsk) \ -- .vtime_seqlock = __SEQLOCK_UNLOCKED(tsk.vtime_seqlock), \ -+ .vtime_lock = __RAW_SPIN_LOCK_UNLOCKED(tsk.vtime_lock), \ -+ .vtime_seq = SEQCNT_ZERO(tsk.vtime_seq), \ - .vtime_snap = 0, \ - .vtime_snap_whence = VTIME_SYS, - #else -@@ -238,6 +245,7 @@ - .cpu_timers = INIT_CPU_TIMERS(tsk.cpu_timers), \ - .pi_lock = __RAW_SPIN_LOCK_UNLOCKED(tsk.pi_lock), \ - .timer_slack_ns = 50000, /* 50 usec default slack */ \ -+ INIT_TIMER_LIST \ - .pids = { \ - [PIDTYPE_PID] = INIT_PID_LINK(PIDTYPE_PID), \ - [PIDTYPE_PGID] = INIT_PID_LINK(PIDTYPE_PGID), \ -diff -Nur linux-4.1.10.orig/include/linux/interrupt.h linux-4.1.10/include/linux/interrupt.h ---- linux-4.1.10.orig/include/linux/interrupt.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/interrupt.h 2015-10-07 18:00:08.000000000 +0200 -@@ -61,6 +61,7 @@ - * interrupt handler after suspending interrupts. For system - * wakeup devices users need to implement wakeup detection in - * their interrupt handlers. -+ * IRQF_NO_SOFTIRQ_CALL - Do not process softirqs in the irq thread context (RT) - */ - #define IRQF_SHARED 0x00000080 - #define IRQF_PROBE_SHARED 0x00000100 -@@ -74,6 +75,7 @@ - #define IRQF_NO_THREAD 0x00010000 - #define IRQF_EARLY_RESUME 0x00020000 - #define IRQF_COND_SUSPEND 0x00040000 -+#define IRQF_NO_SOFTIRQ_CALL 0x00080000 - - #define IRQF_TIMER (__IRQF_TIMER | IRQF_NO_SUSPEND | IRQF_NO_THREAD) - -@@ -102,6 +104,7 @@ - * @flags: flags (see IRQF_* above) - * @thread_fn: interrupt handler function for threaded interrupts - * @thread: thread pointer for threaded interrupts -+ * @secondary: pointer to secondary irqaction (force threading) - * @thread_flags: flags related to @thread - * @thread_mask: bitmask for keeping track of @thread activity - * @dir: pointer to the proc/irq/NN/name entry -@@ -113,6 +116,7 @@ - struct irqaction *next; - irq_handler_t thread_fn; - struct task_struct *thread; -+ struct irqaction *secondary; - unsigned int irq; - unsigned int flags; - unsigned long thread_flags; -@@ -184,7 +188,7 @@ - #ifdef CONFIG_LOCKDEP - # define local_irq_enable_in_hardirq() do { } while (0) - #else --# define local_irq_enable_in_hardirq() local_irq_enable() -+# define local_irq_enable_in_hardirq() local_irq_enable_nort() - #endif - - extern void disable_irq_nosync(unsigned int irq); -@@ -215,6 +219,7 @@ - unsigned int irq; - struct kref kref; - struct work_struct work; -+ struct list_head list; - void (*notify)(struct irq_affinity_notify *, const cpumask_t *mask); - void (*release)(struct kref *ref); - }; -@@ -377,9 +382,13 @@ - bool state); - - #ifdef CONFIG_IRQ_FORCED_THREADING -+# ifndef CONFIG_PREEMPT_RT_BASE - extern bool force_irqthreads; -+# else -+# define force_irqthreads (true) -+# endif - #else --#define force_irqthreads (0) -+#define force_irqthreads (false) - #endif - - #ifndef __ARCH_SET_SOFTIRQ_PENDING -@@ -435,9 +444,10 @@ - void (*action)(struct softirq_action *); - }; ++void lg_global_trylock_relax(struct lglock *lg) ++{ ++ int i; ++ ++ lock_acquire_exclusive(&lg->lock_dep_map, 0, 0, NULL, _RET_IP_); ++ for_each_possible_cpu(i) { ++ lg_lock_ptr *lock; ++ lock = per_cpu_ptr(lg->lock, i); ++ lg_do_trylock_relax(lock); ++ } ++} ++#endif +diff -Nur linux-4.1.10.orig/kernel/locking/lockdep.c linux-4.1.10/kernel/locking/lockdep.c +--- linux-4.1.10.orig/kernel/locking/lockdep.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/locking/lockdep.c 2015-10-12 22:33:32.296675432 +0200 +@@ -3563,6 +3563,7 @@ + } + } +#ifndef CONFIG_PREEMPT_RT_FULL - asmlinkage void do_softirq(void); - asmlinkage void __do_softirq(void); -- -+static inline void thread_do_softirq(void) { do_softirq(); } - #ifdef __ARCH_HAS_DO_SOFTIRQ - void do_softirq_own_stack(void); - #else -@@ -446,6 +456,9 @@ - __do_softirq(); - } - #endif -+#else -+extern void thread_do_softirq(void); + /* + * We dont accurately track softirq state in e.g. + * hardirq contexts (such as on 4KSTACKS), so only +@@ -3577,6 +3578,7 @@ + DEBUG_LOCKS_WARN_ON(!current->softirqs_enabled); + } + } +#endif - extern void open_softirq(int nr, void (*action)(struct softirq_action *)); - extern void softirq_init(void); -@@ -453,6 +466,7 @@ + if (!debug_locks) + print_irqtrace_events(current); +diff -Nur linux-4.1.10.orig/kernel/locking/locktorture.c linux-4.1.10/kernel/locking/locktorture.c +--- linux-4.1.10.orig/kernel/locking/locktorture.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/locking/locktorture.c 2015-10-12 22:33:32.296675432 +0200 +@@ -24,7 +24,6 @@ + #include + #include + #include +-#include + #include + #include + #include +diff -Nur linux-4.1.10.orig/kernel/locking/Makefile linux-4.1.10/kernel/locking/Makefile +--- linux-4.1.10.orig/kernel/locking/Makefile 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/locking/Makefile 2015-10-12 22:33:32.296675432 +0200 +@@ -1,5 +1,5 @@ - extern void raise_softirq_irqoff(unsigned int nr); - extern void raise_softirq(unsigned int nr); -+extern void softirq_check_pending_idle(void); +-obj-y += mutex.o semaphore.o rwsem.o ++obj-y += semaphore.o - DECLARE_PER_CPU(struct task_struct *, ksoftirqd); + ifdef CONFIG_FUNCTION_TRACER + CFLAGS_REMOVE_lockdep.o = $(CC_FLAGS_FTRACE) +@@ -8,7 +8,11 @@ + CFLAGS_REMOVE_rtmutex-debug.o = $(CC_FLAGS_FTRACE) + endif -@@ -474,8 +488,9 @@ - to be executed on some cpu at least once after this. - * If the tasklet is already scheduled, but its execution is still not - started, it will be executed only once. -- * If this tasklet is already running on another CPU (or schedule is called -- from tasklet itself), it is rescheduled for later. -+ * If this tasklet is already running on another CPU, it is rescheduled -+ for later. -+ * Schedule must not be called from the tasklet itself (a lockup occurs) - * Tasklet is strictly serialized wrt itself, but not - wrt another tasklets. If client needs some intertask synchronization, - he makes it with spinlocks. -@@ -500,27 +515,36 @@ - enum - { - TASKLET_STATE_SCHED, /* Tasklet is scheduled for execution */ -- TASKLET_STATE_RUN /* Tasklet is running (SMP only) */ -+ TASKLET_STATE_RUN, /* Tasklet is running (SMP only) */ -+ TASKLET_STATE_PENDING /* Tasklet is pending */ - }; - --#ifdef CONFIG_SMP -+#define TASKLET_STATEF_SCHED (1 << TASKLET_STATE_SCHED) -+#define TASKLET_STATEF_RUN (1 << TASKLET_STATE_RUN) -+#define TASKLET_STATEF_PENDING (1 << TASKLET_STATE_PENDING) ++ifneq ($(CONFIG_PREEMPT_RT_FULL),y) ++obj-y += mutex.o + obj-$(CONFIG_DEBUG_MUTEXES) += mutex-debug.o ++obj-y += rwsem.o ++endif + obj-$(CONFIG_LOCKDEP) += lockdep.o + ifeq ($(CONFIG_PROC_FS),y) + obj-$(CONFIG_LOCKDEP) += lockdep_proc.o +@@ -22,8 +26,11 @@ + obj-$(CONFIG_RT_MUTEX_TESTER) += rtmutex-tester.o + obj-$(CONFIG_DEBUG_SPINLOCK) += spinlock.o + obj-$(CONFIG_DEBUG_SPINLOCK) += spinlock_debug.o ++ifneq ($(CONFIG_PREEMPT_RT_FULL),y) + obj-$(CONFIG_RWSEM_GENERIC_SPINLOCK) += rwsem-spinlock.o + obj-$(CONFIG_RWSEM_XCHGADD_ALGORITHM) += rwsem-xadd.o ++endif + obj-$(CONFIG_PERCPU_RWSEM) += percpu-rwsem.o ++obj-$(CONFIG_PREEMPT_RT_FULL) += rt.o + obj-$(CONFIG_QUEUE_RWLOCK) += qrwlock.o + obj-$(CONFIG_LOCK_TORTURE_TEST) += locktorture.o +diff -Nur linux-4.1.10.orig/kernel/locking/rt.c linux-4.1.10/kernel/locking/rt.c +--- linux-4.1.10.orig/kernel/locking/rt.c 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/kernel/locking/rt.c 2015-10-12 22:33:32.296675432 +0200 +@@ -0,0 +1,461 @@ ++/* ++ * kernel/rt.c ++ * ++ * Real-Time Preemption Support ++ * ++ * started by Ingo Molnar: ++ * ++ * Copyright (C) 2004-2006 Red Hat, Inc., Ingo Molnar ++ * Copyright (C) 2006, Timesys Corp., Thomas Gleixner ++ * ++ * historic credit for proving that Linux spinlocks can be implemented via ++ * RT-aware mutexes goes to many people: The Pmutex project (Dirk Grambow ++ * and others) who prototyped it on 2.4 and did lots of comparative ++ * research and analysis; TimeSys, for proving that you can implement a ++ * fully preemptible kernel via the use of IRQ threading and mutexes; ++ * Bill Huey for persuasively arguing on lkml that the mutex model is the ++ * right one; and to MontaVista, who ported pmutexes to 2.6. ++ * ++ * This code is a from-scratch implementation and is not based on pmutexes, ++ * but the idea of converting spinlocks to mutexes is used here too. ++ * ++ * lock debugging, locking tree, deadlock detection: ++ * ++ * Copyright (C) 2004, LynuxWorks, Inc., Igor Manyilov, Bill Huey ++ * Released under the General Public License (GPL). ++ * ++ * Includes portions of the generic R/W semaphore implementation from: ++ * ++ * Copyright (c) 2001 David Howells (dhowells@redhat.com). ++ * - Derived partially from idea by Andrea Arcangeli ++ * - Derived also from comments by Linus ++ * ++ * Pending ownership of locks and ownership stealing: ++ * ++ * Copyright (C) 2005, Kihon Technologies Inc., Steven Rostedt ++ * ++ * (also by Steven Rostedt) ++ * - Converted single pi_lock to individual task locks. ++ * ++ * By Esben Nielsen: ++ * Doing priority inheritance with help of the scheduler. ++ * ++ * Copyright (C) 2006, Timesys Corp., Thomas Gleixner ++ * - major rework based on Esben Nielsens initial patch ++ * - replaced thread_info references by task_struct refs ++ * - removed task->pending_owner dependency ++ * - BKL drop/reacquire for semaphore style locks to avoid deadlocks ++ * in the scheduler return path as discussed with Steven Rostedt ++ * ++ * Copyright (C) 2006, Kihon Technologies Inc. ++ * Steven Rostedt ++ * - debugged and patched Thomas Gleixner's rework. ++ * - added back the cmpxchg to the rework. ++ * - turned atomic require back on for SMP. ++ */ + -+#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT_FULL) - static inline int tasklet_trylock(struct tasklet_struct *t) - { - return !test_and_set_bit(TASKLET_STATE_RUN, &(t)->state); - } - -+static inline int tasklet_tryunlock(struct tasklet_struct *t) ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "rtmutex_common.h" ++ ++/* ++ * struct mutex functions ++ */ ++void __mutex_do_init(struct mutex *mutex, const char *name, ++ struct lock_class_key *key) +{ -+ return cmpxchg(&t->state, TASKLET_STATEF_RUN, 0) == TASKLET_STATEF_RUN; ++#ifdef CONFIG_DEBUG_LOCK_ALLOC ++ /* ++ * Make sure we are not reinitializing a held lock: ++ */ ++ debug_check_no_locks_freed((void *)mutex, sizeof(*mutex)); ++ lockdep_init_map(&mutex->dep_map, name, key, 0); ++#endif ++ mutex->lock.save_state = 0; +} ++EXPORT_SYMBOL(__mutex_do_init); + - static inline void tasklet_unlock(struct tasklet_struct *t) - { - smp_mb__before_atomic(); - clear_bit(TASKLET_STATE_RUN, &(t)->state); - } - --static inline void tasklet_unlock_wait(struct tasklet_struct *t) --{ -- while (test_bit(TASKLET_STATE_RUN, &(t)->state)) { barrier(); } --} -+extern void tasklet_unlock_wait(struct tasklet_struct *t); ++void __lockfunc _mutex_lock(struct mutex *lock) ++{ ++ mutex_acquire(&lock->dep_map, 0, 0, _RET_IP_); ++ rt_mutex_lock(&lock->lock); ++} ++EXPORT_SYMBOL(_mutex_lock); + - #else - #define tasklet_trylock(t) 1 -+#define tasklet_tryunlock(t) 1 - #define tasklet_unlock_wait(t) do { } while (0) - #define tasklet_unlock(t) do { } while (0) - #endif -@@ -569,12 +593,7 @@ - smp_mb(); - } - --static inline void tasklet_enable(struct tasklet_struct *t) --{ -- smp_mb__before_atomic(); -- atomic_dec(&t->count); --} -- -+extern void tasklet_enable(struct tasklet_struct *t); - extern void tasklet_kill(struct tasklet_struct *t); - extern void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu); - extern void tasklet_init(struct tasklet_struct *t, -@@ -605,6 +624,12 @@ - tasklet_kill(&ttimer->tasklet); - } - -+#ifdef CONFIG_PREEMPT_RT_FULL -+extern void softirq_early_init(void); -+#else -+static inline void softirq_early_init(void) { } -+#endif ++int __lockfunc _mutex_lock_interruptible(struct mutex *lock) ++{ ++ int ret; + - /* - * Autoprobing for irqs: - * -diff -Nur linux-4.1.10.orig/include/linux/io-mapping.h linux-4.1.10/include/linux/io-mapping.h ---- linux-4.1.10.orig/include/linux/io-mapping.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/io-mapping.h 2015-10-07 18:00:08.000000000 +0200 -@@ -141,6 +141,7 @@ - io_mapping_map_atomic_wc(struct io_mapping *mapping, - unsigned long offset) - { -+ preempt_disable(); - pagefault_disable(); - return ((char __force __iomem *) mapping) + offset; - } -@@ -149,6 +150,7 @@ - io_mapping_unmap_atomic(void __iomem *vaddr) - { - pagefault_enable(); -+ preempt_enable(); - } - - /* Non-atomic map/unmap */ -diff -Nur linux-4.1.10.orig/include/linux/irqdesc.h linux-4.1.10/include/linux/irqdesc.h ---- linux-4.1.10.orig/include/linux/irqdesc.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/irqdesc.h 2015-10-07 18:00:08.000000000 +0200 -@@ -63,6 +63,7 @@ - unsigned int irqs_unhandled; - atomic_t threads_handled; - int threads_handled_last; -+ u64 random_ip; - raw_spinlock_t lock; - struct cpumask *percpu_enabled; - #ifdef CONFIG_SMP -diff -Nur linux-4.1.10.orig/include/linux/irqflags.h linux-4.1.10/include/linux/irqflags.h ---- linux-4.1.10.orig/include/linux/irqflags.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/irqflags.h 2015-10-07 18:00:08.000000000 +0200 -@@ -25,8 +25,6 @@ - # define trace_softirqs_enabled(p) ((p)->softirqs_enabled) - # define trace_hardirq_enter() do { current->hardirq_context++; } while (0) - # define trace_hardirq_exit() do { current->hardirq_context--; } while (0) --# define lockdep_softirq_enter() do { current->softirq_context++; } while (0) --# define lockdep_softirq_exit() do { current->softirq_context--; } while (0) - # define INIT_TRACE_IRQFLAGS .softirqs_enabled = 1, - #else - # define trace_hardirqs_on() do { } while (0) -@@ -39,9 +37,15 @@ - # define trace_softirqs_enabled(p) 0 - # define trace_hardirq_enter() do { } while (0) - # define trace_hardirq_exit() do { } while (0) -+# define INIT_TRACE_IRQFLAGS ++ mutex_acquire(&lock->dep_map, 0, 0, _RET_IP_); ++ ret = rt_mutex_lock_interruptible(&lock->lock); ++ if (ret) ++ mutex_release(&lock->dep_map, 1, _RET_IP_); ++ return ret; ++} ++EXPORT_SYMBOL(_mutex_lock_interruptible); ++ ++int __lockfunc _mutex_lock_killable(struct mutex *lock) ++{ ++ int ret; ++ ++ mutex_acquire(&lock->dep_map, 0, 0, _RET_IP_); ++ ret = rt_mutex_lock_killable(&lock->lock); ++ if (ret) ++ mutex_release(&lock->dep_map, 1, _RET_IP_); ++ return ret; ++} ++EXPORT_SYMBOL(_mutex_lock_killable); ++ ++#ifdef CONFIG_DEBUG_LOCK_ALLOC ++void __lockfunc _mutex_lock_nested(struct mutex *lock, int subclass) ++{ ++ mutex_acquire_nest(&lock->dep_map, subclass, 0, NULL, _RET_IP_); ++ rt_mutex_lock(&lock->lock); ++} ++EXPORT_SYMBOL(_mutex_lock_nested); ++ ++void __lockfunc _mutex_lock_nest_lock(struct mutex *lock, struct lockdep_map *nest) ++{ ++ mutex_acquire_nest(&lock->dep_map, 0, 0, nest, _RET_IP_); ++ rt_mutex_lock(&lock->lock); ++} ++EXPORT_SYMBOL(_mutex_lock_nest_lock); ++ ++int __lockfunc _mutex_lock_interruptible_nested(struct mutex *lock, int subclass) ++{ ++ int ret; ++ ++ mutex_acquire_nest(&lock->dep_map, subclass, 0, NULL, _RET_IP_); ++ ret = rt_mutex_lock_interruptible(&lock->lock); ++ if (ret) ++ mutex_release(&lock->dep_map, 1, _RET_IP_); ++ return ret; ++} ++EXPORT_SYMBOL(_mutex_lock_interruptible_nested); ++ ++int __lockfunc _mutex_lock_killable_nested(struct mutex *lock, int subclass) ++{ ++ int ret; ++ ++ mutex_acquire(&lock->dep_map, subclass, 0, _RET_IP_); ++ ret = rt_mutex_lock_killable(&lock->lock); ++ if (ret) ++ mutex_release(&lock->dep_map, 1, _RET_IP_); ++ return ret; ++} ++EXPORT_SYMBOL(_mutex_lock_killable_nested); +#endif + -+#if defined(CONFIG_TRACE_IRQFLAGS) && !defined(CONFIG_PREEMPT_RT_FULL) -+# define lockdep_softirq_enter() do { current->softirq_context++; } while (0) -+# define lockdep_softirq_exit() do { current->softirq_context--; } while (0) -+#else - # define lockdep_softirq_enter() do { } while (0) - # define lockdep_softirq_exit() do { } while (0) --# define INIT_TRACE_IRQFLAGS - #endif - - #if defined(CONFIG_IRQSOFF_TRACER) || \ -@@ -148,4 +152,23 @@ - - #define irqs_disabled_flags(flags) raw_irqs_disabled_flags(flags) - ++int __lockfunc _mutex_trylock(struct mutex *lock) ++{ ++ int ret = rt_mutex_trylock(&lock->lock); ++ ++ if (ret) ++ mutex_acquire(&lock->dep_map, 0, 1, _RET_IP_); ++ ++ return ret; ++} ++EXPORT_SYMBOL(_mutex_trylock); ++ ++void __lockfunc _mutex_unlock(struct mutex *lock) ++{ ++ mutex_release(&lock->dep_map, 1, _RET_IP_); ++ rt_mutex_unlock(&lock->lock); ++} ++EXPORT_SYMBOL(_mutex_unlock); ++ +/* -+ * local_irq* variants depending on RT/!RT ++ * rwlock_t functions + */ -+#ifdef CONFIG_PREEMPT_RT_FULL -+# define local_irq_disable_nort() do { } while (0) -+# define local_irq_enable_nort() do { } while (0) -+# define local_irq_save_nort(flags) local_save_flags(flags) -+# define local_irq_restore_nort(flags) (void)(flags) -+# define local_irq_disable_rt() local_irq_disable() -+# define local_irq_enable_rt() local_irq_enable() -+#else -+# define local_irq_disable_nort() local_irq_disable() -+# define local_irq_enable_nort() local_irq_enable() -+# define local_irq_save_nort(flags) local_irq_save(flags) -+# define local_irq_restore_nort(flags) local_irq_restore(flags) -+# define local_irq_disable_rt() do { } while (0) -+# define local_irq_enable_rt() do { } while (0) -+#endif ++int __lockfunc rt_write_trylock(rwlock_t *rwlock) ++{ ++ int ret; + - #endif -diff -Nur linux-4.1.10.orig/include/linux/irq.h linux-4.1.10/include/linux/irq.h ---- linux-4.1.10.orig/include/linux/irq.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/irq.h 2015-10-07 18:00:08.000000000 +0200 -@@ -72,6 +72,7 @@ - * IRQ_IS_POLLED - Always polled by another interrupt. Exclude - * it from the spurious interrupt detection - * mechanism and from core side polling. -+ * IRQ_NO_SOFTIRQ_CALL - No softirq processing in the irq thread context (RT) - */ - enum { - IRQ_TYPE_NONE = 0x00000000, -@@ -97,13 +98,14 @@ - IRQ_NOTHREAD = (1 << 16), - IRQ_PER_CPU_DEVID = (1 << 17), - IRQ_IS_POLLED = (1 << 18), -+ IRQ_NO_SOFTIRQ_CALL = (1 << 19), - }; - - #define IRQF_MODIFY_MASK \ - (IRQ_TYPE_SENSE_MASK | IRQ_NOPROBE | IRQ_NOREQUEST | \ - IRQ_NOAUTOEN | IRQ_MOVE_PCNTXT | IRQ_LEVEL | IRQ_NO_BALANCING | \ - IRQ_PER_CPU | IRQ_NESTED_THREAD | IRQ_NOTHREAD | IRQ_PER_CPU_DEVID | \ -- IRQ_IS_POLLED) -+ IRQ_IS_POLLED | IRQ_NO_SOFTIRQ_CALL) - - #define IRQ_NO_BALANCING_MASK (IRQ_PER_CPU | IRQ_NO_BALANCING) - -diff -Nur linux-4.1.10.orig/include/linux/irq_work.h linux-4.1.10/include/linux/irq_work.h ---- linux-4.1.10.orig/include/linux/irq_work.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/irq_work.h 2015-10-07 18:00:08.000000000 +0200 -@@ -16,6 +16,7 @@ - #define IRQ_WORK_BUSY 2UL - #define IRQ_WORK_FLAGS 3UL - #define IRQ_WORK_LAZY 4UL /* Doesn't want IPI, wait for tick */ -+#define IRQ_WORK_HARD_IRQ 8UL /* Run hard IRQ context, even on RT */ - - struct irq_work { - unsigned long flags; -diff -Nur linux-4.1.10.orig/include/linux/jbd_common.h linux-4.1.10/include/linux/jbd_common.h ---- linux-4.1.10.orig/include/linux/jbd_common.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/jbd_common.h 2015-10-07 18:00:08.000000000 +0200 -@@ -15,32 +15,56 @@ - - static inline void jbd_lock_bh_state(struct buffer_head *bh) - { -+#ifndef CONFIG_PREEMPT_RT_BASE - bit_spin_lock(BH_State, &bh->b_state); -+#else -+ spin_lock(&bh->b_state_lock); -+#endif - } - - static inline int jbd_trylock_bh_state(struct buffer_head *bh) - { -+#ifndef CONFIG_PREEMPT_RT_BASE - return bit_spin_trylock(BH_State, &bh->b_state); -+#else -+ return spin_trylock(&bh->b_state_lock); -+#endif - } - - static inline int jbd_is_locked_bh_state(struct buffer_head *bh) - { -+#ifndef CONFIG_PREEMPT_RT_BASE - return bit_spin_is_locked(BH_State, &bh->b_state); -+#else -+ return spin_is_locked(&bh->b_state_lock); -+#endif - } - - static inline void jbd_unlock_bh_state(struct buffer_head *bh) - { -+#ifndef CONFIG_PREEMPT_RT_BASE - bit_spin_unlock(BH_State, &bh->b_state); -+#else -+ spin_unlock(&bh->b_state_lock); -+#endif - } - - static inline void jbd_lock_bh_journal_head(struct buffer_head *bh) - { -+#ifndef CONFIG_PREEMPT_RT_BASE - bit_spin_lock(BH_JournalHead, &bh->b_state); -+#else -+ spin_lock(&bh->b_journal_head_lock); -+#endif - } - - static inline void jbd_unlock_bh_journal_head(struct buffer_head *bh) - { -+#ifndef CONFIG_PREEMPT_RT_BASE - bit_spin_unlock(BH_JournalHead, &bh->b_state); -+#else -+ spin_unlock(&bh->b_journal_head_lock); -+#endif - } - - #endif -diff -Nur linux-4.1.10.orig/include/linux/kdb.h linux-4.1.10/include/linux/kdb.h ---- linux-4.1.10.orig/include/linux/kdb.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/kdb.h 2015-10-07 18:00:08.000000000 +0200 -@@ -167,6 +167,7 @@ - extern __printf(1, 2) int kdb_printf(const char *, ...); - typedef __printf(1, 2) int (*kdb_printf_t)(const char *, ...); - -+#define in_kdb_printk() (kdb_trap_printk) - extern void kdb_init(int level); - - /* Access to kdb specific polling devices */ -@@ -201,6 +202,7 @@ - extern int kdb_unregister(char *); - #else /* ! CONFIG_KGDB_KDB */ - static inline __printf(1, 2) int kdb_printf(const char *fmt, ...) { return 0; } -+#define in_kdb_printk() (0) - static inline void kdb_init(int level) {} - static inline int kdb_register(char *cmd, kdb_func_t func, char *usage, - char *help, short minlen) { return 0; } -diff -Nur linux-4.1.10.orig/include/linux/kernel.h linux-4.1.10/include/linux/kernel.h ---- linux-4.1.10.orig/include/linux/kernel.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/kernel.h 2015-10-07 18:00:08.000000000 +0200 -@@ -188,6 +188,9 @@ - */ - # define might_sleep() \ - do { __might_sleep(__FILE__, __LINE__, 0); might_resched(); } while (0) ++ migrate_disable(); ++ ret = rt_mutex_trylock(&rwlock->lock); ++ if (ret) ++ rwlock_acquire(&rwlock->dep_map, 0, 1, _RET_IP_); ++ else ++ migrate_enable(); + -+# define might_sleep_no_state_check() \ -+ do { ___might_sleep(__FILE__, __LINE__, 0); might_resched(); } while (0) - # define sched_annotate_sleep() (current->task_state_change = 0) - #else - static inline void ___might_sleep(const char *file, int line, -@@ -195,6 +198,7 @@ - static inline void __might_sleep(const char *file, int line, - int preempt_offset) { } - # define might_sleep() do { might_resched(); } while (0) -+# define might_sleep_no_state_check() do { might_resched(); } while (0) - # define sched_annotate_sleep() do { } while (0) - #endif - -@@ -244,7 +248,8 @@ - - #if defined(CONFIG_MMU) && \ - (defined(CONFIG_PROVE_LOCKING) || defined(CONFIG_DEBUG_ATOMIC_SLEEP)) --void might_fault(void); -+#define might_fault() __might_fault(__FILE__, __LINE__) -+void __might_fault(const char *file, int line); - #else - static inline void might_fault(void) { } - #endif -@@ -466,6 +471,7 @@ - SYSTEM_HALT, - SYSTEM_POWER_OFF, - SYSTEM_RESTART, -+ SYSTEM_SUSPEND, - } system_state; - - #define TAINT_PROPRIETARY_MODULE 0 -diff -Nur linux-4.1.10.orig/include/linux/kvm_host.h linux-4.1.10/include/linux/kvm_host.h ---- linux-4.1.10.orig/include/linux/kvm_host.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/kvm_host.h 2015-10-07 18:00:08.000000000 +0200 -@@ -230,7 +230,7 @@ - - int fpu_active; - int guest_fpu_loaded, guest_xcr0_loaded; -- wait_queue_head_t wq; -+ struct swait_head wq; - struct pid *pid; - int sigset_active; - sigset_t sigset; -@@ -690,7 +690,7 @@ - } - #endif - --static inline wait_queue_head_t *kvm_arch_vcpu_wq(struct kvm_vcpu *vcpu) -+static inline struct swait_head *kvm_arch_vcpu_wq(struct kvm_vcpu *vcpu) - { - #ifdef __KVM_HAVE_ARCH_WQP - return vcpu->arch.wqp; -diff -Nur linux-4.1.10.orig/include/linux/lglock.h linux-4.1.10/include/linux/lglock.h ---- linux-4.1.10.orig/include/linux/lglock.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/lglock.h 2015-10-07 18:00:08.000000000 +0200 -@@ -34,22 +34,39 @@ - #endif - - struct lglock { -+#ifndef CONFIG_PREEMPT_RT_FULL - arch_spinlock_t __percpu *lock; -+#else -+ struct rt_mutex __percpu *lock; -+#endif - #ifdef CONFIG_DEBUG_LOCK_ALLOC - struct lock_class_key lock_key; - struct lockdep_map lock_dep_map; - #endif - }; - --#define DEFINE_LGLOCK(name) \ -+#ifndef CONFIG_PREEMPT_RT_FULL -+# define DEFINE_LGLOCK(name) \ - static DEFINE_PER_CPU(arch_spinlock_t, name ## _lock) \ - = __ARCH_SPIN_LOCK_UNLOCKED; \ - struct lglock name = { .lock = &name ## _lock } - --#define DEFINE_STATIC_LGLOCK(name) \ -+# define DEFINE_STATIC_LGLOCK(name) \ - static DEFINE_PER_CPU(arch_spinlock_t, name ## _lock) \ - = __ARCH_SPIN_LOCK_UNLOCKED; \ - static struct lglock name = { .lock = &name ## _lock } -+#else -+ -+# define DEFINE_LGLOCK(name) \ -+ static DEFINE_PER_CPU(struct rt_mutex, name ## _lock) \ -+ = __RT_MUTEX_INITIALIZER( name ## _lock); \ -+ struct lglock name = { .lock = &name ## _lock } -+ -+# define DEFINE_STATIC_LGLOCK(name) \ -+ static DEFINE_PER_CPU(struct rt_mutex, name ## _lock) \ -+ = __RT_MUTEX_INITIALIZER( name ## _lock); \ -+ static struct lglock name = { .lock = &name ## _lock } -+#endif - - void lg_lock_init(struct lglock *lg, char *name); - void lg_local_lock(struct lglock *lg); -@@ -59,6 +76,12 @@ - void lg_global_lock(struct lglock *lg); - void lg_global_unlock(struct lglock *lg); - -+#ifndef CONFIG_PREEMPT_RT_FULL -+#define lg_global_trylock_relax(name) lg_global_lock(name) -+#else -+void lg_global_trylock_relax(struct lglock *lg); -+#endif -+ - #else - /* When !CONFIG_SMP, map lglock to spinlock */ - #define lglock spinlock -diff -Nur linux-4.1.10.orig/include/linux/list_bl.h linux-4.1.10/include/linux/list_bl.h ---- linux-4.1.10.orig/include/linux/list_bl.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/list_bl.h 2015-10-07 18:00:08.000000000 +0200 -@@ -2,6 +2,7 @@ - #define _LINUX_LIST_BL_H - - #include -+#include - #include - - /* -@@ -32,13 +33,22 @@ - - struct hlist_bl_head { - struct hlist_bl_node *first; -+#ifdef CONFIG_PREEMPT_RT_BASE -+ raw_spinlock_t lock; -+#endif - }; - - struct hlist_bl_node { - struct hlist_bl_node *next, **pprev; - }; --#define INIT_HLIST_BL_HEAD(ptr) \ -- ((ptr)->first = NULL) ++ return ret; ++} ++EXPORT_SYMBOL(rt_write_trylock); + -+static inline void INIT_HLIST_BL_HEAD(struct hlist_bl_head *h) ++int __lockfunc rt_write_trylock_irqsave(rwlock_t *rwlock, unsigned long *flags) +{ -+ h->first = NULL; -+#ifdef CONFIG_PREEMPT_RT_BASE -+ raw_spin_lock_init(&h->lock); -+#endif ++ int ret; ++ ++ *flags = 0; ++ ret = rt_write_trylock(rwlock); ++ return ret; +} - - static inline void INIT_HLIST_BL_NODE(struct hlist_bl_node *h) - { -@@ -117,12 +127,26 @@ - - static inline void hlist_bl_lock(struct hlist_bl_head *b) - { -+#ifndef CONFIG_PREEMPT_RT_BASE - bit_spin_lock(0, (unsigned long *)b); -+#else -+ raw_spin_lock(&b->lock); -+#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK) -+ __set_bit(0, (unsigned long *)b); -+#endif -+#endif - } - - static inline void hlist_bl_unlock(struct hlist_bl_head *b) - { -+#ifndef CONFIG_PREEMPT_RT_BASE - __bit_spin_unlock(0, (unsigned long *)b); -+#else -+#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK) -+ __clear_bit(0, (unsigned long *)b); -+#endif -+ raw_spin_unlock(&b->lock); -+#endif - } - - static inline bool hlist_bl_is_locked(struct hlist_bl_head *b) -diff -Nur linux-4.1.10.orig/include/linux/locallock.h linux-4.1.10/include/linux/locallock.h ---- linux-4.1.10.orig/include/linux/locallock.h 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.10/include/linux/locallock.h 2015-10-07 18:00:08.000000000 +0200 -@@ -0,0 +1,270 @@ -+#ifndef _LINUX_LOCALLOCK_H -+#define _LINUX_LOCALLOCK_H ++EXPORT_SYMBOL(rt_write_trylock_irqsave); + -+#include -+#include ++int __lockfunc rt_read_trylock(rwlock_t *rwlock) ++{ ++ struct rt_mutex *lock = &rwlock->lock; ++ int ret = 1; + -+#ifdef CONFIG_PREEMPT_RT_BASE ++ /* ++ * recursive read locks succeed when current owns the lock, ++ * but not when read_depth == 0 which means that the lock is ++ * write locked. ++ */ ++ if (rt_mutex_owner(lock) != current) { ++ migrate_disable(); ++ ret = rt_mutex_trylock(lock); ++ if (ret) ++ rwlock_acquire(&rwlock->dep_map, 0, 1, _RET_IP_); ++ else ++ migrate_enable(); + -+#ifdef CONFIG_DEBUG_SPINLOCK -+# define LL_WARN(cond) WARN_ON(cond) -+#else -+# define LL_WARN(cond) do { } while (0) -+#endif ++ } else if (!rwlock->read_depth) { ++ ret = 0; ++ } + -+/* -+ * per cpu lock based substitute for local_irq_*() -+ */ -+struct local_irq_lock { -+ spinlock_t lock; -+ struct task_struct *owner; -+ int nestcnt; -+ unsigned long flags; -+}; ++ if (ret) ++ rwlock->read_depth++; + -+#define DEFINE_LOCAL_IRQ_LOCK(lvar) \ -+ DEFINE_PER_CPU(struct local_irq_lock, lvar) = { \ -+ .lock = __SPIN_LOCK_UNLOCKED((lvar).lock) } ++ return ret; ++} ++EXPORT_SYMBOL(rt_read_trylock); + -+#define DECLARE_LOCAL_IRQ_LOCK(lvar) \ -+ DECLARE_PER_CPU(struct local_irq_lock, lvar) ++void __lockfunc rt_write_lock(rwlock_t *rwlock) ++{ ++ rwlock_acquire(&rwlock->dep_map, 0, 0, _RET_IP_); ++ migrate_disable(); ++ __rt_spin_lock(&rwlock->lock); ++} ++EXPORT_SYMBOL(rt_write_lock); + -+#define local_irq_lock_init(lvar) \ -+ do { \ -+ int __cpu; \ -+ for_each_possible_cpu(__cpu) \ -+ spin_lock_init(&per_cpu(lvar, __cpu).lock); \ -+ } while (0) ++void __lockfunc rt_read_lock(rwlock_t *rwlock) ++{ ++ struct rt_mutex *lock = &rwlock->lock; + -+/* -+ * spin_lock|trylock|unlock_local flavour that does not migrate disable -+ * used for __local_lock|trylock|unlock where get_local_var/put_local_var -+ * already takes care of the migrate_disable/enable -+ * for CONFIG_PREEMPT_BASE map to the normal spin_* calls. -+ */ -+#ifdef CONFIG_PREEMPT_RT_FULL -+# define spin_lock_local(lock) rt_spin_lock(lock) -+# define spin_trylock_local(lock) rt_spin_trylock(lock) -+# define spin_unlock_local(lock) rt_spin_unlock(lock) -+#else -+# define spin_lock_local(lock) spin_lock(lock) -+# define spin_trylock_local(lock) spin_trylock(lock) -+# define spin_unlock_local(lock) spin_unlock(lock) -+#endif + -+static inline void __local_lock(struct local_irq_lock *lv) -+{ -+ if (lv->owner != current) { -+ spin_lock_local(&lv->lock); -+ LL_WARN(lv->owner); -+ LL_WARN(lv->nestcnt); -+ lv->owner = current; ++ /* ++ * recursive read locks succeed when current owns the lock ++ */ ++ if (rt_mutex_owner(lock) != current) { ++ migrate_disable(); ++ rwlock_acquire(&rwlock->dep_map, 0, 0, _RET_IP_); ++ __rt_spin_lock(lock); + } -+ lv->nestcnt++; ++ rwlock->read_depth++; +} + -+#define local_lock(lvar) \ -+ do { __local_lock(&get_local_var(lvar)); } while (0) ++EXPORT_SYMBOL(rt_read_lock); + -+static inline int __local_trylock(struct local_irq_lock *lv) ++void __lockfunc rt_write_unlock(rwlock_t *rwlock) +{ -+ if (lv->owner != current && spin_trylock_local(&lv->lock)) { -+ LL_WARN(lv->owner); -+ LL_WARN(lv->nestcnt); -+ lv->owner = current; -+ lv->nestcnt = 1; -+ return 1; -+ } -+ return 0; ++ /* NOTE: we always pass in '1' for nested, for simplicity */ ++ rwlock_release(&rwlock->dep_map, 1, _RET_IP_); ++ __rt_spin_unlock(&rwlock->lock); ++ migrate_enable(); +} ++EXPORT_SYMBOL(rt_write_unlock); + -+#define local_trylock(lvar) \ -+ ({ \ -+ int __locked; \ -+ __locked = __local_trylock(&get_local_var(lvar)); \ -+ if (!__locked) \ -+ put_local_var(lvar); \ -+ __locked; \ -+ }) -+ -+static inline void __local_unlock(struct local_irq_lock *lv) ++void __lockfunc rt_read_unlock(rwlock_t *rwlock) +{ -+ LL_WARN(lv->nestcnt == 0); -+ LL_WARN(lv->owner != current); -+ if (--lv->nestcnt) -+ return; -+ -+ lv->owner = NULL; -+ spin_unlock_local(&lv->lock); ++ /* Release the lock only when read_depth is down to 0 */ ++ if (--rwlock->read_depth == 0) { ++ rwlock_release(&rwlock->dep_map, 1, _RET_IP_); ++ __rt_spin_unlock(&rwlock->lock); ++ migrate_enable(); ++ } +} ++EXPORT_SYMBOL(rt_read_unlock); + -+#define local_unlock(lvar) \ -+ do { \ -+ __local_unlock(this_cpu_ptr(&lvar)); \ -+ put_local_var(lvar); \ -+ } while (0) -+ -+static inline void __local_lock_irq(struct local_irq_lock *lv) ++unsigned long __lockfunc rt_write_lock_irqsave(rwlock_t *rwlock) +{ -+ spin_lock_irqsave(&lv->lock, lv->flags); -+ LL_WARN(lv->owner); -+ LL_WARN(lv->nestcnt); -+ lv->owner = current; -+ lv->nestcnt = 1; ++ rt_write_lock(rwlock); ++ ++ return 0; +} ++EXPORT_SYMBOL(rt_write_lock_irqsave); + -+#define local_lock_irq(lvar) \ -+ do { __local_lock_irq(&get_local_var(lvar)); } while (0) ++unsigned long __lockfunc rt_read_lock_irqsave(rwlock_t *rwlock) ++{ ++ rt_read_lock(rwlock); + -+#define local_lock_irq_on(lvar, cpu) \ -+ do { __local_lock_irq(&per_cpu(lvar, cpu)); } while (0) ++ return 0; ++} ++EXPORT_SYMBOL(rt_read_lock_irqsave); + -+static inline void __local_unlock_irq(struct local_irq_lock *lv) ++void __rt_rwlock_init(rwlock_t *rwlock, char *name, struct lock_class_key *key) +{ -+ LL_WARN(!lv->nestcnt); -+ LL_WARN(lv->owner != current); -+ lv->owner = NULL; -+ lv->nestcnt = 0; -+ spin_unlock_irq(&lv->lock); ++#ifdef CONFIG_DEBUG_LOCK_ALLOC ++ /* ++ * Make sure we are not reinitializing a held lock: ++ */ ++ debug_check_no_locks_freed((void *)rwlock, sizeof(*rwlock)); ++ lockdep_init_map(&rwlock->dep_map, name, key, 0); ++#endif ++ rwlock->lock.save_state = 1; ++ rwlock->read_depth = 0; +} ++EXPORT_SYMBOL(__rt_rwlock_init); + -+#define local_unlock_irq(lvar) \ -+ do { \ -+ __local_unlock_irq(this_cpu_ptr(&lvar)); \ -+ put_local_var(lvar); \ -+ } while (0) ++/* ++ * rw_semaphores ++ */ + -+#define local_unlock_irq_on(lvar, cpu) \ -+ do { \ -+ __local_unlock_irq(&per_cpu(lvar, cpu)); \ -+ } while (0) ++void rt_up_write(struct rw_semaphore *rwsem) ++{ ++ rwsem_release(&rwsem->dep_map, 1, _RET_IP_); ++ rt_mutex_unlock(&rwsem->lock); ++} ++EXPORT_SYMBOL(rt_up_write); + -+static inline int __local_lock_irqsave(struct local_irq_lock *lv) ++void __rt_up_read(struct rw_semaphore *rwsem) +{ -+ if (lv->owner != current) { -+ __local_lock_irq(lv); -+ return 0; -+ } else { -+ lv->nestcnt++; -+ return 1; -+ } ++ if (--rwsem->read_depth == 0) ++ rt_mutex_unlock(&rwsem->lock); +} + -+#define local_lock_irqsave(lvar, _flags) \ -+ do { \ -+ if (__local_lock_irqsave(&get_local_var(lvar))) \ -+ put_local_var(lvar); \ -+ _flags = __this_cpu_read(lvar.flags); \ -+ } while (0) ++void rt_up_read(struct rw_semaphore *rwsem) ++{ ++ rwsem_release(&rwsem->dep_map, 1, _RET_IP_); ++ __rt_up_read(rwsem); ++} ++EXPORT_SYMBOL(rt_up_read); + -+#define local_lock_irqsave_on(lvar, _flags, cpu) \ -+ do { \ -+ __local_lock_irqsave(&per_cpu(lvar, cpu)); \ -+ _flags = per_cpu(lvar, cpu).flags; \ -+ } while (0) ++/* ++ * downgrade a write lock into a read lock ++ * - just wake up any readers at the front of the queue ++ */ ++void rt_downgrade_write(struct rw_semaphore *rwsem) ++{ ++ BUG_ON(rt_mutex_owner(&rwsem->lock) != current); ++ rwsem->read_depth = 1; ++} ++EXPORT_SYMBOL(rt_downgrade_write); + -+static inline int __local_unlock_irqrestore(struct local_irq_lock *lv, -+ unsigned long flags) ++int rt_down_write_trylock(struct rw_semaphore *rwsem) +{ -+ LL_WARN(!lv->nestcnt); -+ LL_WARN(lv->owner != current); -+ if (--lv->nestcnt) -+ return 0; ++ int ret = rt_mutex_trylock(&rwsem->lock); + -+ lv->owner = NULL; -+ spin_unlock_irqrestore(&lv->lock, lv->flags); -+ return 1; ++ if (ret) ++ rwsem_acquire(&rwsem->dep_map, 0, 1, _RET_IP_); ++ return ret; +} ++EXPORT_SYMBOL(rt_down_write_trylock); + -+#define local_unlock_irqrestore(lvar, flags) \ -+ do { \ -+ if (__local_unlock_irqrestore(this_cpu_ptr(&lvar), flags)) \ -+ put_local_var(lvar); \ -+ } while (0) ++void rt_down_write(struct rw_semaphore *rwsem) ++{ ++ rwsem_acquire(&rwsem->dep_map, 0, 0, _RET_IP_); ++ rt_mutex_lock(&rwsem->lock); ++} ++EXPORT_SYMBOL(rt_down_write); + -+#define local_unlock_irqrestore_on(lvar, flags, cpu) \ -+ do { \ -+ __local_unlock_irqrestore(&per_cpu(lvar, cpu), flags); \ -+ } while (0) ++void rt_down_write_nested(struct rw_semaphore *rwsem, int subclass) ++{ ++ rwsem_acquire(&rwsem->dep_map, subclass, 0, _RET_IP_); ++ rt_mutex_lock(&rwsem->lock); ++} ++EXPORT_SYMBOL(rt_down_write_nested); + -+#define local_spin_trylock_irq(lvar, lock) \ -+ ({ \ -+ int __locked; \ -+ local_lock_irq(lvar); \ -+ __locked = spin_trylock(lock); \ -+ if (!__locked) \ -+ local_unlock_irq(lvar); \ -+ __locked; \ -+ }) ++void rt_down_write_nested_lock(struct rw_semaphore *rwsem, ++ struct lockdep_map *nest) ++{ ++ rwsem_acquire_nest(&rwsem->dep_map, 0, 0, nest, _RET_IP_); ++ rt_mutex_lock(&rwsem->lock); ++} ++EXPORT_SYMBOL(rt_down_write_nested_lock); + -+#define local_spin_lock_irq(lvar, lock) \ -+ do { \ -+ local_lock_irq(lvar); \ -+ spin_lock(lock); \ -+ } while (0) ++int rt_down_read_trylock(struct rw_semaphore *rwsem) ++{ ++ struct rt_mutex *lock = &rwsem->lock; ++ int ret = 1; + -+#define local_spin_unlock_irq(lvar, lock) \ -+ do { \ -+ spin_unlock(lock); \ -+ local_unlock_irq(lvar); \ -+ } while (0) ++ /* ++ * recursive read locks succeed when current owns the rwsem, ++ * but not when read_depth == 0 which means that the rwsem is ++ * write locked. ++ */ ++ if (rt_mutex_owner(lock) != current) ++ ret = rt_mutex_trylock(&rwsem->lock); ++ else if (!rwsem->read_depth) ++ ret = 0; + -+#define local_spin_lock_irqsave(lvar, lock, flags) \ -+ do { \ -+ local_lock_irqsave(lvar, flags); \ -+ spin_lock(lock); \ -+ } while (0) ++ if (ret) { ++ rwsem->read_depth++; ++ rwsem_acquire(&rwsem->dep_map, 0, 1, _RET_IP_); ++ } ++ return ret; ++} ++EXPORT_SYMBOL(rt_down_read_trylock); + -+#define local_spin_unlock_irqrestore(lvar, lock, flags) \ -+ do { \ -+ spin_unlock(lock); \ -+ local_unlock_irqrestore(lvar, flags); \ -+ } while (0) ++static void __rt_down_read(struct rw_semaphore *rwsem, int subclass) ++{ ++ struct rt_mutex *lock = &rwsem->lock; + -+#define get_locked_var(lvar, var) \ -+ (*({ \ -+ local_lock(lvar); \ -+ this_cpu_ptr(&var); \ -+ })) ++ rwsem_acquire_read(&rwsem->dep_map, subclass, 0, _RET_IP_); + -+#define put_locked_var(lvar, var) local_unlock(lvar); ++ if (rt_mutex_owner(lock) != current) ++ rt_mutex_lock(&rwsem->lock); ++ rwsem->read_depth++; ++} + -+#define local_lock_cpu(lvar) \ -+ ({ \ -+ local_lock(lvar); \ -+ smp_processor_id(); \ -+ }) ++void rt_down_read(struct rw_semaphore *rwsem) ++{ ++ __rt_down_read(rwsem, 0); ++} ++EXPORT_SYMBOL(rt_down_read); + -+#define local_unlock_cpu(lvar) local_unlock(lvar) ++void rt_down_read_nested(struct rw_semaphore *rwsem, int subclass) ++{ ++ __rt_down_read(rwsem, subclass); ++} ++EXPORT_SYMBOL(rt_down_read_nested); + -+#else /* PREEMPT_RT_BASE */ ++void __rt_rwsem_init(struct rw_semaphore *rwsem, const char *name, ++ struct lock_class_key *key) ++{ ++#ifdef CONFIG_DEBUG_LOCK_ALLOC ++ /* ++ * Make sure we are not reinitializing a held lock: ++ */ ++ debug_check_no_locks_freed((void *)rwsem, sizeof(*rwsem)); ++ lockdep_init_map(&rwsem->dep_map, name, key, 0); ++#endif ++ rwsem->read_depth = 0; ++ rwsem->lock.save_state = 0; ++} ++EXPORT_SYMBOL(__rt_rwsem_init); + -+#define DEFINE_LOCAL_IRQ_LOCK(lvar) __typeof__(const int) lvar -+#define DECLARE_LOCAL_IRQ_LOCK(lvar) extern __typeof__(const int) lvar -+ -+static inline void local_irq_lock_init(int lvar) { } -+ -+#define local_lock(lvar) preempt_disable() -+#define local_unlock(lvar) preempt_enable() -+#define local_lock_irq(lvar) local_irq_disable() -+#define local_unlock_irq(lvar) local_irq_enable() -+#define local_lock_irqsave(lvar, flags) local_irq_save(flags) -+#define local_unlock_irqrestore(lvar, flags) local_irq_restore(flags) -+ -+#define local_spin_trylock_irq(lvar, lock) spin_trylock_irq(lock) -+#define local_spin_lock_irq(lvar, lock) spin_lock_irq(lock) -+#define local_spin_unlock_irq(lvar, lock) spin_unlock_irq(lock) -+#define local_spin_lock_irqsave(lvar, lock, flags) \ -+ spin_lock_irqsave(lock, flags) -+#define local_spin_unlock_irqrestore(lvar, lock, flags) \ -+ spin_unlock_irqrestore(lock, flags) -+ -+#define get_locked_var(lvar, var) get_cpu_var(var) -+#define put_locked_var(lvar, var) put_cpu_var(var) -+ -+#define local_lock_cpu(lvar) get_cpu() -+#define local_unlock_cpu(lvar) put_cpu() -+ -+#endif -+ -+#endif -diff -Nur linux-4.1.10.orig/include/linux/mm_types.h linux-4.1.10/include/linux/mm_types.h ---- linux-4.1.10.orig/include/linux/mm_types.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/mm_types.h 2015-10-07 18:00:08.000000000 +0200 -@@ -11,6 +11,7 @@ - #include - #include - #include -+#include - #include - #include - #include -@@ -453,6 +454,9 @@ - bool tlb_flush_pending; - #endif - struct uprobes_state uprobes_state; -+#ifdef CONFIG_PREEMPT_RT_BASE -+ struct rcu_head delayed_drop; -+#endif - #ifdef CONFIG_X86_INTEL_MPX - /* address of the bounds directory */ - void __user *bd_addr; -diff -Nur linux-4.1.10.orig/include/linux/mm_types.h.orig linux-4.1.10/include/linux/mm_types.h.orig ---- linux-4.1.10.orig/include/linux/mm_types.h.orig 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.10/include/linux/mm_types.h.orig 2015-10-03 13:49:38.000000000 +0200 -@@ -0,0 +1,539 @@ -+#ifndef _LINUX_MM_TYPES_H -+#define _LINUX_MM_TYPES_H -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#ifndef AT_VECTOR_SIZE_ARCH -+#define AT_VECTOR_SIZE_ARCH 0 -+#endif -+#define AT_VECTOR_SIZE (2*(AT_VECTOR_SIZE_ARCH + AT_VECTOR_SIZE_BASE + 1)) -+ -+struct address_space; -+struct mem_cgroup; -+ -+#define USE_SPLIT_PTE_PTLOCKS (NR_CPUS >= CONFIG_SPLIT_PTLOCK_CPUS) -+#define USE_SPLIT_PMD_PTLOCKS (USE_SPLIT_PTE_PTLOCKS && \ -+ IS_ENABLED(CONFIG_ARCH_ENABLE_SPLIT_PMD_PTLOCK)) -+#define ALLOC_SPLIT_PTLOCKS (SPINLOCK_SIZE > BITS_PER_LONG/8) -+ -+typedef void compound_page_dtor(struct page *); -+ -+/* -+ * Each physical page in the system has a struct page associated with -+ * it to keep track of whatever it is we are using the page for at the -+ * moment. Note that we have no way to track which tasks are using -+ * a page, though if it is a pagecache page, rmap structures can tell us -+ * who is mapping it. ++/** ++ * atomic_dec_and_mutex_lock - return holding mutex if we dec to 0 ++ * @cnt: the atomic which we are to dec ++ * @lock: the mutex to return holding if we dec to 0 + * -+ * The objects in struct page are organized in double word blocks in -+ * order to allows us to use atomic double word operations on portions -+ * of struct page. That is currently only used by slub but the arrangement -+ * allows the use of atomic double word operations on the flags/mapping -+ * and lru list pointers also. ++ * return true and hold lock if we dec to 0, return false otherwise + */ -+struct page { -+ /* First double word block */ -+ unsigned long flags; /* Atomic flags, some possibly -+ * updated asynchronously */ -+ union { -+ struct address_space *mapping; /* If low bit clear, points to -+ * inode address_space, or NULL. -+ * If page mapped as anonymous -+ * memory, low bit is set, and -+ * it points to anon_vma object: -+ * see PAGE_MAPPING_ANON below. -+ */ -+ void *s_mem; /* slab first object */ -+ }; -+ -+ /* Second double word */ -+ struct { -+ union { -+ pgoff_t index; /* Our offset within mapping. */ -+ void *freelist; /* sl[aou]b first free object */ -+ }; -+ -+ union { -+#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \ -+ defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE) -+ /* Used for cmpxchg_double in slub */ -+ unsigned long counters; -+#else -+ /* -+ * Keep _count separate from slub cmpxchg_double data. -+ * As the rest of the double word is protected by -+ * slab_lock but _count is not. -+ */ -+ unsigned counters; -+#endif -+ -+ struct { -+ -+ union { -+ /* -+ * Count of ptes mapped in -+ * mms, to show when page is -+ * mapped & limit reverse map -+ * searches. -+ * -+ * Used also for tail pages -+ * refcounting instead of -+ * _count. Tail pages cannot -+ * be mapped and keeping the -+ * tail page _count zero at -+ * all times guarantees -+ * get_page_unless_zero() will -+ * never succeed on tail -+ * pages. -+ */ -+ atomic_t _mapcount; -+ -+ struct { /* SLUB */ -+ unsigned inuse:16; -+ unsigned objects:15; -+ unsigned frozen:1; -+ }; -+ int units; /* SLOB */ -+ }; -+ atomic_t _count; /* Usage count, see below. */ -+ }; -+ unsigned int active; /* SLAB */ -+ }; -+ }; ++int atomic_dec_and_mutex_lock(atomic_t *cnt, struct mutex *lock) ++{ ++ /* dec if we can't possibly hit 0 */ ++ if (atomic_add_unless(cnt, -1, 1)) ++ return 0; ++ /* we might hit 0, so take the lock */ ++ mutex_lock(lock); ++ if (!atomic_dec_and_test(cnt)) { ++ /* when we actually did the dec, we didn't hit 0 */ ++ mutex_unlock(lock); ++ return 0; ++ } ++ /* we hit 0, and we hold the lock */ ++ return 1; ++} ++EXPORT_SYMBOL(atomic_dec_and_mutex_lock); +diff -Nur linux-4.1.10.orig/kernel/locking/rtmutex.c linux-4.1.10/kernel/locking/rtmutex.c +--- linux-4.1.10.orig/kernel/locking/rtmutex.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/locking/rtmutex.c 2015-10-12 22:33:32.296675432 +0200 +@@ -7,6 +7,11 @@ + * Copyright (C) 2005-2006 Timesys Corp., Thomas Gleixner + * Copyright (C) 2005 Kihon Technologies Inc., Steven Rostedt + * Copyright (C) 2006 Esben Nielsen ++ * Adaptive Spinlocks: ++ * Copyright (C) 2008 Novell, Inc., Gregory Haskins, Sven Dietrich, ++ * and Peter Morreale, ++ * Adaptive Spinlocks simplification: ++ * Copyright (C) 2008 Red Hat, Inc., Steven Rostedt + * + * See Documentation/locking/rt-mutex-design.txt for details. + */ +@@ -16,6 +21,7 @@ + #include + #include + #include ++#include + + #include "rtmutex_common.h" + +@@ -69,6 +75,12 @@ + clear_rt_mutex_waiters(lock); + } + ++static int rt_mutex_real_waiter(struct rt_mutex_waiter *waiter) ++{ ++ return waiter && waiter != PI_WAKEUP_INPROGRESS && ++ waiter != PI_REQUEUE_INPROGRESS; ++} + -+ /* Third double word block */ -+ union { -+ struct list_head lru; /* Pageout list, eg. active_list -+ * protected by zone->lru_lock ! -+ * Can be used as a generic list -+ * by the page owner. -+ */ -+ struct { /* slub per cpu partial pages */ -+ struct page *next; /* Next partial slab */ -+#ifdef CONFIG_64BIT -+ int pages; /* Nr of partial slabs left */ -+ int pobjects; /* Approximate # of objects */ -+#else -+ short int pages; -+ short int pobjects; -+#endif -+ }; + /* + * We can speed up the acquire/release, if the architecture + * supports cmpxchg and if there's no debugging state to be set up +@@ -300,7 +312,7 @@ + * of task. We do not use the spin_xx_mutex() variants here as we are + * outside of the debug path.) + */ +-static void rt_mutex_adjust_prio(struct task_struct *task) ++void rt_mutex_adjust_prio(struct task_struct *task) + { + unsigned long flags; + +@@ -335,6 +347,14 @@ + return debug_rt_mutex_detect_deadlock(waiter, chwalk); + } + ++static void rt_mutex_wake_waiter(struct rt_mutex_waiter *waiter) ++{ ++ if (waiter->savestate) ++ wake_up_lock_sleeper(waiter->task); ++ else ++ wake_up_process(waiter->task); ++} + -+ struct slab *slab_page; /* slab fields */ -+ struct rcu_head rcu_head; /* Used by SLAB -+ * when destroying via RCU -+ */ -+ /* First tail page of compound page */ -+ struct { -+ compound_page_dtor *compound_dtor; -+ unsigned long compound_order; -+ }; + /* + * Max number of times we'll walk the boosting chain: + */ +@@ -342,7 +362,8 @@ + + static inline struct rt_mutex *task_blocked_on_lock(struct task_struct *p) + { +- return p->pi_blocked_on ? p->pi_blocked_on->lock : NULL; ++ return rt_mutex_real_waiter(p->pi_blocked_on) ? ++ p->pi_blocked_on->lock : NULL; + } + + /* +@@ -479,7 +500,7 @@ + * reached or the state of the chain has changed while we + * dropped the locks. + */ +- if (!waiter) ++ if (!rt_mutex_real_waiter(waiter)) + goto out_unlock_pi; + + /* +@@ -641,13 +662,16 @@ + * follow here. This is the end of the chain we are walking. + */ + if (!rt_mutex_owner(lock)) { ++ struct rt_mutex_waiter *lock_top_waiter; + -+#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && USE_SPLIT_PMD_PTLOCKS -+ pgtable_t pmd_huge_pte; /* protected by page->ptl */ -+#endif -+ }; + /* + * If the requeue [7] above changed the top waiter, + * then we need to wake the new top waiter up to try + * to get the lock. + */ +- if (prerequeue_top_waiter != rt_mutex_top_waiter(lock)) +- wake_up_process(rt_mutex_top_waiter(lock)->task); ++ lock_top_waiter = rt_mutex_top_waiter(lock); ++ if (prerequeue_top_waiter != lock_top_waiter) ++ rt_mutex_wake_waiter(lock_top_waiter); + raw_spin_unlock(&lock->wait_lock); + return 0; + } +@@ -740,6 +764,25 @@ + return ret; + } + + -+ /* Remainder is not double word aligned */ -+ union { -+ unsigned long private; /* Mapping-private opaque data: -+ * usually used for buffer_heads -+ * if PagePrivate set; used for -+ * swp_entry_t if PageSwapCache; -+ * indicates order in the buddy -+ * system if PG_buddy is set. -+ */ -+#if USE_SPLIT_PTE_PTLOCKS -+#if ALLOC_SPLIT_PTLOCKS -+ spinlock_t *ptl; -+#else -+ spinlock_t ptl; -+#endif -+#endif -+ struct kmem_cache *slab_cache; /* SL[AU]B: Pointer to slab */ -+ struct page *first_page; /* Compound tail pages */ -+ }; ++#define STEAL_NORMAL 0 ++#define STEAL_LATERAL 1 + -+#ifdef CONFIG_MEMCG -+ struct mem_cgroup *mem_cgroup; -+#endif ++/* ++ * Note that RT tasks are excluded from lateral-steals to prevent the ++ * introduction of an unbounded latency ++ */ ++static inline int lock_is_stealable(struct task_struct *task, ++ struct task_struct *pendowner, int mode) ++{ ++ if (mode == STEAL_NORMAL || rt_task(task)) { ++ if (task->prio >= pendowner->prio) ++ return 0; ++ } else if (task->prio > pendowner->prio) ++ return 0; ++ return 1; ++} + -+ /* -+ * On machines where all RAM is mapped into kernel address space, -+ * we can simply calculate the virtual address. On machines with -+ * highmem some memory is mapped into kernel virtual memory -+ * dynamically, so we need a place to store that address. -+ * Note that this field could be 16 bits on x86 ... ;) -+ * -+ * Architectures with slow multiplication can define -+ * WANT_PAGE_VIRTUAL in asm/page.h -+ */ -+#if defined(WANT_PAGE_VIRTUAL) -+ void *virtual; /* Kernel virtual address (NULL if -+ not kmapped, ie. highmem) */ -+#endif /* WANT_PAGE_VIRTUAL */ + /* + * Try to take an rt-mutex + * +@@ -750,8 +793,9 @@ + * @waiter: The waiter that is queued to the lock's wait list if the + * callsite called task_blocked_on_lock(), otherwise NULL + */ +-static int try_to_take_rt_mutex(struct rt_mutex *lock, struct task_struct *task, +- struct rt_mutex_waiter *waiter) ++static int __try_to_take_rt_mutex(struct rt_mutex *lock, ++ struct task_struct *task, ++ struct rt_mutex_waiter *waiter, int mode) + { + unsigned long flags; + +@@ -790,8 +834,10 @@ + * If waiter is not the highest priority waiter of + * @lock, give up. + */ +- if (waiter != rt_mutex_top_waiter(lock)) ++ if (waiter != rt_mutex_top_waiter(lock)) { ++ /* XXX lock_is_stealable() ? */ + return 0; ++ } + + /* + * We can acquire the lock. Remove the waiter from the +@@ -809,14 +855,10 @@ + * not need to be dequeued. + */ + if (rt_mutex_has_waiters(lock)) { +- /* +- * If @task->prio is greater than or equal to +- * the top waiter priority (kernel view), +- * @task lost. +- */ +- if (task->prio >= rt_mutex_top_waiter(lock)->prio) +- return 0; ++ struct task_struct *pown = rt_mutex_top_waiter(lock)->task; + ++ if (task != pown && !lock_is_stealable(task, pown, mode)) ++ return 0; + /* + * The current top waiter stays enqueued. We + * don't have to change anything in the lock +@@ -865,6 +907,347 @@ + return 1; + } + ++#ifdef CONFIG_PREEMPT_RT_FULL ++/* ++ * preemptible spin_lock functions: ++ */ ++static inline void rt_spin_lock_fastlock(struct rt_mutex *lock, ++ void (*slowfn)(struct rt_mutex *lock)) ++{ ++ might_sleep_no_state_check(); + -+#ifdef CONFIG_KMEMCHECK -+ /* -+ * kmemcheck wants to track the status of each byte in a page; this -+ * is a pointer to such a status block. NULL if not tracked. -+ */ -+ void *shadow; -+#endif ++ if (likely(rt_mutex_cmpxchg(lock, NULL, current))) ++ rt_mutex_deadlock_account_lock(lock, current); ++ else ++ slowfn(lock); ++} + -+#ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS -+ int _last_cpupid; -+#endif ++static inline void rt_spin_lock_fastunlock(struct rt_mutex *lock, ++ void (*slowfn)(struct rt_mutex *lock)) ++{ ++ if (likely(rt_mutex_cmpxchg(lock, current, NULL))) ++ rt_mutex_deadlock_account_unlock(current); ++ else ++ slowfn(lock); +} ++#ifdef CONFIG_SMP +/* -+ * The struct page can be forced to be double word aligned so that atomic ops -+ * on double words work. The SLUB allocator can make use of such a feature. ++ * Note that owner is a speculative pointer and dereferencing relies ++ * on rcu_read_lock() and the check against the lock owner. + */ -+#ifdef CONFIG_HAVE_ALIGNED_STRUCT_PAGE -+ __aligned(2 * sizeof(unsigned long)) -+#endif -+; ++static int adaptive_wait(struct rt_mutex *lock, ++ struct task_struct *owner) ++{ ++ int res = 0; + -+struct page_frag { -+ struct page *page; -+#if (BITS_PER_LONG > 32) || (PAGE_SIZE >= 65536) -+ __u32 offset; -+ __u32 size; ++ rcu_read_lock(); ++ for (;;) { ++ if (owner != rt_mutex_owner(lock)) ++ break; ++ /* ++ * Ensure that owner->on_cpu is dereferenced _after_ ++ * checking the above to be valid. ++ */ ++ barrier(); ++ if (!owner->on_cpu) { ++ res = 1; ++ break; ++ } ++ cpu_relax(); ++ } ++ rcu_read_unlock(); ++ return res; ++} +#else -+ __u16 offset; -+ __u16 size; ++static int adaptive_wait(struct rt_mutex *lock, ++ struct task_struct *orig_owner) ++{ ++ return 1; ++} +#endif -+}; + -+typedef unsigned long __nocast vm_flags_t; -+ -+/* -+ * A region containing a mapping of a non-memory backed file under NOMMU -+ * conditions. These are held in a global tree and are pinned by the VMAs that -+ * map parts of them. -+ */ -+struct vm_region { -+ struct rb_node vm_rb; /* link in global region tree */ -+ vm_flags_t vm_flags; /* VMA vm_flags */ -+ unsigned long vm_start; /* start address of region */ -+ unsigned long vm_end; /* region initialised to here */ -+ unsigned long vm_top; /* region allocated to here */ -+ unsigned long vm_pgoff; /* the offset in vm_file corresponding to vm_start */ -+ struct file *vm_file; /* the backing file or NULL */ -+ -+ int vm_usage; /* region usage count (access under nommu_region_sem) */ -+ bool vm_icache_flushed : 1; /* true if the icache has been flushed for -+ * this region */ -+}; ++# define pi_lock(lock) raw_spin_lock_irq(lock) ++# define pi_unlock(lock) raw_spin_unlock_irq(lock) + ++static int task_blocks_on_rt_mutex(struct rt_mutex *lock, ++ struct rt_mutex_waiter *waiter, ++ struct task_struct *task, ++ enum rtmutex_chainwalk chwalk); +/* -+ * This struct defines a memory VMM memory area. There is one of these -+ * per VM-area/task. A VM area is any part of the process virtual memory -+ * space that has a special rule for the page-fault handlers (ie a shared -+ * library, the executable area etc). ++ * Slow path lock function spin_lock style: this variant is very ++ * careful not to miss any non-lock wakeups. ++ * ++ * We store the current state under p->pi_lock in p->saved_state and ++ * the try_to_wake_up() code handles this accordingly. + */ -+struct vm_area_struct { -+ /* The first cache line has the info for VMA tree walking. */ -+ -+ unsigned long vm_start; /* Our start address within vm_mm. */ -+ unsigned long vm_end; /* The first byte after our end address -+ within vm_mm. */ -+ -+ /* linked list of VM areas per task, sorted by address */ -+ struct vm_area_struct *vm_next, *vm_prev; -+ -+ struct rb_node vm_rb; ++static void noinline __sched rt_spin_lock_slowlock(struct rt_mutex *lock) ++{ ++ struct task_struct *lock_owner, *self = current; ++ struct rt_mutex_waiter waiter, *top_waiter; ++ int ret; + -+ /* -+ * Largest free memory gap in bytes to the left of this VMA. -+ * Either between this VMA and vma->vm_prev, or between one of the -+ * VMAs below us in the VMA rbtree and its ->vm_prev. This helps -+ * get_unmapped_area find a free area of the right size. -+ */ -+ unsigned long rb_subtree_gap; ++ rt_mutex_init_waiter(&waiter, true); + -+ /* Second cache line starts here. */ ++ raw_spin_lock(&lock->wait_lock); + -+ struct mm_struct *vm_mm; /* The address space we belong to. */ -+ pgprot_t vm_page_prot; /* Access permissions of this VMA. */ -+ unsigned long vm_flags; /* Flags, see mm.h. */ ++ if (__try_to_take_rt_mutex(lock, self, NULL, STEAL_LATERAL)) { ++ raw_spin_unlock(&lock->wait_lock); ++ return; ++ } + -+ /* -+ * For areas with an address space and backing store, -+ * linkage into the address_space->i_mmap interval tree. -+ */ -+ struct { -+ struct rb_node rb; -+ unsigned long rb_subtree_last; -+ } shared; ++ BUG_ON(rt_mutex_owner(lock) == self); + + /* -+ * A file's MAP_PRIVATE vma can be in both i_mmap tree and anon_vma -+ * list, after a COW of one of the file pages. A MAP_SHARED vma -+ * can only be in the i_mmap tree. An anonymous MAP_PRIVATE, stack -+ * or brk vma (with NULL file) can only be in an anon_vma list. ++ * We save whatever state the task is in and we'll restore it ++ * after acquiring the lock taking real wakeups into account ++ * as well. We are serialized via pi_lock against wakeups. See ++ * try_to_wake_up(). + */ -+ struct list_head anon_vma_chain; /* Serialized by mmap_sem & -+ * page_table_lock */ -+ struct anon_vma *anon_vma; /* Serialized by page_table_lock */ -+ -+ /* Function pointers to deal with this struct. */ -+ const struct vm_operations_struct *vm_ops; ++ pi_lock(&self->pi_lock); ++ self->saved_state = self->state; ++ __set_current_state_no_track(TASK_UNINTERRUPTIBLE); ++ pi_unlock(&self->pi_lock); + -+ /* Information about our backing store: */ -+ unsigned long vm_pgoff; /* Offset (within vm_file) in PAGE_SIZE -+ units, *not* PAGE_CACHE_SIZE */ -+ struct file * vm_file; /* File we map to (can be NULL). */ -+ void * vm_private_data; /* was vm_pte (shared mem) */ ++ ret = task_blocks_on_rt_mutex(lock, &waiter, self, 0); ++ BUG_ON(ret); + -+#ifndef CONFIG_MMU -+ struct vm_region *vm_region; /* NOMMU mapping region */ -+#endif -+#ifdef CONFIG_NUMA -+ struct mempolicy *vm_policy; /* NUMA policy for the VMA */ -+#endif -+}; ++ for (;;) { ++ /* Try to acquire the lock again. */ ++ if (__try_to_take_rt_mutex(lock, self, &waiter, STEAL_LATERAL)) ++ break; + -+struct core_thread { -+ struct task_struct *task; -+ struct core_thread *next; -+}; ++ top_waiter = rt_mutex_top_waiter(lock); ++ lock_owner = rt_mutex_owner(lock); + -+struct core_state { -+ atomic_t nr_threads; -+ struct core_thread dumper; -+ struct completion startup; -+}; ++ raw_spin_unlock(&lock->wait_lock); + -+enum { -+ MM_FILEPAGES, -+ MM_ANONPAGES, -+ MM_SWAPENTS, -+ NR_MM_COUNTERS -+}; ++ debug_rt_mutex_print_deadlock(&waiter); + -+#if USE_SPLIT_PTE_PTLOCKS && defined(CONFIG_MMU) -+#define SPLIT_RSS_COUNTING -+/* per-thread cached information, */ -+struct task_rss_stat { -+ int events; /* for synchronization threshold */ -+ int count[NR_MM_COUNTERS]; -+}; -+#endif /* USE_SPLIT_PTE_PTLOCKS */ ++ if (top_waiter != &waiter || adaptive_wait(lock, lock_owner)) ++ schedule_rt_mutex(lock); + -+struct mm_rss_stat { -+ atomic_long_t count[NR_MM_COUNTERS]; -+}; ++ raw_spin_lock(&lock->wait_lock); + -+struct kioctx_table; -+struct mm_struct { -+ struct vm_area_struct *mmap; /* list of VMAs */ -+ struct rb_root mm_rb; -+ u32 vmacache_seqnum; /* per-thread vmacache */ -+#ifdef CONFIG_MMU -+ unsigned long (*get_unmapped_area) (struct file *filp, -+ unsigned long addr, unsigned long len, -+ unsigned long pgoff, unsigned long flags); -+#endif -+ unsigned long mmap_base; /* base of mmap area */ -+ unsigned long mmap_legacy_base; /* base of mmap area in bottom-up allocations */ -+ unsigned long task_size; /* size of task vm space */ -+ unsigned long highest_vm_end; /* highest vma end address */ -+ pgd_t * pgd; -+ atomic_t mm_users; /* How many users with user space? */ -+ atomic_t mm_count; /* How many references to "struct mm_struct" (users count as 1) */ -+ atomic_long_t nr_ptes; /* PTE page table pages */ -+#if CONFIG_PGTABLE_LEVELS > 2 -+ atomic_long_t nr_pmds; /* PMD page table pages */ -+#endif -+ int map_count; /* number of VMAs */ ++ pi_lock(&self->pi_lock); ++ __set_current_state_no_track(TASK_UNINTERRUPTIBLE); ++ pi_unlock(&self->pi_lock); ++ } + -+ spinlock_t page_table_lock; /* Protects page tables and some counters */ -+ struct rw_semaphore mmap_sem; ++ /* ++ * Restore the task state to current->saved_state. We set it ++ * to the original state above and the try_to_wake_up() code ++ * has possibly updated it when a real (non-rtmutex) wakeup ++ * happened while we were blocked. Clear saved_state so ++ * try_to_wakeup() does not get confused. ++ */ ++ pi_lock(&self->pi_lock); ++ __set_current_state_no_track(self->saved_state); ++ self->saved_state = TASK_RUNNING; ++ pi_unlock(&self->pi_lock); + -+ struct list_head mmlist; /* List of maybe swapped mm's. These are globally strung -+ * together off init_mm.mmlist, and are protected -+ * by mmlist_lock -+ */ ++ /* ++ * try_to_take_rt_mutex() sets the waiter bit ++ * unconditionally. We might have to fix that up: ++ */ ++ fixup_rt_mutex_waiters(lock); + ++ BUG_ON(rt_mutex_has_waiters(lock) && &waiter == rt_mutex_top_waiter(lock)); ++ BUG_ON(!RB_EMPTY_NODE(&waiter.tree_entry)); + -+ unsigned long hiwater_rss; /* High-watermark of RSS usage */ -+ unsigned long hiwater_vm; /* High-water virtual memory usage */ ++ raw_spin_unlock(&lock->wait_lock); + -+ unsigned long total_vm; /* Total pages mapped */ -+ unsigned long locked_vm; /* Pages that have PG_mlocked set */ -+ unsigned long pinned_vm; /* Refcount permanently increased */ -+ unsigned long shared_vm; /* Shared pages (files) */ -+ unsigned long exec_vm; /* VM_EXEC & ~VM_WRITE */ -+ unsigned long stack_vm; /* VM_GROWSUP/DOWN */ -+ unsigned long def_flags; -+ unsigned long start_code, end_code, start_data, end_data; -+ unsigned long start_brk, brk, start_stack; -+ unsigned long arg_start, arg_end, env_start, env_end; ++ debug_rt_mutex_free_waiter(&waiter); ++} + -+ unsigned long saved_auxv[AT_VECTOR_SIZE]; /* for /proc/PID/auxv */ ++static void wakeup_next_waiter(struct rt_mutex *lock); ++/* ++ * Slow path to release a rt_mutex spin_lock style ++ */ ++static void noinline __sched rt_spin_lock_slowunlock(struct rt_mutex *lock) ++{ ++ raw_spin_lock(&lock->wait_lock); + -+ /* -+ * Special counters, in some configurations protected by the -+ * page_table_lock, in other configurations by being atomic. -+ */ -+ struct mm_rss_stat rss_stat; ++ debug_rt_mutex_unlock(lock); + -+ struct linux_binfmt *binfmt; ++ rt_mutex_deadlock_account_unlock(current); + -+ cpumask_var_t cpu_vm_mask_var; ++ if (!rt_mutex_has_waiters(lock)) { ++ lock->owner = NULL; ++ raw_spin_unlock(&lock->wait_lock); ++ return; ++ } + -+ /* Architecture-specific MM context */ -+ mm_context_t context; ++ wakeup_next_waiter(lock); + -+ unsigned long flags; /* Must use atomic bitops to access the bits */ ++ raw_spin_unlock(&lock->wait_lock); + -+ struct core_state *core_state; /* coredumping support */ -+#ifdef CONFIG_AIO -+ spinlock_t ioctx_lock; -+ struct kioctx_table __rcu *ioctx_table; -+#endif -+#ifdef CONFIG_MEMCG -+ /* -+ * "owner" points to a task that is regarded as the canonical -+ * user/owner of this mm. All of the following must be true in -+ * order for it to be changed: -+ * -+ * current == mm->owner -+ * current->mm != mm -+ * new_owner->mm == mm -+ * new_owner->alloc_lock is held -+ */ -+ struct task_struct __rcu *owner; -+#endif ++ /* Undo pi boosting.when necessary */ ++ rt_mutex_adjust_prio(current); ++} + -+ /* store ref to file /proc//exe symlink points to */ -+ struct file __rcu *exe_file; -+#ifdef CONFIG_MMU_NOTIFIER -+ struct mmu_notifier_mm *mmu_notifier_mm; -+#endif -+#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS -+ pgtable_t pmd_huge_pte; /* protected by page_table_lock */ -+#endif -+#ifdef CONFIG_CPUMASK_OFFSTACK -+ struct cpumask cpumask_allocation; -+#endif -+#ifdef CONFIG_NUMA_BALANCING -+ /* -+ * numa_next_scan is the next time that the PTEs will be marked -+ * pte_numa. NUMA hinting faults will gather statistics and migrate -+ * pages to new nodes if necessary. -+ */ -+ unsigned long numa_next_scan; ++void __lockfunc rt_spin_lock(spinlock_t *lock) ++{ ++ rt_spin_lock_fastlock(&lock->lock, rt_spin_lock_slowlock); ++ spin_acquire(&lock->dep_map, 0, 0, _RET_IP_); ++} ++EXPORT_SYMBOL(rt_spin_lock); + -+ /* Restart point for scanning and setting pte_numa */ -+ unsigned long numa_scan_offset; ++void __lockfunc __rt_spin_lock(struct rt_mutex *lock) ++{ ++ rt_spin_lock_fastlock(lock, rt_spin_lock_slowlock); ++} ++EXPORT_SYMBOL(__rt_spin_lock); + -+ /* numa_scan_seq prevents two threads setting pte_numa */ -+ int numa_scan_seq; -+#endif -+#if defined(CONFIG_NUMA_BALANCING) || defined(CONFIG_COMPACTION) -+ /* -+ * An operation with batched TLB flushing is going on. Anything that -+ * can move process memory needs to flush the TLB when moving a -+ * PROT_NONE or PROT_NUMA mapped page. -+ */ -+ bool tlb_flush_pending; -+#endif -+ struct uprobes_state uprobes_state; -+#ifdef CONFIG_X86_INTEL_MPX -+ /* address of the bounds directory */ -+ void __user *bd_addr; ++#ifdef CONFIG_DEBUG_LOCK_ALLOC ++void __lockfunc rt_spin_lock_nested(spinlock_t *lock, int subclass) ++{ ++ rt_spin_lock_fastlock(&lock->lock, rt_spin_lock_slowlock); ++ spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_); ++} ++EXPORT_SYMBOL(rt_spin_lock_nested); +#endif -+}; + -+static inline void mm_init_cpumask(struct mm_struct *mm) ++void __lockfunc rt_spin_unlock(spinlock_t *lock) +{ -+#ifdef CONFIG_CPUMASK_OFFSTACK -+ mm->cpu_vm_mask_var = &mm->cpumask_allocation; -+#endif -+ cpumask_clear(mm->cpu_vm_mask_var); ++ /* NOTE: we always pass in '1' for nested, for simplicity */ ++ spin_release(&lock->dep_map, 1, _RET_IP_); ++ rt_spin_lock_fastunlock(&lock->lock, rt_spin_lock_slowunlock); +} ++EXPORT_SYMBOL(rt_spin_unlock); + -+/* Future-safe accessor for struct mm_struct's cpu_vm_mask. */ -+static inline cpumask_t *mm_cpumask(struct mm_struct *mm) ++void __lockfunc __rt_spin_unlock(struct rt_mutex *lock) +{ -+ return mm->cpu_vm_mask_var; ++ rt_spin_lock_fastunlock(lock, rt_spin_lock_slowunlock); +} ++EXPORT_SYMBOL(__rt_spin_unlock); + -+#if defined(CONFIG_NUMA_BALANCING) || defined(CONFIG_COMPACTION) +/* -+ * Memory barriers to keep this state in sync are graciously provided by -+ * the page table locks, outside of which no page table modifications happen. -+ * The barriers below prevent the compiler from re-ordering the instructions -+ * around the memory barriers that are already present in the code. ++ * Wait for the lock to get unlocked: instead of polling for an unlock ++ * (like raw spinlocks do), we lock and unlock, to force the kernel to ++ * schedule if there's contention: + */ -+static inline bool mm_tlb_flush_pending(struct mm_struct *mm) ++void __lockfunc rt_spin_unlock_wait(spinlock_t *lock) +{ -+ barrier(); -+ return mm->tlb_flush_pending; ++ spin_lock(lock); ++ spin_unlock(lock); +} -+static inline void set_tlb_flush_pending(struct mm_struct *mm) -+{ -+ mm->tlb_flush_pending = true; ++EXPORT_SYMBOL(rt_spin_unlock_wait); + -+ /* -+ * Guarantee that the tlb_flush_pending store does not leak into the -+ * critical section updating the page tables -+ */ -+ smp_mb__before_spinlock(); -+} -+/* Clearing is done after a TLB flush, which also provides a barrier. */ -+static inline void clear_tlb_flush_pending(struct mm_struct *mm) -+{ -+ barrier(); -+ mm->tlb_flush_pending = false; -+} -+#else -+static inline bool mm_tlb_flush_pending(struct mm_struct *mm) -+{ -+ return false; -+} -+static inline void set_tlb_flush_pending(struct mm_struct *mm) ++int __lockfunc __rt_spin_trylock(struct rt_mutex *lock) +{ ++ return rt_mutex_trylock(lock); +} -+static inline void clear_tlb_flush_pending(struct mm_struct *mm) ++ ++int __lockfunc rt_spin_trylock(spinlock_t *lock) +{ ++ int ret = rt_mutex_trylock(&lock->lock); ++ ++ if (ret) ++ spin_acquire(&lock->dep_map, 0, 1, _RET_IP_); ++ return ret; +} -+#endif ++EXPORT_SYMBOL(rt_spin_trylock); + -+struct vm_special_mapping ++int __lockfunc rt_spin_trylock_bh(spinlock_t *lock) +{ -+ const char *name; -+ struct page **pages; -+}; ++ int ret; + -+enum tlb_flush_reason { -+ TLB_FLUSH_ON_TASK_SWITCH, -+ TLB_REMOTE_SHOOTDOWN, -+ TLB_LOCAL_SHOOTDOWN, -+ TLB_LOCAL_MM_SHOOTDOWN, -+ NR_TLB_FLUSH_REASONS, -+}; ++ local_bh_disable(); ++ ret = rt_mutex_trylock(&lock->lock); ++ if (ret) { ++ migrate_disable(); ++ spin_acquire(&lock->dep_map, 0, 1, _RET_IP_); ++ } else ++ local_bh_enable(); ++ return ret; ++} ++EXPORT_SYMBOL(rt_spin_trylock_bh); + -+ /* -+ * A swap entry has to fit into a "unsigned long", as the entry is hidden -+ * in the "index" field of the swapper address space. -+ */ -+typedef struct { -+ unsigned long val; -+} swp_entry_t; ++int __lockfunc rt_spin_trylock_irqsave(spinlock_t *lock, unsigned long *flags) ++{ ++ int ret; + -+#endif /* _LINUX_MM_TYPES_H */ -diff -Nur linux-4.1.10.orig/include/linux/mutex.h linux-4.1.10/include/linux/mutex.h ---- linux-4.1.10.orig/include/linux/mutex.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/mutex.h 2015-10-07 18:00:08.000000000 +0200 -@@ -19,6 +19,17 @@ - #include - #include - -+#ifdef CONFIG_DEBUG_LOCK_ALLOC -+# define __DEP_MAP_MUTEX_INITIALIZER(lockname) \ -+ , .dep_map = { .name = #lockname } -+#else -+# define __DEP_MAP_MUTEX_INITIALIZER(lockname) -+#endif ++ *flags = 0; ++ ret = rt_mutex_trylock(&lock->lock); ++ if (ret) { ++ migrate_disable(); ++ spin_acquire(&lock->dep_map, 0, 1, _RET_IP_); ++ } ++ return ret; ++} ++EXPORT_SYMBOL(rt_spin_trylock_irqsave); + -+#ifdef CONFIG_PREEMPT_RT_FULL -+# include -+#else ++int atomic_dec_and_spin_lock(atomic_t *atomic, spinlock_t *lock) ++{ ++ /* Subtract 1 from counter unless that drops it to 0 (ie. it was 1) */ ++ if (atomic_add_unless(atomic, -1, 1)) ++ return 0; ++ migrate_disable(); ++ rt_spin_lock(lock); ++ if (atomic_dec_and_test(atomic)) ++ return 1; ++ rt_spin_unlock(lock); ++ migrate_enable(); ++ return 0; ++} ++EXPORT_SYMBOL(atomic_dec_and_spin_lock); + - /* - * Simple, straightforward mutexes with strict semantics: - * -@@ -99,13 +110,6 @@ - static inline void mutex_destroy(struct mutex *lock) {} - #endif - --#ifdef CONFIG_DEBUG_LOCK_ALLOC --# define __DEP_MAP_MUTEX_INITIALIZER(lockname) \ -- , .dep_map = { .name = #lockname } --#else --# define __DEP_MAP_MUTEX_INITIALIZER(lockname) --#endif -- - #define __MUTEX_INITIALIZER(lockname) \ - { .count = ATOMIC_INIT(1) \ - , .wait_lock = __SPIN_LOCK_UNLOCKED(lockname.wait_lock) \ -@@ -173,6 +177,8 @@ - extern int mutex_trylock(struct mutex *lock); - extern void mutex_unlock(struct mutex *lock); - -+#endif /* !PREEMPT_RT_FULL */ ++ void ++__rt_spin_lock_init(spinlock_t *lock, char *name, struct lock_class_key *key) ++{ ++#ifdef CONFIG_DEBUG_LOCK_ALLOC ++ /* ++ * Make sure we are not reinitializing a held lock: ++ */ ++ debug_check_no_locks_freed((void *)lock, sizeof(*lock)); ++ lockdep_init_map(&lock->dep_map, name, key, 0); ++#endif ++} ++EXPORT_SYMBOL(__rt_spin_lock_init); + - extern int atomic_dec_and_mutex_lock(atomic_t *cnt, struct mutex *lock); - - #endif /* __LINUX_MUTEX_H */ -diff -Nur linux-4.1.10.orig/include/linux/mutex_rt.h linux-4.1.10/include/linux/mutex_rt.h ---- linux-4.1.10.orig/include/linux/mutex_rt.h 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.10/include/linux/mutex_rt.h 2015-10-07 18:00:08.000000000 +0200 -@@ -0,0 +1,84 @@ -+#ifndef __LINUX_MUTEX_RT_H -+#define __LINUX_MUTEX_RT_H ++#endif /* PREEMPT_RT_FULL */ + -+#ifndef __LINUX_MUTEX_H -+#error "Please include mutex.h" -+#endif ++#ifdef CONFIG_PREEMPT_RT_FULL ++ static inline int __sched ++__mutex_lock_check_stamp(struct rt_mutex *lock, struct ww_acquire_ctx *ctx) ++{ ++ struct ww_mutex *ww = container_of(lock, struct ww_mutex, base.lock); ++ struct ww_acquire_ctx *hold_ctx = ACCESS_ONCE(ww->ctx); + -+#include ++ if (!hold_ctx) ++ return 0; + -+/* FIXME: Just for __lockfunc */ -+#include ++ if (unlikely(ctx == hold_ctx)) ++ return -EALREADY; + -+struct mutex { -+ struct rt_mutex lock; -+#ifdef CONFIG_DEBUG_LOCK_ALLOC -+ struct lockdep_map dep_map; ++ if (ctx->stamp - hold_ctx->stamp <= LONG_MAX && ++ (ctx->stamp != hold_ctx->stamp || ctx > hold_ctx)) { ++#ifdef CONFIG_DEBUG_MUTEXES ++ DEBUG_LOCKS_WARN_ON(ctx->contending_lock); ++ ctx->contending_lock = ww; +#endif -+}; -+ -+#define __MUTEX_INITIALIZER(mutexname) \ -+ { \ -+ .lock = __RT_MUTEX_INITIALIZER(mutexname.lock) \ -+ __DEP_MAP_MUTEX_INITIALIZER(mutexname) \ ++ return -EDEADLK; + } + -+#define DEFINE_MUTEX(mutexname) \ -+ struct mutex mutexname = __MUTEX_INITIALIZER(mutexname) -+ -+extern void __mutex_do_init(struct mutex *lock, const char *name, struct lock_class_key *key); -+extern void __lockfunc _mutex_lock(struct mutex *lock); -+extern int __lockfunc _mutex_lock_interruptible(struct mutex *lock); -+extern int __lockfunc _mutex_lock_killable(struct mutex *lock); -+extern void __lockfunc _mutex_lock_nested(struct mutex *lock, int subclass); -+extern void __lockfunc _mutex_lock_nest_lock(struct mutex *lock, struct lockdep_map *nest_lock); -+extern int __lockfunc _mutex_lock_interruptible_nested(struct mutex *lock, int subclass); -+extern int __lockfunc _mutex_lock_killable_nested(struct mutex *lock, int subclass); -+extern int __lockfunc _mutex_trylock(struct mutex *lock); -+extern void __lockfunc _mutex_unlock(struct mutex *lock); -+ -+#define mutex_is_locked(l) rt_mutex_is_locked(&(l)->lock) -+#define mutex_lock(l) _mutex_lock(l) -+#define mutex_lock_interruptible(l) _mutex_lock_interruptible(l) -+#define mutex_lock_killable(l) _mutex_lock_killable(l) -+#define mutex_trylock(l) _mutex_trylock(l) -+#define mutex_unlock(l) _mutex_unlock(l) -+#define mutex_destroy(l) rt_mutex_destroy(&(l)->lock) ++ return 0; ++} ++#else ++ static inline int __sched ++__mutex_lock_check_stamp(struct rt_mutex *lock, struct ww_acquire_ctx *ctx) ++{ ++ BUG(); ++ return 0; ++} + -+#ifdef CONFIG_DEBUG_LOCK_ALLOC -+# define mutex_lock_nested(l, s) _mutex_lock_nested(l, s) -+# define mutex_lock_interruptible_nested(l, s) \ -+ _mutex_lock_interruptible_nested(l, s) -+# define mutex_lock_killable_nested(l, s) \ -+ _mutex_lock_killable_nested(l, s) ++#endif + -+# define mutex_lock_nest_lock(lock, nest_lock) \ -+do { \ -+ typecheck(struct lockdep_map *, &(nest_lock)->dep_map); \ -+ _mutex_lock_nest_lock(lock, &(nest_lock)->dep_map); \ -+} while (0) ++static inline int ++try_to_take_rt_mutex(struct rt_mutex *lock, struct task_struct *task, ++ struct rt_mutex_waiter *waiter) ++{ ++ return __try_to_take_rt_mutex(lock, task, waiter, STEAL_NORMAL); ++} + -+#else -+# define mutex_lock_nested(l, s) _mutex_lock(l) -+# define mutex_lock_interruptible_nested(l, s) \ -+ _mutex_lock_interruptible(l) -+# define mutex_lock_killable_nested(l, s) \ -+ _mutex_lock_killable(l) -+# define mutex_lock_nest_lock(lock, nest_lock) mutex_lock(lock) -+#endif + /* + * Task blocks on lock. + * +@@ -896,6 +1279,23 @@ + return -EDEADLK; + + raw_spin_lock_irqsave(&task->pi_lock, flags); + -+# define mutex_init(mutex) \ -+do { \ -+ static struct lock_class_key __key; \ -+ \ -+ rt_mutex_init(&(mutex)->lock); \ -+ __mutex_do_init((mutex), #mutex, &__key); \ -+} while (0) ++ /* ++ * In the case of futex requeue PI, this will be a proxy ++ * lock. The task will wake unaware that it is enqueueed on ++ * this lock. Avoid blocking on two locks and corrupting ++ * pi_blocked_on via the PI_WAKEUP_INPROGRESS ++ * flag. futex_wait_requeue_pi() sets this when it wakes up ++ * before requeue (due to a signal or timeout). Do not enqueue ++ * the task if PI_WAKEUP_INPROGRESS is set. ++ */ ++ if (task != current && task->pi_blocked_on == PI_WAKEUP_INPROGRESS) { ++ raw_spin_unlock_irqrestore(&task->pi_lock, flags); ++ return -EAGAIN; ++ } + -+# define __mutex_init(mutex, name, key) \ -+do { \ -+ rt_mutex_init(&(mutex)->lock); \ -+ __mutex_do_init((mutex), name, key); \ -+} while (0) ++ BUG_ON(rt_mutex_real_waiter(task->pi_blocked_on)); + -+#endif -diff -Nur linux-4.1.10.orig/include/linux/netdevice.h linux-4.1.10/include/linux/netdevice.h ---- linux-4.1.10.orig/include/linux/netdevice.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/netdevice.h 2015-10-07 18:00:08.000000000 +0200 -@@ -2469,6 +2469,7 @@ - unsigned int dropped; - struct sk_buff_head input_pkt_queue; - struct napi_struct backlog; -+ struct sk_buff_head tofree_queue; + __rt_mutex_adjust_prio(task); + waiter->task = task; + waiter->lock = lock; +@@ -919,7 +1319,7 @@ + rt_mutex_enqueue_pi(owner, waiter); - }; + __rt_mutex_adjust_prio(owner); +- if (owner->pi_blocked_on) ++ if (rt_mutex_real_waiter(owner->pi_blocked_on)) + chain_walk = 1; + } else if (rt_mutex_cond_detect_deadlock(waiter, chwalk)) { + chain_walk = 1; +@@ -957,8 +1357,9 @@ + /* + * Wake up the next waiter on the lock. + * +- * Remove the top waiter from the current tasks pi waiter list and +- * wake it up. ++ * Remove the top waiter from the current tasks pi waiter list, ++ * wake it up and return whether the current task needs to undo ++ * a potential priority boosting. + * + * Called with lock->wait_lock held. + */ +@@ -996,7 +1397,7 @@ + * long as we hold lock->wait_lock. The waiter task needs to + * acquire it in order to dequeue the waiter. + */ +- wake_up_process(waiter->task); ++ rt_mutex_wake_waiter(waiter); + } -diff -Nur linux-4.1.10.orig/include/linux/netfilter/x_tables.h linux-4.1.10/include/linux/netfilter/x_tables.h ---- linux-4.1.10.orig/include/linux/netfilter/x_tables.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/netfilter/x_tables.h 2015-10-07 18:00:08.000000000 +0200 -@@ -3,6 +3,7 @@ + /* +@@ -1010,7 +1411,7 @@ + { + bool is_top_waiter = (waiter == rt_mutex_top_waiter(lock)); + struct task_struct *owner = rt_mutex_owner(lock); +- struct rt_mutex *next_lock; ++ struct rt_mutex *next_lock = NULL; + unsigned long flags; + raw_spin_lock_irqsave(¤t->pi_lock, flags); +@@ -1035,7 +1436,8 @@ + __rt_mutex_adjust_prio(owner); - #include -+#include - #include + /* Store the lock on which owner is blocked or NULL */ +- next_lock = task_blocked_on_lock(owner); ++ if (rt_mutex_real_waiter(owner->pi_blocked_on)) ++ next_lock = task_blocked_on_lock(owner); - /** -@@ -282,6 +283,8 @@ - */ - DECLARE_PER_CPU(seqcount_t, xt_recseq); + raw_spin_unlock_irqrestore(&owner->pi_lock, flags); -+DECLARE_LOCAL_IRQ_LOCK(xt_write_lock); -+ - /** - * xt_write_recseq_begin - start of a write section - * -@@ -296,6 +299,9 @@ - { - unsigned int addend; +@@ -1071,17 +1473,17 @@ + raw_spin_lock_irqsave(&task->pi_lock, flags); -+ /* RT protection */ -+ local_lock(xt_write_lock); -+ - /* - * Low order bit of sequence is set if we already - * called xt_write_recseq_begin(). -@@ -326,6 +332,7 @@ - /* this is kind of a write_seqcount_end(), but addend is 0 or 1 */ - smp_wmb(); - __this_cpu_add(xt_recseq.sequence, addend); -+ local_unlock(xt_write_lock); - } + waiter = task->pi_blocked_on; +- if (!waiter || (waiter->prio == task->prio && ++ if (!rt_mutex_real_waiter(waiter) || (waiter->prio == task->prio && + !dl_prio(task->prio))) { + raw_spin_unlock_irqrestore(&task->pi_lock, flags); + return; + } + next_lock = waiter->lock; +- raw_spin_unlock_irqrestore(&task->pi_lock, flags); - /* -diff -Nur linux-4.1.10.orig/include/linux/notifier.h linux-4.1.10/include/linux/notifier.h ---- linux-4.1.10.orig/include/linux/notifier.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/notifier.h 2015-10-07 18:00:08.000000000 +0200 -@@ -6,7 +6,7 @@ - * - * Alan Cox - */ -- -+ - #ifndef _LINUX_NOTIFIER_H - #define _LINUX_NOTIFIER_H - #include -@@ -42,9 +42,7 @@ - * in srcu_notifier_call_chain(): no cache bounces and no memory barriers. - * As compensation, srcu_notifier_chain_unregister() is rather expensive. - * SRCU notifier chains should be used when the chain will be called very -- * often but notifier_blocks will seldom be removed. Also, SRCU notifier -- * chains are slightly more difficult to use because they require special -- * runtime initialization. -+ * often but notifier_blocks will seldom be removed. - */ + /* gets dropped in rt_mutex_adjust_prio_chain()! */ + get_task_struct(task); - typedef int (*notifier_fn_t)(struct notifier_block *nb, -@@ -88,7 +86,7 @@ - (name)->head = NULL; \ - } while (0) ++ raw_spin_unlock_irqrestore(&task->pi_lock, flags); + rt_mutex_adjust_prio_chain(task, RT_MUTEX_MIN_CHAINWALK, NULL, + next_lock, NULL, task); + } +@@ -1099,7 +1501,8 @@ + static int __sched + __rt_mutex_slowlock(struct rt_mutex *lock, int state, + struct hrtimer_sleeper *timeout, +- struct rt_mutex_waiter *waiter) ++ struct rt_mutex_waiter *waiter, ++ struct ww_acquire_ctx *ww_ctx) + { + int ret = 0; --/* srcu_notifier_heads must be initialized and cleaned up dynamically */ -+/* srcu_notifier_heads must be cleaned up dynamically */ - extern void srcu_init_notifier_head(struct srcu_notifier_head *nh); - #define srcu_cleanup_notifier_head(name) \ - cleanup_srcu_struct(&(name)->srcu); -@@ -101,7 +99,13 @@ - .head = NULL } - #define RAW_NOTIFIER_INIT(name) { \ - .head = NULL } --/* srcu_notifier_heads cannot be initialized statically */ +@@ -1122,6 +1525,12 @@ + break; + } + ++ if (ww_ctx && ww_ctx->acquired > 0) { ++ ret = __mutex_lock_check_stamp(lock, ww_ctx); ++ if (ret) ++ break; ++ } + -+#define SRCU_NOTIFIER_INIT(name, pcpu) \ -+ { \ -+ .mutex = __MUTEX_INITIALIZER(name.mutex), \ -+ .head = NULL, \ -+ .srcu = __SRCU_STRUCT_INIT(name.srcu, pcpu), \ -+ } + raw_spin_unlock(&lock->wait_lock); - #define ATOMIC_NOTIFIER_HEAD(name) \ - struct atomic_notifier_head name = \ -@@ -113,6 +117,18 @@ - struct raw_notifier_head name = \ - RAW_NOTIFIER_INIT(name) + debug_rt_mutex_print_deadlock(waiter); +@@ -1156,25 +1565,102 @@ + } + } -+#define _SRCU_NOTIFIER_HEAD(name, mod) \ -+ static DEFINE_PER_CPU(struct srcu_struct_array, \ -+ name##_head_srcu_array); \ -+ mod struct srcu_notifier_head name = \ -+ SRCU_NOTIFIER_INIT(name, name##_head_srcu_array) ++static __always_inline void ww_mutex_lock_acquired(struct ww_mutex *ww, ++ struct ww_acquire_ctx *ww_ctx) ++{ ++#ifdef CONFIG_DEBUG_MUTEXES ++ /* ++ * If this WARN_ON triggers, you used ww_mutex_lock to acquire, ++ * but released with a normal mutex_unlock in this call. ++ * ++ * This should never happen, always use ww_mutex_unlock. ++ */ ++ DEBUG_LOCKS_WARN_ON(ww->ctx); + -+#define SRCU_NOTIFIER_HEAD(name) \ -+ _SRCU_NOTIFIER_HEAD(name, ) ++ /* ++ * Not quite done after calling ww_acquire_done() ? ++ */ ++ DEBUG_LOCKS_WARN_ON(ww_ctx->done_acquire); + -+#define SRCU_NOTIFIER_HEAD_STATIC(name) \ -+ _SRCU_NOTIFIER_HEAD(name, static) ++ if (ww_ctx->contending_lock) { ++ /* ++ * After -EDEADLK you tried to ++ * acquire a different ww_mutex? Bad! ++ */ ++ DEBUG_LOCKS_WARN_ON(ww_ctx->contending_lock != ww); + - #ifdef __KERNEL__ - - extern int atomic_notifier_chain_register(struct atomic_notifier_head *nh, -@@ -182,12 +198,12 @@ - - /* - * Declared notifiers so far. I can imagine quite a few more chains -- * over time (eg laptop power reset chains, reboot chain (to clean -+ * over time (eg laptop power reset chains, reboot chain (to clean - * device units up), device [un]mount chain, module load/unload chain, -- * low memory chain, screenblank chain (for plug in modular screenblankers) -+ * low memory chain, screenblank chain (for plug in modular screenblankers) - * VC switch chains (for loadable kernel svgalib VC switch helpers) etc... - */ -- ++ /* ++ * You called ww_mutex_lock after receiving -EDEADLK, ++ * but 'forgot' to unlock everything else first? ++ */ ++ DEBUG_LOCKS_WARN_ON(ww_ctx->acquired > 0); ++ ww_ctx->contending_lock = NULL; ++ } + - /* CPU notfiers are defined in include/linux/cpu.h. */ - - /* netdevice notifiers are defined in include/linux/netdevice.h */ -diff -Nur linux-4.1.10.orig/include/linux/percpu.h linux-4.1.10/include/linux/percpu.h ---- linux-4.1.10.orig/include/linux/percpu.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/percpu.h 2015-10-07 18:00:08.000000000 +0200 -@@ -24,6 +24,35 @@ - PERCPU_MODULE_RESERVE) - #endif - -+#ifdef CONFIG_PREEMPT_RT_FULL ++ /* ++ * Naughty, using a different class will lead to undefined behavior! ++ */ ++ DEBUG_LOCKS_WARN_ON(ww_ctx->ww_class != ww->ww_class); ++#endif ++ ww_ctx->acquired++; ++} + -+#define get_local_var(var) (*({ \ -+ migrate_disable(); \ -+ this_cpu_ptr(&var); })) ++#ifdef CONFIG_PREEMPT_RT_FULL ++static void ww_mutex_account_lock(struct rt_mutex *lock, ++ struct ww_acquire_ctx *ww_ctx) ++{ ++ struct ww_mutex *ww = container_of(lock, struct ww_mutex, base.lock); ++ struct rt_mutex_waiter *waiter, *n; + -+#define put_local_var(var) do { \ -+ (void)&(var); \ -+ migrate_enable(); \ -+} while (0) ++ /* ++ * This branch gets optimized out for the common case, ++ * and is only important for ww_mutex_lock. ++ */ ++ ww_mutex_lock_acquired(ww, ww_ctx); ++ ww->ctx = ww_ctx; + -+# define get_local_ptr(var) ({ \ -+ migrate_disable(); \ -+ this_cpu_ptr(var); }) ++ /* ++ * Give any possible sleeping processes the chance to wake up, ++ * so they can recheck if they have to back off. ++ */ ++ rbtree_postorder_for_each_entry_safe(waiter, n, &lock->waiters, ++ tree_entry) { ++ /* XXX debug rt mutex waiter wakeup */ + -+# define put_local_ptr(var) do { \ -+ (void)(var); \ -+ migrate_enable(); \ -+} while (0) ++ BUG_ON(waiter->lock != lock); ++ rt_mutex_wake_waiter(waiter); ++ } ++} + +#else + -+#define get_local_var(var) get_cpu_var(var) -+#define put_local_var(var) put_cpu_var(var) -+#define get_local_ptr(var) get_cpu_ptr(var) -+#define put_local_ptr(var) put_cpu_ptr(var) -+ ++static void ww_mutex_account_lock(struct rt_mutex *lock, ++ struct ww_acquire_ctx *ww_ctx) ++{ ++ BUG(); ++} +#endif + - /* minimum unit size, also is the maximum supported allocation size */ - #define PCPU_MIN_UNIT_SIZE PFN_ALIGN(32 << 10) - -diff -Nur linux-4.1.10.orig/include/linux/pid.h linux-4.1.10/include/linux/pid.h ---- linux-4.1.10.orig/include/linux/pid.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/pid.h 2015-10-07 18:00:08.000000000 +0200 -@@ -2,6 +2,7 @@ - #define _LINUX_PID_H - - #include -+#include - - enum pid_type + /* + * Slow path lock function: + */ + static int __sched + rt_mutex_slowlock(struct rt_mutex *lock, int state, + struct hrtimer_sleeper *timeout, +- enum rtmutex_chainwalk chwalk) ++ enum rtmutex_chainwalk chwalk, ++ struct ww_acquire_ctx *ww_ctx) { -diff -Nur linux-4.1.10.orig/include/linux/preempt.h linux-4.1.10/include/linux/preempt.h ---- linux-4.1.10.orig/include/linux/preempt.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/preempt.h 2015-10-07 18:00:08.000000000 +0200 -@@ -33,6 +33,20 @@ - #define preempt_count_inc() preempt_count_add(1) - #define preempt_count_dec() preempt_count_sub(1) - -+#ifdef CONFIG_PREEMPT_LAZY -+#define add_preempt_lazy_count(val) do { preempt_lazy_count() += (val); } while (0) -+#define sub_preempt_lazy_count(val) do { preempt_lazy_count() -= (val); } while (0) -+#define inc_preempt_lazy_count() add_preempt_lazy_count(1) -+#define dec_preempt_lazy_count() sub_preempt_lazy_count(1) -+#define preempt_lazy_count() (current_thread_info()->preempt_lazy_count) -+#else -+#define add_preempt_lazy_count(val) do { } while (0) -+#define sub_preempt_lazy_count(val) do { } while (0) -+#define inc_preempt_lazy_count() do { } while (0) -+#define dec_preempt_lazy_count() do { } while (0) -+#define preempt_lazy_count() (0) -+#endif -+ - #ifdef CONFIG_PREEMPT_COUNT - - #define preempt_disable() \ -@@ -41,13 +55,25 @@ - barrier(); \ - } while (0) - -+#define preempt_lazy_disable() \ -+do { \ -+ inc_preempt_lazy_count(); \ -+ barrier(); \ -+} while (0) -+ - #define sched_preempt_enable_no_resched() \ - do { \ - barrier(); \ - preempt_count_dec(); \ - } while (0) - --#define preempt_enable_no_resched() sched_preempt_enable_no_resched() -+#ifdef CONFIG_PREEMPT_RT_BASE -+# define preempt_enable_no_resched() sched_preempt_enable_no_resched() -+# define preempt_check_resched_rt() preempt_check_resched() -+#else -+# define preempt_enable_no_resched() preempt_enable() -+# define preempt_check_resched_rt() barrier(); -+#endif - - #ifdef CONFIG_PREEMPT - #define preempt_enable() \ -@@ -63,6 +89,13 @@ - __preempt_schedule(); \ - } while (0) + struct rt_mutex_waiter waiter; + int ret = 0; -+#define preempt_lazy_enable() \ -+do { \ -+ dec_preempt_lazy_count(); \ -+ barrier(); \ -+ preempt_check_resched(); \ -+} while (0) -+ - #else - #define preempt_enable() \ - do { \ -@@ -121,6 +154,7 @@ - #define preempt_disable_notrace() barrier() - #define preempt_enable_no_resched_notrace() barrier() - #define preempt_enable_notrace() barrier() -+#define preempt_check_resched_rt() barrier() +- debug_rt_mutex_init_waiter(&waiter); +- RB_CLEAR_NODE(&waiter.pi_tree_entry); +- RB_CLEAR_NODE(&waiter.tree_entry); ++ rt_mutex_init_waiter(&waiter, false); - #endif /* CONFIG_PREEMPT_COUNT */ + raw_spin_lock(&lock->wait_lock); -@@ -140,10 +174,31 @@ - } while (0) - #define preempt_fold_need_resched() \ - do { \ -- if (tif_need_resched()) \ -+ if (tif_need_resched_now()) \ - set_preempt_need_resched(); \ - } while (0) + /* Try to acquire the lock again: */ + if (try_to_take_rt_mutex(lock, current, NULL)) { ++ if (ww_ctx) ++ ww_mutex_account_lock(lock, ww_ctx); + raw_spin_unlock(&lock->wait_lock); + return 0; + } +@@ -1192,13 +1678,23 @@ -+#ifdef CONFIG_PREEMPT_RT_FULL -+# define preempt_disable_rt() preempt_disable() -+# define preempt_enable_rt() preempt_enable() -+# define preempt_disable_nort() barrier() -+# define preempt_enable_nort() barrier() -+# ifdef CONFIG_SMP -+ extern void migrate_disable(void); -+ extern void migrate_enable(void); -+# else /* CONFIG_SMP */ -+# define migrate_disable() barrier() -+# define migrate_enable() barrier() -+# endif /* CONFIG_SMP */ -+#else -+# define preempt_disable_rt() barrier() -+# define preempt_enable_rt() barrier() -+# define preempt_disable_nort() preempt_disable() -+# define preempt_enable_nort() preempt_enable() -+# define migrate_disable() preempt_disable() -+# define migrate_enable() preempt_enable() -+#endif -+ - #ifdef CONFIG_PREEMPT_NOTIFIERS + if (likely(!ret)) + /* sleep on the mutex */ +- ret = __rt_mutex_slowlock(lock, state, timeout, &waiter); ++ ret = __rt_mutex_slowlock(lock, state, timeout, &waiter, ++ ww_ctx); ++ else if (ww_ctx) { ++ /* ww_mutex received EDEADLK, let it become EALREADY */ ++ ret = __mutex_lock_check_stamp(lock, ww_ctx); ++ BUG_ON(!ret); ++ } - struct preempt_notifier; -diff -Nur linux-4.1.10.orig/include/linux/preempt_mask.h linux-4.1.10/include/linux/preempt_mask.h ---- linux-4.1.10.orig/include/linux/preempt_mask.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/preempt_mask.h 2015-10-07 18:00:08.000000000 +0200 -@@ -44,16 +44,26 @@ - #define HARDIRQ_OFFSET (1UL << HARDIRQ_SHIFT) - #define NMI_OFFSET (1UL << NMI_SHIFT) + if (unlikely(ret)) { + __set_current_state(TASK_RUNNING); + if (rt_mutex_has_waiters(lock)) + remove_waiter(lock, &waiter); +- rt_mutex_handle_deadlock(ret, chwalk, &waiter); ++ /* ww_mutex want to report EDEADLK/EALREADY, let them */ ++ if (!ww_ctx) ++ rt_mutex_handle_deadlock(ret, chwalk, &waiter); ++ } else if (ww_ctx) { ++ ww_mutex_account_lock(lock, ww_ctx); + } --#define SOFTIRQ_DISABLE_OFFSET (2 * SOFTIRQ_OFFSET) -+#ifndef CONFIG_PREEMPT_RT_FULL -+# define SOFTIRQ_DISABLE_OFFSET (2 * SOFTIRQ_OFFSET) -+#else -+# define SOFTIRQ_DISABLE_OFFSET (0) -+#endif + /* +@@ -1255,7 +1751,7 @@ + /* + * Slow path to release a rt-mutex: + */ +-static void __sched ++static bool __sched + rt_mutex_slowunlock(struct rt_mutex *lock) + { + raw_spin_lock(&lock->wait_lock); +@@ -1298,7 +1794,7 @@ + while (!rt_mutex_has_waiters(lock)) { + /* Drops lock->wait_lock ! */ + if (unlock_rt_mutex_safe(lock) == true) +- return; ++ return false; + /* Relock the rtmutex and try again */ + raw_spin_lock(&lock->wait_lock); + } +@@ -1311,8 +1807,7 @@ - #define PREEMPT_ACTIVE_BITS 1 - #define PREEMPT_ACTIVE_SHIFT (NMI_SHIFT + NMI_BITS) - #define PREEMPT_ACTIVE (__IRQ_MASK(PREEMPT_ACTIVE_BITS) << PREEMPT_ACTIVE_SHIFT) + raw_spin_unlock(&lock->wait_lock); - #define hardirq_count() (preempt_count() & HARDIRQ_MASK) --#define softirq_count() (preempt_count() & SOFTIRQ_MASK) - #define irq_count() (preempt_count() & (HARDIRQ_MASK | SOFTIRQ_MASK \ - | NMI_MASK)) -+#ifndef CONFIG_PREEMPT_RT_FULL -+# define softirq_count() (preempt_count() & SOFTIRQ_MASK) -+# define in_serving_softirq() (softirq_count() & SOFTIRQ_OFFSET) -+#else -+# define softirq_count() (0UL) -+extern int in_serving_softirq(void); -+#endif +- /* Undo pi boosting if necessary: */ +- rt_mutex_adjust_prio(current); ++ return true; + } /* - * Are we doing bottom half or hardware interrupt processing? -@@ -64,7 +74,6 @@ - #define in_irq() (hardirq_count()) - #define in_softirq() (softirq_count()) - #define in_interrupt() (irq_count()) --#define in_serving_softirq() (softirq_count() & SOFTIRQ_OFFSET) +@@ -1323,31 +1818,36 @@ + */ + static inline int + rt_mutex_fastlock(struct rt_mutex *lock, int state, ++ struct ww_acquire_ctx *ww_ctx, + int (*slowfn)(struct rt_mutex *lock, int state, + struct hrtimer_sleeper *timeout, +- enum rtmutex_chainwalk chwalk)) ++ enum rtmutex_chainwalk chwalk, ++ struct ww_acquire_ctx *ww_ctx)) + { + if (likely(rt_mutex_cmpxchg(lock, NULL, current))) { + rt_mutex_deadlock_account_lock(lock, current); + return 0; + } else +- return slowfn(lock, state, NULL, RT_MUTEX_MIN_CHAINWALK); ++ return slowfn(lock, state, NULL, RT_MUTEX_MIN_CHAINWALK, ++ ww_ctx); + } - /* - * Are we in NMI context? -diff -Nur linux-4.1.10.orig/include/linux/printk.h linux-4.1.10/include/linux/printk.h ---- linux-4.1.10.orig/include/linux/printk.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/printk.h 2015-10-07 18:00:08.000000000 +0200 -@@ -115,9 +115,11 @@ - #ifdef CONFIG_EARLY_PRINTK - extern asmlinkage __printf(1, 2) - void early_printk(const char *fmt, ...); -+extern void printk_kill(void); - #else - static inline __printf(1, 2) __cold - void early_printk(const char *s, ...) { } -+static inline void printk_kill(void) { } - #endif + static inline int + rt_mutex_timed_fastlock(struct rt_mutex *lock, int state, + struct hrtimer_sleeper *timeout, + enum rtmutex_chainwalk chwalk, ++ struct ww_acquire_ctx *ww_ctx, + int (*slowfn)(struct rt_mutex *lock, int state, + struct hrtimer_sleeper *timeout, +- enum rtmutex_chainwalk chwalk)) ++ enum rtmutex_chainwalk chwalk, ++ struct ww_acquire_ctx *ww_ctx)) + { + if (chwalk == RT_MUTEX_MIN_CHAINWALK && + likely(rt_mutex_cmpxchg(lock, NULL, current))) { + rt_mutex_deadlock_account_lock(lock, current); + return 0; + } else +- return slowfn(lock, state, timeout, chwalk); ++ return slowfn(lock, state, timeout, chwalk, ww_ctx); + } - typedef int(*printk_func_t)(const char *fmt, va_list args); -diff -Nur linux-4.1.10.orig/include/linux/radix-tree.h linux-4.1.10/include/linux/radix-tree.h ---- linux-4.1.10.orig/include/linux/radix-tree.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/radix-tree.h 2015-10-07 18:00:08.000000000 +0200 -@@ -277,8 +277,13 @@ - unsigned int radix_tree_gang_lookup_slot(struct radix_tree_root *root, - void ***results, unsigned long *indices, - unsigned long first_index, unsigned int max_items); -+#ifndef CONFIG_PREEMPT_RT_FULL - int radix_tree_preload(gfp_t gfp_mask); - int radix_tree_maybe_preload(gfp_t gfp_mask); -+#else -+static inline int radix_tree_preload(gfp_t gm) { return 0; } -+static inline int radix_tree_maybe_preload(gfp_t gfp_mask) { return 0; } -+#endif - void radix_tree_init(void); - void *radix_tree_tag_set(struct radix_tree_root *root, - unsigned long index, unsigned int tag); -@@ -303,7 +308,7 @@ + static inline int +@@ -1363,12 +1863,14 @@ - static inline void radix_tree_preload_end(void) + static inline void + rt_mutex_fastunlock(struct rt_mutex *lock, +- void (*slowfn)(struct rt_mutex *lock)) ++ bool (*slowfn)(struct rt_mutex *lock)) { -- preempt_enable(); -+ preempt_enable_nort(); +- if (likely(rt_mutex_cmpxchg(lock, current, NULL))) ++ if (likely(rt_mutex_cmpxchg(lock, current, NULL))) { + rt_mutex_deadlock_account_unlock(current); +- else +- slowfn(lock); ++ } else if (slowfn(lock)) { ++ /* Undo pi boosting if necessary: */ ++ rt_mutex_adjust_prio(current); ++ } } /** -diff -Nur linux-4.1.10.orig/include/linux/random.h linux-4.1.10/include/linux/random.h ---- linux-4.1.10.orig/include/linux/random.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/random.h 2015-10-07 18:00:08.000000000 +0200 -@@ -11,7 +11,7 @@ - extern void add_device_randomness(const void *, unsigned int); - extern void add_input_randomness(unsigned int type, unsigned int code, - unsigned int value); --extern void add_interrupt_randomness(int irq, int irq_flags); -+extern void add_interrupt_randomness(int irq, int irq_flags, __u64 ip); - - extern void get_random_bytes(void *buf, int nbytes); - extern void get_random_bytes_arch(void *buf, int nbytes); -diff -Nur linux-4.1.10.orig/include/linux/rcupdate.h linux-4.1.10/include/linux/rcupdate.h ---- linux-4.1.10.orig/include/linux/rcupdate.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/rcupdate.h 2015-10-07 18:00:08.000000000 +0200 -@@ -167,6 +167,9 @@ +@@ -1380,7 +1882,7 @@ + { + might_sleep(); - #endif /* #else #ifdef CONFIG_PREEMPT_RCU */ +- rt_mutex_fastlock(lock, TASK_UNINTERRUPTIBLE, rt_mutex_slowlock); ++ rt_mutex_fastlock(lock, TASK_UNINTERRUPTIBLE, NULL, rt_mutex_slowlock); + } + EXPORT_SYMBOL_GPL(rt_mutex_lock); -+#ifdef CONFIG_PREEMPT_RT_FULL -+#define call_rcu_bh call_rcu -+#else - /** - * call_rcu_bh() - Queue an RCU for invocation after a quicker grace period. - * @head: structure to be used for queueing the RCU updates. -@@ -190,6 +193,7 @@ - */ - void call_rcu_bh(struct rcu_head *head, - void (*func)(struct rcu_head *head)); -+#endif +@@ -1397,7 +1899,7 @@ + { + might_sleep(); - /** - * call_rcu_sched() - Queue an RCU for invocation after sched grace period. -@@ -260,6 +264,11 @@ - * types of kernel builds, the rcu_read_lock() nesting depth is unknowable. - */ - #define rcu_preempt_depth() (current->rcu_read_lock_nesting) -+#ifndef CONFIG_PREEMPT_RT_FULL -+#define sched_rcu_preempt_depth() rcu_preempt_depth() -+#else -+static inline int sched_rcu_preempt_depth(void) { return 0; } -+#endif +- return rt_mutex_fastlock(lock, TASK_INTERRUPTIBLE, rt_mutex_slowlock); ++ return rt_mutex_fastlock(lock, TASK_INTERRUPTIBLE, NULL, rt_mutex_slowlock); + } + EXPORT_SYMBOL_GPL(rt_mutex_lock_interruptible); - #else /* #ifdef CONFIG_PREEMPT_RCU */ +@@ -1410,11 +1912,30 @@ + might_sleep(); -@@ -283,6 +292,8 @@ - return 0; + return rt_mutex_timed_fastlock(lock, TASK_INTERRUPTIBLE, timeout, +- RT_MUTEX_FULL_CHAINWALK, ++ RT_MUTEX_FULL_CHAINWALK, NULL, + rt_mutex_slowlock); } -+#define sched_rcu_preempt_depth() rcu_preempt_depth() + /** ++ * rt_mutex_lock_killable - lock a rt_mutex killable ++ * ++ * @lock: the rt_mutex to be locked ++ * @detect_deadlock: deadlock detection on/off ++ * ++ * Returns: ++ * 0 on success ++ * -EINTR when interrupted by a signal ++ * -EDEADLK when the lock would deadlock (when deadlock detection is on) ++ */ ++int __sched rt_mutex_lock_killable(struct rt_mutex *lock) ++{ ++ might_sleep(); + - #endif /* #else #ifdef CONFIG_PREEMPT_RCU */ ++ return rt_mutex_fastlock(lock, TASK_KILLABLE, NULL, rt_mutex_slowlock); ++} ++EXPORT_SYMBOL_GPL(rt_mutex_lock_killable); ++ ++/** + * rt_mutex_timed_lock - lock a rt_mutex interruptible + * the timeout structure is provided + * by the caller +@@ -1434,6 +1955,7 @@ - /* Internal to kernel */ -@@ -463,7 +474,14 @@ - int debug_lockdep_rcu_enabled(void); + return rt_mutex_timed_fastlock(lock, TASK_INTERRUPTIBLE, timeout, + RT_MUTEX_MIN_CHAINWALK, ++ NULL, + rt_mutex_slowlock); + } + EXPORT_SYMBOL_GPL(rt_mutex_timed_lock); +@@ -1463,6 +1985,22 @@ + EXPORT_SYMBOL_GPL(rt_mutex_unlock); - int rcu_read_lock_held(void); -+#ifdef CONFIG_PREEMPT_RT_FULL -+static inline int rcu_read_lock_bh_held(void) + /** ++ * rt_mutex_futex_unlock - Futex variant of rt_mutex_unlock ++ * @lock: the rt_mutex to be unlocked ++ * ++ * Returns: true/false indicating whether priority adjustment is ++ * required or not. ++ */ ++bool __sched rt_mutex_futex_unlock(struct rt_mutex *lock) +{ -+ return rcu_read_lock_held(); ++ if (likely(rt_mutex_cmpxchg(lock, current, NULL))) { ++ rt_mutex_deadlock_account_unlock(current); ++ return false; ++ } ++ return rt_mutex_slowunlock(lock); +} -+#else - int rcu_read_lock_bh_held(void); -+#endif - - /** - * rcu_read_lock_sched_held() - might we be in RCU-sched read-side critical section? -@@ -990,10 +1008,14 @@ - static inline void rcu_read_lock_bh(void) ++ ++/** + * rt_mutex_destroy - mark a mutex unusable + * @lock: the mutex to be destroyed + * +@@ -1492,13 +2030,12 @@ + void __rt_mutex_init(struct rt_mutex *lock, const char *name) { - local_bh_disable(); -+#ifdef CONFIG_PREEMPT_RT_FULL -+ rcu_read_lock(); -+#else - __acquire(RCU_BH); - rcu_lock_acquire(&rcu_bh_lock_map); - rcu_lockdep_assert(rcu_is_watching(), - "rcu_read_lock_bh() used illegally while idle"); -+#endif - } + lock->owner = NULL; +- raw_spin_lock_init(&lock->wait_lock); + lock->waiters = RB_ROOT; + lock->waiters_leftmost = NULL; - /* -@@ -1003,10 +1025,14 @@ - */ - static inline void rcu_read_unlock_bh(void) - { -+#ifdef CONFIG_PREEMPT_RT_FULL -+ rcu_read_unlock(); -+#else - rcu_lockdep_assert(rcu_is_watching(), - "rcu_read_unlock_bh() used illegally while idle"); - rcu_lock_release(&rcu_bh_lock_map); - __release(RCU_BH); -+#endif - local_bh_enable(); + debug_rt_mutex_init(lock, name); } +-EXPORT_SYMBOL_GPL(__rt_mutex_init); ++EXPORT_SYMBOL(__rt_mutex_init); -diff -Nur linux-4.1.10.orig/include/linux/rcutree.h linux-4.1.10/include/linux/rcutree.h ---- linux-4.1.10.orig/include/linux/rcutree.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/rcutree.h 2015-10-07 18:00:08.000000000 +0200 -@@ -46,7 +46,11 @@ - rcu_note_context_switch(); - } + /** + * rt_mutex_init_proxy_locked - initialize and lock a rt_mutex on behalf of a +@@ -1513,7 +2050,7 @@ + void rt_mutex_init_proxy_locked(struct rt_mutex *lock, + struct task_struct *proxy_owner) + { +- __rt_mutex_init(lock, NULL); ++ rt_mutex_init(lock); + debug_rt_mutex_proxy_lock(lock, proxy_owner); + rt_mutex_set_owner(lock, proxy_owner); + rt_mutex_deadlock_account_lock(lock, proxy_owner); +@@ -1561,6 +2098,35 @@ + return 1; + } +#ifdef CONFIG_PREEMPT_RT_FULL -+# define synchronize_rcu_bh synchronize_rcu -+#else - void synchronize_rcu_bh(void); ++ /* ++ * In PREEMPT_RT there's an added race. ++ * If the task, that we are about to requeue, times out, ++ * it can set the PI_WAKEUP_INPROGRESS. This tells the requeue ++ * to skip this task. But right after the task sets ++ * its pi_blocked_on to PI_WAKEUP_INPROGRESS it can then ++ * block on the spin_lock(&hb->lock), which in RT is an rtmutex. ++ * This will replace the PI_WAKEUP_INPROGRESS with the actual ++ * lock that it blocks on. We *must not* place this task ++ * on this proxy lock in that case. ++ * ++ * To prevent this race, we first take the task's pi_lock ++ * and check if it has updated its pi_blocked_on. If it has, ++ * we assume that it woke up and we return -EAGAIN. ++ * Otherwise, we set the task's pi_blocked_on to ++ * PI_REQUEUE_INPROGRESS, so that if the task is waking up ++ * it will know that we are in the process of requeuing it. ++ */ ++ raw_spin_lock_irq(&task->pi_lock); ++ if (task->pi_blocked_on) { ++ raw_spin_unlock_irq(&task->pi_lock); ++ raw_spin_unlock(&lock->wait_lock); ++ return -EAGAIN; ++ } ++ task->pi_blocked_on = PI_REQUEUE_INPROGRESS; ++ raw_spin_unlock_irq(&task->pi_lock); +#endif - void synchronize_sched_expedited(void); - void synchronize_rcu_expedited(void); - -@@ -74,7 +78,11 @@ - } ++ + /* We enforce deadlock detection for futexes */ + ret = task_blocks_on_rt_mutex(lock, waiter, task, + RT_MUTEX_FULL_CHAINWALK); +@@ -1631,7 +2197,7 @@ + set_current_state(TASK_INTERRUPTIBLE); - void rcu_barrier(void); -+#ifdef CONFIG_PREEMPT_RT_FULL -+# define rcu_barrier_bh rcu_barrier -+#else - void rcu_barrier_bh(void); -+#endif - void rcu_barrier_sched(void); - unsigned long get_state_synchronize_rcu(void); - void cond_synchronize_rcu(unsigned long oldstate); -@@ -85,12 +93,10 @@ - unsigned long rcu_batches_started_bh(void); - unsigned long rcu_batches_started_sched(void); - unsigned long rcu_batches_completed(void); --unsigned long rcu_batches_completed_bh(void); - unsigned long rcu_batches_completed_sched(void); - void show_rcu_gp_kthreads(void); - - void rcu_force_quiescent_state(void); --void rcu_bh_force_quiescent_state(void); - void rcu_sched_force_quiescent_state(void); - - void exit_rcu(void); -@@ -100,6 +106,14 @@ - - bool rcu_is_watching(void); - -+#ifndef CONFIG_PREEMPT_RT_FULL -+void rcu_bh_force_quiescent_state(void); -+unsigned long rcu_batches_completed_bh(void); -+#else -+# define rcu_bh_force_quiescent_state rcu_force_quiescent_state -+# define rcu_batches_completed_bh rcu_batches_completed -+#endif -+ - void rcu_all_qs(void); - - #endif /* __LINUX_RCUTREE_H */ -diff -Nur linux-4.1.10.orig/include/linux/rtmutex.h linux-4.1.10/include/linux/rtmutex.h ---- linux-4.1.10.orig/include/linux/rtmutex.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/rtmutex.h 2015-10-07 18:00:08.000000000 +0200 -@@ -14,10 +14,14 @@ - - #include - #include --#include -+#include + /* sleep on the mutex */ +- ret = __rt_mutex_slowlock(lock, TASK_INTERRUPTIBLE, to, waiter); ++ ret = __rt_mutex_slowlock(lock, TASK_INTERRUPTIBLE, to, waiter, NULL); - extern int max_lock_depth; /* for sysctl */ + if (unlikely(ret)) + remove_waiter(lock, waiter); +@@ -1646,3 +2212,89 @@ -+#ifdef CONFIG_DEBUG_MUTEXES -+#include -+#endif + return ret; + } + - /** - * The rt_mutex structure - * -@@ -31,8 +35,8 @@ - struct rb_root waiters; - struct rb_node *waiters_leftmost; - struct task_struct *owner; --#ifdef CONFIG_DEBUG_RT_MUTEXES - int save_state; -+#ifdef CONFIG_DEBUG_RT_MUTEXES - const char *name, *file; - int line; - void *magic; -@@ -55,22 +59,33 @@ - # define rt_mutex_debug_check_no_locks_held(task) do { } while (0) - #endif - -+# define rt_mutex_init(mutex) \ -+ do { \ -+ raw_spin_lock_init(&(mutex)->wait_lock); \ -+ __rt_mutex_init(mutex, #mutex); \ -+ } while (0) ++static inline int ++ww_mutex_deadlock_injection(struct ww_mutex *lock, struct ww_acquire_ctx *ctx) ++{ ++#ifdef CONFIG_DEBUG_WW_MUTEX_SLOWPATH ++ unsigned tmp; + - #ifdef CONFIG_DEBUG_RT_MUTEXES - # define __DEBUG_RT_MUTEX_INITIALIZER(mutexname) \ - , .name = #mutexname, .file = __FILE__, .line = __LINE__ --# define rt_mutex_init(mutex) __rt_mutex_init(mutex, __func__) - extern void rt_mutex_debug_task_free(struct task_struct *tsk); - #else - # define __DEBUG_RT_MUTEX_INITIALIZER(mutexname) --# define rt_mutex_init(mutex) __rt_mutex_init(mutex, NULL) - # define rt_mutex_debug_task_free(t) do { } while (0) - #endif - --#define __RT_MUTEX_INITIALIZER(mutexname) \ -- { .wait_lock = __RAW_SPIN_LOCK_UNLOCKED(mutexname.wait_lock) \ -+#define __RT_MUTEX_INITIALIZER_PLAIN(mutexname) \ -+ .wait_lock = __RAW_SPIN_LOCK_UNLOCKED(mutexname.wait_lock) \ - , .waiters = RB_ROOT \ - , .owner = NULL \ -- __DEBUG_RT_MUTEX_INITIALIZER(mutexname)} -+ __DEBUG_RT_MUTEX_INITIALIZER(mutexname) ++ if (ctx->deadlock_inject_countdown-- == 0) { ++ tmp = ctx->deadlock_inject_interval; ++ if (tmp > UINT_MAX/4) ++ tmp = UINT_MAX; ++ else ++ tmp = tmp*2 + tmp + tmp/2; + -+#define __RT_MUTEX_INITIALIZER(mutexname) \ -+ { __RT_MUTEX_INITIALIZER_PLAIN(mutexname) } ++ ctx->deadlock_inject_interval = tmp; ++ ctx->deadlock_inject_countdown = tmp; ++ ctx->contending_lock = lock; + -+#define __RT_MUTEX_INITIALIZER_SAVE_STATE(mutexname) \ -+ { __RT_MUTEX_INITIALIZER_PLAIN(mutexname) \ -+ , .save_state = 1 } - - #define DEFINE_RT_MUTEX(mutexname) \ - struct rt_mutex mutexname = __RT_MUTEX_INITIALIZER(mutexname) -@@ -91,6 +106,7 @@ - - extern void rt_mutex_lock(struct rt_mutex *lock); - extern int rt_mutex_lock_interruptible(struct rt_mutex *lock); -+extern int rt_mutex_lock_killable(struct rt_mutex *lock); - extern int rt_mutex_timed_lock(struct rt_mutex *lock, - struct hrtimer_sleeper *timeout); - -diff -Nur linux-4.1.10.orig/include/linux/rwlock_rt.h linux-4.1.10/include/linux/rwlock_rt.h ---- linux-4.1.10.orig/include/linux/rwlock_rt.h 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.10/include/linux/rwlock_rt.h 2015-10-07 18:00:08.000000000 +0200 -@@ -0,0 +1,99 @@ -+#ifndef __LINUX_RWLOCK_RT_H -+#define __LINUX_RWLOCK_RT_H ++ ww_mutex_unlock(lock); + -+#ifndef __LINUX_SPINLOCK_H -+#error Do not include directly. Use spinlock.h ++ return -EDEADLK; ++ } +#endif + -+#define rwlock_init(rwl) \ -+do { \ -+ static struct lock_class_key __key; \ -+ \ -+ rt_mutex_init(&(rwl)->lock); \ -+ __rt_rwlock_init(rwl, #rwl, &__key); \ -+} while (0) -+ -+extern void __lockfunc rt_write_lock(rwlock_t *rwlock); -+extern void __lockfunc rt_read_lock(rwlock_t *rwlock); -+extern int __lockfunc rt_write_trylock(rwlock_t *rwlock); -+extern int __lockfunc rt_write_trylock_irqsave(rwlock_t *trylock, unsigned long *flags); -+extern int __lockfunc rt_read_trylock(rwlock_t *rwlock); -+extern void __lockfunc rt_write_unlock(rwlock_t *rwlock); -+extern void __lockfunc rt_read_unlock(rwlock_t *rwlock); -+extern unsigned long __lockfunc rt_write_lock_irqsave(rwlock_t *rwlock); -+extern unsigned long __lockfunc rt_read_lock_irqsave(rwlock_t *rwlock); -+extern void __rt_rwlock_init(rwlock_t *rwlock, char *name, struct lock_class_key *key); -+ -+#define read_trylock(lock) __cond_lock(lock, rt_read_trylock(lock)) -+#define write_trylock(lock) __cond_lock(lock, rt_write_trylock(lock)) -+ -+#define write_trylock_irqsave(lock, flags) \ -+ __cond_lock(lock, rt_write_trylock_irqsave(lock, &flags)) -+ -+#define read_lock_irqsave(lock, flags) \ -+ do { \ -+ typecheck(unsigned long, flags); \ -+ flags = rt_read_lock_irqsave(lock); \ -+ } while (0) ++ return 0; ++} + -+#define write_lock_irqsave(lock, flags) \ -+ do { \ -+ typecheck(unsigned long, flags); \ -+ flags = rt_write_lock_irqsave(lock); \ -+ } while (0) ++#ifdef CONFIG_PREEMPT_RT_FULL ++int __sched ++__ww_mutex_lock_interruptible(struct ww_mutex *lock, struct ww_acquire_ctx *ww_ctx) ++{ ++ int ret; + -+#define read_lock(lock) rt_read_lock(lock) ++ might_sleep(); + -+#define read_lock_bh(lock) \ -+ do { \ -+ local_bh_disable(); \ -+ rt_read_lock(lock); \ -+ } while (0) ++ mutex_acquire_nest(&lock->base.dep_map, 0, 0, &ww_ctx->dep_map, _RET_IP_); ++ ret = rt_mutex_slowlock(&lock->base.lock, TASK_INTERRUPTIBLE, NULL, 0, ww_ctx); ++ if (ret) ++ mutex_release(&lock->base.dep_map, 1, _RET_IP_); ++ else if (!ret && ww_ctx->acquired > 1) ++ return ww_mutex_deadlock_injection(lock, ww_ctx); + -+#define read_lock_irq(lock) read_lock(lock) ++ return ret; ++} ++EXPORT_SYMBOL_GPL(__ww_mutex_lock_interruptible); + -+#define write_lock(lock) rt_write_lock(lock) ++int __sched ++__ww_mutex_lock(struct ww_mutex *lock, struct ww_acquire_ctx *ww_ctx) ++{ ++ int ret; + -+#define write_lock_bh(lock) \ -+ do { \ -+ local_bh_disable(); \ -+ rt_write_lock(lock); \ -+ } while (0) ++ might_sleep(); + -+#define write_lock_irq(lock) write_lock(lock) ++ mutex_acquire_nest(&lock->base.dep_map, 0, 0, &ww_ctx->dep_map, _RET_IP_); ++ ret = rt_mutex_slowlock(&lock->base.lock, TASK_UNINTERRUPTIBLE, NULL, 0, ww_ctx); ++ if (ret) ++ mutex_release(&lock->base.dep_map, 1, _RET_IP_); ++ else if (!ret && ww_ctx->acquired > 1) ++ return ww_mutex_deadlock_injection(lock, ww_ctx); + -+#define read_unlock(lock) rt_read_unlock(lock) ++ return ret; ++} ++EXPORT_SYMBOL_GPL(__ww_mutex_lock); + -+#define read_unlock_bh(lock) \ -+ do { \ -+ rt_read_unlock(lock); \ -+ local_bh_enable(); \ -+ } while (0) ++void __sched ww_mutex_unlock(struct ww_mutex *lock) ++{ ++ int nest = !!lock->ctx; + -+#define read_unlock_irq(lock) read_unlock(lock) ++ /* ++ * The unlocking fastpath is the 0->1 transition from 'locked' ++ * into 'unlocked' state: ++ */ ++ if (nest) { ++#ifdef CONFIG_DEBUG_MUTEXES ++ DEBUG_LOCKS_WARN_ON(!lock->ctx->acquired); ++#endif ++ if (lock->ctx->acquired > 0) ++ lock->ctx->acquired--; ++ lock->ctx = NULL; ++ } + -+#define write_unlock(lock) rt_write_unlock(lock) ++ mutex_release(&lock->base.dep_map, nest, _RET_IP_); ++ rt_mutex_unlock(&lock->base.lock); ++} ++EXPORT_SYMBOL(ww_mutex_unlock); ++#endif +diff -Nur linux-4.1.10.orig/kernel/locking/rtmutex_common.h linux-4.1.10/kernel/locking/rtmutex_common.h +--- linux-4.1.10.orig/kernel/locking/rtmutex_common.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/locking/rtmutex_common.h 2015-10-12 22:33:32.296675432 +0200 +@@ -49,6 +49,7 @@ + struct rb_node pi_tree_entry; + struct task_struct *task; + struct rt_mutex *lock; ++ bool savestate; + #ifdef CONFIG_DEBUG_RT_MUTEXES + unsigned long ip; + struct pid *deadlock_task_pid; +@@ -119,6 +120,9 @@ + /* + * PI-futex support (proxy locking functions, etc.): + */ ++#define PI_WAKEUP_INPROGRESS ((struct rt_mutex_waiter *) 1) ++#define PI_REQUEUE_INPROGRESS ((struct rt_mutex_waiter *) 2) + -+#define write_unlock_bh(lock) \ -+ do { \ -+ rt_write_unlock(lock); \ -+ local_bh_enable(); \ -+ } while (0) + extern struct task_struct *rt_mutex_next_owner(struct rt_mutex *lock); + extern void rt_mutex_init_proxy_locked(struct rt_mutex *lock, + struct task_struct *proxy_owner); +@@ -132,10 +136,24 @@ + struct rt_mutex_waiter *waiter); + extern int rt_mutex_timed_futex_lock(struct rt_mutex *l, struct hrtimer_sleeper *to); + ++extern bool rt_mutex_futex_unlock(struct rt_mutex *lock); + -+#define write_unlock_irq(lock) write_unlock(lock) ++extern void rt_mutex_adjust_prio(struct task_struct *task); + -+#define read_unlock_irqrestore(lock, flags) \ -+ do { \ -+ typecheck(unsigned long, flags); \ -+ (void) flags; \ -+ rt_read_unlock(lock); \ -+ } while (0) + #ifdef CONFIG_DEBUG_RT_MUTEXES + # include "rtmutex-debug.h" + #else + # include "rtmutex.h" + #endif + ++static inline void ++rt_mutex_init_waiter(struct rt_mutex_waiter *waiter, bool savestate) ++{ ++ debug_rt_mutex_init_waiter(waiter); ++ waiter->task = NULL; ++ waiter->savestate = savestate; ++ RB_CLEAR_NODE(&waiter->pi_tree_entry); ++ RB_CLEAR_NODE(&waiter->tree_entry); ++} + -+#define write_unlock_irqrestore(lock, flags) \ -+ do { \ -+ typecheck(unsigned long, flags); \ -+ (void) flags; \ -+ rt_write_unlock(lock); \ -+ } while (0) + #endif +diff -Nur linux-4.1.10.orig/kernel/locking/spinlock.c linux-4.1.10/kernel/locking/spinlock.c +--- linux-4.1.10.orig/kernel/locking/spinlock.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/locking/spinlock.c 2015-10-12 22:33:32.300675168 +0200 +@@ -124,8 +124,11 @@ + * __[spin|read|write]_lock_bh() + */ + BUILD_LOCK_OPS(spin, raw_spinlock); + ++#ifndef CONFIG_PREEMPT_RT_FULL + BUILD_LOCK_OPS(read, rwlock); + BUILD_LOCK_OPS(write, rwlock); +#endif -diff -Nur linux-4.1.10.orig/include/linux/rwlock_types.h linux-4.1.10/include/linux/rwlock_types.h ---- linux-4.1.10.orig/include/linux/rwlock_types.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/rwlock_types.h 2015-10-07 18:00:08.000000000 +0200 -@@ -1,6 +1,10 @@ - #ifndef __LINUX_RWLOCK_TYPES_H - #define __LINUX_RWLOCK_TYPES_H -+#if !defined(__LINUX_SPINLOCK_TYPES_H) -+# error "Do not include directly, include spinlock_types.h" -+#endif -+ - /* - * include/linux/rwlock_types.h - generic rwlock type definitions - * and initializers -@@ -43,6 +47,7 @@ - RW_DEP_MAP_INIT(lockname) } #endif --#define DEFINE_RWLOCK(x) rwlock_t x = __RW_LOCK_UNLOCKED(x) -+#define DEFINE_RWLOCK(name) \ -+ rwlock_t name __cacheline_aligned_in_smp = __RW_LOCK_UNLOCKED(name) +@@ -209,6 +212,8 @@ + EXPORT_SYMBOL(_raw_spin_unlock_bh); + #endif - #endif /* __LINUX_RWLOCK_TYPES_H */ -diff -Nur linux-4.1.10.orig/include/linux/rwlock_types_rt.h linux-4.1.10/include/linux/rwlock_types_rt.h ---- linux-4.1.10.orig/include/linux/rwlock_types_rt.h 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.10/include/linux/rwlock_types_rt.h 2015-10-07 18:00:08.000000000 +0200 -@@ -0,0 +1,33 @@ -+#ifndef __LINUX_RWLOCK_TYPES_RT_H -+#define __LINUX_RWLOCK_TYPES_RT_H ++#ifndef CONFIG_PREEMPT_RT_FULL + -+#ifndef __LINUX_SPINLOCK_TYPES_H -+#error "Do not include directly. Include spinlock_types.h instead" -+#endif + #ifndef CONFIG_INLINE_READ_TRYLOCK + int __lockfunc _raw_read_trylock(rwlock_t *lock) + { +@@ -353,6 +358,8 @@ + EXPORT_SYMBOL(_raw_write_unlock_bh); + #endif + ++#endif /* !PREEMPT_RT_FULL */ + -+/* -+ * rwlocks - rtmutex which allows single reader recursion -+ */ -+typedef struct { -+ struct rt_mutex lock; -+ int read_depth; -+ unsigned int break_lock; -+#ifdef CONFIG_DEBUG_LOCK_ALLOC -+ struct lockdep_map dep_map; + #ifdef CONFIG_DEBUG_LOCK_ALLOC + + void __lockfunc _raw_spin_lock_nested(raw_spinlock_t *lock, int subclass) +diff -Nur linux-4.1.10.orig/kernel/locking/spinlock_debug.c linux-4.1.10/kernel/locking/spinlock_debug.c +--- linux-4.1.10.orig/kernel/locking/spinlock_debug.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/locking/spinlock_debug.c 2015-10-12 22:33:32.300675168 +0200 +@@ -31,6 +31,7 @@ + + EXPORT_SYMBOL(__raw_spin_lock_init); + ++#ifndef CONFIG_PREEMPT_RT_FULL + void __rwlock_init(rwlock_t *lock, const char *name, + struct lock_class_key *key) + { +@@ -48,6 +49,7 @@ + } + + EXPORT_SYMBOL(__rwlock_init); +#endif -+} rwlock_t; + + static void spin_dump(raw_spinlock_t *lock, const char *msg) + { +@@ -159,6 +161,7 @@ + arch_spin_unlock(&lock->raw_lock); + } + ++#ifndef CONFIG_PREEMPT_RT_FULL + static void rwlock_bug(rwlock_t *lock, const char *msg) + { + if (!debug_locks_off()) +@@ -300,3 +303,5 @@ + debug_write_unlock(lock); + arch_write_unlock(&lock->raw_lock); + } + -+#ifdef CONFIG_DEBUG_LOCK_ALLOC -+# define RW_DEP_MAP_INIT(lockname) .dep_map = { .name = #lockname } -+#else -+# define RW_DEP_MAP_INIT(lockname) +#endif -+ -+#define __RW_LOCK_UNLOCKED(name) \ -+ { .lock = __RT_MUTEX_INITIALIZER_SAVE_STATE(name.lock), \ -+ RW_DEP_MAP_INIT(name) } -+ -+#define DEFINE_RWLOCK(name) \ -+ rwlock_t name __cacheline_aligned_in_smp = __RW_LOCK_UNLOCKED(name) -+ +diff -Nur linux-4.1.10.orig/kernel/panic.c linux-4.1.10/kernel/panic.c +--- linux-4.1.10.orig/kernel/panic.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/panic.c 2015-10-12 22:33:32.300675168 +0200 +@@ -387,9 +387,11 @@ + + static int init_oops_id(void) + { ++#ifndef CONFIG_PREEMPT_RT_FULL + if (!oops_id) + get_random_bytes(&oops_id, sizeof(oops_id)); + else +#endif -diff -Nur linux-4.1.10.orig/include/linux/rwsem.h linux-4.1.10/include/linux/rwsem.h ---- linux-4.1.10.orig/include/linux/rwsem.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/rwsem.h 2015-10-07 18:00:08.000000000 +0200 -@@ -18,6 +18,10 @@ - #include - #endif + oops_id++; -+#ifdef CONFIG_PREEMPT_RT_FULL -+#include -+#else /* PREEMPT_RT_FULL */ -+ - struct rw_semaphore; + return 0; +diff -Nur linux-4.1.10.orig/kernel/power/hibernate.c linux-4.1.10/kernel/power/hibernate.c +--- linux-4.1.10.orig/kernel/power/hibernate.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/power/hibernate.c 2015-10-12 22:33:32.300675168 +0200 +@@ -285,6 +285,8 @@ - #ifdef CONFIG_RWSEM_GENERIC_SPINLOCK -@@ -177,4 +181,6 @@ - # define up_read_non_owner(sem) up_read(sem) - #endif + local_irq_disable(); -+#endif /* !PREEMPT_RT_FULL */ ++ system_state = SYSTEM_SUSPEND; + - #endif /* _LINUX_RWSEM_H */ -diff -Nur linux-4.1.10.orig/include/linux/rwsem_rt.h linux-4.1.10/include/linux/rwsem_rt.h ---- linux-4.1.10.orig/include/linux/rwsem_rt.h 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.10/include/linux/rwsem_rt.h 2015-10-07 18:00:08.000000000 +0200 -@@ -0,0 +1,140 @@ -+#ifndef _LINUX_RWSEM_RT_H -+#define _LINUX_RWSEM_RT_H + error = syscore_suspend(); + if (error) { + printk(KERN_ERR "PM: Some system devices failed to power down, " +@@ -314,6 +316,7 @@ + syscore_resume(); + + Enable_irqs: ++ system_state = SYSTEM_RUNNING; + local_irq_enable(); + + Enable_cpus: +@@ -437,6 +440,7 @@ + goto Enable_cpus; + + local_irq_disable(); ++ system_state = SYSTEM_SUSPEND; + + error = syscore_suspend(); + if (error) +@@ -470,6 +474,7 @@ + syscore_resume(); + + Enable_irqs: ++ system_state = SYSTEM_RUNNING; + local_irq_enable(); + + Enable_cpus: +@@ -555,6 +560,7 @@ + goto Platform_finish; + + local_irq_disable(); ++ system_state = SYSTEM_SUSPEND; + syscore_suspend(); + if (pm_wakeup_pending()) { + error = -EAGAIN; +@@ -567,6 +573,7 @@ + + Power_up: + syscore_resume(); ++ system_state = SYSTEM_RUNNING; + local_irq_enable(); + enable_nonboot_cpus(); + +diff -Nur linux-4.1.10.orig/kernel/power/suspend.c linux-4.1.10/kernel/power/suspend.c +--- linux-4.1.10.orig/kernel/power/suspend.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/power/suspend.c 2015-10-12 22:33:32.300675168 +0200 +@@ -356,6 +356,8 @@ + arch_suspend_disable_irqs(); + BUG_ON(!irqs_disabled()); + ++ system_state = SYSTEM_SUSPEND; + -+#ifndef _LINUX_RWSEM_H -+#error "Include rwsem.h" -+#endif + error = syscore_suspend(); + if (!error) { + *wakeup = pm_wakeup_pending(); +@@ -370,6 +372,8 @@ + syscore_resume(); + } + ++ system_state = SYSTEM_RUNNING; + -+/* -+ * RW-semaphores are a spinlock plus a reader-depth count. -+ * -+ * Note that the semantics are different from the usual -+ * Linux rw-sems, in PREEMPT_RT mode we do not allow -+ * multiple readers to hold the lock at once, we only allow -+ * a read-lock owner to read-lock recursively. This is -+ * better for latency, makes the implementation inherently -+ * fair and makes it simpler as well. -+ */ -+ -+#include -+ -+struct rw_semaphore { -+ struct rt_mutex lock; -+ int read_depth; -+#ifdef CONFIG_DEBUG_LOCK_ALLOC -+ struct lockdep_map dep_map; + arch_suspend_enable_irqs(); + BUG_ON(irqs_disabled()); + +diff -Nur linux-4.1.10.orig/kernel/printk/printk.c linux-4.1.10/kernel/printk/printk.c +--- linux-4.1.10.orig/kernel/printk/printk.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/printk/printk.c 2015-10-12 22:33:32.300675168 +0200 +@@ -1163,6 +1163,7 @@ + { + char *text; + int len = 0; ++ int attempts = 0; + + text = kmalloc(LOG_LINE_MAX + PREFIX_MAX, GFP_KERNEL); + if (!text) +@@ -1174,7 +1175,14 @@ + u64 seq; + u32 idx; + enum log_flags prev; +- ++ int num_msg; ++try_again: ++ attempts++; ++ if (attempts > 10) { ++ len = -EBUSY; ++ goto out; ++ } ++ num_msg = 0; + if (clear_seq < log_first_seq) { + /* messages are gone, move to first available one */ + clear_seq = log_first_seq; +@@ -1195,6 +1203,14 @@ + prev = msg->flags; + idx = log_next(idx); + seq++; ++ num_msg++; ++ if (num_msg > 5) { ++ num_msg = 0; ++ raw_spin_unlock_irq(&logbuf_lock); ++ raw_spin_lock_irq(&logbuf_lock); ++ if (clear_seq < log_first_seq) ++ goto try_again; ++ } + } + + /* move first record forward until length fits into the buffer */ +@@ -1208,6 +1224,14 @@ + prev = msg->flags; + idx = log_next(idx); + seq++; ++ num_msg++; ++ if (num_msg > 5) { ++ num_msg = 0; ++ raw_spin_unlock_irq(&logbuf_lock); ++ raw_spin_lock_irq(&logbuf_lock); ++ if (clear_seq < log_first_seq) ++ goto try_again; ++ } + } + + /* last message fitting into this dump */ +@@ -1248,6 +1272,7 @@ + clear_seq = log_next_seq; + clear_idx = log_next_idx; + } ++out: + raw_spin_unlock_irq(&logbuf_lock); + + kfree(text); +@@ -1401,6 +1426,7 @@ + if (!console_drivers) + return; + ++ migrate_disable(); + for_each_console(con) { + if (exclusive_console && con != exclusive_console) + continue; +@@ -1413,6 +1439,7 @@ + continue; + con->write(con, text, len); + } ++ migrate_enable(); + } + + /* +@@ -1473,6 +1500,15 @@ + static int console_trylock_for_printk(void) + { + unsigned int cpu = smp_processor_id(); ++#ifdef CONFIG_PREEMPT_RT_FULL ++ int lock = !early_boot_irqs_disabled && (preempt_count() == 0) && ++ !irqs_disabled(); ++#else ++ int lock = 1; +#endif -+}; -+ -+#define __RWSEM_INITIALIZER(name) \ -+ { .lock = __RT_MUTEX_INITIALIZER(name.lock), \ -+ RW_DEP_MAP_INIT(name) } -+ -+#define DECLARE_RWSEM(lockname) \ -+ struct rw_semaphore lockname = __RWSEM_INITIALIZER(lockname) -+ -+extern void __rt_rwsem_init(struct rw_semaphore *rwsem, const char *name, -+ struct lock_class_key *key); -+ -+#define __rt_init_rwsem(sem, name, key) \ -+ do { \ -+ rt_mutex_init(&(sem)->lock); \ -+ __rt_rwsem_init((sem), (name), (key));\ -+ } while (0) -+ -+#define __init_rwsem(sem, name, key) __rt_init_rwsem(sem, name, key) -+ -+# define rt_init_rwsem(sem) \ -+do { \ -+ static struct lock_class_key __key; \ -+ \ -+ __rt_init_rwsem((sem), #sem, &__key); \ -+} while (0) -+ -+extern void rt_down_write(struct rw_semaphore *rwsem); -+extern void rt_down_read_nested(struct rw_semaphore *rwsem, int subclass); -+extern void rt_down_write_nested(struct rw_semaphore *rwsem, int subclass); -+extern void rt_down_write_nested_lock(struct rw_semaphore *rwsem, -+ struct lockdep_map *nest); -+extern void rt_down_read(struct rw_semaphore *rwsem); -+extern int rt_down_write_trylock(struct rw_semaphore *rwsem); -+extern int rt_down_read_trylock(struct rw_semaphore *rwsem); -+extern void __rt_up_read(struct rw_semaphore *rwsem); -+extern void rt_up_read(struct rw_semaphore *rwsem); -+extern void rt_up_write(struct rw_semaphore *rwsem); -+extern void rt_downgrade_write(struct rw_semaphore *rwsem); -+ -+#define init_rwsem(sem) rt_init_rwsem(sem) -+#define rwsem_is_locked(s) rt_mutex_is_locked(&(s)->lock) -+ -+static inline int rwsem_is_contended(struct rw_semaphore *sem) -+{ -+ /* rt_mutex_has_waiters() */ -+ return !RB_EMPTY_ROOT(&sem->lock.waiters); -+} -+ -+static inline void down_read(struct rw_semaphore *sem) -+{ -+ rt_down_read(sem); -+} + -+static inline int down_read_trylock(struct rw_semaphore *sem) -+{ -+ return rt_down_read_trylock(sem); -+} ++ if (!lock) ++ return 0; + + if (!console_trylock()) + return 0; +@@ -1607,6 +1643,62 @@ + return textlen; + } + ++#ifdef CONFIG_EARLY_PRINTK ++struct console *early_console; + -+static inline void down_write(struct rw_semaphore *sem) ++static void early_vprintk(const char *fmt, va_list ap) +{ -+ rt_down_write(sem); -+} ++ if (early_console) { ++ char buf[512]; ++ int n = vscnprintf(buf, sizeof(buf), fmt, ap); + -+static inline int down_write_trylock(struct rw_semaphore *sem) -+{ -+ return rt_down_write_trylock(sem); ++ early_console->write(early_console, buf, n); ++ } +} + -+static inline void __up_read(struct rw_semaphore *sem) ++asmlinkage void early_printk(const char *fmt, ...) +{ -+ __rt_up_read(sem); -+} ++ va_list ap; + -+static inline void up_read(struct rw_semaphore *sem) -+{ -+ rt_up_read(sem); ++ va_start(ap, fmt); ++ early_vprintk(fmt, ap); ++ va_end(ap); +} + -+static inline void up_write(struct rw_semaphore *sem) -+{ -+ rt_up_write(sem); -+} ++/* ++ * This is independent of any log levels - a global ++ * kill switch that turns off all of printk. ++ * ++ * Used by the NMI watchdog if early-printk is enabled. ++ */ ++static bool __read_mostly printk_killswitch; + -+static inline void downgrade_write(struct rw_semaphore *sem) ++static int __init force_early_printk_setup(char *str) +{ -+ rt_downgrade_write(sem); ++ printk_killswitch = true; ++ return 0; +} ++early_param("force_early_printk", force_early_printk_setup); + -+static inline void down_read_nested(struct rw_semaphore *sem, int subclass) ++void printk_kill(void) +{ -+ return rt_down_read_nested(sem, subclass); ++ printk_killswitch = true; +} + -+static inline void down_write_nested(struct rw_semaphore *sem, int subclass) -+{ -+ rt_down_write_nested(sem, subclass); -+} -+#ifdef CONFIG_DEBUG_LOCK_ALLOC -+static inline void down_write_nest_lock(struct rw_semaphore *sem, -+ struct rw_semaphore *nest_lock) ++static int forced_early_printk(const char *fmt, va_list ap) +{ -+ rt_down_write_nested_lock(sem, &nest_lock->dep_map); ++ if (!printk_killswitch) ++ return 0; ++ early_vprintk(fmt, ap); ++ return 1; +} -+ +#else -+ -+static inline void down_write_nest_lock(struct rw_semaphore *sem, -+ struct rw_semaphore *nest_lock) ++static inline int forced_early_printk(const char *fmt, va_list ap) +{ -+ rt_down_write_nested_lock(sem, NULL); ++ return 0; +} +#endif -+#endif -diff -Nur linux-4.1.10.orig/include/linux/sched.h linux-4.1.10/include/linux/sched.h ---- linux-4.1.10.orig/include/linux/sched.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/sched.h 2015-10-07 18:00:08.000000000 +0200 -@@ -26,6 +26,7 @@ - #include - #include - #include -+#include - - #include - #include -@@ -234,10 +235,7 @@ - TASK_UNINTERRUPTIBLE | __TASK_STOPPED | \ - __TASK_TRACED | EXIT_ZOMBIE | EXIT_DEAD) - --#define task_is_traced(task) ((task->state & __TASK_TRACED) != 0) - #define task_is_stopped(task) ((task->state & __TASK_STOPPED) != 0) --#define task_is_stopped_or_traced(task) \ -- ((task->state & (__TASK_STOPPED | __TASK_TRACED)) != 0) - #define task_contributes_to_load(task) \ - ((task->state & TASK_UNINTERRUPTIBLE) != 0 && \ - (task->flags & PF_FROZEN) == 0) -@@ -302,6 +300,11 @@ - - #endif ++ + asmlinkage int vprintk_emit(int facility, int level, + const char *dict, size_t dictlen, + const char *fmt, va_list args) +@@ -1623,6 +1715,13 @@ + /* cpu currently holding logbuf_lock in this function */ + static unsigned int logbuf_cpu = UINT_MAX; -+#define __set_current_state_no_track(state_value) \ -+ do { current->state = (state_value); } while (0) -+#define set_current_state_no_track(state_value) \ -+ set_mb(current->state, (state_value)) ++ /* ++ * Fall back to early_printk if a debugging subsystem has ++ * killed printk output ++ */ ++ if (unlikely(forced_early_printk(fmt, args))) ++ return 1; + - /* Task command name length */ - #define TASK_COMM_LEN 16 + if (level == LOGLEVEL_SCHED) { + level = LOGLEVEL_DEFAULT; + in_sched = true; +@@ -1764,8 +1863,7 @@ + * console_sem which would prevent anyone from printing to + * console + */ +- preempt_disable(); +- ++ migrate_disable(); + /* + * Try to acquire and then immediately release the console + * semaphore. The release will print out buffers and wake up +@@ -1773,7 +1871,7 @@ + */ + if (console_trylock_for_printk()) + console_unlock(); +- preempt_enable(); ++ migrate_enable(); + lockdep_on(); + } -@@ -900,6 +903,50 @@ - #define SCHED_CAPACITY_SCALE (1L << SCHED_CAPACITY_SHIFT) +@@ -1902,26 +2000,6 @@ - /* -+ * Wake-queues are lists of tasks with a pending wakeup, whose -+ * callers have already marked the task as woken internally, -+ * and can thus carry on. A common use case is being able to -+ * do the wakeups once the corresponding user lock as been -+ * released. -+ * -+ * We hold reference to each task in the list across the wakeup, -+ * thus guaranteeing that the memory is still valid by the time -+ * the actual wakeups are performed in wake_up_q(). -+ * -+ * One per task suffices, because there's never a need for a task to be -+ * in two wake queues simultaneously; it is forbidden to abandon a task -+ * in a wake queue (a call to wake_up_q() _must_ follow), so if a task is -+ * already in a wake queue, the wakeup will happen soon and the second -+ * waker can just skip it. -+ * -+ * The WAKE_Q macro declares and initializes the list head. -+ * wake_up_q() does NOT reinitialize the list; it's expected to be -+ * called near the end of a function, where the fact that the queue is -+ * not used again will be easy to see by inspection. -+ * -+ * Note that this can cause spurious wakeups. schedule() callers -+ * must ensure the call is done inside a loop, confirming that the -+ * wakeup condition has in fact occurred. -+ */ -+struct wake_q_node { -+ struct wake_q_node *next; -+}; -+ -+struct wake_q_head { -+ struct wake_q_node *first; -+ struct wake_q_node **lastp; -+}; -+ -+#define WAKE_Q_TAIL ((struct wake_q_node *) 0x01) -+ -+#define WAKE_Q(name) \ -+ struct wake_q_head name = { WAKE_Q_TAIL, &name.first } -+ -+extern void wake_q_add(struct wake_q_head *head, -+ struct task_struct *task); -+extern void wake_up_q(struct wake_q_head *head); -+ -+/* - * sched-domains (multiprocessor balancing) declarations: - */ - #ifdef CONFIG_SMP -@@ -1291,6 +1338,7 @@ + #endif /* CONFIG_PRINTK */ - struct task_struct { - volatile long state; /* -1 unrunnable, 0 runnable, >0 stopped */ -+ volatile long saved_state; /* saved state for "spinlock sleepers" */ - void *stack; - atomic_t usage; - unsigned int flags; /* per process flags, defined below */ -@@ -1327,6 +1375,12 @@ - #endif +-#ifdef CONFIG_EARLY_PRINTK +-struct console *early_console; +- +-asmlinkage __visible void early_printk(const char *fmt, ...) +-{ +- va_list ap; +- char buf[512]; +- int n; +- +- if (!early_console) +- return; +- +- va_start(ap, fmt); +- n = vscnprintf(buf, sizeof(buf), fmt, ap); +- va_end(ap); +- +- early_console->write(early_console, buf, n); +-} +-#endif +- + static int __add_preferred_console(char *name, int idx, char *options, + char *brl_options) + { +@@ -2143,11 +2221,16 @@ + goto out; - unsigned int policy; + len = cont_print_text(text, size); ++#ifndef CONFIG_PREEMPT_RT_FULL + raw_spin_unlock(&logbuf_lock); + stop_critical_timings(); + call_console_drivers(cont.level, text, len); + start_critical_timings(); + local_irq_restore(flags); ++#else ++ raw_spin_unlock_irqrestore(&logbuf_lock, flags); ++ call_console_drivers(cont.level, text, len); ++#endif + return; + out: + raw_spin_unlock_irqrestore(&logbuf_lock, flags); +@@ -2235,12 +2318,17 @@ + console_idx = log_next(console_idx); + console_seq++; + console_prev = msg->flags; +#ifdef CONFIG_PREEMPT_RT_FULL -+ int migrate_disable; -+# ifdef CONFIG_SCHED_DEBUG -+ int migrate_disable_atomic; -+# endif ++ raw_spin_unlock_irqrestore(&logbuf_lock, flags); ++ call_console_drivers(level, text, len); ++#else + raw_spin_unlock(&logbuf_lock); + + stop_critical_timings(); /* don't trace print latency */ + call_console_drivers(level, text, len); + start_critical_timings(); + local_irq_restore(flags); +#endif - int nr_cpus_allowed; - cpumask_t cpus_allowed; + } + console_locked = 0; -@@ -1434,7 +1488,8 @@ - struct cputime prev_cputime; - #endif - #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN -- seqlock_t vtime_seqlock; -+ raw_spinlock_t vtime_lock; -+ seqcount_t vtime_seq; - unsigned long long vtime_snap; - enum { - VTIME_SLEEPING = 0, -@@ -1450,6 +1505,9 @@ +diff -Nur linux-4.1.10.orig/kernel/ptrace.c linux-4.1.10/kernel/ptrace.c +--- linux-4.1.10.orig/kernel/ptrace.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/ptrace.c 2015-10-12 22:33:32.300675168 +0200 +@@ -129,7 +129,12 @@ - struct task_cputime cputime_expires; - struct list_head cpu_timers[3]; -+#ifdef CONFIG_PREEMPT_RT_BASE -+ struct task_struct *posix_timer_list; -+#endif + spin_lock_irq(&task->sighand->siglock); + if (task_is_traced(task) && !__fatal_signal_pending(task)) { +- task->state = __TASK_TRACED; ++ raw_spin_lock_irq(&task->pi_lock); ++ if (task->state & __TASK_TRACED) ++ task->state = __TASK_TRACED; ++ else ++ task->saved_state = __TASK_TRACED; ++ raw_spin_unlock_irq(&task->pi_lock); + ret = true; + } + spin_unlock_irq(&task->sighand->siglock); +diff -Nur linux-4.1.10.orig/kernel/rcu/tree.c linux-4.1.10/kernel/rcu/tree.c +--- linux-4.1.10.orig/kernel/rcu/tree.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/rcu/tree.c 2015-10-12 22:33:32.300675168 +0200 +@@ -56,6 +56,11 @@ + #include + #include + #include ++#include ++#include ++#include ++#include ++#include "../time/tick-internal.h" - /* process credentials */ - const struct cred __rcu *real_cred; /* objective and real subjective task -@@ -1482,10 +1540,15 @@ - /* signal handlers */ - struct signal_struct *signal; - struct sighand_struct *sighand; -+ struct sigqueue *sigqueue_cache; + #include "tree.h" + #include "rcu.h" +@@ -220,6 +225,19 @@ + } + } - sigset_t blocked, real_blocked; - sigset_t saved_sigmask; /* restored if set_restore_sigmask() was used */ - struct sigpending pending; +#ifdef CONFIG_PREEMPT_RT_FULL -+ /* TODO: move me into ->restart_block ? */ -+ struct siginfo forced_info; ++static void rcu_preempt_qs(void); ++ ++void rcu_bh_qs(void) ++{ ++ unsigned long flags; ++ ++ /* Callers to this function, rcu_preempt_qs(), must disable irqs. */ ++ local_irq_save(flags); ++ rcu_preempt_qs(); ++ local_irq_restore(flags); ++} ++#else + void rcu_bh_qs(void) + { + if (!__this_cpu_read(rcu_bh_data.passed_quiesce)) { +@@ -229,6 +247,7 @@ + __this_cpu_write(rcu_bh_data.passed_quiesce, 1); + } + } +#endif - unsigned long sas_ss_sp; - size_t sas_ss_size; -@@ -1511,6 +1574,8 @@ - /* Protection of the PI data structures: */ - raw_spinlock_t pi_lock; + static DEFINE_PER_CPU(int, rcu_sched_qs_mask); -+ struct wake_q_node wake_q; -+ - #ifdef CONFIG_RT_MUTEXES - /* PI waiters blocked on a rt_mutex held by this task */ - struct rb_root pi_waiters; -@@ -1705,6 +1770,12 @@ - unsigned long trace; - /* bitmask and counter of trace recursion */ - unsigned long trace_recursion; -+#ifdef CONFIG_WAKEUP_LATENCY_HIST -+ u64 preempt_timestamp_hist; -+#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST -+ long timer_offset; -+#endif -+#endif - #endif /* CONFIG_TRACING */ - #ifdef CONFIG_MEMCG - struct memcg_oom_info { -@@ -1721,14 +1792,23 @@ - unsigned int sequential_io; - unsigned int sequential_io_avg; - #endif -+#ifdef CONFIG_PREEMPT_RT_BASE -+ struct rcu_head put_rcu; -+ int softirq_nestcnt; -+ unsigned int softirqs_raised; -+#endif -+#ifdef CONFIG_PREEMPT_RT_FULL -+# if defined CONFIG_HIGHMEM || defined CONFIG_X86_32 -+ int kmap_idx; -+ pte_t kmap_pte[KM_TYPE_NR]; -+# endif -+#endif - #ifdef CONFIG_DEBUG_ATOMIC_SLEEP - unsigned long task_state_change; - #endif -+ int pagefault_disabled; - }; - --/* Future-safe accessor for struct task_struct's cpus_allowed. */ --#define tsk_cpus_allowed(tsk) (&(tsk)->cpus_allowed) -- - #define TNF_MIGRATED 0x01 - #define TNF_NO_GROUP 0x02 - #define TNF_SHARED 0x04 -@@ -1917,6 +1997,15 @@ - extern void free_task(struct task_struct *tsk); - #define get_task_struct(tsk) do { atomic_inc(&(tsk)->usage); } while(0) - -+#ifdef CONFIG_PREEMPT_RT_BASE -+extern void __put_task_struct_cb(struct rcu_head *rhp); -+ -+static inline void put_task_struct(struct task_struct *t) -+{ -+ if (atomic_dec_and_test(&t->usage)) -+ call_rcu(&t->put_rcu, __put_task_struct_cb); -+} -+#else - extern void __put_task_struct(struct task_struct *t); - - static inline void put_task_struct(struct task_struct *t) -@@ -1924,6 +2013,7 @@ - if (atomic_dec_and_test(&t->usage)) - __put_task_struct(t); +@@ -404,6 +423,7 @@ } -+#endif + EXPORT_SYMBOL_GPL(rcu_batches_completed_sched); - #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN - extern void task_cputime(struct task_struct *t, -@@ -1962,6 +2052,7 @@ ++#ifndef CONFIG_PREEMPT_RT_FULL /* - * Per process flags + * Return the number of RCU BH batches completed thus far for debug & stats. */ -+#define PF_IN_SOFTIRQ 0x00000001 /* Task is serving softirq */ - #define PF_EXITING 0x00000004 /* getting shut down */ - #define PF_EXITPIDONE 0x00000008 /* pi exit done on shut down */ - #define PF_VCPU 0x00000010 /* I'm a virtual CPU */ -@@ -2126,6 +2217,10 @@ - - extern int set_cpus_allowed_ptr(struct task_struct *p, - const struct cpumask *new_mask); -+int migrate_me(void); -+void tell_sched_cpu_down_begin(int cpu); -+void tell_sched_cpu_down_done(int cpu); -+ - #else - static inline void do_set_cpus_allowed(struct task_struct *p, - const struct cpumask *new_mask) -@@ -2138,6 +2233,9 @@ - return -EINVAL; - return 0; - } -+static inline int migrate_me(void) { return 0; } -+static inline void tell_sched_cpu_down_begin(int cpu) { } -+static inline void tell_sched_cpu_down_done(int cpu) { } - #endif - - #ifdef CONFIG_NO_HZ_COMMON -@@ -2354,6 +2452,7 @@ - - extern int wake_up_state(struct task_struct *tsk, unsigned int state); - extern int wake_up_process(struct task_struct *tsk); -+extern int wake_up_lock_sleeper(struct task_struct * tsk); - extern void wake_up_new_task(struct task_struct *tsk); - #ifdef CONFIG_SMP - extern void kick_process(struct task_struct *tsk); -@@ -2470,12 +2569,24 @@ - - /* mmdrop drops the mm and the page tables */ - extern void __mmdrop(struct mm_struct *); -+ - static inline void mmdrop(struct mm_struct * mm) - { - if (unlikely(atomic_dec_and_test(&mm->mm_count))) - __mmdrop(mm); +@@ -431,6 +451,13 @@ } + EXPORT_SYMBOL_GPL(rcu_bh_force_quiescent_state); -+#ifdef CONFIG_PREEMPT_RT_BASE -+extern void __mmdrop_delayed(struct rcu_head *rhp); -+static inline void mmdrop_delayed(struct mm_struct *mm) ++#else ++void rcu_force_quiescent_state(void) +{ -+ if (atomic_dec_and_test(&mm->mm_count)) -+ call_rcu(&mm->delayed_drop, __mmdrop_delayed); +} -+#else -+# define mmdrop_delayed(mm) mmdrop(mm) ++EXPORT_SYMBOL_GPL(rcu_force_quiescent_state); +#endif + - /* mmput gets rid of the mappings and all user-space */ - extern void mmput(struct mm_struct *); - /* Grab a reference to a task's mm, if it is not already going away */ -@@ -2787,6 +2898,43 @@ - return unlikely(test_tsk_thread_flag(tsk,TIF_NEED_RESCHED)); + /* + * Force a quiescent state for RCU-sched. + */ +@@ -1545,7 +1572,7 @@ + !ACCESS_ONCE(rsp->gp_flags) || + !rsp->gp_kthread) + return; +- wake_up(&rsp->gp_wq); ++ swait_wake(&rsp->gp_wq); } -+#ifdef CONFIG_PREEMPT_LAZY -+static inline void set_tsk_need_resched_lazy(struct task_struct *tsk) -+{ -+ set_tsk_thread_flag(tsk,TIF_NEED_RESCHED_LAZY); -+} -+ -+static inline void clear_tsk_need_resched_lazy(struct task_struct *tsk) -+{ -+ clear_tsk_thread_flag(tsk,TIF_NEED_RESCHED_LAZY); -+} -+ -+static inline int test_tsk_need_resched_lazy(struct task_struct *tsk) -+{ -+ return unlikely(test_tsk_thread_flag(tsk,TIF_NEED_RESCHED_LAZY)); -+} -+ -+static inline int need_resched_lazy(void) -+{ -+ return test_thread_flag(TIF_NEED_RESCHED_LAZY); -+} -+ -+static inline int need_resched_now(void) -+{ -+ return test_thread_flag(TIF_NEED_RESCHED); -+} -+ -+#else -+static inline void clear_tsk_need_resched_lazy(struct task_struct *tsk) { } -+static inline int need_resched_lazy(void) { return 0; } -+ -+static inline int need_resched_now(void) -+{ -+ return test_thread_flag(TIF_NEED_RESCHED); -+} -+ -+#endif -+ - static inline int restart_syscall(void) + /* +@@ -1986,7 +2013,7 @@ + ACCESS_ONCE(rsp->gpnum), + TPS("reqwait")); + rsp->gp_state = RCU_GP_WAIT_GPS; +- wait_event_interruptible(rsp->gp_wq, ++ swait_event_interruptible(rsp->gp_wq, + ACCESS_ONCE(rsp->gp_flags) & + RCU_GP_FLAG_INIT); + /* Locking provides needed memory barrier. */ +@@ -2015,7 +2042,7 @@ + ACCESS_ONCE(rsp->gpnum), + TPS("fqswait")); + rsp->gp_state = RCU_GP_WAIT_FQS; +- ret = wait_event_interruptible_timeout(rsp->gp_wq, ++ ret = swait_event_interruptible_timeout(rsp->gp_wq, + ((gf = ACCESS_ONCE(rsp->gp_flags)) & + RCU_GP_FLAG_FQS) || + (!ACCESS_ONCE(rnp->qsmask) && +@@ -2860,18 +2887,17 @@ + /* + * Do RCU core processing for the current CPU. + */ +-static void rcu_process_callbacks(struct softirq_action *unused) ++static void rcu_process_callbacks(void) { - set_tsk_thread_flag(current, TIF_SIGPENDING); -@@ -2818,6 +2966,51 @@ - return (state & TASK_INTERRUPTIBLE) || __fatal_signal_pending(p); + struct rcu_state *rsp; + + if (cpu_is_offline(smp_processor_id())) + return; +- trace_rcu_utilization(TPS("Start RCU core")); + for_each_rcu_flavor(rsp) + __rcu_process_callbacks(rsp); +- trace_rcu_utilization(TPS("End RCU core")); } -+static inline bool __task_is_stopped_or_traced(struct task_struct *task) -+{ -+ if (task->state & (__TASK_STOPPED | __TASK_TRACED)) -+ return true; -+#ifdef CONFIG_PREEMPT_RT_FULL -+ if (task->saved_state & (__TASK_STOPPED | __TASK_TRACED)) -+ return true; -+#endif -+ return false; ++static DEFINE_PER_CPU(struct task_struct *, rcu_cpu_kthread_task); + /* + * Schedule RCU callback invocation. If the specified type of RCU + * does not support RCU priority boosting, just do a direct call, +@@ -2883,18 +2909,105 @@ + { + if (unlikely(!ACCESS_ONCE(rcu_scheduler_fully_active))) + return; +- if (likely(!rsp->boost)) { +- rcu_do_batch(rsp, rdp); ++ rcu_do_batch(rsp, rdp); +} + -+static inline bool task_is_stopped_or_traced(struct task_struct *task) ++static void rcu_wake_cond(struct task_struct *t, int status) +{ -+ bool traced_stopped; ++ /* ++ * If the thread is yielding, only wake it when this ++ * is invoked from idle ++ */ ++ if (t && (status != RCU_KTHREAD_YIELDING || is_idle_task(current))) ++ wake_up_process(t); ++} + -+#ifdef CONFIG_PREEMPT_RT_FULL ++/* ++ * Wake up this CPU's rcuc kthread to do RCU core processing. ++ */ ++static void invoke_rcu_core(void) ++{ + unsigned long flags; ++ struct task_struct *t; + -+ raw_spin_lock_irqsave(&task->pi_lock, flags); -+ traced_stopped = __task_is_stopped_or_traced(task); -+ raw_spin_unlock_irqrestore(&task->pi_lock, flags); -+#else -+ traced_stopped = __task_is_stopped_or_traced(task); -+#endif -+ return traced_stopped; ++ if (!cpu_online(smp_processor_id())) + return; ++ local_irq_save(flags); ++ __this_cpu_write(rcu_cpu_has_work, 1); ++ t = __this_cpu_read(rcu_cpu_kthread_task); ++ if (t != NULL && current != t) ++ rcu_wake_cond(t, __this_cpu_read(rcu_cpu_kthread_status)); ++ local_irq_restore(flags); +} + -+static inline bool task_is_traced(struct task_struct *task) ++static void rcu_cpu_kthread_park(unsigned int cpu) +{ -+ bool traced = false; -+ -+ if (task->state & __TASK_TRACED) -+ return true; -+#ifdef CONFIG_PREEMPT_RT_FULL -+ /* in case the task is sleeping on tasklist_lock */ -+ raw_spin_lock_irq(&task->pi_lock); -+ if (task->state & __TASK_TRACED) -+ traced = true; -+ else if (task->saved_state & __TASK_TRACED) -+ traced = true; -+ raw_spin_unlock_irq(&task->pi_lock); -+#endif -+ return traced; ++ per_cpu(rcu_cpu_kthread_status, cpu) = RCU_KTHREAD_OFFCPU; +} + - /* - * cond_resched() and cond_resched_lock(): latency reduction via - * explicit rescheduling in places that are safe. The return -@@ -2834,7 +3027,7 @@ - - extern int __cond_resched_lock(spinlock_t *lock); - --#ifdef CONFIG_PREEMPT_COUNT -+#if defined(CONFIG_PREEMPT_COUNT) && !defined(CONFIG_PREEMPT_RT_FULL) - #define PREEMPT_LOCK_OFFSET PREEMPT_OFFSET - #else - #define PREEMPT_LOCK_OFFSET 0 -@@ -2845,12 +3038,16 @@ - __cond_resched_lock(lock); \ - }) - -+#ifndef CONFIG_PREEMPT_RT_FULL - extern int __cond_resched_softirq(void); - - #define cond_resched_softirq() ({ \ - ___might_sleep(__FILE__, __LINE__, SOFTIRQ_DISABLE_OFFSET); \ - __cond_resched_softirq(); \ - }) -+#else -+# define cond_resched_softirq() cond_resched() -+#endif - - static inline void cond_resched_rcu(void) - { -@@ -3017,6 +3214,26 @@ - - #endif /* CONFIG_SMP */ - -+static inline int __migrate_disabled(struct task_struct *p) ++static int rcu_cpu_kthread_should_run(unsigned int cpu) +{ -+#ifdef CONFIG_PREEMPT_RT_FULL -+ return p->migrate_disable; -+#else -+ return 0; -+#endif ++ return __this_cpu_read(rcu_cpu_has_work); +} + -+/* Future-safe accessor for struct task_struct's cpus_allowed. */ -+static inline const struct cpumask *tsk_cpus_allowed(struct task_struct *p) ++/* ++ * Per-CPU kernel thread that invokes RCU callbacks. This replaces the ++ * RCU softirq used in flavors and configurations of RCU that do not ++ * support RCU priority boosting. ++ */ ++static void rcu_cpu_kthread(unsigned int cpu) +{ -+#ifdef CONFIG_PREEMPT_RT_FULL -+ if (p->migrate_disable) -+ return cpumask_of(task_cpu(p)); -+#endif -+ -+ return &p->cpus_allowed; -+} ++ unsigned int *statusp = this_cpu_ptr(&rcu_cpu_kthread_status); ++ char work, *workp = this_cpu_ptr(&rcu_cpu_has_work); ++ int spincnt; + - extern long sched_setaffinity(pid_t pid, const struct cpumask *new_mask); - extern long sched_getaffinity(pid_t pid, struct cpumask *mask); - -diff -Nur linux-4.1.10.orig/include/linux/seqlock.h linux-4.1.10/include/linux/seqlock.h ---- linux-4.1.10.orig/include/linux/seqlock.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/seqlock.h 2015-10-07 18:00:08.000000000 +0200 -@@ -219,20 +219,30 @@ - return __read_seqcount_retry(s, start); - } - -- -- --static inline void raw_write_seqcount_begin(seqcount_t *s) -+static inline void __raw_write_seqcount_begin(seqcount_t *s) - { - s->sequence++; - smp_wmb(); ++ for (spincnt = 0; spincnt < 10; spincnt++) { ++ trace_rcu_utilization(TPS("Start CPU kthread@rcu_wait")); ++ local_bh_disable(); ++ *statusp = RCU_KTHREAD_RUNNING; ++ this_cpu_inc(rcu_cpu_kthread_loops); ++ local_irq_disable(); ++ work = *workp; ++ *workp = 0; ++ local_irq_enable(); ++ if (work) ++ rcu_process_callbacks(); ++ local_bh_enable(); ++ if (*workp == 0) { ++ trace_rcu_utilization(TPS("End CPU kthread@rcu_wait")); ++ *statusp = RCU_KTHREAD_WAITING; ++ return; ++ } + } +- invoke_rcu_callbacks_kthread(); ++ *statusp = RCU_KTHREAD_YIELDING; ++ trace_rcu_utilization(TPS("Start CPU kthread@rcu_yield")); ++ schedule_timeout_interruptible(2); ++ trace_rcu_utilization(TPS("End CPU kthread@rcu_yield")); ++ *statusp = RCU_KTHREAD_WAITING; } --static inline void raw_write_seqcount_end(seqcount_t *s) -+static inline void raw_write_seqcount_begin(seqcount_t *s) -+{ -+ preempt_disable_rt(); -+ __raw_write_seqcount_begin(s); -+} +-static void invoke_rcu_core(void) ++static struct smp_hotplug_thread rcu_cpu_thread_spec = { ++ .store = &rcu_cpu_kthread_task, ++ .thread_should_run = rcu_cpu_kthread_should_run, ++ .thread_fn = rcu_cpu_kthread, ++ .thread_comm = "rcuc/%u", ++ .setup = rcu_cpu_kthread_setup, ++ .park = rcu_cpu_kthread_park, ++}; + -+static inline void __raw_write_seqcount_end(seqcount_t *s) ++/* ++ * Spawn per-CPU RCU core processing kthreads. ++ */ ++static int __init rcu_spawn_core_kthreads(void) { - smp_wmb(); - s->sequence++; +- if (cpu_online(smp_processor_id())) +- raise_softirq(RCU_SOFTIRQ); ++ int cpu; ++ ++ for_each_possible_cpu(cpu) ++ per_cpu(rcu_cpu_has_work, cpu) = 0; ++ BUG_ON(smpboot_register_percpu_thread(&rcu_cpu_thread_spec)); ++ return 0; } ++early_initcall(rcu_spawn_core_kthreads); -+static inline void raw_write_seqcount_end(seqcount_t *s) -+{ -+ __raw_write_seqcount_end(s); -+ preempt_enable_rt(); -+} -+ /* - * raw_write_seqcount_latch - redirect readers to even/odd copy - * @s: pointer to seqcount_t -@@ -305,10 +315,32 @@ + * Handle any core-RCU processing required by a call_rcu() invocation. +@@ -3040,6 +3153,7 @@ + } + EXPORT_SYMBOL_GPL(call_rcu_sched); + ++#ifndef CONFIG_PREEMPT_RT_FULL /* - * Read side functions for starting and finalizing a read side section. + * Queue an RCU callback for invocation after a quicker grace period. */ -+#ifndef CONFIG_PREEMPT_RT_FULL - static inline unsigned read_seqbegin(const seqlock_t *sl) - { - return read_seqcount_begin(&sl->seqcount); +@@ -3048,6 +3162,7 @@ + __call_rcu(head, func, &rcu_bh_state, -1, 0); } -+#else -+/* -+ * Starvation safe read side for RT -+ */ -+static inline unsigned read_seqbegin(seqlock_t *sl) -+{ -+ unsigned ret; -+ -+repeat: -+ ret = ACCESS_ONCE(sl->seqcount.sequence); -+ if (unlikely(ret & 1)) { -+ /* -+ * Take the lock and let the writer proceed (i.e. evtl -+ * boost it), otherwise we could loop here forever. -+ */ -+ spin_unlock_wait(&sl->lock); -+ goto repeat; -+ } -+ return ret; -+} + EXPORT_SYMBOL_GPL(call_rcu_bh); +#endif - static inline unsigned read_seqretry(const seqlock_t *sl, unsigned start) - { -@@ -323,36 +355,36 @@ - static inline void write_seqlock(seqlock_t *sl) - { - spin_lock(&sl->lock); -- write_seqcount_begin(&sl->seqcount); -+ __raw_write_seqcount_begin(&sl->seqcount); + /* + * Queue an RCU callback for lazy invocation after a grace period. +@@ -3139,6 +3254,7 @@ } + EXPORT_SYMBOL_GPL(synchronize_sched); - static inline void write_sequnlock(seqlock_t *sl) - { -- write_seqcount_end(&sl->seqcount); -+ __raw_write_seqcount_end(&sl->seqcount); - spin_unlock(&sl->lock); ++#ifndef CONFIG_PREEMPT_RT_FULL + /** + * synchronize_rcu_bh - wait until an rcu_bh grace period has elapsed. + * +@@ -3165,6 +3281,7 @@ + wait_rcu_gp(call_rcu_bh); } + EXPORT_SYMBOL_GPL(synchronize_rcu_bh); ++#endif - static inline void write_seqlock_bh(seqlock_t *sl) - { - spin_lock_bh(&sl->lock); -- write_seqcount_begin(&sl->seqcount); -+ __raw_write_seqcount_begin(&sl->seqcount); + /** + * get_state_synchronize_rcu - Snapshot current RCU state +@@ -3677,6 +3794,7 @@ + mutex_unlock(&rsp->barrier_mutex); } - static inline void write_sequnlock_bh(seqlock_t *sl) - { -- write_seqcount_end(&sl->seqcount); -+ __raw_write_seqcount_end(&sl->seqcount); - spin_unlock_bh(&sl->lock); ++#ifndef CONFIG_PREEMPT_RT_FULL + /** + * rcu_barrier_bh - Wait until all in-flight call_rcu_bh() callbacks complete. + */ +@@ -3685,6 +3803,7 @@ + _rcu_barrier(&rcu_bh_state); } + EXPORT_SYMBOL_GPL(rcu_barrier_bh); ++#endif - static inline void write_seqlock_irq(seqlock_t *sl) - { - spin_lock_irq(&sl->lock); -- write_seqcount_begin(&sl->seqcount); -+ __raw_write_seqcount_begin(&sl->seqcount); - } + /** + * rcu_barrier_sched - Wait for in-flight call_rcu_sched() callbacks. +@@ -4021,7 +4140,7 @@ + } + } - static inline void write_sequnlock_irq(seqlock_t *sl) - { -- write_seqcount_end(&sl->seqcount); -+ __raw_write_seqcount_end(&sl->seqcount); - spin_unlock_irq(&sl->lock); - } +- init_waitqueue_head(&rsp->gp_wq); ++ init_swait_head(&rsp->gp_wq); + rnp = rsp->level[rcu_num_lvls - 1]; + for_each_possible_cpu(i) { + while (i > rnp->grphi) +@@ -4120,7 +4239,6 @@ + rcu_init_one(&rcu_bh_state, &rcu_bh_data); + rcu_init_one(&rcu_sched_state, &rcu_sched_data); + __rcu_init_preempt(); +- open_softirq(RCU_SOFTIRQ, rcu_process_callbacks); -@@ -361,7 +393,7 @@ - unsigned long flags; + /* + * We don't need protection against CPU-hotplug here because +diff -Nur linux-4.1.10.orig/kernel/rcu/tree.h linux-4.1.10/kernel/rcu/tree.h +--- linux-4.1.10.orig/kernel/rcu/tree.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/rcu/tree.h 2015-10-12 22:33:32.300675168 +0200 +@@ -27,6 +27,7 @@ + #include + #include + #include ++#include - spin_lock_irqsave(&sl->lock, flags); -- write_seqcount_begin(&sl->seqcount); -+ __raw_write_seqcount_begin(&sl->seqcount); - return flags; - } + /* + * Define shape of hierarchy based on NR_CPUS, CONFIG_RCU_FANOUT, and +@@ -210,7 +211,7 @@ + /* This can happen due to race conditions. */ + #endif /* #ifdef CONFIG_RCU_BOOST */ + #ifdef CONFIG_RCU_NOCB_CPU +- wait_queue_head_t nocb_gp_wq[2]; ++ struct swait_head nocb_gp_wq[2]; + /* Place for rcu_nocb_kthread() to wait GP. */ + #endif /* #ifdef CONFIG_RCU_NOCB_CPU */ + int need_future_gp[2]; +@@ -349,7 +350,7 @@ + atomic_long_t nocb_q_count_lazy; /* invocation (all stages). */ + struct rcu_head *nocb_follower_head; /* CBs ready to invoke. */ + struct rcu_head **nocb_follower_tail; +- wait_queue_head_t nocb_wq; /* For nocb kthreads to sleep on. */ ++ struct swait_head nocb_wq; /* For nocb kthreads to sleep on. */ + struct task_struct *nocb_kthread; + int nocb_defer_wakeup; /* Defer wakeup of nocb_kthread. */ -@@ -371,7 +403,7 @@ - static inline void - write_sequnlock_irqrestore(seqlock_t *sl, unsigned long flags) - { -- write_seqcount_end(&sl->seqcount); -+ __raw_write_seqcount_end(&sl->seqcount); - spin_unlock_irqrestore(&sl->lock, flags); - } +@@ -438,7 +439,7 @@ + unsigned long gpnum; /* Current gp number. */ + unsigned long completed; /* # of last completed gp. */ + struct task_struct *gp_kthread; /* Task for grace periods. */ +- wait_queue_head_t gp_wq; /* Where GP task waits. */ ++ struct swait_head gp_wq; /* Where GP task waits. */ + short gp_flags; /* Commands for GP task. */ + short gp_state; /* GP kthread sleep state. */ -diff -Nur linux-4.1.10.orig/include/linux/signal.h linux-4.1.10/include/linux/signal.h ---- linux-4.1.10.orig/include/linux/signal.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/signal.h 2015-10-07 18:00:08.000000000 +0200 -@@ -218,6 +218,7 @@ - } +@@ -529,12 +530,10 @@ + DECLARE_PER_CPU(struct rcu_data, rcu_preempt_data); + #endif /* #ifdef CONFIG_PREEMPT_RCU */ - extern void flush_sigqueue(struct sigpending *queue); -+extern void flush_task_sigqueue(struct task_struct *tsk); +-#ifdef CONFIG_RCU_BOOST + DECLARE_PER_CPU(unsigned int, rcu_cpu_kthread_status); + DECLARE_PER_CPU(int, rcu_cpu_kthread_cpu); + DECLARE_PER_CPU(unsigned int, rcu_cpu_kthread_loops); + DECLARE_PER_CPU(char, rcu_cpu_has_work); +-#endif /* #ifdef CONFIG_RCU_BOOST */ - /* Test if 'sig' is valid signal. Use this instead of testing _NSIG directly */ - static inline int valid_signal(unsigned long sig) -diff -Nur linux-4.1.10.orig/include/linux/skbuff.h linux-4.1.10/include/linux/skbuff.h ---- linux-4.1.10.orig/include/linux/skbuff.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/skbuff.h 2015-10-07 18:00:08.000000000 +0200 -@@ -187,6 +187,7 @@ + #ifndef RCU_TREE_NONCORE - __u32 qlen; - spinlock_t lock; -+ raw_spinlock_t raw_lock; - }; +@@ -553,10 +552,9 @@ + static void __init __rcu_init_preempt(void); + static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags); + static void rcu_preempt_boost_start_gp(struct rcu_node *rnp); +-static void invoke_rcu_callbacks_kthread(void); + static bool rcu_is_callbacks_kthread(void); ++static void rcu_cpu_kthread_setup(unsigned int cpu); + #ifdef CONFIG_RCU_BOOST +-static void rcu_preempt_do_callbacks(void); + static int rcu_spawn_one_boost_kthread(struct rcu_state *rsp, + struct rcu_node *rnp); + #endif /* #ifdef CONFIG_RCU_BOOST */ +diff -Nur linux-4.1.10.orig/kernel/rcu/tree_plugin.h linux-4.1.10/kernel/rcu/tree_plugin.h +--- linux-4.1.10.orig/kernel/rcu/tree_plugin.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/rcu/tree_plugin.h 2015-10-12 22:33:32.300675168 +0200 +@@ -24,27 +24,20 @@ + * Paul E. McKenney + */ - struct sk_buff; -@@ -1336,6 +1337,12 @@ - __skb_queue_head_init(list); - } +-#include +-#include +-#include +-#include +-#include "../time/tick-internal.h" +- + #ifdef CONFIG_RCU_BOOST -+static inline void skb_queue_head_init_raw(struct sk_buff_head *list) -+{ -+ raw_spin_lock_init(&list->raw_lock); -+ __skb_queue_head_init(list); -+} -+ - static inline void skb_queue_head_init_class(struct sk_buff_head *list, - struct lock_class_key *class) - { -diff -Nur linux-4.1.10.orig/include/linux/smp.h linux-4.1.10/include/linux/smp.h ---- linux-4.1.10.orig/include/linux/smp.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/smp.h 2015-10-07 18:00:08.000000000 +0200 -@@ -185,6 +185,9 @@ - #define get_cpu() ({ preempt_disable(); smp_processor_id(); }) - #define put_cpu() preempt_enable() + #include "../locking/rtmutex_common.h" -+#define get_cpu_light() ({ migrate_disable(); smp_processor_id(); }) -+#define put_cpu_light() migrate_enable() ++#endif /* #ifdef CONFIG_RCU_BOOST */ + /* - * Callback to arch code if there's nosmp or maxcpus=0 on the - * boot command line: -diff -Nur linux-4.1.10.orig/include/linux/spinlock_api_smp.h linux-4.1.10/include/linux/spinlock_api_smp.h ---- linux-4.1.10.orig/include/linux/spinlock_api_smp.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/spinlock_api_smp.h 2015-10-07 18:00:08.000000000 +0200 -@@ -189,6 +189,8 @@ - return 0; - } - --#include -+#ifndef CONFIG_PREEMPT_RT_FULL -+# include -+#endif - - #endif /* __LINUX_SPINLOCK_API_SMP_H */ -diff -Nur linux-4.1.10.orig/include/linux/spinlock.h linux-4.1.10/include/linux/spinlock.h ---- linux-4.1.10.orig/include/linux/spinlock.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/spinlock.h 2015-10-07 18:00:08.000000000 +0200 -@@ -281,7 +281,11 @@ - #define raw_spin_can_lock(lock) (!raw_spin_is_locked(lock)) + * Control variables for per-CPU and per-rcu_node kthreads. These + * handle all flavors of RCU. + */ +-static DEFINE_PER_CPU(struct task_struct *, rcu_cpu_kthread_task); + DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_status); + DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_loops); + DEFINE_PER_CPU(char, rcu_cpu_has_work); - /* Include rwlock functions */ --#include -+#ifdef CONFIG_PREEMPT_RT_FULL -+# include -+#else -+# include -+#endif +-#endif /* #ifdef CONFIG_RCU_BOOST */ +- + #ifdef CONFIG_RCU_NOCB_CPU + static cpumask_var_t rcu_nocb_mask; /* CPUs to have callbacks offloaded. */ + static bool have_rcu_nocb_mask; /* Was rcu_nocb_mask allocated? */ +@@ -291,7 +284,7 @@ + } - /* - * Pull the _spin_*()/_read_*()/_write_*() functions/declarations: -@@ -292,6 +296,10 @@ - # include - #endif + /* Hardware IRQ handlers cannot block, complain if they get here. */ +- if (in_irq() || in_serving_softirq()) { ++ if (preempt_count() & (HARDIRQ_MASK | SOFTIRQ_OFFSET)) { + lockdep_rcu_suspicious(__FILE__, __LINE__, + "rcu_read_unlock() from irq or softirq with blocking in critical section!!!\n"); + pr_alert("->rcu_read_unlock_special: %#x (b: %d, nq: %d)\n", +@@ -496,15 +489,6 @@ + t->rcu_read_unlock_special.b.need_qs = true; + } -+#ifdef CONFIG_PREEMPT_RT_FULL -+# include -+#else /* PREEMPT_RT_FULL */ -+ +-#ifdef CONFIG_RCU_BOOST +- +-static void rcu_preempt_do_callbacks(void) +-{ +- rcu_do_batch(&rcu_preempt_state, this_cpu_ptr(&rcu_preempt_data)); +-} +- +-#endif /* #ifdef CONFIG_RCU_BOOST */ +- /* - * Map the spin_lock functions to the raw variants for PREEMPT_RT=n + * Queue a preemptible-RCU callback for invocation after a grace period. */ -@@ -426,4 +434,6 @@ - #define atomic_dec_and_lock(atomic, lock) \ - __cond_lock(lock, _atomic_dec_and_lock(atomic, lock)) +@@ -939,6 +923,19 @@ + + #endif /* #else #ifdef CONFIG_PREEMPT_RCU */ -+#endif /* !PREEMPT_RT_FULL */ -+ - #endif /* __LINUX_SPINLOCK_H */ -diff -Nur linux-4.1.10.orig/include/linux/spinlock_rt.h linux-4.1.10/include/linux/spinlock_rt.h ---- linux-4.1.10.orig/include/linux/spinlock_rt.h 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.10/include/linux/spinlock_rt.h 2015-10-07 18:00:08.000000000 +0200 -@@ -0,0 +1,174 @@ -+#ifndef __LINUX_SPINLOCK_RT_H -+#define __LINUX_SPINLOCK_RT_H -+ -+#ifndef __LINUX_SPINLOCK_H -+#error Do not include directly. Use spinlock.h -+#endif -+ -+#include -+ -+extern void -+__rt_spin_lock_init(spinlock_t *lock, char *name, struct lock_class_key *key); -+ -+#define spin_lock_init(slock) \ -+do { \ -+ static struct lock_class_key __key; \ -+ \ -+ rt_mutex_init(&(slock)->lock); \ -+ __rt_spin_lock_init(slock, #slock, &__key); \ -+} while (0) -+ -+extern void __lockfunc rt_spin_lock(spinlock_t *lock); -+extern unsigned long __lockfunc rt_spin_lock_trace_flags(spinlock_t *lock); -+extern void __lockfunc rt_spin_lock_nested(spinlock_t *lock, int subclass); -+extern void __lockfunc rt_spin_unlock(spinlock_t *lock); -+extern void __lockfunc rt_spin_unlock_wait(spinlock_t *lock); -+extern int __lockfunc rt_spin_trylock_irqsave(spinlock_t *lock, unsigned long *flags); -+extern int __lockfunc rt_spin_trylock_bh(spinlock_t *lock); -+extern int __lockfunc rt_spin_trylock(spinlock_t *lock); -+extern int atomic_dec_and_spin_lock(atomic_t *atomic, spinlock_t *lock); -+ +/* -+ * lockdep-less calls, for derived types like rwlock: -+ * (for trylock they can use rt_mutex_trylock() directly. ++ * If boosting, set rcuc kthreads to realtime priority. + */ -+extern void __lockfunc __rt_spin_lock(struct rt_mutex *lock); -+extern void __lockfunc __rt_spin_unlock(struct rt_mutex *lock); -+extern int __lockfunc __rt_spin_trylock(struct rt_mutex *lock); -+ -+#define spin_lock(lock) \ -+ do { \ -+ migrate_disable(); \ -+ rt_spin_lock(lock); \ -+ } while (0) -+ -+#define spin_lock_bh(lock) \ -+ do { \ -+ local_bh_disable(); \ -+ migrate_disable(); \ -+ rt_spin_lock(lock); \ -+ } while (0) -+ -+#define spin_lock_irq(lock) spin_lock(lock) -+ -+#define spin_do_trylock(lock) __cond_lock(lock, rt_spin_trylock(lock)) -+ -+#define spin_trylock(lock) \ -+({ \ -+ int __locked; \ -+ migrate_disable(); \ -+ __locked = spin_do_trylock(lock); \ -+ if (!__locked) \ -+ migrate_enable(); \ -+ __locked; \ -+}) -+ -+#ifdef CONFIG_LOCKDEP -+# define spin_lock_nested(lock, subclass) \ -+ do { \ -+ migrate_disable(); \ -+ rt_spin_lock_nested(lock, subclass); \ -+ } while (0) -+ -+#define spin_lock_bh_nested(lock, subclass) \ -+ do { \ -+ local_bh_disable(); \ -+ migrate_disable(); \ -+ rt_spin_lock_nested(lock, subclass); \ -+ } while (0) -+ -+# define spin_lock_irqsave_nested(lock, flags, subclass) \ -+ do { \ -+ typecheck(unsigned long, flags); \ -+ flags = 0; \ -+ migrate_disable(); \ -+ rt_spin_lock_nested(lock, subclass); \ -+ } while (0) -+#else -+# define spin_lock_nested(lock, subclass) spin_lock(lock) -+# define spin_lock_bh_nested(lock, subclass) spin_lock_bh(lock) -+ -+# define spin_lock_irqsave_nested(lock, flags, subclass) \ -+ do { \ -+ typecheck(unsigned long, flags); \ -+ flags = 0; \ -+ spin_lock(lock); \ -+ } while (0) -+#endif -+ -+#define spin_lock_irqsave(lock, flags) \ -+ do { \ -+ typecheck(unsigned long, flags); \ -+ flags = 0; \ -+ spin_lock(lock); \ -+ } while (0) -+ -+static inline unsigned long spin_lock_trace_flags(spinlock_t *lock) -+{ -+ unsigned long flags = 0; -+#ifdef CONFIG_TRACE_IRQFLAGS -+ flags = rt_spin_lock_trace_flags(lock); -+#else -+ spin_lock(lock); /* lock_local */ -+#endif -+ return flags; -+} -+ -+/* FIXME: we need rt_spin_lock_nest_lock */ -+#define spin_lock_nest_lock(lock, nest_lock) spin_lock_nested(lock, 0) -+ -+#define spin_unlock(lock) \ -+ do { \ -+ rt_spin_unlock(lock); \ -+ migrate_enable(); \ -+ } while (0) -+ -+#define spin_unlock_bh(lock) \ -+ do { \ -+ rt_spin_unlock(lock); \ -+ migrate_enable(); \ -+ local_bh_enable(); \ -+ } while (0) -+ -+#define spin_unlock_irq(lock) spin_unlock(lock) -+ -+#define spin_unlock_irqrestore(lock, flags) \ -+ do { \ -+ typecheck(unsigned long, flags); \ -+ (void) flags; \ -+ spin_unlock(lock); \ -+ } while (0) -+ -+#define spin_trylock_bh(lock) __cond_lock(lock, rt_spin_trylock_bh(lock)) -+#define spin_trylock_irq(lock) spin_trylock(lock) -+ -+#define spin_trylock_irqsave(lock, flags) \ -+ rt_spin_trylock_irqsave(lock, &(flags)) -+ -+#define spin_unlock_wait(lock) rt_spin_unlock_wait(lock) -+ -+#ifdef CONFIG_GENERIC_LOCKBREAK -+# define spin_is_contended(lock) ((lock)->break_lock) -+#else -+# define spin_is_contended(lock) (((void)(lock), 0)) -+#endif -+ -+static inline int spin_can_lock(spinlock_t *lock) -+{ -+ return !rt_mutex_is_locked(&lock->lock); -+} -+ -+static inline int spin_is_locked(spinlock_t *lock) ++static void rcu_cpu_kthread_setup(unsigned int cpu) +{ -+ return rt_mutex_is_locked(&lock->lock); -+} ++#ifdef CONFIG_RCU_BOOST ++ struct sched_param sp; + -+static inline void assert_spin_locked(spinlock_t *lock) -+{ -+ BUG_ON(!spin_is_locked(lock)); ++ sp.sched_priority = kthread_prio; ++ sched_setscheduler_nocheck(current, SCHED_FIFO, &sp); ++#endif /* #ifdef CONFIG_RCU_BOOST */ +} + -+#define atomic_dec_and_lock(atomic, lock) \ -+ atomic_dec_and_spin_lock(atomic, lock) -+ -+#endif -diff -Nur linux-4.1.10.orig/include/linux/spinlock_types.h linux-4.1.10/include/linux/spinlock_types.h ---- linux-4.1.10.orig/include/linux/spinlock_types.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/spinlock_types.h 2015-10-07 18:00:08.000000000 +0200 -@@ -9,80 +9,15 @@ - * Released under the General Public License (GPL). - */ + #ifdef CONFIG_RCU_BOOST --#if defined(CONFIG_SMP) --# include --#else --# include --#endif -- --#include -- --typedef struct raw_spinlock { -- arch_spinlock_t raw_lock; --#ifdef CONFIG_GENERIC_LOCKBREAK -- unsigned int break_lock; --#endif --#ifdef CONFIG_DEBUG_SPINLOCK -- unsigned int magic, owner_cpu; -- void *owner; --#endif --#ifdef CONFIG_DEBUG_LOCK_ALLOC -- struct lockdep_map dep_map; --#endif --} raw_spinlock_t; + #include "../locking/rtmutex_common.h" +@@ -970,16 +967,6 @@ + + #endif /* #else #ifdef CONFIG_RCU_TRACE */ + +-static void rcu_wake_cond(struct task_struct *t, int status) +-{ +- /* +- * If the thread is yielding, only wake it when this +- * is invoked from idle +- */ +- if (status != RCU_KTHREAD_YIELDING || is_idle_task(current)) +- wake_up_process(t); +-} - --#define SPINLOCK_MAGIC 0xdead4ead + /* + * Carry out RCU priority boosting on the task indicated by ->exp_tasks + * or ->boost_tasks, advancing the pointer to the next task in the +@@ -1125,23 +1112,6 @@ + } + + /* +- * Wake up the per-CPU kthread to invoke RCU callbacks. +- */ +-static void invoke_rcu_callbacks_kthread(void) +-{ +- unsigned long flags; - --#define SPINLOCK_OWNER_INIT ((void *)-1L) +- local_irq_save(flags); +- __this_cpu_write(rcu_cpu_has_work, 1); +- if (__this_cpu_read(rcu_cpu_kthread_task) != NULL && +- current != __this_cpu_read(rcu_cpu_kthread_task)) { +- rcu_wake_cond(__this_cpu_read(rcu_cpu_kthread_task), +- __this_cpu_read(rcu_cpu_kthread_status)); +- } +- local_irq_restore(flags); +-} - --#ifdef CONFIG_DEBUG_LOCK_ALLOC --# define SPIN_DEP_MAP_INIT(lockname) .dep_map = { .name = #lockname } --#else --# define SPIN_DEP_MAP_INIT(lockname) --#endif -+#include - --#ifdef CONFIG_DEBUG_SPINLOCK --# define SPIN_DEBUG_INIT(lockname) \ -- .magic = SPINLOCK_MAGIC, \ -- .owner_cpu = -1, \ -- .owner = SPINLOCK_OWNER_INIT, -+#ifndef CONFIG_PREEMPT_RT_FULL -+# include -+# include - #else --# define SPIN_DEBUG_INIT(lockname) -+# include -+# include -+# include - #endif +-/* + * Is the current CPU running the RCU-callbacks kthread? + * Caller must have preemption disabled. + */ +@@ -1196,67 +1166,6 @@ + return 0; + } --#define __RAW_SPIN_LOCK_INITIALIZER(lockname) \ -- { \ -- .raw_lock = __ARCH_SPIN_LOCK_UNLOCKED, \ -- SPIN_DEBUG_INIT(lockname) \ -- SPIN_DEP_MAP_INIT(lockname) } +-static void rcu_kthread_do_work(void) +-{ +- rcu_do_batch(&rcu_sched_state, this_cpu_ptr(&rcu_sched_data)); +- rcu_do_batch(&rcu_bh_state, this_cpu_ptr(&rcu_bh_data)); +- rcu_preempt_do_callbacks(); +-} - --#define __RAW_SPIN_LOCK_UNLOCKED(lockname) \ -- (raw_spinlock_t) __RAW_SPIN_LOCK_INITIALIZER(lockname) +-static void rcu_cpu_kthread_setup(unsigned int cpu) +-{ +- struct sched_param sp; - --#define DEFINE_RAW_SPINLOCK(x) raw_spinlock_t x = __RAW_SPIN_LOCK_UNLOCKED(x) +- sp.sched_priority = kthread_prio; +- sched_setscheduler_nocheck(current, SCHED_FIFO, &sp); +-} - --typedef struct spinlock { -- union { -- struct raw_spinlock rlock; +-static void rcu_cpu_kthread_park(unsigned int cpu) +-{ +- per_cpu(rcu_cpu_kthread_status, cpu) = RCU_KTHREAD_OFFCPU; +-} - --#ifdef CONFIG_DEBUG_LOCK_ALLOC --# define LOCK_PADSIZE (offsetof(struct raw_spinlock, dep_map)) -- struct { -- u8 __padding[LOCK_PADSIZE]; -- struct lockdep_map dep_map; -- }; --#endif -- }; --} spinlock_t; -- --#define __SPIN_LOCK_INITIALIZER(lockname) \ -- { { .rlock = __RAW_SPIN_LOCK_INITIALIZER(lockname) } } -- --#define __SPIN_LOCK_UNLOCKED(lockname) \ -- (spinlock_t ) __SPIN_LOCK_INITIALIZER(lockname) +-static int rcu_cpu_kthread_should_run(unsigned int cpu) +-{ +- return __this_cpu_read(rcu_cpu_has_work); +-} - --#define DEFINE_SPINLOCK(x) spinlock_t x = __SPIN_LOCK_UNLOCKED(x) +-/* +- * Per-CPU kernel thread that invokes RCU callbacks. This replaces the +- * RCU softirq used in flavors and configurations of RCU that do not +- * support RCU priority boosting. +- */ +-static void rcu_cpu_kthread(unsigned int cpu) +-{ +- unsigned int *statusp = this_cpu_ptr(&rcu_cpu_kthread_status); +- char work, *workp = this_cpu_ptr(&rcu_cpu_has_work); +- int spincnt; - --#include +- for (spincnt = 0; spincnt < 10; spincnt++) { +- trace_rcu_utilization(TPS("Start CPU kthread@rcu_wait")); +- local_bh_disable(); +- *statusp = RCU_KTHREAD_RUNNING; +- this_cpu_inc(rcu_cpu_kthread_loops); +- local_irq_disable(); +- work = *workp; +- *workp = 0; +- local_irq_enable(); +- if (work) +- rcu_kthread_do_work(); +- local_bh_enable(); +- if (*workp == 0) { +- trace_rcu_utilization(TPS("End CPU kthread@rcu_wait")); +- *statusp = RCU_KTHREAD_WAITING; +- return; +- } +- } +- *statusp = RCU_KTHREAD_YIELDING; +- trace_rcu_utilization(TPS("Start CPU kthread@rcu_yield")); +- schedule_timeout_interruptible(2); +- trace_rcu_utilization(TPS("End CPU kthread@rcu_yield")); +- *statusp = RCU_KTHREAD_WAITING; +-} - - #endif /* __LINUX_SPINLOCK_TYPES_H */ -diff -Nur linux-4.1.10.orig/include/linux/spinlock_types_nort.h linux-4.1.10/include/linux/spinlock_types_nort.h ---- linux-4.1.10.orig/include/linux/spinlock_types_nort.h 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.10/include/linux/spinlock_types_nort.h 2015-10-07 18:00:08.000000000 +0200 -@@ -0,0 +1,33 @@ -+#ifndef __LINUX_SPINLOCK_TYPES_NORT_H -+#define __LINUX_SPINLOCK_TYPES_NORT_H -+ -+#ifndef __LINUX_SPINLOCK_TYPES_H -+#error "Do not include directly. Include spinlock_types.h instead" -+#endif -+ -+/* -+ * The non RT version maps spinlocks to raw_spinlocks -+ */ -+typedef struct spinlock { -+ union { -+ struct raw_spinlock rlock; -+ -+#ifdef CONFIG_DEBUG_LOCK_ALLOC -+# define LOCK_PADSIZE (offsetof(struct raw_spinlock, dep_map)) -+ struct { -+ u8 __padding[LOCK_PADSIZE]; -+ struct lockdep_map dep_map; -+ }; -+#endif -+ }; -+} spinlock_t; -+ -+#define __SPIN_LOCK_INITIALIZER(lockname) \ -+ { { .rlock = __RAW_SPIN_LOCK_INITIALIZER(lockname) } } -+ -+#define __SPIN_LOCK_UNLOCKED(lockname) \ -+ (spinlock_t ) __SPIN_LOCK_INITIALIZER(lockname) -+ -+#define DEFINE_SPINLOCK(x) spinlock_t x = __SPIN_LOCK_UNLOCKED(x) -+ -+#endif -diff -Nur linux-4.1.10.orig/include/linux/spinlock_types_raw.h linux-4.1.10/include/linux/spinlock_types_raw.h ---- linux-4.1.10.orig/include/linux/spinlock_types_raw.h 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.10/include/linux/spinlock_types_raw.h 2015-10-07 18:00:08.000000000 +0200 -@@ -0,0 +1,56 @@ -+#ifndef __LINUX_SPINLOCK_TYPES_RAW_H -+#define __LINUX_SPINLOCK_TYPES_RAW_H -+ -+#if defined(CONFIG_SMP) -+# include -+#else -+# include -+#endif -+ -+#include -+ -+typedef struct raw_spinlock { -+ arch_spinlock_t raw_lock; -+#ifdef CONFIG_GENERIC_LOCKBREAK -+ unsigned int break_lock; -+#endif -+#ifdef CONFIG_DEBUG_SPINLOCK -+ unsigned int magic, owner_cpu; -+ void *owner; -+#endif -+#ifdef CONFIG_DEBUG_LOCK_ALLOC -+ struct lockdep_map dep_map; -+#endif -+} raw_spinlock_t; -+ -+#define SPINLOCK_MAGIC 0xdead4ead -+ -+#define SPINLOCK_OWNER_INIT ((void *)-1L) -+ -+#ifdef CONFIG_DEBUG_LOCK_ALLOC -+# define SPIN_DEP_MAP_INIT(lockname) .dep_map = { .name = #lockname } -+#else -+# define SPIN_DEP_MAP_INIT(lockname) -+#endif -+ -+#ifdef CONFIG_DEBUG_SPINLOCK -+# define SPIN_DEBUG_INIT(lockname) \ -+ .magic = SPINLOCK_MAGIC, \ -+ .owner_cpu = -1, \ -+ .owner = SPINLOCK_OWNER_INIT, -+#else -+# define SPIN_DEBUG_INIT(lockname) -+#endif -+ -+#define __RAW_SPIN_LOCK_INITIALIZER(lockname) \ -+ { \ -+ .raw_lock = __ARCH_SPIN_LOCK_UNLOCKED, \ -+ SPIN_DEBUG_INIT(lockname) \ -+ SPIN_DEP_MAP_INIT(lockname) } -+ -+#define __RAW_SPIN_LOCK_UNLOCKED(lockname) \ -+ (raw_spinlock_t) __RAW_SPIN_LOCK_INITIALIZER(lockname) -+ -+#define DEFINE_RAW_SPINLOCK(x) raw_spinlock_t x = __RAW_SPIN_LOCK_UNLOCKED(x) -+ -+#endif -diff -Nur linux-4.1.10.orig/include/linux/spinlock_types_rt.h linux-4.1.10/include/linux/spinlock_types_rt.h ---- linux-4.1.10.orig/include/linux/spinlock_types_rt.h 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.10/include/linux/spinlock_types_rt.h 2015-10-07 18:00:08.000000000 +0200 -@@ -0,0 +1,51 @@ -+#ifndef __LINUX_SPINLOCK_TYPES_RT_H -+#define __LINUX_SPINLOCK_TYPES_RT_H -+ -+#ifndef __LINUX_SPINLOCK_TYPES_H -+#error "Do not include directly. Include spinlock_types.h instead" -+#endif -+ -+#include -+ -+/* -+ * PREEMPT_RT: spinlocks - an RT mutex plus lock-break field: -+ */ -+typedef struct spinlock { -+ struct rt_mutex lock; -+ unsigned int break_lock; -+#ifdef CONFIG_DEBUG_LOCK_ALLOC -+ struct lockdep_map dep_map; -+#endif -+} spinlock_t; -+ -+#ifdef CONFIG_DEBUG_RT_MUTEXES -+# define __RT_SPIN_INITIALIZER(name) \ -+ { \ -+ .wait_lock = __RAW_SPIN_LOCK_UNLOCKED(name.wait_lock), \ -+ .save_state = 1, \ -+ .file = __FILE__, \ -+ .line = __LINE__ , \ -+ } -+#else -+# define __RT_SPIN_INITIALIZER(name) \ -+ { \ -+ .wait_lock = __RAW_SPIN_LOCK_UNLOCKED(name.wait_lock), \ -+ .save_state = 1, \ -+ } -+#endif -+ -+/* -+.wait_list = PLIST_HEAD_INIT_RAW((name).lock.wait_list, (name).lock.wait_lock) -+*/ -+ -+#define __SPIN_LOCK_UNLOCKED(name) \ -+ { .lock = __RT_SPIN_INITIALIZER(name.lock), \ -+ SPIN_DEP_MAP_INIT(name) } -+ -+#define __DEFINE_SPINLOCK(name) \ -+ spinlock_t name = __SPIN_LOCK_UNLOCKED(name) -+ -+#define DEFINE_SPINLOCK(name) \ -+ spinlock_t name __cacheline_aligned_in_smp = __SPIN_LOCK_UNLOCKED(name) -+ -+#endif -diff -Nur linux-4.1.10.orig/include/linux/srcu.h linux-4.1.10/include/linux/srcu.h ---- linux-4.1.10.orig/include/linux/srcu.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/srcu.h 2015-10-07 18:00:08.000000000 +0200 -@@ -84,10 +84,10 @@ - - void process_srcu(struct work_struct *work); + /* + * Set the per-rcu_node kthread's affinity to cover all CPUs that are + * served by the rcu_node in question. The CPU hotplug lock is still +@@ -1286,26 +1195,12 @@ + free_cpumask_var(cm); + } --#define __SRCU_STRUCT_INIT(name) \ -+#define __SRCU_STRUCT_INIT(name, pcpu_name) \ - { \ - .completed = -300, \ -- .per_cpu_ref = &name##_srcu_array, \ -+ .per_cpu_ref = &pcpu_name, \ - .queue_lock = __SPIN_LOCK_UNLOCKED(name.queue_lock), \ - .running = false, \ - .batch_queue = RCU_BATCH_INIT(name.batch_queue), \ -@@ -104,7 +104,7 @@ +-static struct smp_hotplug_thread rcu_cpu_thread_spec = { +- .store = &rcu_cpu_kthread_task, +- .thread_should_run = rcu_cpu_kthread_should_run, +- .thread_fn = rcu_cpu_kthread, +- .thread_comm = "rcuc/%u", +- .setup = rcu_cpu_kthread_setup, +- .park = rcu_cpu_kthread_park, +-}; +- + /* + * Spawn boost kthreads -- called as soon as the scheduler is running. */ - #define __DEFINE_SRCU(name, is_static) \ - static DEFINE_PER_CPU(struct srcu_struct_array, name##_srcu_array);\ -- is_static struct srcu_struct name = __SRCU_STRUCT_INIT(name) -+ is_static struct srcu_struct name = __SRCU_STRUCT_INIT(name, name##_srcu_array) - #define DEFINE_SRCU(name) __DEFINE_SRCU(name, /* not static */) - #define DEFINE_STATIC_SRCU(name) __DEFINE_SRCU(name, static) - -diff -Nur linux-4.1.10.orig/include/linux/swap.h linux-4.1.10/include/linux/swap.h ---- linux-4.1.10.orig/include/linux/swap.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/swap.h 2015-10-07 18:00:08.000000000 +0200 -@@ -11,6 +11,7 @@ - #include - #include - #include -+#include - #include - - struct notifier_block; -@@ -252,7 +253,8 @@ - void *workingset_eviction(struct address_space *mapping, struct page *page); - bool workingset_refault(void *shadow); - void workingset_activation(struct page *page); --extern struct list_lru workingset_shadow_nodes; -+extern struct list_lru __workingset_shadow_nodes; -+DECLARE_LOCAL_IRQ_LOCK(workingset_shadow_lock); - - static inline unsigned int workingset_node_pages(struct radix_tree_node *node) + static void __init rcu_spawn_boost_kthreads(void) { -@@ -296,6 +298,7 @@ + struct rcu_node *rnp; +- int cpu; +- +- for_each_possible_cpu(cpu) +- per_cpu(rcu_cpu_has_work, cpu) = 0; +- BUG_ON(smpboot_register_percpu_thread(&rcu_cpu_thread_spec)); + rcu_for_each_leaf_node(rcu_state_p, rnp) + (void)rcu_spawn_one_boost_kthread(rcu_state_p, rnp); + } +@@ -1328,11 +1223,6 @@ + raw_spin_unlock_irqrestore(&rnp->lock, flags); + } +-static void invoke_rcu_callbacks_kthread(void) +-{ +- WARN_ON_ONCE(1); +-} +- + static bool rcu_is_callbacks_kthread(void) + { + return false; +@@ -1356,7 +1246,7 @@ - /* linux/mm/swap.c */ -+DECLARE_LOCAL_IRQ_LOCK(swapvec_lock); - extern void lru_cache_add(struct page *); - extern void lru_cache_add_anon(struct page *page); - extern void lru_cache_add_file(struct page *page); -diff -Nur linux-4.1.10.orig/include/linux/thread_info.h linux-4.1.10/include/linux/thread_info.h ---- linux-4.1.10.orig/include/linux/thread_info.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/thread_info.h 2015-10-07 18:00:08.000000000 +0200 -@@ -102,7 +102,17 @@ - #define test_thread_flag(flag) \ - test_ti_thread_flag(current_thread_info(), flag) + #endif /* #else #ifdef CONFIG_RCU_BOOST */ --#define tif_need_resched() test_thread_flag(TIF_NEED_RESCHED) -+#ifdef CONFIG_PREEMPT_LAZY -+#define tif_need_resched() (test_thread_flag(TIF_NEED_RESCHED) || \ -+ test_thread_flag(TIF_NEED_RESCHED_LAZY)) -+#define tif_need_resched_now() (test_thread_flag(TIF_NEED_RESCHED)) -+#define tif_need_resched_lazy() test_thread_flag(TIF_NEED_RESCHED_LAZY)) -+ -+#else -+#define tif_need_resched() test_thread_flag(TIF_NEED_RESCHED) -+#define tif_need_resched_now() test_thread_flag(TIF_NEED_RESCHED) -+#define tif_need_resched_lazy() 0 -+#endif +-#if !defined(CONFIG_RCU_FAST_NO_HZ) ++#if !defined(CONFIG_RCU_FAST_NO_HZ) || defined(CONFIG_PREEMPT_RT_FULL) - #if defined TIF_RESTORE_SIGMASK && !defined HAVE_SET_RESTORE_SIGMASK /* -diff -Nur linux-4.1.10.orig/include/linux/timer.h linux-4.1.10/include/linux/timer.h ---- linux-4.1.10.orig/include/linux/timer.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/timer.h 2015-10-07 18:00:08.000000000 +0200 -@@ -241,7 +241,7 @@ - - extern int try_to_del_timer_sync(struct timer_list *timer); - --#ifdef CONFIG_SMP -+#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT_FULL) - extern int del_timer_sync(struct timer_list *timer); - #else - # define del_timer_sync(t) del_timer(t) -diff -Nur linux-4.1.10.orig/include/linux/uaccess.h linux-4.1.10/include/linux/uaccess.h ---- linux-4.1.10.orig/include/linux/uaccess.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/uaccess.h 2015-10-07 18:00:08.000000000 +0200 -@@ -1,21 +1,31 @@ - #ifndef __LINUX_UACCESS_H__ - #define __LINUX_UACCESS_H__ + * Check to see if any future RCU-related work will need to be done +@@ -1374,7 +1264,9 @@ + return rcu_cpu_has_callbacks(NULL); + } + #endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */ ++#endif /* !defined(CONFIG_RCU_FAST_NO_HZ) || defined(CONFIG_PREEMPT_RT_FULL) */ --#include -+#include - #include ++#if !defined(CONFIG_RCU_FAST_NO_HZ) + /* + * Because we do not have RCU_FAST_NO_HZ, don't bother cleaning up + * after it. +@@ -1472,6 +1364,8 @@ + return cbs_ready; + } -+static __always_inline void pagefault_disabled_inc(void) -+{ -+ current->pagefault_disabled++; -+} -+ -+static __always_inline void pagefault_disabled_dec(void) -+{ -+ current->pagefault_disabled--; -+ WARN_ON(current->pagefault_disabled < 0); -+} ++#ifndef CONFIG_PREEMPT_RT_FULL + /* -- * These routines enable/disable the pagefault handler in that -- * it will not take any locks and go straight to the fixup table. -+ * These routines enable/disable the pagefault handler. If disabled, it will -+ * not take any locks and go straight to the fixup table. - * -- * They have great resemblance to the preempt_disable/enable calls -- * and in fact they are identical; this is because currently there is -- * no other way to make the pagefault handlers do this. So we do -- * disable preemption but we don't necessarily care about that. -+ * User access methods will not sleep when called from a pagefault_disabled() -+ * environment. + * Allow the CPU to enter dyntick-idle mode unless it has callbacks ready + * to invoke. If the CPU has callbacks, try to advance them. Tell the +@@ -1512,7 +1406,7 @@ + return 0; + } + #endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */ +- ++#endif /* #ifndef CONFIG_PREEMPT_RT_FULL */ + /* + * Prepare a CPU for idle from an RCU perspective. The first major task + * is to sense whether nohz mode has been enabled or disabled via sysfs. +@@ -1859,7 +1753,7 @@ */ - static inline void pagefault_disable(void) + static void rcu_nocb_gp_cleanup(struct rcu_state *rsp, struct rcu_node *rnp) { -- preempt_count_inc(); -+ migrate_disable(); -+ pagefault_disabled_inc(); - /* - * make sure to have issued the store before a pagefault - * can hit. -@@ -25,18 +35,32 @@ +- wake_up_all(&rnp->nocb_gp_wq[rnp->completed & 0x1]); ++ swait_wake_all(&rnp->nocb_gp_wq[rnp->completed & 0x1]); + } - static inline void pagefault_enable(void) + /* +@@ -1877,8 +1771,8 @@ + + static void rcu_init_one_nocb(struct rcu_node *rnp) { --#ifndef CONFIG_PREEMPT - /* - * make sure to issue those last loads/stores before enabling - * the pagefault handler again. - */ - barrier(); -- preempt_count_dec(); --#else -- preempt_enable(); --#endif -+ pagefault_disabled_dec(); -+ migrate_enable(); +- init_waitqueue_head(&rnp->nocb_gp_wq[0]); +- init_waitqueue_head(&rnp->nocb_gp_wq[1]); ++ init_swait_head(&rnp->nocb_gp_wq[0]); ++ init_swait_head(&rnp->nocb_gp_wq[1]); } -+/* -+ * Is the pagefault handler disabled? If so, user access methods will not sleep. -+ */ -+#define pagefault_disabled() (current->pagefault_disabled != 0) -+ -+/* -+ * The pagefault handler is in general disabled by pagefault_disable() or -+ * when in irq context (via in_atomic()). -+ * -+ * This function should only be used by the fault handlers. Other users should -+ * stick to pagefault_disabled(). -+ * Please NEVER use preempt_disable() to disable the fault handler. With -+ * !CONFIG_PREEMPT_COUNT, this is like a NOP. So the handler won't be disabled. -+ * in_atomic() will report different values based on !CONFIG_PREEMPT_COUNT. -+ */ -+#define faulthandler_disabled() (pagefault_disabled() || in_atomic()) -+ - #ifndef ARCH_HAS_NOCACHE_UACCESS + #ifndef CONFIG_RCU_NOCB_CPU_ALL +@@ -1903,7 +1797,7 @@ + if (ACCESS_ONCE(rdp_leader->nocb_leader_sleep) || force) { + /* Prior smp_mb__after_atomic() orders against prior enqueue. */ + ACCESS_ONCE(rdp_leader->nocb_leader_sleep) = false; +- wake_up(&rdp_leader->nocb_wq); ++ swait_wake(&rdp_leader->nocb_wq); + } + } - static inline unsigned long __copy_from_user_inatomic_nocache(void *to, -diff -Nur linux-4.1.10.orig/include/linux/uprobes.h linux-4.1.10/include/linux/uprobes.h ---- linux-4.1.10.orig/include/linux/uprobes.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/uprobes.h 2015-10-07 18:00:08.000000000 +0200 -@@ -27,6 +27,7 @@ - #include - #include - #include -+#include +@@ -2116,7 +2010,7 @@ + */ + trace_rcu_future_gp(rnp, rdp, c, TPS("StartWait")); + for (;;) { +- wait_event_interruptible( ++ swait_event_interruptible( + rnp->nocb_gp_wq[c & 0x1], + (d = ULONG_CMP_GE(ACCESS_ONCE(rnp->completed), c))); + if (likely(d)) +@@ -2144,7 +2038,7 @@ + /* Wait for callbacks to appear. */ + if (!rcu_nocb_poll) { + trace_rcu_nocb_wake(my_rdp->rsp->name, my_rdp->cpu, "Sleep"); +- wait_event_interruptible(my_rdp->nocb_wq, ++ swait_event_interruptible(my_rdp->nocb_wq, + !ACCESS_ONCE(my_rdp->nocb_leader_sleep)); + /* Memory barrier handled by smp_mb() calls below and repoll. */ + } else if (firsttime) { +@@ -2219,7 +2113,7 @@ + * List was empty, wake up the follower. + * Memory barriers supplied by atomic_long_add(). + */ +- wake_up(&rdp->nocb_wq); ++ swait_wake(&rdp->nocb_wq); + } + } - struct vm_area_struct; - struct mm_struct; -diff -Nur linux-4.1.10.orig/include/linux/vmstat.h linux-4.1.10/include/linux/vmstat.h ---- linux-4.1.10.orig/include/linux/vmstat.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/vmstat.h 2015-10-07 18:00:08.000000000 +0200 -@@ -33,7 +33,9 @@ - */ - static inline void __count_vm_event(enum vm_event_item item) +@@ -2240,7 +2134,7 @@ + if (!rcu_nocb_poll) { + trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, + "FollowerSleep"); +- wait_event_interruptible(rdp->nocb_wq, ++ swait_event_interruptible(rdp->nocb_wq, + ACCESS_ONCE(rdp->nocb_follower_head)); + } else if (firsttime) { + /* Don't drown trace log with "Poll"! */ +@@ -2399,7 +2293,7 @@ + static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp) { -+ preempt_disable_rt(); - raw_cpu_inc(vm_event_states.event[item]); -+ preempt_enable_rt(); + rdp->nocb_tail = &rdp->nocb_head; +- init_waitqueue_head(&rdp->nocb_wq); ++ init_swait_head(&rdp->nocb_wq); + rdp->nocb_follower_tail = &rdp->nocb_follower_head; } - static inline void count_vm_event(enum vm_event_item item) -@@ -43,7 +45,9 @@ - - static inline void __count_vm_events(enum vm_event_item item, long delta) - { -+ preempt_disable_rt(); - raw_cpu_add(vm_event_states.event[item], delta); -+ preempt_enable_rt(); +diff -Nur linux-4.1.10.orig/kernel/rcu/update.c linux-4.1.10/kernel/rcu/update.c +--- linux-4.1.10.orig/kernel/rcu/update.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/rcu/update.c 2015-10-12 22:33:32.300675168 +0200 +@@ -227,6 +227,7 @@ } + EXPORT_SYMBOL_GPL(rcu_read_lock_held); - static inline void count_vm_events(enum vm_event_item item, long delta) -diff -Nur linux-4.1.10.orig/include/linux/wait.h linux-4.1.10/include/linux/wait.h ---- linux-4.1.10.orig/include/linux/wait.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/linux/wait.h 2015-10-07 18:00:08.000000000 +0200 -@@ -8,6 +8,7 @@ - #include - #include - #include -+#include ++#ifndef CONFIG_PREEMPT_RT_FULL + /** + * rcu_read_lock_bh_held() - might we be in RCU-bh read-side critical section? + * +@@ -253,6 +254,7 @@ + return in_softirq() || irqs_disabled(); + } + EXPORT_SYMBOL_GPL(rcu_read_lock_bh_held); ++#endif - typedef struct __wait_queue wait_queue_t; - typedef int (*wait_queue_func_t)(wait_queue_t *wait, unsigned mode, int flags, void *key); -diff -Nur linux-4.1.10.orig/include/linux/wait-simple.h linux-4.1.10/include/linux/wait-simple.h ---- linux-4.1.10.orig/include/linux/wait-simple.h 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.10/include/linux/wait-simple.h 2015-10-07 18:00:08.000000000 +0200 -@@ -0,0 +1,207 @@ -+#ifndef _LINUX_WAIT_SIMPLE_H -+#define _LINUX_WAIT_SIMPLE_H -+ -+#include -+#include -+ -+#include -+ -+struct swaiter { -+ struct task_struct *task; -+ struct list_head node; -+}; -+ -+#define DEFINE_SWAITER(name) \ -+ struct swaiter name = { \ -+ .task = current, \ -+ .node = LIST_HEAD_INIT((name).node), \ -+ } -+ -+struct swait_head { -+ raw_spinlock_t lock; -+ struct list_head list; -+}; -+ -+#define SWAIT_HEAD_INITIALIZER(name) { \ -+ .lock = __RAW_SPIN_LOCK_UNLOCKED(name.lock), \ -+ .list = LIST_HEAD_INIT((name).list), \ -+ } -+ -+#define DEFINE_SWAIT_HEAD(name) \ -+ struct swait_head name = SWAIT_HEAD_INITIALIZER(name) -+ -+extern void __init_swait_head(struct swait_head *h, struct lock_class_key *key); + #endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ + +diff -Nur linux-4.1.10.orig/kernel/relay.c linux-4.1.10/kernel/relay.c +--- linux-4.1.10.orig/kernel/relay.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/relay.c 2015-10-12 22:33:32.304674904 +0200 +@@ -339,6 +339,10 @@ + { + struct rchan_buf *buf = (struct rchan_buf *)data; + wake_up_interruptible(&buf->read_wait); ++ /* ++ * Stupid polling for now: ++ */ ++ mod_timer(&buf->timer, jiffies + 1); + } + + /** +@@ -356,6 +360,7 @@ + init_waitqueue_head(&buf->read_wait); + kref_init(&buf->kref); + setup_timer(&buf->timer, wakeup_readers, (unsigned long)buf); ++ mod_timer(&buf->timer, jiffies + 1); + } else + del_timer_sync(&buf->timer); + +@@ -739,15 +744,6 @@ + else + buf->early_bytes += buf->chan->subbuf_size - + buf->padding[old_subbuf]; +- smp_mb(); +- if (waitqueue_active(&buf->read_wait)) +- /* +- * Calling wake_up_interruptible() from here +- * will deadlock if we happen to be logging +- * from the scheduler (trying to re-grab +- * rq->lock), so defer it. +- */ +- mod_timer(&buf->timer, jiffies + 1); + } + + old = buf->data; +diff -Nur linux-4.1.10.orig/kernel/sched/completion.c linux-4.1.10/kernel/sched/completion.c +--- linux-4.1.10.orig/kernel/sched/completion.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/sched/completion.c 2015-10-12 22:33:32.304674904 +0200 +@@ -30,10 +30,10 @@ + { + unsigned long flags; + +- spin_lock_irqsave(&x->wait.lock, flags); ++ raw_spin_lock_irqsave(&x->wait.lock, flags); + x->done++; +- __wake_up_locked(&x->wait, TASK_NORMAL, 1); +- spin_unlock_irqrestore(&x->wait.lock, flags); ++ __swait_wake_locked(&x->wait, TASK_NORMAL, 1); ++ raw_spin_unlock_irqrestore(&x->wait.lock, flags); + } + EXPORT_SYMBOL(complete); + +@@ -50,10 +50,10 @@ + { + unsigned long flags; + +- spin_lock_irqsave(&x->wait.lock, flags); ++ raw_spin_lock_irqsave(&x->wait.lock, flags); + x->done += UINT_MAX/2; +- __wake_up_locked(&x->wait, TASK_NORMAL, 0); +- spin_unlock_irqrestore(&x->wait.lock, flags); ++ __swait_wake_locked(&x->wait, TASK_NORMAL, 0); ++ raw_spin_unlock_irqrestore(&x->wait.lock, flags); + } + EXPORT_SYMBOL(complete_all); + +@@ -62,20 +62,20 @@ + long (*action)(long), long timeout, int state) + { + if (!x->done) { +- DECLARE_WAITQUEUE(wait, current); ++ DEFINE_SWAITER(wait); + +- __add_wait_queue_tail_exclusive(&x->wait, &wait); ++ swait_prepare_locked(&x->wait, &wait); + do { + if (signal_pending_state(state, current)) { + timeout = -ERESTARTSYS; + break; + } + __set_current_state(state); +- spin_unlock_irq(&x->wait.lock); ++ raw_spin_unlock_irq(&x->wait.lock); + timeout = action(timeout); +- spin_lock_irq(&x->wait.lock); ++ raw_spin_lock_irq(&x->wait.lock); + } while (!x->done && timeout); +- __remove_wait_queue(&x->wait, &wait); ++ swait_finish_locked(&x->wait, &wait); + if (!x->done) + return timeout; + } +@@ -89,9 +89,9 @@ + { + might_sleep(); + +- spin_lock_irq(&x->wait.lock); ++ raw_spin_lock_irq(&x->wait.lock); + timeout = do_wait_for_common(x, action, timeout, state); +- spin_unlock_irq(&x->wait.lock); ++ raw_spin_unlock_irq(&x->wait.lock); + return timeout; + } + +@@ -277,12 +277,12 @@ + if (!READ_ONCE(x->done)) + return 0; + +- spin_lock_irqsave(&x->wait.lock, flags); ++ raw_spin_lock_irqsave(&x->wait.lock, flags); + if (!x->done) + ret = 0; + else + x->done--; +- spin_unlock_irqrestore(&x->wait.lock, flags); ++ raw_spin_unlock_irqrestore(&x->wait.lock, flags); + return ret; + } + EXPORT_SYMBOL(try_wait_for_completion); +@@ -311,7 +311,7 @@ + * after it's acquired the lock. + */ + smp_rmb(); +- spin_unlock_wait(&x->wait.lock); ++ raw_spin_unlock_wait(&x->wait.lock); + return true; + } + EXPORT_SYMBOL(completion_done); +diff -Nur linux-4.1.10.orig/kernel/sched/core.c linux-4.1.10/kernel/sched/core.c +--- linux-4.1.10.orig/kernel/sched/core.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/sched/core.c 2015-10-12 22:33:32.304674904 +0200 +@@ -282,7 +282,11 @@ + * Number of tasks to iterate in a single balance run. + * Limited because this is done with IRQs disabled. + */ ++#ifndef CONFIG_PREEMPT_RT_FULL + const_debug unsigned int sysctl_sched_nr_migrate = 32; ++#else ++const_debug unsigned int sysctl_sched_nr_migrate = 8; ++#endif + + /* + * period over which we average the RT time consumption, measured +@@ -461,6 +465,7 @@ + + hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + rq->hrtick_timer.function = hrtick; ++ rq->hrtick_timer.irqsafe = 1; + } + #else /* CONFIG_SCHED_HRTICK */ + static inline void hrtick_clear(struct rq *rq) +@@ -541,6 +546,52 @@ + #endif + #endif + ++void wake_q_add(struct wake_q_head *head, struct task_struct *task) ++{ ++ struct wake_q_node *node = &task->wake_q; + -+#define init_swait_head(swh) \ -+ do { \ -+ static struct lock_class_key __key; \ -+ \ -+ __init_swait_head((swh), &__key); \ -+ } while (0) ++ /* ++ * Atomically grab the task, if ->wake_q is !nil already it means ++ * its already queued (either by us or someone else) and will get the ++ * wakeup due to that. ++ * ++ * This cmpxchg() implies a full barrier, which pairs with the write ++ * barrier implied by the wakeup in wake_up_list(). ++ */ ++ if (cmpxchg(&node->next, NULL, WAKE_Q_TAIL)) ++ return; + -+/* -+ * Waiter functions -+ */ -+extern void swait_prepare_locked(struct swait_head *head, struct swaiter *w); -+extern void swait_prepare(struct swait_head *head, struct swaiter *w, int state); -+extern void swait_finish_locked(struct swait_head *head, struct swaiter *w); -+extern void swait_finish(struct swait_head *head, struct swaiter *w); ++ get_task_struct(task); + -+/* Check whether a head has waiters enqueued */ -+static inline bool swaitqueue_active(struct swait_head *h) -+{ -+ /* Make sure the condition is visible before checking list_empty() */ -+ smp_mb(); -+ return !list_empty(&h->list); ++ /* ++ * The head is context local, there can be no concurrency. ++ */ ++ *head->lastp = node; ++ head->lastp = &node->next; +} + -+/* -+ * Wakeup functions -+ */ -+extern unsigned int __swait_wake(struct swait_head *head, unsigned int state, unsigned int num); -+extern unsigned int __swait_wake_locked(struct swait_head *head, unsigned int state, unsigned int num); -+ -+#define swait_wake(head) __swait_wake(head, TASK_NORMAL, 1) -+#define swait_wake_interruptible(head) __swait_wake(head, TASK_INTERRUPTIBLE, 1) -+#define swait_wake_all(head) __swait_wake(head, TASK_NORMAL, 0) -+#define swait_wake_all_interruptible(head) __swait_wake(head, TASK_INTERRUPTIBLE, 0) -+ -+/* -+ * Event API -+ */ -+#define __swait_event(wq, condition) \ -+do { \ -+ DEFINE_SWAITER(__wait); \ -+ \ -+ for (;;) { \ -+ swait_prepare(&wq, &__wait, TASK_UNINTERRUPTIBLE); \ -+ if (condition) \ -+ break; \ -+ schedule(); \ -+ } \ -+ swait_finish(&wq, &__wait); \ -+} while (0) ++void wake_up_q(struct wake_q_head *head) ++{ ++ struct wake_q_node *node = head->first; + -+/** -+ * swait_event - sleep until a condition gets true -+ * @wq: the waitqueue to wait on -+ * @condition: a C expression for the event to wait for -+ * -+ * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the -+ * @condition evaluates to true. The @condition is checked each time -+ * the waitqueue @wq is woken up. -+ * -+ * wake_up() has to be called after changing any variable that could -+ * change the result of the wait condition. -+ */ -+#define swait_event(wq, condition) \ -+do { \ -+ if (condition) \ -+ break; \ -+ __swait_event(wq, condition); \ -+} while (0) ++ while (node != WAKE_Q_TAIL) { ++ struct task_struct *task; + -+#define __swait_event_interruptible(wq, condition, ret) \ -+do { \ -+ DEFINE_SWAITER(__wait); \ -+ \ -+ for (;;) { \ -+ swait_prepare(&wq, &__wait, TASK_INTERRUPTIBLE); \ -+ if (condition) \ -+ break; \ -+ if (signal_pending(current)) { \ -+ ret = -ERESTARTSYS; \ -+ break; \ -+ } \ -+ schedule(); \ -+ } \ -+ swait_finish(&wq, &__wait); \ -+} while (0) ++ task = container_of(node, struct task_struct, wake_q); ++ BUG_ON(!task); ++ /* task can safely be re-inserted now */ ++ node = node->next; ++ task->wake_q.next = NULL; + -+#define __swait_event_interruptible_timeout(wq, condition, ret) \ -+do { \ -+ DEFINE_SWAITER(__wait); \ -+ \ -+ for (;;) { \ -+ swait_prepare(&wq, &__wait, TASK_INTERRUPTIBLE); \ -+ if (condition) \ -+ break; \ -+ if (signal_pending(current)) { \ -+ ret = -ERESTARTSYS; \ -+ break; \ -+ } \ -+ ret = schedule_timeout(ret); \ -+ if (!ret) \ -+ break; \ -+ } \ -+ swait_finish(&wq, &__wait); \ -+} while (0) ++ /* ++ * wake_up_process() implies a wmb() to pair with the queueing ++ * in wake_q_add() so as not to miss wakeups. ++ */ ++ wake_up_process(task); ++ put_task_struct(task); ++ } ++} + -+/** -+ * swait_event_interruptible - sleep until a condition gets true -+ * @wq: the waitqueue to wait on -+ * @condition: a C expression for the event to wait for -+ * -+ * The process is put to sleep (TASK_INTERRUPTIBLE) until the -+ * @condition evaluates to true. The @condition is checked each time -+ * the waitqueue @wq is woken up. -+ * -+ * wake_up() has to be called after changing any variable that could -+ * change the result of the wait condition. -+ */ -+#define swait_event_interruptible(wq, condition) \ -+({ \ -+ int __ret = 0; \ -+ if (!(condition)) \ -+ __swait_event_interruptible(wq, condition, __ret); \ -+ __ret; \ -+}) + /* + * resched_curr - mark rq's current task 'to be rescheduled now'. + * +@@ -572,6 +623,38 @@ + trace_sched_wake_idle_without_ipi(cpu); + } + ++#ifdef CONFIG_PREEMPT_LAZY ++void resched_curr_lazy(struct rq *rq) ++{ ++ struct task_struct *curr = rq->curr; ++ int cpu; + -+#define swait_event_interruptible_timeout(wq, condition, timeout) \ -+({ \ -+ int __ret = timeout; \ -+ if (!(condition)) \ -+ __swait_event_interruptible_timeout(wq, condition, __ret); \ -+ __ret; \ -+}) ++ if (!sched_feat(PREEMPT_LAZY)) { ++ resched_curr(rq); ++ return; ++ } + -+#define __swait_event_timeout(wq, condition, ret) \ -+do { \ -+ DEFINE_SWAITER(__wait); \ -+ \ -+ for (;;) { \ -+ swait_prepare(&wq, &__wait, TASK_UNINTERRUPTIBLE); \ -+ if (condition) \ -+ break; \ -+ ret = schedule_timeout(ret); \ -+ if (!ret) \ -+ break; \ -+ } \ -+ swait_finish(&wq, &__wait); \ -+} while (0) ++ lockdep_assert_held(&rq->lock); + -+/** -+ * swait_event_timeout - sleep until a condition gets true or a timeout elapses -+ * @wq: the waitqueue to wait on -+ * @condition: a C expression for the event to wait for -+ * @timeout: timeout, in jiffies -+ * -+ * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the -+ * @condition evaluates to true. The @condition is checked each time -+ * the waitqueue @wq is woken up. -+ * -+ * wake_up() has to be called after changing any variable that could -+ * change the result of the wait condition. -+ * -+ * The function returns 0 if the @timeout elapsed, and the remaining -+ * jiffies if the condition evaluated to true before the timeout elapsed. -+ */ -+#define swait_event_timeout(wq, condition, timeout) \ -+({ \ -+ long __ret = timeout; \ -+ if (!(condition)) \ -+ __swait_event_timeout(wq, condition, __ret); \ -+ __ret; \ -+}) ++ if (test_tsk_need_resched(curr)) ++ return; + -+#endif -diff -Nur linux-4.1.10.orig/include/linux/work-simple.h linux-4.1.10/include/linux/work-simple.h ---- linux-4.1.10.orig/include/linux/work-simple.h 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.10/include/linux/work-simple.h 2015-10-07 18:00:08.000000000 +0200 -@@ -0,0 +1,24 @@ -+#ifndef _LINUX_SWORK_H -+#define _LINUX_SWORK_H ++ if (test_tsk_need_resched_lazy(curr)) ++ return; + -+#include ++ set_tsk_need_resched_lazy(curr); + -+struct swork_event { -+ struct list_head item; -+ unsigned long flags; -+ void (*func)(struct swork_event *); -+}; ++ cpu = cpu_of(rq); ++ if (cpu == smp_processor_id()) ++ return; + -+static inline void INIT_SWORK(struct swork_event *event, -+ void (*func)(struct swork_event *)) -+{ -+ event->flags = 0; -+ event->func = func; ++ /* NEED_RESCHED_LAZY must be visible before we test polling */ ++ smp_mb(); ++ if (!tsk_is_polling(curr)) ++ smp_send_reschedule(cpu); +} ++#endif + -+bool swork_queue(struct swork_event *sev); -+ -+int swork_get(void); -+void swork_put(void); -+ -+#endif /* _LINUX_SWORK_H */ -diff -Nur linux-4.1.10.orig/include/net/dst.h linux-4.1.10/include/net/dst.h ---- linux-4.1.10.orig/include/net/dst.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/net/dst.h 2015-10-07 18:00:08.000000000 +0200 -@@ -403,7 +403,7 @@ - static inline int dst_neigh_output(struct dst_entry *dst, struct neighbour *n, - struct sk_buff *skb) + void resched_cpu(int cpu) { -- const struct hh_cache *hh; -+ struct hh_cache *hh; + struct rq *rq = cpu_rq(cpu); +@@ -595,12 +678,14 @@ + */ + int get_nohz_timer_target(int pinned) + { +- int cpu = smp_processor_id(); ++ int cpu; + int i; + struct sched_domain *sd; - if (dst->pending_confirm) { - unsigned long now = jiffies; -diff -Nur linux-4.1.10.orig/include/net/neighbour.h linux-4.1.10/include/net/neighbour.h ---- linux-4.1.10.orig/include/net/neighbour.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/net/neighbour.h 2015-10-07 18:00:08.000000000 +0200 -@@ -445,7 +445,7 @@ - } - #endif ++ preempt_disable_rt(); ++ cpu = smp_processor_id(); + if (pinned || !get_sysctl_timer_migration() || !idle_cpu(cpu)) +- return cpu; ++ goto preempt_en_rt; --static inline int neigh_hh_output(const struct hh_cache *hh, struct sk_buff *skb) -+static inline int neigh_hh_output(struct hh_cache *hh, struct sk_buff *skb) - { - unsigned int seq; - int hh_len; -@@ -500,7 +500,7 @@ + rcu_read_lock(); + for_each_domain(cpu, sd) { +@@ -613,6 +698,8 @@ + } + unlock: + rcu_read_unlock(); ++preempt_en_rt: ++ preempt_enable_rt(); + return cpu; + } + /* +@@ -1164,6 +1251,18 @@ - #define NEIGH_CB(skb) ((struct neighbour_cb *)(skb)->cb) + static int migration_cpu_stop(void *data); --static inline void neigh_ha_snapshot(char *dst, const struct neighbour *n, -+static inline void neigh_ha_snapshot(char *dst, struct neighbour *n, - const struct net_device *dev) - { - unsigned int seq; -diff -Nur linux-4.1.10.orig/include/net/netns/ipv4.h linux-4.1.10/include/net/netns/ipv4.h ---- linux-4.1.10.orig/include/net/netns/ipv4.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/include/net/netns/ipv4.h 2015-10-07 18:00:08.000000000 +0200 -@@ -69,6 +69,7 @@ ++static bool check_task_state(struct task_struct *p, long match_state) ++{ ++ bool match = false; ++ ++ raw_spin_lock_irq(&p->pi_lock); ++ if (p->state == match_state || p->saved_state == match_state) ++ match = true; ++ raw_spin_unlock_irq(&p->pi_lock); ++ ++ return match; ++} ++ + /* + * wait_task_inactive - wait for a thread to unschedule. + * +@@ -1208,7 +1307,7 @@ + * is actually now running somewhere else! + */ + while (task_running(rq, p)) { +- if (match_state && unlikely(p->state != match_state)) ++ if (match_state && !check_task_state(p, match_state)) + return 0; + cpu_relax(); + } +@@ -1223,7 +1322,8 @@ + running = task_running(rq, p); + queued = task_on_rq_queued(p); + ncsw = 0; +- if (!match_state || p->state == match_state) ++ if (!match_state || p->state == match_state || ++ p->saved_state == match_state) + ncsw = p->nvcsw | LONG_MIN; /* sets MSB */ + task_rq_unlock(rq, p, &flags); - int sysctl_icmp_echo_ignore_all; - int sysctl_icmp_echo_ignore_broadcasts; -+ int sysctl_icmp_echo_sysrq; - int sysctl_icmp_ignore_bogus_error_responses; - int sysctl_icmp_ratelimit; - int sysctl_icmp_ratemask; -diff -Nur linux-4.1.10.orig/include/trace/events/hist.h linux-4.1.10/include/trace/events/hist.h ---- linux-4.1.10.orig/include/trace/events/hist.h 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.10/include/trace/events/hist.h 2015-10-07 18:00:08.000000000 +0200 -@@ -0,0 +1,72 @@ -+#undef TRACE_SYSTEM -+#define TRACE_SYSTEM hist +@@ -1449,10 +1549,6 @@ + { + activate_task(rq, p, en_flags); + p->on_rq = TASK_ON_RQ_QUEUED; +- +- /* if a worker is waking up, notify workqueue */ +- if (p->flags & PF_WQ_WORKER) +- wq_worker_waking_up(p, cpu_of(rq)); + } + + /* +@@ -1666,8 +1762,27 @@ + */ + smp_mb__before_spinlock(); + raw_spin_lock_irqsave(&p->pi_lock, flags); +- if (!(p->state & state)) ++ if (!(p->state & state)) { ++ /* ++ * The task might be running due to a spinlock sleeper ++ * wakeup. Check the saved state and set it to running ++ * if the wakeup condition is true. ++ */ ++ if (!(wake_flags & WF_LOCK_SLEEPER)) { ++ if (p->saved_state & state) { ++ p->saved_state = TASK_RUNNING; ++ success = 1; ++ } ++ } + goto out; ++ } + -+#if !defined(_TRACE_HIST_H) || defined(TRACE_HEADER_MULTI_READ) -+#define _TRACE_HIST_H ++ /* ++ * If this is a regular wakeup, then we can unconditionally ++ * clear the saved state of a "lock sleeper". ++ */ ++ if (!(wake_flags & WF_LOCK_SLEEPER)) ++ p->saved_state = TASK_RUNNING; + + success = 1; /* we're going to change ->state */ + cpu = task_cpu(p); +@@ -1710,42 +1825,6 @@ + } + + /** +- * try_to_wake_up_local - try to wake up a local task with rq lock held +- * @p: the thread to be awakened +- * +- * Put @p on the run-queue if it's not already there. The caller must +- * ensure that this_rq() is locked, @p is bound to this_rq() and not +- * the current task. +- */ +-static void try_to_wake_up_local(struct task_struct *p) +-{ +- struct rq *rq = task_rq(p); +- +- if (WARN_ON_ONCE(rq != this_rq()) || +- WARN_ON_ONCE(p == current)) +- return; +- +- lockdep_assert_held(&rq->lock); +- +- if (!raw_spin_trylock(&p->pi_lock)) { +- raw_spin_unlock(&rq->lock); +- raw_spin_lock(&p->pi_lock); +- raw_spin_lock(&rq->lock); +- } +- +- if (!(p->state & TASK_NORMAL)) +- goto out; +- +- if (!task_on_rq_queued(p)) +- ttwu_activate(rq, p, ENQUEUE_WAKEUP); +- +- ttwu_do_wakeup(rq, p, 0); +- ttwu_stat(p, smp_processor_id(), 0); +-out: +- raw_spin_unlock(&p->pi_lock); +-} +- +-/** + * wake_up_process - Wake up a specific process + * @p: The process to be woken up. + * +@@ -1759,11 +1838,23 @@ + */ + int wake_up_process(struct task_struct *p) + { +- WARN_ON(task_is_stopped_or_traced(p)); ++ WARN_ON(__task_is_stopped_or_traced(p)); + return try_to_wake_up(p, TASK_NORMAL, 0); + } + EXPORT_SYMBOL(wake_up_process); + ++/** ++ * wake_up_lock_sleeper - Wake up a specific process blocked on a "sleeping lock" ++ * @p: The process to be woken up. ++ * ++ * Same as wake_up_process() above, but wake_flags=WF_LOCK_SLEEPER to indicate ++ * the nature of the wakeup. ++ */ ++int wake_up_lock_sleeper(struct task_struct *p) ++{ ++ return try_to_wake_up(p, TASK_ALL, WF_LOCK_SLEEPER); ++} + -+#include "latency_hist.h" -+#include + int wake_up_state(struct task_struct *p, unsigned int state) + { + return try_to_wake_up(p, state, 0); +@@ -1959,6 +2050,9 @@ + p->on_cpu = 0; + #endif + init_task_preempt_count(p); ++#ifdef CONFIG_HAVE_PREEMPT_LAZY ++ task_thread_info(p)->preempt_lazy_count = 0; ++#endif + #ifdef CONFIG_SMP + plist_node_init(&p->pushable_tasks, MAX_PRIO); + RB_CLEAR_NODE(&p->pushable_dl_tasks); +@@ -2231,8 +2325,12 @@ + finish_arch_post_lock_switch(); + + fire_sched_in_preempt_notifiers(current); ++ /* ++ * We use mmdrop_delayed() here so we don't have to do the ++ * full __mmdrop() when we are the last user. ++ */ + if (mm) +- mmdrop(mm); ++ mmdrop_delayed(mm); + if (unlikely(prev_state == TASK_DEAD)) { + if (prev->sched_class->task_dead) + prev->sched_class->task_dead(prev); +@@ -2650,6 +2748,133 @@ + schedstat_inc(this_rq(), sched_count); + } + ++#if defined(CONFIG_PREEMPT_RT_FULL) && defined(CONFIG_SMP) ++#define MIGRATE_DISABLE_SET_AFFIN (1<<30) /* Can't make a negative */ ++#define migrate_disabled_updated(p) ((p)->migrate_disable & MIGRATE_DISABLE_SET_AFFIN) ++#define migrate_disable_count(p) ((p)->migrate_disable & ~MIGRATE_DISABLE_SET_AFFIN) + -+#if !defined(CONFIG_PREEMPT_OFF_HIST) && !defined(CONFIG_INTERRUPT_OFF_HIST) -+#define trace_preemptirqsoff_hist(a, b) -+#else -+TRACE_EVENT(preemptirqsoff_hist, ++static inline void update_migrate_disable(struct task_struct *p) ++{ ++ const struct cpumask *mask; + -+ TP_PROTO(int reason, int starthist), ++ if (likely(!p->migrate_disable)) ++ return; + -+ TP_ARGS(reason, starthist), ++ /* Did we already update affinity? */ ++ if (unlikely(migrate_disabled_updated(p))) ++ return; + -+ TP_STRUCT__entry( -+ __field(int, reason) -+ __field(int, starthist) -+ ), ++ /* ++ * Since this is always current we can get away with only locking ++ * rq->lock, the ->cpus_allowed value can normally only be changed ++ * while holding both p->pi_lock and rq->lock, but seeing that this ++ * is current, we cannot actually be waking up, so all code that ++ * relies on serialization against p->pi_lock is out of scope. ++ * ++ * Having rq->lock serializes us against things like ++ * set_cpus_allowed_ptr() that can still happen concurrently. ++ */ ++ mask = tsk_cpus_allowed(p); + -+ TP_fast_assign( -+ __entry->reason = reason; -+ __entry->starthist = starthist; -+ ), ++ if (p->sched_class->set_cpus_allowed) ++ p->sched_class->set_cpus_allowed(p, mask); ++ /* mask==cpumask_of(task_cpu(p)) which has a cpumask_weight==1 */ ++ p->nr_cpus_allowed = 1; + -+ TP_printk("reason=%s starthist=%s", getaction(__entry->reason), -+ __entry->starthist ? "start" : "stop") -+); -+#endif ++ /* Let migrate_enable know to fix things back up */ ++ p->migrate_disable |= MIGRATE_DISABLE_SET_AFFIN; ++} + -+#ifndef CONFIG_MISSED_TIMER_OFFSETS_HIST -+#define trace_hrtimer_interrupt(a, b, c, d) -+#else -+TRACE_EVENT(hrtimer_interrupt, ++void migrate_disable(void) ++{ ++ struct task_struct *p = current; + -+ TP_PROTO(int cpu, long long offset, struct task_struct *curr, -+ struct task_struct *task), ++ if (in_atomic()) { ++#ifdef CONFIG_SCHED_DEBUG ++ p->migrate_disable_atomic++; ++#endif ++ return; ++ } + -+ TP_ARGS(cpu, offset, curr, task), ++#ifdef CONFIG_SCHED_DEBUG ++ if (unlikely(p->migrate_disable_atomic)) { ++ tracing_off(); ++ WARN_ON_ONCE(1); ++ } ++#endif + -+ TP_STRUCT__entry( -+ __field(int, cpu) -+ __field(long long, offset) -+ __array(char, ccomm, TASK_COMM_LEN) -+ __field(int, cprio) -+ __array(char, tcomm, TASK_COMM_LEN) -+ __field(int, tprio) -+ ), ++ if (p->migrate_disable) { ++ p->migrate_disable++; ++ return; ++ } + -+ TP_fast_assign( -+ __entry->cpu = cpu; -+ __entry->offset = offset; -+ memcpy(__entry->ccomm, curr->comm, TASK_COMM_LEN); -+ __entry->cprio = curr->prio; -+ memcpy(__entry->tcomm, task != NULL ? task->comm : "", -+ task != NULL ? TASK_COMM_LEN : 7); -+ __entry->tprio = task != NULL ? task->prio : -1; -+ ), ++ preempt_disable(); ++ preempt_lazy_disable(); ++ pin_current_cpu(); ++ p->migrate_disable = 1; ++ preempt_enable(); ++} ++EXPORT_SYMBOL(migrate_disable); + -+ TP_printk("cpu=%d offset=%lld curr=%s[%d] thread=%s[%d]", -+ __entry->cpu, __entry->offset, __entry->ccomm, -+ __entry->cprio, __entry->tcomm, __entry->tprio) -+); ++void migrate_enable(void) ++{ ++ struct task_struct *p = current; ++ const struct cpumask *mask; ++ unsigned long flags; ++ struct rq *rq; ++ ++ if (in_atomic()) { ++#ifdef CONFIG_SCHED_DEBUG ++ p->migrate_disable_atomic--; +#endif ++ return; ++ } + -+#endif /* _TRACE_HIST_H */ ++#ifdef CONFIG_SCHED_DEBUG ++ if (unlikely(p->migrate_disable_atomic)) { ++ tracing_off(); ++ WARN_ON_ONCE(1); ++ } ++#endif ++ WARN_ON_ONCE(p->migrate_disable <= 0); + -+/* This part must be outside protection */ -+#include -diff -Nur linux-4.1.10.orig/include/trace/events/latency_hist.h linux-4.1.10/include/trace/events/latency_hist.h ---- linux-4.1.10.orig/include/trace/events/latency_hist.h 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.10/include/trace/events/latency_hist.h 2015-10-07 18:00:08.000000000 +0200 -@@ -0,0 +1,29 @@ -+#ifndef _LATENCY_HIST_H -+#define _LATENCY_HIST_H ++ if (migrate_disable_count(p) > 1) { ++ p->migrate_disable--; ++ return; ++ } + -+enum hist_action { -+ IRQS_ON, -+ PREEMPT_ON, -+ TRACE_STOP, -+ IRQS_OFF, -+ PREEMPT_OFF, -+ TRACE_START, -+}; ++ preempt_disable(); ++ if (unlikely(migrate_disabled_updated(p))) { ++ /* ++ * Undo whatever update_migrate_disable() did, also see there ++ * about locking. ++ */ ++ rq = this_rq(); ++ raw_spin_lock_irqsave(&rq->lock, flags); + -+static char *actions[] = { -+ "IRQS_ON", -+ "PREEMPT_ON", -+ "TRACE_STOP", -+ "IRQS_OFF", -+ "PREEMPT_OFF", -+ "TRACE_START", -+}; ++ /* ++ * Clearing migrate_disable causes tsk_cpus_allowed to ++ * show the tasks original cpu affinity. ++ */ ++ p->migrate_disable = 0; ++ mask = tsk_cpus_allowed(p); ++ if (p->sched_class->set_cpus_allowed) ++ p->sched_class->set_cpus_allowed(p, mask); ++ p->nr_cpus_allowed = cpumask_weight(mask); ++ raw_spin_unlock_irqrestore(&rq->lock, flags); ++ } else ++ p->migrate_disable = 0; + -+static inline char *getaction(int action) -+{ -+ if (action >= 0 && action <= sizeof(actions)/sizeof(actions[0])) -+ return actions[action]; -+ return "unknown"; ++ unpin_current_cpu(); ++ preempt_enable(); ++ preempt_lazy_enable(); +} ++EXPORT_SYMBOL(migrate_enable); ++#else ++static inline void update_migrate_disable(struct task_struct *p) { } ++#define migrate_disabled_updated(p) 0 ++#endif + -+#endif /* _LATENCY_HIST_H */ -diff -Nur linux-4.1.10.orig/init/Kconfig linux-4.1.10/init/Kconfig ---- linux-4.1.10.orig/init/Kconfig 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/init/Kconfig 2015-10-07 18:00:08.000000000 +0200 -@@ -637,7 +637,7 @@ - - config RCU_FAST_NO_HZ - bool "Accelerate last non-dyntick-idle CPU's grace periods" -- depends on NO_HZ_COMMON && SMP -+ depends on NO_HZ_COMMON && SMP && !PREEMPT_RT_FULL - default n - help - This option permits CPUs to enter dynticks-idle state even if -@@ -664,7 +664,7 @@ - config RCU_BOOST - bool "Enable RCU priority boosting" - depends on RT_MUTEXES && PREEMPT_RCU -- default n -+ default y if PREEMPT_RT_FULL - help - This option boosts the priority of preempted RCU readers that - block the current preemptible RCU grace period for too long. -@@ -1101,6 +1101,7 @@ - config RT_GROUP_SCHED - bool "Group scheduling for SCHED_RR/FIFO" - depends on CGROUP_SCHED -+ depends on !PREEMPT_RT_FULL - default n - help - This feature lets you explicitly allocate real CPU bandwidth -@@ -1688,6 +1689,7 @@ - - config SLAB - bool "SLAB" -+ depends on !PREEMPT_RT_FULL - help - The regular slab allocator that is established and known to work - well in all environments. It organizes cache hot objects in -@@ -1706,6 +1708,7 @@ - config SLOB - depends on EXPERT - bool "SLOB (Simple Allocator)" -+ depends on !PREEMPT_RT_FULL - help - SLOB replaces the stock allocator with a drastically simpler - allocator. SLOB is generally more space efficient but -@@ -1715,7 +1718,7 @@ + /* + * Pick up the highest-prio task: + */ +@@ -2756,6 +2981,8 @@ + smp_mb__before_spinlock(); + raw_spin_lock_irq(&rq->lock); - config SLUB_CPU_PARTIAL - default y -- depends on SLUB && SMP -+ depends on SLUB && SMP && !PREEMPT_RT_FULL - bool "SLUB per cpu partial cache" - help - Per cpu partial caches accellerate objects allocation and freeing -diff -Nur linux-4.1.10.orig/init/main.c linux-4.1.10/init/main.c ---- linux-4.1.10.orig/init/main.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/init/main.c 2015-10-07 18:00:08.000000000 +0200 -@@ -525,6 +525,7 @@ - setup_command_line(command_line); - setup_nr_cpu_ids(); - setup_per_cpu_areas(); -+ softirq_early_init(); - smp_prepare_boot_cpu(); /* arch-specific boot-cpu hooks */ ++ update_migrate_disable(prev); ++ + rq->clock_skip_update <<= 1; /* promote REQ to ACT */ - build_all_zonelists(NULL, NULL); -diff -Nur linux-4.1.10.orig/init/Makefile linux-4.1.10/init/Makefile ---- linux-4.1.10.orig/init/Makefile 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/init/Makefile 2015-10-07 18:00:08.000000000 +0200 -@@ -33,4 +33,4 @@ - include/generated/compile.h: FORCE - @$($(quiet)chk_compile.h) - $(Q)$(CONFIG_SHELL) $(srctree)/scripts/mkcompile_h $@ \ -- "$(UTS_MACHINE)" "$(CONFIG_SMP)" "$(CONFIG_PREEMPT)" "$(CC) $(KBUILD_CFLAGS)" -+ "$(UTS_MACHINE)" "$(CONFIG_SMP)" "$(CONFIG_PREEMPT)" "$(CONFIG_PREEMPT_RT_FULL)" "$(CC) $(KBUILD_CFLAGS)" -diff -Nur linux-4.1.10.orig/ipc/mqueue.c linux-4.1.10/ipc/mqueue.c ---- linux-4.1.10.orig/ipc/mqueue.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/ipc/mqueue.c 2015-10-07 18:00:08.000000000 +0200 -@@ -47,8 +47,7 @@ - #define RECV 1 + switch_count = &prev->nivcsw; +@@ -2765,19 +2992,6 @@ + } else { + deactivate_task(rq, prev, DEQUEUE_SLEEP); + prev->on_rq = 0; +- +- /* +- * If a worker went to sleep, notify and ask workqueue +- * whether it wants to wake up a task to maintain +- * concurrency. +- */ +- if (prev->flags & PF_WQ_WORKER) { +- struct task_struct *to_wakeup; +- +- to_wakeup = wq_worker_sleeping(prev, cpu); +- if (to_wakeup) +- try_to_wake_up_local(to_wakeup); +- } + } + switch_count = &prev->nvcsw; + } +@@ -2787,6 +3001,7 @@ - #define STATE_NONE 0 --#define STATE_PENDING 1 --#define STATE_READY 2 -+#define STATE_READY 1 + next = pick_next_task(rq, prev); + clear_tsk_need_resched(prev); ++ clear_tsk_need_resched_lazy(prev); + clear_preempt_need_resched(); + rq->clock_skip_update = 0; - struct posix_msg_tree_node { - struct rb_node rb_node; -@@ -568,15 +567,12 @@ - wq_add(info, sr, ewp); +@@ -2807,8 +3022,19 @@ - for (;;) { -- set_current_state(TASK_INTERRUPTIBLE); -+ __set_current_state(TASK_INTERRUPTIBLE); + static inline void sched_submit_work(struct task_struct *tsk) + { +- if (!tsk->state || tsk_is_pi_blocked(tsk)) ++ if (!tsk->state) ++ return; ++ /* ++ * If a worker went to sleep, notify and ask workqueue whether ++ * it wants to wake up a task to maintain concurrency. ++ */ ++ if (tsk->flags & PF_WQ_WORKER) ++ wq_worker_sleeping(tsk); ++ ++ ++ if (tsk_is_pi_blocked(tsk)) + return; ++ + /* + * If we are going to sleep and we have plugged IO queued, + * make sure to submit it to avoid deadlocks. +@@ -2817,6 +3043,12 @@ + blk_schedule_flush_plug(tsk); + } - spin_unlock(&info->lock); - time = schedule_hrtimeout_range_clock(timeout, 0, - HRTIMER_MODE_ABS, CLOCK_REALTIME); - -- while (ewp->state == STATE_PENDING) -- cpu_relax(); -- - if (ewp->state == STATE_READY) { - retval = 0; - goto out; -@@ -904,11 +900,15 @@ - * list of waiting receivers. A sender checks that list before adding the new - * message into the message array. If there is a waiting receiver, then it - * bypasses the message array and directly hands the message over to the -- * receiver. -- * The receiver accepts the message and returns without grabbing the queue -- * spinlock. Therefore an intermediate STATE_PENDING state and memory barriers -- * are necessary. The same algorithm is used for sysv semaphores, see -- * ipc/sem.c for more details. -+ * receiver. The receiver accepts the message and returns without grabbing the -+ * queue spinlock: -+ * -+ * - Set pointer to message. -+ * - Queue the receiver task for later wakeup (without the info->lock). -+ * - Update its state to STATE_READY. Now the receiver can continue. -+ * - Wake up the process after the lock is dropped. Should the process wake up -+ * before this wakeup (due to a timeout or a signal) it will either see -+ * STATE_READY and continue or acquire the lock to check the state again. - * - * The same algorithm is used for senders. - */ -@@ -916,21 +916,29 @@ - /* pipelined_send() - send a message directly to the task waiting in - * sys_mq_timedreceive() (without inserting message into a queue). - */ --static inline void pipelined_send(struct mqueue_inode_info *info, -+static inline void pipelined_send(struct wake_q_head *wake_q, -+ struct mqueue_inode_info *info, - struct msg_msg *message, - struct ext_wait_queue *receiver) ++static void sched_update_worker(struct task_struct *tsk) ++{ ++ if (tsk->flags & PF_WQ_WORKER) ++ wq_worker_running(tsk); ++} ++ + asmlinkage __visible void __sched schedule(void) { - receiver->msg = message; - list_del(&receiver->list); -- receiver->state = STATE_PENDING; -- wake_up_process(receiver->task); -- smp_wmb(); -+ wake_q_add(wake_q, receiver->task); -+ /* -+ * Rely on the implicit cmpxchg barrier from wake_q_add such -+ * that we can ensure that updating receiver->state is the last -+ * write operation: As once set, the receiver can continue, -+ * and if we don't have the reference count from the wake_q, -+ * yet, at that point we can later have a use-after-free -+ * condition and bogus wakeup. -+ */ - receiver->state = STATE_READY; + struct task_struct *tsk = current; +@@ -2825,6 +3057,7 @@ + do { + __schedule(); + } while (need_resched()); ++ sched_update_worker(tsk); } + EXPORT_SYMBOL(schedule); - /* pipelined_receive() - if there is task waiting in sys_mq_timedsend() - * gets its message and put to the queue (we have one free place for sure). */ --static inline void pipelined_receive(struct mqueue_inode_info *info) -+static inline void pipelined_receive(struct wake_q_head *wake_q, -+ struct mqueue_inode_info *info) - { - struct ext_wait_queue *sender = wq_get_first_waiter(info, SEND); - -@@ -941,10 +949,9 @@ - } - if (msg_insert(sender->msg, info)) +@@ -2916,6 +3149,14 @@ + if (likely(!preemptible())) return; -+ - list_del(&sender->list); -- sender->state = STATE_PENDING; -- wake_up_process(sender->task); -- smp_wmb(); -+ wake_q_add(wake_q, sender->task); - sender->state = STATE_READY; - } - -@@ -962,6 +969,7 @@ - struct timespec ts; - struct posix_msg_tree_node *new_leaf = NULL; - int ret = 0; -+ WAKE_Q(wake_q); - - if (u_abs_timeout) { - int res = prepare_timeout(u_abs_timeout, &expires, &ts); -@@ -1045,7 +1053,7 @@ - } else { - receiver = wq_get_first_waiter(info, RECV); - if (receiver) { -- pipelined_send(info, msg_ptr, receiver); -+ pipelined_send(&wake_q, info, msg_ptr, receiver); - } else { - /* adds message to the queue */ - ret = msg_insert(msg_ptr, info); -@@ -1058,6 +1066,7 @@ - } - out_unlock: - spin_unlock(&info->lock); -+ wake_up_q(&wake_q); - out_free: - if (ret) - free_msg(msg_ptr); -@@ -1144,14 +1153,17 @@ - msg_ptr = wait.msg; - } - } else { -+ WAKE_Q(wake_q); -+ - msg_ptr = msg_get(info); - - inode->i_atime = inode->i_mtime = inode->i_ctime = - CURRENT_TIME; - - /* There is now free space in queue. */ -- pipelined_receive(info); -+ pipelined_receive(&wake_q, info); - spin_unlock(&info->lock); -+ wake_up_q(&wake_q); - ret = 0; - } - if (ret == 0) { -diff -Nur linux-4.1.10.orig/ipc/msg.c linux-4.1.10/ipc/msg.c ---- linux-4.1.10.orig/ipc/msg.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/ipc/msg.c 2015-10-07 18:00:08.000000000 +0200 -@@ -188,6 +188,12 @@ - struct msg_receiver *msr, *t; - list_for_each_entry_safe(msr, t, &msq->q_receivers, r_list) { -+ /* -+ * Make sure that the wakeup doesnt preempt -+ * this CPU prematurely. (on PREEMPT_RT) -+ */ -+ preempt_disable_rt(); -+ - msr->r_msg = NULL; /* initialize expunge ordering */ - wake_up_process(msr->r_tsk); ++#ifdef CONFIG_PREEMPT_LAZY ++ /* ++ * Check for lazy preemption ++ */ ++ if (current_thread_info()->preempt_lazy_count && ++ !test_thread_flag(TIF_NEED_RESCHED)) ++ return; ++#endif + do { + __preempt_count_add(PREEMPT_ACTIVE); /* -@@ -198,6 +204,8 @@ +@@ -2924,7 +3165,16 @@ + * an infinite recursion. */ - smp_mb(); - msr->r_msg = ERR_PTR(res); -+ -+ preempt_enable_rt(); - } - } - -@@ -574,6 +582,11 @@ - if (testmsg(msg, msr->r_msgtype, msr->r_mode) && - !security_msg_queue_msgrcv(msq, msg, msr->r_tsk, - msr->r_msgtype, msr->r_mode)) { -+ /* -+ * Make sure that the wakeup doesnt preempt -+ * this CPU prematurely. (on PREEMPT_RT) -+ */ -+ preempt_disable_rt(); - - list_del(&msr->r_list); - if (msr->r_maxsize < msg->m_ts) { -@@ -595,12 +608,13 @@ - */ - smp_mb(); - msr->r_msg = msg; -+ preempt_enable_rt(); + prev_ctx = exception_enter(); ++ /* ++ * The add/subtract must not be traced by the function ++ * tracer. But we still want to account for the ++ * preempt off latency tracer. Since the _notrace versions ++ * of add/subtract skip the accounting for latency tracer ++ * we must force it manually. ++ */ ++ start_critical_timings(); + __schedule(); ++ stop_critical_timings(); + exception_exit(prev_ctx); - return 1; - } -+ preempt_enable_rt(); - } - } -- - return 0; + __preempt_count_sub(PREEMPT_ACTIVE); +@@ -4261,6 +4511,7 @@ } + EXPORT_SYMBOL(__cond_resched_lock); -diff -Nur linux-4.1.10.orig/ipc/sem.c linux-4.1.10/ipc/sem.c ---- linux-4.1.10.orig/ipc/sem.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/ipc/sem.c 2015-10-07 18:00:08.000000000 +0200 -@@ -690,6 +690,13 @@ - static void wake_up_sem_queue_prepare(struct list_head *pt, - struct sem_queue *q, int error) ++#ifndef CONFIG_PREEMPT_RT_FULL + int __sched __cond_resched_softirq(void) { -+#ifdef CONFIG_PREEMPT_RT_BASE -+ struct task_struct *p = q->sleeper; -+ get_task_struct(p); -+ q->status = error; -+ wake_up_process(p); -+ put_task_struct(p); -+#else - if (list_empty(pt)) { - /* - * Hold preempt off so that we don't get preempted and have the -@@ -701,6 +708,7 @@ - q->pid = error; - - list_add_tail(&q->list, pt); -+#endif + BUG_ON(!in_softirq()); +@@ -4274,6 +4525,7 @@ + return 0; } + EXPORT_SYMBOL(__cond_resched_softirq); ++#endif /** -@@ -714,6 +722,7 @@ - */ - static void wake_up_sem_queue_do(struct list_head *pt) - { -+#ifndef CONFIG_PREEMPT_RT_BASE - struct sem_queue *q, *t; - int did_something; + * yield - yield the current processor to other threads. +@@ -4628,7 +4880,9 @@ -@@ -726,6 +735,7 @@ - } - if (did_something) - preempt_enable(); + /* Set the preempt count _outside_ the spinlocks! */ + init_idle_preempt_count(idle, cpu); +- ++#ifdef CONFIG_HAVE_PREEMPT_LAZY ++ task_thread_info(idle)->preempt_lazy_count = 0; +#endif - } - - static void unlink_queue(struct sem_array *sma, struct sem_queue *q) -diff -Nur linux-4.1.10.orig/kernel/cgroup.c linux-4.1.10/kernel/cgroup.c ---- linux-4.1.10.orig/kernel/cgroup.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/cgroup.c 2015-10-07 18:00:08.000000000 +0200 -@@ -4422,10 +4422,10 @@ - queue_work(cgroup_destroy_wq, &css->destroy_work); - } + /* + * The idle tasks have their own, simple scheduling class: + */ +@@ -4748,11 +5002,91 @@ --static void css_release_work_fn(struct work_struct *work) -+static void css_release_work_fn(struct swork_event *sev) + void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask) { - struct cgroup_subsys_state *css = -- container_of(work, struct cgroup_subsys_state, destroy_work); -+ container_of(sev, struct cgroup_subsys_state, destroy_swork); - struct cgroup_subsys *ss = css->ss; - struct cgroup *cgrp = css->cgroup; - -@@ -4464,8 +4464,8 @@ - struct cgroup_subsys_state *css = - container_of(ref, struct cgroup_subsys_state, refcnt); - -- INIT_WORK(&css->destroy_work, css_release_work_fn); -- queue_work(cgroup_destroy_wq, &css->destroy_work); -+ INIT_SWORK(&css->destroy_swork, css_release_work_fn); -+ swork_queue(&css->destroy_swork); - } - - static void init_and_link_css(struct cgroup_subsys_state *css, -@@ -5069,6 +5069,7 @@ - */ - cgroup_destroy_wq = alloc_workqueue("cgroup_destroy", 0, 1); - BUG_ON(!cgroup_destroy_wq); -+ BUG_ON(swork_get()); +- if (p->sched_class->set_cpus_allowed) +- p->sched_class->set_cpus_allowed(p, new_mask); ++ if (!migrate_disabled_updated(p)) { ++ if (p->sched_class->set_cpus_allowed) ++ p->sched_class->set_cpus_allowed(p, new_mask); ++ p->nr_cpus_allowed = cpumask_weight(new_mask); ++ } - /* - * Used to destroy pidlists and separate to serve as flush domain. -diff -Nur linux-4.1.10.orig/kernel/cgroup.c.orig linux-4.1.10/kernel/cgroup.c.orig ---- linux-4.1.10.orig/kernel/cgroup.c.orig 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.10/kernel/cgroup.c.orig 2015-10-03 13:49:38.000000000 +0200 -@@ -0,0 +1,5602 @@ -+/* -+ * Generic process-grouping system. -+ * -+ * Based originally on the cpuset system, extracted by Paul Menage -+ * Copyright (C) 2006 Google, Inc -+ * -+ * Notifications support -+ * Copyright (C) 2009 Nokia Corporation -+ * Author: Kirill A. Shutemov -+ * -+ * Copyright notices from the original cpuset code: -+ * -------------------------------------------------- -+ * Copyright (C) 2003 BULL SA. -+ * Copyright (C) 2004-2006 Silicon Graphics, Inc. -+ * -+ * Portions derived from Patrick Mochel's sysfs code. -+ * sysfs is Copyright (c) 2001-3 Patrick Mochel -+ * -+ * 2003-10-10 Written by Simon Derr. -+ * 2003-10-22 Updates by Stephen Hemminger. -+ * 2004 May-July Rework by Paul Jackson. -+ * --------------------------------------------------- -+ * -+ * This file is subject to the terms and conditions of the GNU General Public -+ * License. See the file COPYING in the main directory of the Linux -+ * distribution for more details. -+ */ -+ -+#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include /* TODO: replace with more sophisticated array */ -+#include -+#include -+ -+#include -+ -+/* -+ * pidlists linger the following amount before being destroyed. The goal -+ * is avoiding frequent destruction in the middle of consecutive read calls -+ * Expiring in the middle is a performance problem not a correctness one. -+ * 1 sec should be enough. -+ */ -+#define CGROUP_PIDLIST_DESTROY_DELAY HZ -+ -+#define CGROUP_FILE_NAME_MAX (MAX_CGROUP_TYPE_NAMELEN + \ -+ MAX_CFTYPE_NAME + 2) -+ -+/* -+ * cgroup_mutex is the master lock. Any modification to cgroup or its -+ * hierarchy must be performed while holding it. -+ * -+ * css_set_rwsem protects task->cgroups pointer, the list of css_set -+ * objects, and the chain of tasks off each css_set. -+ * -+ * These locks are exported if CONFIG_PROVE_RCU so that accessors in -+ * cgroup.h can use them for lockdep annotations. -+ */ -+#ifdef CONFIG_PROVE_RCU -+DEFINE_MUTEX(cgroup_mutex); -+DECLARE_RWSEM(css_set_rwsem); -+EXPORT_SYMBOL_GPL(cgroup_mutex); -+EXPORT_SYMBOL_GPL(css_set_rwsem); -+#else -+static DEFINE_MUTEX(cgroup_mutex); -+static DECLARE_RWSEM(css_set_rwsem); -+#endif -+ -+/* -+ * Protects cgroup_idr and css_idr so that IDs can be released without -+ * grabbing cgroup_mutex. -+ */ -+static DEFINE_SPINLOCK(cgroup_idr_lock); -+ -+/* -+ * Protects cgroup_subsys->release_agent_path. Modifying it also requires -+ * cgroup_mutex. Reading requires either cgroup_mutex or this spinlock. -+ */ -+static DEFINE_SPINLOCK(release_agent_path_lock); -+ -+#define cgroup_assert_mutex_or_rcu_locked() \ -+ rcu_lockdep_assert(rcu_read_lock_held() || \ -+ lockdep_is_held(&cgroup_mutex), \ -+ "cgroup_mutex or RCU read lock required"); -+ -+/* -+ * cgroup destruction makes heavy use of work items and there can be a lot -+ * of concurrent destructions. Use a separate workqueue so that cgroup -+ * destruction work items don't end up filling up max_active of system_wq -+ * which may lead to deadlock. -+ */ -+static struct workqueue_struct *cgroup_destroy_wq; -+ -+/* -+ * pidlist destructions need to be flushed on cgroup destruction. Use a -+ * separate workqueue as flush domain. -+ */ -+static struct workqueue_struct *cgroup_pidlist_destroy_wq; -+ -+/* generate an array of cgroup subsystem pointers */ -+#define SUBSYS(_x) [_x ## _cgrp_id] = &_x ## _cgrp_subsys, -+static struct cgroup_subsys *cgroup_subsys[] = { -+#include -+}; -+#undef SUBSYS -+ -+/* array of cgroup subsystem names */ -+#define SUBSYS(_x) [_x ## _cgrp_id] = #_x, -+static const char *cgroup_subsys_name[] = { -+#include -+}; -+#undef SUBSYS -+ -+/* -+ * The default hierarchy, reserved for the subsystems that are otherwise -+ * unattached - it never has more than a single cgroup, and all tasks are -+ * part of that cgroup. -+ */ -+struct cgroup_root cgrp_dfl_root; -+ -+/* -+ * The default hierarchy always exists but is hidden until mounted for the -+ * first time. This is for backward compatibility. -+ */ -+static bool cgrp_dfl_root_visible; -+ -+/* -+ * Set by the boot param of the same name and makes subsystems with NULL -+ * ->dfl_files to use ->legacy_files on the default hierarchy. -+ */ -+static bool cgroup_legacy_files_on_dfl; -+ -+/* some controllers are not supported in the default hierarchy */ -+static unsigned int cgrp_dfl_root_inhibit_ss_mask; -+ -+/* The list of hierarchy roots */ -+ -+static LIST_HEAD(cgroup_roots); -+static int cgroup_root_count; -+ -+/* hierarchy ID allocation and mapping, protected by cgroup_mutex */ -+static DEFINE_IDR(cgroup_hierarchy_idr); -+ -+/* -+ * Assign a monotonically increasing serial number to csses. It guarantees -+ * cgroups with bigger numbers are newer than those with smaller numbers. -+ * Also, as csses are always appended to the parent's ->children list, it -+ * guarantees that sibling csses are always sorted in the ascending serial -+ * number order on the list. Protected by cgroup_mutex. -+ */ -+static u64 css_serial_nr_next = 1; -+ -+/* This flag indicates whether tasks in the fork and exit paths should -+ * check for fork/exit handlers to call. This avoids us having to do -+ * extra work in the fork/exit path if none of the subsystems need to -+ * be called. -+ */ -+static int need_forkexit_callback __read_mostly; -+ -+static struct cftype cgroup_dfl_base_files[]; -+static struct cftype cgroup_legacy_base_files[]; -+ -+static int rebind_subsystems(struct cgroup_root *dst_root, -+ unsigned int ss_mask); -+static int cgroup_destroy_locked(struct cgroup *cgrp); -+static int create_css(struct cgroup *cgrp, struct cgroup_subsys *ss, -+ bool visible); -+static void css_release(struct percpu_ref *ref); -+static void kill_css(struct cgroup_subsys_state *css); -+static int cgroup_addrm_files(struct cgroup *cgrp, struct cftype cfts[], -+ bool is_add); -+ -+/* IDR wrappers which synchronize using cgroup_idr_lock */ -+static int cgroup_idr_alloc(struct idr *idr, void *ptr, int start, int end, -+ gfp_t gfp_mask) -+{ -+ int ret; -+ -+ idr_preload(gfp_mask); -+ spin_lock_bh(&cgroup_idr_lock); -+ ret = idr_alloc(idr, ptr, start, end, gfp_mask); -+ spin_unlock_bh(&cgroup_idr_lock); -+ idr_preload_end(); -+ return ret; + cpumask_copy(&p->cpus_allowed, new_mask); +- p->nr_cpus_allowed = cpumask_weight(new_mask); +} + -+static void *cgroup_idr_replace(struct idr *idr, void *ptr, int id) -+{ -+ void *ret; -+ -+ spin_lock_bh(&cgroup_idr_lock); -+ ret = idr_replace(idr, ptr, id); -+ spin_unlock_bh(&cgroup_idr_lock); -+ return ret; -+} ++static DEFINE_PER_CPU(struct cpumask, sched_cpumasks); ++static DEFINE_MUTEX(sched_down_mutex); ++static cpumask_t sched_down_cpumask; + -+static void cgroup_idr_remove(struct idr *idr, int id) ++void tell_sched_cpu_down_begin(int cpu) +{ -+ spin_lock_bh(&cgroup_idr_lock); -+ idr_remove(idr, id); -+ spin_unlock_bh(&cgroup_idr_lock); ++ mutex_lock(&sched_down_mutex); ++ cpumask_set_cpu(cpu, &sched_down_cpumask); ++ mutex_unlock(&sched_down_mutex); +} + -+static struct cgroup *cgroup_parent(struct cgroup *cgrp) ++void tell_sched_cpu_down_done(int cpu) +{ -+ struct cgroup_subsys_state *parent_css = cgrp->self.parent; -+ -+ if (parent_css) -+ return container_of(parent_css, struct cgroup, self); -+ return NULL; ++ mutex_lock(&sched_down_mutex); ++ cpumask_clear_cpu(cpu, &sched_down_cpumask); ++ mutex_unlock(&sched_down_mutex); +} + +/** -+ * cgroup_css - obtain a cgroup's css for the specified subsystem -+ * @cgrp: the cgroup of interest -+ * @ss: the subsystem of interest (%NULL returns @cgrp->self) ++ * migrate_me - try to move the current task off this cpu + * -+ * Return @cgrp's css (cgroup_subsys_state) associated with @ss. This -+ * function must be called either under cgroup_mutex or rcu_read_lock() and -+ * the caller is responsible for pinning the returned css if it wants to -+ * keep accessing it outside the said locks. This function may return -+ * %NULL if @cgrp doesn't have @subsys_id enabled. -+ */ -+static struct cgroup_subsys_state *cgroup_css(struct cgroup *cgrp, -+ struct cgroup_subsys *ss) -+{ -+ if (ss) -+ return rcu_dereference_check(cgrp->subsys[ss->id], -+ lockdep_is_held(&cgroup_mutex)); -+ else -+ return &cgrp->self; -+} -+ -+/** -+ * cgroup_e_css - obtain a cgroup's effective css for the specified subsystem -+ * @cgrp: the cgroup of interest -+ * @ss: the subsystem of interest (%NULL returns @cgrp->self) ++ * Used by the pin_current_cpu() code to try to get tasks ++ * to move off the current CPU as it is going down. ++ * It will only move the task if the task isn't pinned to ++ * the CPU (with migrate_disable, affinity or NO_SETAFFINITY) ++ * and the task has to be in a RUNNING state. Otherwise the ++ * movement of the task will wake it up (change its state ++ * to running) when the task did not expect it. + * -+ * Similar to cgroup_css() but returns the effctive css, which is defined -+ * as the matching css of the nearest ancestor including self which has @ss -+ * enabled. If @ss is associated with the hierarchy @cgrp is on, this -+ * function is guaranteed to return non-NULL css. ++ * Returns 1 if it succeeded in moving the current task ++ * 0 otherwise. + */ -+static struct cgroup_subsys_state *cgroup_e_css(struct cgroup *cgrp, -+ struct cgroup_subsys *ss) ++int migrate_me(void) +{ -+ lockdep_assert_held(&cgroup_mutex); -+ -+ if (!ss) -+ return &cgrp->self; -+ -+ if (!(cgrp->root->subsys_mask & (1 << ss->id))) -+ return NULL; ++ struct task_struct *p = current; ++ struct migration_arg arg; ++ struct cpumask *cpumask; ++ struct cpumask *mask; ++ unsigned long flags; ++ unsigned int dest_cpu; ++ struct rq *rq; + + /* -+ * This function is used while updating css associations and thus -+ * can't test the csses directly. Use ->child_subsys_mask. ++ * We can not migrate tasks bounded to a CPU or tasks not ++ * running. The movement of the task will wake it up. + */ -+ while (cgroup_parent(cgrp) && -+ !(cgroup_parent(cgrp)->child_subsys_mask & (1 << ss->id))) -+ cgrp = cgroup_parent(cgrp); -+ -+ return cgroup_css(cgrp, ss); -+} -+ -+/** -+ * cgroup_get_e_css - get a cgroup's effective css for the specified subsystem -+ * @cgrp: the cgroup of interest -+ * @ss: the subsystem of interest -+ * -+ * Find and get the effective css of @cgrp for @ss. The effective css is -+ * defined as the matching css of the nearest ancestor including self which -+ * has @ss enabled. If @ss is not mounted on the hierarchy @cgrp is on, -+ * the root css is returned, so this function always returns a valid css. -+ * The returned css must be put using css_put(). -+ */ -+struct cgroup_subsys_state *cgroup_get_e_css(struct cgroup *cgrp, -+ struct cgroup_subsys *ss) -+{ -+ struct cgroup_subsys_state *css; -+ -+ rcu_read_lock(); -+ -+ do { -+ css = cgroup_css(cgrp, ss); -+ -+ if (css && css_tryget_online(css)) -+ goto out_unlock; -+ cgrp = cgroup_parent(cgrp); -+ } while (cgrp); -+ -+ css = init_css_set.subsys[ss->id]; -+ css_get(css); -+out_unlock: -+ rcu_read_unlock(); -+ return css; -+} ++ if (p->flags & PF_NO_SETAFFINITY || p->state) ++ return 0; + -+/* convenient tests for these bits */ -+static inline bool cgroup_is_dead(const struct cgroup *cgrp) -+{ -+ return !(cgrp->self.flags & CSS_ONLINE); -+} ++ mutex_lock(&sched_down_mutex); ++ rq = task_rq_lock(p, &flags); + -+struct cgroup_subsys_state *of_css(struct kernfs_open_file *of) -+{ -+ struct cgroup *cgrp = of->kn->parent->priv; -+ struct cftype *cft = of_cft(of); ++ cpumask = this_cpu_ptr(&sched_cpumasks); ++ mask = &p->cpus_allowed; + -+ /* -+ * This is open and unprotected implementation of cgroup_css(). -+ * seq_css() is only called from a kernfs file operation which has -+ * an active reference on the file. Because all the subsystem -+ * files are drained before a css is disassociated with a cgroup, -+ * the matching css from the cgroup's subsys table is guaranteed to -+ * be and stay valid until the enclosing operation is complete. -+ */ -+ if (cft->ss) -+ return rcu_dereference_raw(cgrp->subsys[cft->ss->id]); -+ else -+ return &cgrp->self; -+} -+EXPORT_SYMBOL_GPL(of_css); ++ cpumask_andnot(cpumask, mask, &sched_down_cpumask); + -+/** -+ * cgroup_is_descendant - test ancestry -+ * @cgrp: the cgroup to be tested -+ * @ancestor: possible ancestor of @cgrp -+ * -+ * Test whether @cgrp is a descendant of @ancestor. It also returns %true -+ * if @cgrp == @ancestor. This function is safe to call as long as @cgrp -+ * and @ancestor are accessible. -+ */ -+bool cgroup_is_descendant(struct cgroup *cgrp, struct cgroup *ancestor) -+{ -+ while (cgrp) { -+ if (cgrp == ancestor) -+ return true; -+ cgrp = cgroup_parent(cgrp); ++ if (!cpumask_weight(cpumask)) { ++ /* It's only on this CPU? */ ++ task_rq_unlock(rq, p, &flags); ++ mutex_unlock(&sched_down_mutex); ++ return 0; + } -+ return false; -+} -+ -+static int notify_on_release(const struct cgroup *cgrp) -+{ -+ return test_bit(CGRP_NOTIFY_ON_RELEASE, &cgrp->flags); -+} -+ -+/** -+ * for_each_css - iterate all css's of a cgroup -+ * @css: the iteration cursor -+ * @ssid: the index of the subsystem, CGROUP_SUBSYS_COUNT after reaching the end -+ * @cgrp: the target cgroup to iterate css's of -+ * -+ * Should be called under cgroup_[tree_]mutex. -+ */ -+#define for_each_css(css, ssid, cgrp) \ -+ for ((ssid) = 0; (ssid) < CGROUP_SUBSYS_COUNT; (ssid)++) \ -+ if (!((css) = rcu_dereference_check( \ -+ (cgrp)->subsys[(ssid)], \ -+ lockdep_is_held(&cgroup_mutex)))) { } \ -+ else -+ -+/** -+ * for_each_e_css - iterate all effective css's of a cgroup -+ * @css: the iteration cursor -+ * @ssid: the index of the subsystem, CGROUP_SUBSYS_COUNT after reaching the end -+ * @cgrp: the target cgroup to iterate css's of -+ * -+ * Should be called under cgroup_[tree_]mutex. -+ */ -+#define for_each_e_css(css, ssid, cgrp) \ -+ for ((ssid) = 0; (ssid) < CGROUP_SUBSYS_COUNT; (ssid)++) \ -+ if (!((css) = cgroup_e_css(cgrp, cgroup_subsys[(ssid)]))) \ -+ ; \ -+ else -+ -+/** -+ * for_each_subsys - iterate all enabled cgroup subsystems -+ * @ss: the iteration cursor -+ * @ssid: the index of @ss, CGROUP_SUBSYS_COUNT after reaching the end -+ */ -+#define for_each_subsys(ss, ssid) \ -+ for ((ssid) = 0; (ssid) < CGROUP_SUBSYS_COUNT && \ -+ (((ss) = cgroup_subsys[ssid]) || true); (ssid)++) -+ -+/* iterate across the hierarchies */ -+#define for_each_root(root) \ -+ list_for_each_entry((root), &cgroup_roots, root_list) -+ -+/* iterate over child cgrps, lock should be held throughout iteration */ -+#define cgroup_for_each_live_child(child, cgrp) \ -+ list_for_each_entry((child), &(cgrp)->self.children, self.sibling) \ -+ if (({ lockdep_assert_held(&cgroup_mutex); \ -+ cgroup_is_dead(child); })) \ -+ ; \ -+ else -+ -+static void cgroup_release_agent(struct work_struct *work); -+static void check_for_release(struct cgroup *cgrp); + -+/* -+ * A cgroup can be associated with multiple css_sets as different tasks may -+ * belong to different cgroups on different hierarchies. In the other -+ * direction, a css_set is naturally associated with multiple cgroups. -+ * This M:N relationship is represented by the following link structure -+ * which exists for each association and allows traversing the associations -+ * from both sides. -+ */ -+struct cgrp_cset_link { -+ /* the cgroup and css_set this link associates */ -+ struct cgroup *cgrp; -+ struct css_set *cset; -+ -+ /* list of cgrp_cset_links anchored at cgrp->cset_links */ -+ struct list_head cset_link; -+ -+ /* list of cgrp_cset_links anchored at css_set->cgrp_links */ -+ struct list_head cgrp_link; -+}; ++ dest_cpu = cpumask_any_and(cpu_active_mask, cpumask); + -+/* -+ * The default css_set - used by init and its children prior to any -+ * hierarchies being mounted. It contains a pointer to the root state -+ * for each subsystem. Also used to anchor the list of css_sets. Not -+ * reference-counted, to improve performance when child cgroups -+ * haven't been created. -+ */ -+struct css_set init_css_set = { -+ .refcount = ATOMIC_INIT(1), -+ .cgrp_links = LIST_HEAD_INIT(init_css_set.cgrp_links), -+ .tasks = LIST_HEAD_INIT(init_css_set.tasks), -+ .mg_tasks = LIST_HEAD_INIT(init_css_set.mg_tasks), -+ .mg_preload_node = LIST_HEAD_INIT(init_css_set.mg_preload_node), -+ .mg_node = LIST_HEAD_INIT(init_css_set.mg_node), -+}; ++ arg.task = p; ++ arg.dest_cpu = dest_cpu; + -+static int css_set_count = 1; /* 1 for init_css_set */ ++ task_rq_unlock(rq, p, &flags); + -+/** -+ * cgroup_update_populated - updated populated count of a cgroup -+ * @cgrp: the target cgroup -+ * @populated: inc or dec populated count -+ * -+ * @cgrp is either getting the first task (css_set) or losing the last. -+ * Update @cgrp->populated_cnt accordingly. The count is propagated -+ * towards root so that a given cgroup's populated_cnt is zero iff the -+ * cgroup and all its descendants are empty. -+ * -+ * @cgrp's interface file "cgroup.populated" is zero if -+ * @cgrp->populated_cnt is zero and 1 otherwise. When @cgrp->populated_cnt -+ * changes from or to zero, userland is notified that the content of the -+ * interface file has changed. This can be used to detect when @cgrp and -+ * its descendants become populated or empty. -+ */ -+static void cgroup_update_populated(struct cgroup *cgrp, bool populated) -+{ -+ lockdep_assert_held(&css_set_rwsem); ++ stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg); ++ tlb_migrate_finish(p->mm); ++ mutex_unlock(&sched_down_mutex); + -+ do { -+ bool trigger; ++ return 1; + } + + /* +@@ -4798,7 +5132,7 @@ + do_set_cpus_allowed(p, new_mask); + + /* Can the task run on the task's current CPU? If so, we're done */ +- if (cpumask_test_cpu(task_cpu(p), new_mask)) ++ if (cpumask_test_cpu(task_cpu(p), new_mask) || __migrate_disabled(p)) + goto out; + + dest_cpu = cpumask_any_and(cpu_active_mask, new_mask); +@@ -4938,6 +5272,8 @@ + + #ifdef CONFIG_HOTPLUG_CPU + ++static DEFINE_PER_CPU(struct mm_struct *, idle_last_mm); + -+ if (populated) -+ trigger = !cgrp->populated_cnt++; -+ else -+ trigger = !--cgrp->populated_cnt; -+ -+ if (!trigger) -+ break; + /* + * Ensures that the idle task is using init_mm right before its cpu goes + * offline. +@@ -4952,7 +5288,11 @@ + switch_mm(mm, &init_mm, current); + finish_arch_post_lock_switch(); + } +- mmdrop(mm); ++ /* ++ * Defer the cleanup to an alive cpu. On RT we can neither ++ * call mmdrop() nor mmdrop_delayed() from here. ++ */ ++ per_cpu(idle_last_mm, smp_processor_id()) = mm; + } + + /* +@@ -5295,6 +5635,10 @@ + + case CPU_DEAD: + calc_load_migrate(rq); ++ if (per_cpu(idle_last_mm, cpu)) { ++ mmdrop(per_cpu(idle_last_mm, cpu)); ++ per_cpu(idle_last_mm, cpu) = NULL; ++ } + break; + #endif + } +@@ -7274,7 +7618,8 @@ + #ifdef CONFIG_DEBUG_ATOMIC_SLEEP + static inline int preempt_count_equals(int preempt_offset) + { +- int nested = (preempt_count() & ~PREEMPT_ACTIVE) + rcu_preempt_depth(); ++ int nested = (preempt_count() & ~PREEMPT_ACTIVE) + ++ sched_rcu_preempt_depth(); + + return (nested == preempt_offset); + } +diff -Nur linux-4.1.10.orig/kernel/sched/cputime.c linux-4.1.10/kernel/sched/cputime.c +--- linux-4.1.10.orig/kernel/sched/cputime.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/sched/cputime.c 2015-10-12 22:33:32.308674640 +0200 +@@ -675,37 +675,45 @@ + + void vtime_account_system(struct task_struct *tsk) + { +- write_seqlock(&tsk->vtime_seqlock); ++ raw_spin_lock(&tsk->vtime_lock); ++ write_seqcount_begin(&tsk->vtime_seq); + __vtime_account_system(tsk); +- write_sequnlock(&tsk->vtime_seqlock); ++ write_seqcount_end(&tsk->vtime_seq); ++ raw_spin_unlock(&tsk->vtime_lock); + } + + void vtime_gen_account_irq_exit(struct task_struct *tsk) + { +- write_seqlock(&tsk->vtime_seqlock); ++ raw_spin_lock(&tsk->vtime_lock); ++ write_seqcount_begin(&tsk->vtime_seq); + __vtime_account_system(tsk); + if (context_tracking_in_user()) + tsk->vtime_snap_whence = VTIME_USER; +- write_sequnlock(&tsk->vtime_seqlock); ++ write_seqcount_end(&tsk->vtime_seq); ++ raw_spin_unlock(&tsk->vtime_lock); + } + + void vtime_account_user(struct task_struct *tsk) + { + cputime_t delta_cpu; + +- write_seqlock(&tsk->vtime_seqlock); ++ raw_spin_lock(&tsk->vtime_lock); ++ write_seqcount_begin(&tsk->vtime_seq); + delta_cpu = get_vtime_delta(tsk); + tsk->vtime_snap_whence = VTIME_SYS; + account_user_time(tsk, delta_cpu, cputime_to_scaled(delta_cpu)); +- write_sequnlock(&tsk->vtime_seqlock); ++ write_seqcount_end(&tsk->vtime_seq); ++ raw_spin_unlock(&tsk->vtime_lock); + } + + void vtime_user_enter(struct task_struct *tsk) + { +- write_seqlock(&tsk->vtime_seqlock); ++ raw_spin_lock(&tsk->vtime_lock); ++ write_seqcount_begin(&tsk->vtime_seq); + __vtime_account_system(tsk); + tsk->vtime_snap_whence = VTIME_USER; +- write_sequnlock(&tsk->vtime_seqlock); ++ write_seqcount_end(&tsk->vtime_seq); ++ raw_spin_unlock(&tsk->vtime_lock); + } + + void vtime_guest_enter(struct task_struct *tsk) +@@ -717,19 +725,23 @@ + * synchronization against the reader (task_gtime()) + * that can thus safely catch up with a tickless delta. + */ +- write_seqlock(&tsk->vtime_seqlock); ++ raw_spin_lock(&tsk->vtime_lock); ++ write_seqcount_begin(&tsk->vtime_seq); + __vtime_account_system(tsk); + current->flags |= PF_VCPU; +- write_sequnlock(&tsk->vtime_seqlock); ++ write_seqcount_end(&tsk->vtime_seq); ++ raw_spin_unlock(&tsk->vtime_lock); + } + EXPORT_SYMBOL_GPL(vtime_guest_enter); + + void vtime_guest_exit(struct task_struct *tsk) + { +- write_seqlock(&tsk->vtime_seqlock); ++ raw_spin_lock(&tsk->vtime_lock); ++ write_seqcount_begin(&tsk->vtime_seq); + __vtime_account_system(tsk); + current->flags &= ~PF_VCPU; +- write_sequnlock(&tsk->vtime_seqlock); ++ write_seqcount_end(&tsk->vtime_seq); ++ raw_spin_unlock(&tsk->vtime_lock); + } + EXPORT_SYMBOL_GPL(vtime_guest_exit); + +@@ -742,24 +754,30 @@ + + void arch_vtime_task_switch(struct task_struct *prev) + { +- write_seqlock(&prev->vtime_seqlock); ++ raw_spin_lock(&prev->vtime_lock); ++ write_seqcount_begin(&prev->vtime_seq); + prev->vtime_snap_whence = VTIME_SLEEPING; +- write_sequnlock(&prev->vtime_seqlock); ++ write_seqcount_end(&prev->vtime_seq); ++ raw_spin_unlock(&prev->vtime_lock); + +- write_seqlock(¤t->vtime_seqlock); ++ raw_spin_lock(¤t->vtime_lock); ++ write_seqcount_begin(¤t->vtime_seq); + current->vtime_snap_whence = VTIME_SYS; + current->vtime_snap = sched_clock_cpu(smp_processor_id()); +- write_sequnlock(¤t->vtime_seqlock); ++ write_seqcount_end(¤t->vtime_seq); ++ raw_spin_unlock(¤t->vtime_lock); + } + + void vtime_init_idle(struct task_struct *t, int cpu) + { + unsigned long flags; + +- write_seqlock_irqsave(&t->vtime_seqlock, flags); ++ raw_spin_lock_irqsave(&t->vtime_lock, flags); ++ write_seqcount_begin(&t->vtime_seq); + t->vtime_snap_whence = VTIME_SYS; + t->vtime_snap = sched_clock_cpu(cpu); +- write_sequnlock_irqrestore(&t->vtime_seqlock, flags); ++ write_seqcount_end(&t->vtime_seq); ++ raw_spin_unlock_irqrestore(&t->vtime_lock, flags); + } + + cputime_t task_gtime(struct task_struct *t) +@@ -768,13 +786,13 @@ + cputime_t gtime; + + do { +- seq = read_seqbegin(&t->vtime_seqlock); ++ seq = read_seqcount_begin(&t->vtime_seq); + + gtime = t->gtime; + if (t->flags & PF_VCPU) + gtime += vtime_delta(t); + +- } while (read_seqretry(&t->vtime_seqlock, seq)); ++ } while (read_seqcount_retry(&t->vtime_seq, seq)); + + return gtime; + } +@@ -797,7 +815,7 @@ + *udelta = 0; + *sdelta = 0; + +- seq = read_seqbegin(&t->vtime_seqlock); ++ seq = read_seqcount_begin(&t->vtime_seq); + + if (u_dst) + *u_dst = *u_src; +@@ -821,7 +839,7 @@ + if (t->vtime_snap_whence == VTIME_SYS) + *sdelta = delta; + } +- } while (read_seqretry(&t->vtime_seqlock, seq)); ++ } while (read_seqcount_retry(&t->vtime_seq, seq)); + } + + +diff -Nur linux-4.1.10.orig/kernel/sched/deadline.c linux-4.1.10/kernel/sched/deadline.c +--- linux-4.1.10.orig/kernel/sched/deadline.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/sched/deadline.c 2015-10-12 22:33:32.308674640 +0200 +@@ -637,6 +637,7 @@ + + hrtimer_init(timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + timer->function = dl_task_timer; ++ timer->irqsafe = 1; + } + + static +diff -Nur linux-4.1.10.orig/kernel/sched/debug.c linux-4.1.10/kernel/sched/debug.c +--- linux-4.1.10.orig/kernel/sched/debug.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/sched/debug.c 2015-10-12 22:33:32.308674640 +0200 +@@ -260,6 +260,9 @@ + P(rt_throttled); + PN(rt_time); + PN(rt_runtime); ++#ifdef CONFIG_SMP ++ P(rt_nr_migratory); ++#endif + + #undef PN + #undef P +@@ -648,6 +651,10 @@ + #endif + P(policy); + P(prio); ++#ifdef CONFIG_PREEMPT_RT_FULL ++ P(migrate_disable); ++#endif ++ P(nr_cpus_allowed); + #undef PN + #undef __PN + #undef P +diff -Nur linux-4.1.10.orig/kernel/sched/fair.c linux-4.1.10/kernel/sched/fair.c +--- linux-4.1.10.orig/kernel/sched/fair.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/sched/fair.c 2015-10-12 22:33:32.308674640 +0200 +@@ -3201,7 +3201,7 @@ + ideal_runtime = sched_slice(cfs_rq, curr); + delta_exec = curr->sum_exec_runtime - curr->prev_sum_exec_runtime; + if (delta_exec > ideal_runtime) { +- resched_curr(rq_of(cfs_rq)); ++ resched_curr_lazy(rq_of(cfs_rq)); + /* + * The current task ran long enough, ensure it doesn't get + * re-elected due to buddy favours. +@@ -3225,7 +3225,7 @@ + return; + + if (delta > ideal_runtime) +- resched_curr(rq_of(cfs_rq)); ++ resched_curr_lazy(rq_of(cfs_rq)); + } + + static void +@@ -3366,7 +3366,7 @@ + * validating it and just reschedule. + */ + if (queued) { +- resched_curr(rq_of(cfs_rq)); ++ resched_curr_lazy(rq_of(cfs_rq)); + return; + } + /* +@@ -3557,7 +3557,7 @@ + * hierarchy can be throttled + */ + if (!assign_cfs_rq_runtime(cfs_rq) && likely(cfs_rq->curr)) +- resched_curr(rq_of(cfs_rq)); ++ resched_curr_lazy(rq_of(cfs_rq)); + } + + static __always_inline +@@ -4180,7 +4180,7 @@ + + if (delta < 0) { + if (rq->curr == p) +- resched_curr(rq); ++ resched_curr_lazy(rq); + return; + } + hrtick_start(rq, delta); +@@ -5076,7 +5076,7 @@ + return; + + preempt: +- resched_curr(rq); ++ resched_curr_lazy(rq); + /* + * Only set the backward buddy when the current task is still + * on the rq. This can happen when a wakeup gets interleaved +@@ -7866,7 +7866,7 @@ + * 'current' within the tree based on its new key value. + */ + swap(curr->vruntime, se->vruntime); +- resched_curr(rq); ++ resched_curr_lazy(rq); + } + + se->vruntime -= cfs_rq->min_vruntime; +@@ -7891,7 +7891,7 @@ + */ + if (rq->curr == p) { + if (p->prio > oldprio) +- resched_curr(rq); ++ resched_curr_lazy(rq); + } else + check_preempt_curr(rq, p, 0); + } +diff -Nur linux-4.1.10.orig/kernel/sched/features.h linux-4.1.10/kernel/sched/features.h +--- linux-4.1.10.orig/kernel/sched/features.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/sched/features.h 2015-10-12 22:33:32.308674640 +0200 +@@ -50,11 +50,19 @@ + */ + SCHED_FEAT(NONTASK_CAPACITY, true) + ++#ifdef CONFIG_PREEMPT_RT_FULL ++SCHED_FEAT(TTWU_QUEUE, false) ++# ifdef CONFIG_PREEMPT_LAZY ++SCHED_FEAT(PREEMPT_LAZY, true) ++# endif ++#else + -+ if (cgrp->populated_kn) -+ kernfs_notify(cgrp->populated_kn); -+ cgrp = cgroup_parent(cgrp); -+ } while (cgrp); + /* + * Queue remote wakeups on the target CPU and process them + * using the scheduler IPI. Reduces rq->lock contention/bounces. + */ + SCHED_FEAT(TTWU_QUEUE, true) ++#endif + + #ifdef HAVE_RT_PUSH_IPI + /* +diff -Nur linux-4.1.10.orig/kernel/sched/Makefile linux-4.1.10/kernel/sched/Makefile +--- linux-4.1.10.orig/kernel/sched/Makefile 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/sched/Makefile 2015-10-12 22:33:32.308674640 +0200 +@@ -13,7 +13,7 @@ + + obj-y += core.o proc.o clock.o cputime.o + obj-y += idle_task.o fair.o rt.o deadline.o stop_task.o +-obj-y += wait.o completion.o idle.o ++obj-y += wait.o wait-simple.o work-simple.o completion.o idle.o + obj-$(CONFIG_SMP) += cpupri.o cpudeadline.o + obj-$(CONFIG_SCHED_AUTOGROUP) += auto_group.o + obj-$(CONFIG_SCHEDSTATS) += stats.o +diff -Nur linux-4.1.10.orig/kernel/sched/rt.c linux-4.1.10/kernel/sched/rt.c +--- linux-4.1.10.orig/kernel/sched/rt.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/sched/rt.c 2015-10-12 22:33:32.312674375 +0200 +@@ -44,6 +44,7 @@ + + hrtimer_init(&rt_b->rt_period_timer, + CLOCK_MONOTONIC, HRTIMER_MODE_REL); ++ rt_b->rt_period_timer.irqsafe = 1; + rt_b->rt_period_timer.function = sched_rt_period_timer; + } + +@@ -89,6 +90,7 @@ + rt_rq->push_cpu = nr_cpu_ids; + raw_spin_lock_init(&rt_rq->push_lock); + init_irq_work(&rt_rq->push_work, push_irq_work_func); ++ rt_rq->push_work.flags |= IRQ_WORK_HARD_IRQ; + #endif + #endif /* CONFIG_SMP */ + /* We start is dequeued state, because no RT tasks are queued */ +diff -Nur linux-4.1.10.orig/kernel/sched/sched.h linux-4.1.10/kernel/sched/sched.h +--- linux-4.1.10.orig/kernel/sched/sched.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/sched/sched.h 2015-10-12 22:33:32.312674375 +0200 +@@ -1092,6 +1092,7 @@ + #define WF_SYNC 0x01 /* waker goes to sleep after wakeup */ + #define WF_FORK 0x02 /* child wakeup after fork */ + #define WF_MIGRATED 0x4 /* internal use, task got migrated */ ++#define WF_LOCK_SLEEPER 0x08 /* wakeup spinlock "sleeper" */ + + /* + * To aid in avoiding the subversion of "niceness" due to uneven distribution +@@ -1289,6 +1290,15 @@ + extern void resched_curr(struct rq *rq); + extern void resched_cpu(int cpu); + ++#ifdef CONFIG_PREEMPT_LAZY ++extern void resched_curr_lazy(struct rq *rq); ++#else ++static inline void resched_curr_lazy(struct rq *rq) ++{ ++ resched_curr(rq); +} ++#endif + + extern struct rt_bandwidth def_rt_bandwidth; + extern void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime); + +diff -Nur linux-4.1.10.orig/kernel/sched/wait-simple.c linux-4.1.10/kernel/sched/wait-simple.c +--- linux-4.1.10.orig/kernel/sched/wait-simple.c 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/kernel/sched/wait-simple.c 2015-10-12 22:33:32.312674375 +0200 +@@ -0,0 +1,115 @@ +/* -+ * hash table for cgroup groups. This improves the performance to find -+ * an existing css_set. This hash doesn't (currently) take into -+ * account cgroups in empty hierarchies. ++ * Simple waitqueues without fancy flags and callbacks ++ * ++ * (C) 2011 Thomas Gleixner ++ * ++ * Based on kernel/wait.c ++ * ++ * For licencing details see kernel-base/COPYING + */ -+#define CSS_SET_HASH_BITS 7 -+static DEFINE_HASHTABLE(css_set_table, CSS_SET_HASH_BITS); ++#include ++#include ++#include ++#include + -+static unsigned long css_set_hash(struct cgroup_subsys_state *css[]) ++/* Adds w to head->list. Must be called with head->lock locked. */ ++static inline void __swait_enqueue(struct swait_head *head, struct swaiter *w) +{ -+ unsigned long key = 0UL; -+ struct cgroup_subsys *ss; -+ int i; -+ -+ for_each_subsys(ss, i) -+ key += (unsigned long)css[i]; -+ key = (key >> 16) ^ key; -+ -+ return key; ++ list_add(&w->node, &head->list); ++ /* We can't let the condition leak before the setting of head */ ++ smp_mb(); +} + -+static void put_css_set_locked(struct css_set *cset) ++/* Removes w from head->list. Must be called with head->lock locked. */ ++static inline void __swait_dequeue(struct swaiter *w) +{ -+ struct cgrp_cset_link *link, *tmp_link; -+ struct cgroup_subsys *ss; -+ int ssid; -+ -+ lockdep_assert_held(&css_set_rwsem); -+ -+ if (!atomic_dec_and_test(&cset->refcount)) -+ return; -+ -+ /* This css_set is dead. unlink it and release cgroup refcounts */ -+ for_each_subsys(ss, ssid) -+ list_del(&cset->e_cset_node[ssid]); -+ hash_del(&cset->hlist); -+ css_set_count--; -+ -+ list_for_each_entry_safe(link, tmp_link, &cset->cgrp_links, cgrp_link) { -+ struct cgroup *cgrp = link->cgrp; -+ -+ list_del(&link->cset_link); -+ list_del(&link->cgrp_link); -+ -+ /* @cgrp can't go away while we're holding css_set_rwsem */ -+ if (list_empty(&cgrp->cset_links)) { -+ cgroup_update_populated(cgrp, false); -+ check_for_release(cgrp); -+ } ++ list_del_init(&w->node); ++} + -+ kfree(link); -+ } ++void __init_swait_head(struct swait_head *head, struct lock_class_key *key) ++{ ++ raw_spin_lock_init(&head->lock); ++ lockdep_set_class(&head->lock, key); ++ INIT_LIST_HEAD(&head->list); ++} ++EXPORT_SYMBOL(__init_swait_head); + -+ kfree_rcu(cset, rcu_head); ++void swait_prepare_locked(struct swait_head *head, struct swaiter *w) ++{ ++ w->task = current; ++ if (list_empty(&w->node)) ++ __swait_enqueue(head, w); +} + -+static void put_css_set(struct css_set *cset) ++void swait_prepare(struct swait_head *head, struct swaiter *w, int state) +{ -+ /* -+ * Ensure that the refcount doesn't hit zero while any readers -+ * can see it. Similar to atomic_dec_and_lock(), but for an -+ * rwlock -+ */ -+ if (atomic_add_unless(&cset->refcount, -1, 1)) -+ return; ++ unsigned long flags; + -+ down_write(&css_set_rwsem); -+ put_css_set_locked(cset); -+ up_write(&css_set_rwsem); ++ raw_spin_lock_irqsave(&head->lock, flags); ++ swait_prepare_locked(head, w); ++ __set_current_state(state); ++ raw_spin_unlock_irqrestore(&head->lock, flags); +} ++EXPORT_SYMBOL(swait_prepare); + -+/* -+ * refcounted get/put for css_set objects -+ */ -+static inline void get_css_set(struct css_set *cset) ++void swait_finish_locked(struct swait_head *head, struct swaiter *w) +{ -+ atomic_inc(&cset->refcount); ++ __set_current_state(TASK_RUNNING); ++ if (w->task) ++ __swait_dequeue(w); +} + -+/** -+ * compare_css_sets - helper function for find_existing_css_set(). -+ * @cset: candidate css_set being tested -+ * @old_cset: existing css_set for a task -+ * @new_cgrp: cgroup that's being entered by the task -+ * @template: desired set of css pointers in css_set (pre-calculated) -+ * -+ * Returns true if "cset" matches "old_cset" except for the hierarchy -+ * which "new_cgrp" belongs to, for which it should match "new_cgrp". -+ */ -+static bool compare_css_sets(struct css_set *cset, -+ struct css_set *old_cset, -+ struct cgroup *new_cgrp, -+ struct cgroup_subsys_state *template[]) ++void swait_finish(struct swait_head *head, struct swaiter *w) +{ -+ struct list_head *l1, *l2; -+ -+ /* -+ * On the default hierarchy, there can be csets which are -+ * associated with the same set of cgroups but different csses. -+ * Let's first ensure that csses match. -+ */ -+ if (memcmp(template, cset->subsys, sizeof(cset->subsys))) -+ return false; -+ -+ /* -+ * Compare cgroup pointers in order to distinguish between -+ * different cgroups in hierarchies. As different cgroups may -+ * share the same effective css, this comparison is always -+ * necessary. -+ */ -+ l1 = &cset->cgrp_links; -+ l2 = &old_cset->cgrp_links; -+ while (1) { -+ struct cgrp_cset_link *link1, *link2; -+ struct cgroup *cgrp1, *cgrp2; -+ -+ l1 = l1->next; -+ l2 = l2->next; -+ /* See if we reached the end - both lists are equal length. */ -+ if (l1 == &cset->cgrp_links) { -+ BUG_ON(l2 != &old_cset->cgrp_links); -+ break; -+ } else { -+ BUG_ON(l2 == &old_cset->cgrp_links); -+ } -+ /* Locate the cgroups associated with these links. */ -+ link1 = list_entry(l1, struct cgrp_cset_link, cgrp_link); -+ link2 = list_entry(l2, struct cgrp_cset_link, cgrp_link); -+ cgrp1 = link1->cgrp; -+ cgrp2 = link2->cgrp; -+ /* Hierarchies should be linked in the same order. */ -+ BUG_ON(cgrp1->root != cgrp2->root); ++ unsigned long flags; + -+ /* -+ * If this hierarchy is the hierarchy of the cgroup -+ * that's changing, then we need to check that this -+ * css_set points to the new cgroup; if it's any other -+ * hierarchy, then this css_set should point to the -+ * same cgroup as the old css_set. -+ */ -+ if (cgrp1->root == new_cgrp->root) { -+ if (cgrp1 != new_cgrp) -+ return false; -+ } else { -+ if (cgrp1 != cgrp2) -+ return false; -+ } ++ __set_current_state(TASK_RUNNING); ++ if (w->task) { ++ raw_spin_lock_irqsave(&head->lock, flags); ++ __swait_dequeue(w); ++ raw_spin_unlock_irqrestore(&head->lock, flags); + } -+ return true; +} ++EXPORT_SYMBOL(swait_finish); + -+/** -+ * find_existing_css_set - init css array and find the matching css_set -+ * @old_cset: the css_set that we're using before the cgroup transition -+ * @cgrp: the cgroup that we're moving into -+ * @template: out param for the new set of csses, should be clear on entry -+ */ -+static struct css_set *find_existing_css_set(struct css_set *old_cset, -+ struct cgroup *cgrp, -+ struct cgroup_subsys_state *template[]) ++unsigned int ++__swait_wake_locked(struct swait_head *head, unsigned int state, unsigned int num) +{ -+ struct cgroup_root *root = cgrp->root; -+ struct cgroup_subsys *ss; -+ struct css_set *cset; -+ unsigned long key; -+ int i; ++ struct swaiter *curr, *next; ++ int woken = 0; + -+ /* -+ * Build the set of subsystem state objects that we want to see in the -+ * new css_set. while subsystems can change globally, the entries here -+ * won't change, so no need for locking. -+ */ -+ for_each_subsys(ss, i) { -+ if (root->subsys_mask & (1UL << i)) { -+ /* -+ * @ss is in this hierarchy, so we want the -+ * effective css from @cgrp. -+ */ -+ template[i] = cgroup_e_css(cgrp, ss); -+ } else { ++ list_for_each_entry_safe(curr, next, &head->list, node) { ++ if (wake_up_state(curr->task, state)) { ++ __swait_dequeue(curr); + /* -+ * @ss is not in this hierarchy, so we don't want -+ * to change the css. ++ * The waiting task can free the waiter as ++ * soon as curr->task = NULL is written, ++ * without taking any locks. A memory barrier ++ * is required here to prevent the following ++ * store to curr->task from getting ahead of ++ * the dequeue operation. + */ -+ template[i] = old_cset->subsys[i]; ++ smp_wmb(); ++ curr->task = NULL; ++ if (++woken == num) ++ break; + } + } -+ -+ key = css_set_hash(template); -+ hash_for_each_possible(css_set_table, cset, hlist, key) { -+ if (!compare_css_sets(cset, old_cset, cgrp, template)) -+ continue; -+ -+ /* This css_set matches what we need */ -+ return cset; -+ } -+ -+ /* No existing cgroup group matched */ -+ return NULL; -+} -+ -+static void free_cgrp_cset_links(struct list_head *links_to_free) -+{ -+ struct cgrp_cset_link *link, *tmp_link; -+ -+ list_for_each_entry_safe(link, tmp_link, links_to_free, cset_link) { -+ list_del(&link->cset_link); -+ kfree(link); -+ } ++ return woken; +} + -+/** -+ * allocate_cgrp_cset_links - allocate cgrp_cset_links -+ * @count: the number of links to allocate -+ * @tmp_links: list_head the allocated links are put on -+ * -+ * Allocate @count cgrp_cset_link structures and chain them on @tmp_links -+ * through ->cset_link. Returns 0 on success or -errno. -+ */ -+static int allocate_cgrp_cset_links(int count, struct list_head *tmp_links) ++unsigned int ++__swait_wake(struct swait_head *head, unsigned int state, unsigned int num) +{ -+ struct cgrp_cset_link *link; -+ int i; ++ unsigned long flags; ++ int woken; + -+ INIT_LIST_HEAD(tmp_links); ++ if (!swaitqueue_active(head)) ++ return 0; + -+ for (i = 0; i < count; i++) { -+ link = kzalloc(sizeof(*link), GFP_KERNEL); -+ if (!link) { -+ free_cgrp_cset_links(tmp_links); -+ return -ENOMEM; -+ } -+ list_add(&link->cset_link, tmp_links); -+ } -+ return 0; -+} -+ -+/** -+ * link_css_set - a helper function to link a css_set to a cgroup -+ * @tmp_links: cgrp_cset_link objects allocated by allocate_cgrp_cset_links() -+ * @cset: the css_set to be linked -+ * @cgrp: the destination cgroup -+ */ -+static void link_css_set(struct list_head *tmp_links, struct css_set *cset, -+ struct cgroup *cgrp) -+{ -+ struct cgrp_cset_link *link; -+ -+ BUG_ON(list_empty(tmp_links)); -+ -+ if (cgroup_on_dfl(cgrp)) -+ cset->dfl_cgrp = cgrp; -+ -+ link = list_first_entry(tmp_links, struct cgrp_cset_link, cset_link); -+ link->cset = cset; -+ link->cgrp = cgrp; -+ -+ if (list_empty(&cgrp->cset_links)) -+ cgroup_update_populated(cgrp, true); -+ list_move(&link->cset_link, &cgrp->cset_links); -+ -+ /* -+ * Always add links to the tail of the list so that the list -+ * is sorted by order of hierarchy creation -+ */ -+ list_add_tail(&link->cgrp_link, &cset->cgrp_links); ++ raw_spin_lock_irqsave(&head->lock, flags); ++ woken = __swait_wake_locked(head, state, num); ++ raw_spin_unlock_irqrestore(&head->lock, flags); ++ return woken; +} -+ -+/** -+ * find_css_set - return a new css_set with one cgroup updated -+ * @old_cset: the baseline css_set -+ * @cgrp: the cgroup to be updated ++EXPORT_SYMBOL(__swait_wake); +diff -Nur linux-4.1.10.orig/kernel/sched/work-simple.c linux-4.1.10/kernel/sched/work-simple.c +--- linux-4.1.10.orig/kernel/sched/work-simple.c 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/kernel/sched/work-simple.c 2015-10-12 22:33:32.312674375 +0200 +@@ -0,0 +1,172 @@ ++/* ++ * Copyright (C) 2014 BMW Car IT GmbH, Daniel Wagner daniel.wagner@bmw-carit.de + * -+ * Return a new css_set that's equivalent to @old_cset, but with @cgrp -+ * substituted into the appropriate hierarchy. ++ * Provides a framework for enqueuing callbacks from irq context ++ * PREEMPT_RT_FULL safe. The callbacks are executed in kthread context. + */ -+static struct css_set *find_css_set(struct css_set *old_cset, -+ struct cgroup *cgrp) -+{ -+ struct cgroup_subsys_state *template[CGROUP_SUBSYS_COUNT] = { }; -+ struct css_set *cset; -+ struct list_head tmp_links; -+ struct cgrp_cset_link *link; -+ struct cgroup_subsys *ss; -+ unsigned long key; -+ int ssid; -+ -+ lockdep_assert_held(&cgroup_mutex); -+ -+ /* First see if we already have a cgroup group that matches -+ * the desired set */ -+ down_read(&css_set_rwsem); -+ cset = find_existing_css_set(old_cset, cgrp, template); -+ if (cset) -+ get_css_set(cset); -+ up_read(&css_set_rwsem); -+ -+ if (cset) -+ return cset; -+ -+ cset = kzalloc(sizeof(*cset), GFP_KERNEL); -+ if (!cset) -+ return NULL; + -+ /* Allocate all the cgrp_cset_link objects that we'll need */ -+ if (allocate_cgrp_cset_links(cgroup_root_count, &tmp_links) < 0) { -+ kfree(cset); -+ return NULL; -+ } -+ -+ atomic_set(&cset->refcount, 1); -+ INIT_LIST_HEAD(&cset->cgrp_links); -+ INIT_LIST_HEAD(&cset->tasks); -+ INIT_LIST_HEAD(&cset->mg_tasks); -+ INIT_LIST_HEAD(&cset->mg_preload_node); -+ INIT_LIST_HEAD(&cset->mg_node); -+ INIT_HLIST_NODE(&cset->hlist); -+ -+ /* Copy the set of subsystem state objects generated in -+ * find_existing_css_set() */ -+ memcpy(cset->subsys, template, sizeof(cset->subsys)); -+ -+ down_write(&css_set_rwsem); -+ /* Add reference counts and links from the new css_set. */ -+ list_for_each_entry(link, &old_cset->cgrp_links, cgrp_link) { -+ struct cgroup *c = link->cgrp; -+ -+ if (c->root == cgrp->root) -+ c = cgrp; -+ link_css_set(&tmp_links, cset, c); -+ } -+ -+ BUG_ON(!list_empty(&tmp_links)); -+ -+ css_set_count++; -+ -+ /* Add @cset to the hash table */ -+ key = css_set_hash(cset->subsys); -+ hash_add(css_set_table, &cset->hlist, key); -+ -+ for_each_subsys(ss, ssid) -+ list_add_tail(&cset->e_cset_node[ssid], -+ &cset->subsys[ssid]->cgroup->e_csets[ssid]); -+ -+ up_write(&css_set_rwsem); -+ -+ return cset; -+} -+ -+static struct cgroup_root *cgroup_root_from_kf(struct kernfs_root *kf_root) -+{ -+ struct cgroup *root_cgrp = kf_root->kn->priv; ++#include ++#include ++#include ++#include ++#include + -+ return root_cgrp->root; -+} ++#define SWORK_EVENT_PENDING (1 << 0) + -+static int cgroup_init_root_id(struct cgroup_root *root) -+{ -+ int id; ++static DEFINE_MUTEX(worker_mutex); ++static struct sworker *glob_worker; + -+ lockdep_assert_held(&cgroup_mutex); ++struct sworker { ++ struct list_head events; ++ struct swait_head wq; + -+ id = idr_alloc_cyclic(&cgroup_hierarchy_idr, root, 0, 0, GFP_KERNEL); -+ if (id < 0) -+ return id; ++ raw_spinlock_t lock; + -+ root->hierarchy_id = id; -+ return 0; -+} ++ struct task_struct *task; ++ int refs; ++}; + -+static void cgroup_exit_root_id(struct cgroup_root *root) ++static bool swork_readable(struct sworker *worker) +{ -+ lockdep_assert_held(&cgroup_mutex); ++ bool r; + -+ if (root->hierarchy_id) { -+ idr_remove(&cgroup_hierarchy_idr, root->hierarchy_id); -+ root->hierarchy_id = 0; -+ } -+} ++ if (kthread_should_stop()) ++ return true; + -+static void cgroup_free_root(struct cgroup_root *root) -+{ -+ if (root) { -+ /* hierarhcy ID shoulid already have been released */ -+ WARN_ON_ONCE(root->hierarchy_id); ++ raw_spin_lock_irq(&worker->lock); ++ r = !list_empty(&worker->events); ++ raw_spin_unlock_irq(&worker->lock); + -+ idr_destroy(&root->cgroup_idr); -+ kfree(root); -+ } ++ return r; +} + -+static void cgroup_destroy_root(struct cgroup_root *root) ++static int swork_kthread(void *arg) +{ -+ struct cgroup *cgrp = &root->cgrp; -+ struct cgrp_cset_link *link, *tmp_link; -+ -+ mutex_lock(&cgroup_mutex); -+ -+ BUG_ON(atomic_read(&root->nr_cgrps)); -+ BUG_ON(!list_empty(&cgrp->self.children)); ++ struct sworker *worker = arg; + -+ /* Rebind all subsystems back to the default hierarchy */ -+ rebind_subsystems(&cgrp_dfl_root, root->subsys_mask); ++ for (;;) { ++ swait_event_interruptible(worker->wq, ++ swork_readable(worker)); ++ if (kthread_should_stop()) ++ break; + -+ /* -+ * Release all the links from cset_links to this hierarchy's -+ * root cgroup -+ */ -+ down_write(&css_set_rwsem); ++ raw_spin_lock_irq(&worker->lock); ++ while (!list_empty(&worker->events)) { ++ struct swork_event *sev; + -+ list_for_each_entry_safe(link, tmp_link, &cgrp->cset_links, cset_link) { -+ list_del(&link->cset_link); -+ list_del(&link->cgrp_link); -+ kfree(link); -+ } -+ up_write(&css_set_rwsem); ++ sev = list_first_entry(&worker->events, ++ struct swork_event, item); ++ list_del(&sev->item); ++ raw_spin_unlock_irq(&worker->lock); + -+ if (!list_empty(&root->root_list)) { -+ list_del(&root->root_list); -+ cgroup_root_count--; ++ WARN_ON_ONCE(!test_and_clear_bit(SWORK_EVENT_PENDING, ++ &sev->flags)); ++ sev->func(sev); ++ raw_spin_lock_irq(&worker->lock); ++ } ++ raw_spin_unlock_irq(&worker->lock); + } -+ -+ cgroup_exit_root_id(root); -+ -+ mutex_unlock(&cgroup_mutex); -+ -+ kernfs_destroy_root(root->kf_root); -+ cgroup_free_root(root); ++ return 0; +} + -+/* look up cgroup associated with given css_set on the specified hierarchy */ -+static struct cgroup *cset_cgroup_from_root(struct css_set *cset, -+ struct cgroup_root *root) ++static struct sworker *swork_create(void) +{ -+ struct cgroup *res = NULL; -+ -+ lockdep_assert_held(&cgroup_mutex); -+ lockdep_assert_held(&css_set_rwsem); ++ struct sworker *worker; + -+ if (cset == &init_css_set) { -+ res = &root->cgrp; -+ } else { -+ struct cgrp_cset_link *link; ++ worker = kzalloc(sizeof(*worker), GFP_KERNEL); ++ if (!worker) ++ return ERR_PTR(-ENOMEM); + -+ list_for_each_entry(link, &cset->cgrp_links, cgrp_link) { -+ struct cgroup *c = link->cgrp; ++ INIT_LIST_HEAD(&worker->events); ++ raw_spin_lock_init(&worker->lock); ++ init_swait_head(&worker->wq); + -+ if (c->root == root) { -+ res = c; -+ break; -+ } -+ } ++ worker->task = kthread_run(swork_kthread, worker, "kswork"); ++ if (IS_ERR(worker->task)) { ++ kfree(worker); ++ return ERR_PTR(-ENOMEM); + } + -+ BUG_ON(!res); -+ return res; ++ return worker; +} + -+/* -+ * Return the cgroup for "task" from the given hierarchy. Must be -+ * called with cgroup_mutex and css_set_rwsem held. -+ */ -+static struct cgroup *task_cgroup_from_root(struct task_struct *task, -+ struct cgroup_root *root) ++static void swork_destroy(struct sworker *worker) +{ -+ /* -+ * No need to lock the task - since we hold cgroup_mutex the -+ * task can't change groups, so the only thing that can happen -+ * is that it exits and its css is set back to init_css_set. -+ */ -+ return cset_cgroup_from_root(task_css_set(task), root); -+} -+ -+/* -+ * A task must hold cgroup_mutex to modify cgroups. -+ * -+ * Any task can increment and decrement the count field without lock. -+ * So in general, code holding cgroup_mutex can't rely on the count -+ * field not changing. However, if the count goes to zero, then only -+ * cgroup_attach_task() can increment it again. Because a count of zero -+ * means that no tasks are currently attached, therefore there is no -+ * way a task attached to that cgroup can fork (the other way to -+ * increment the count). So code holding cgroup_mutex can safely -+ * assume that if the count is zero, it will stay zero. Similarly, if -+ * a task holds cgroup_mutex on a cgroup with zero count, it -+ * knows that the cgroup won't be removed, as cgroup_rmdir() -+ * needs that mutex. -+ * -+ * A cgroup can only be deleted if both its 'count' of using tasks -+ * is zero, and its list of 'children' cgroups is empty. Since all -+ * tasks in the system use _some_ cgroup, and since there is always at -+ * least one task in the system (init, pid == 1), therefore, root cgroup -+ * always has either children cgroups and/or using tasks. So we don't -+ * need a special hack to ensure that root cgroup cannot be deleted. -+ * -+ * P.S. One more locking exception. RCU is used to guard the -+ * update of a tasks cgroup pointer by cgroup_attach_task() -+ */ -+ -+static int cgroup_populate_dir(struct cgroup *cgrp, unsigned int subsys_mask); -+static struct kernfs_syscall_ops cgroup_kf_syscall_ops; -+static const struct file_operations proc_cgroupstats_operations; ++ kthread_stop(worker->task); + -+static char *cgroup_file_name(struct cgroup *cgrp, const struct cftype *cft, -+ char *buf) -+{ -+ if (cft->ss && !(cft->flags & CFTYPE_NO_PREFIX) && -+ !(cgrp->root->flags & CGRP_ROOT_NOPREFIX)) -+ snprintf(buf, CGROUP_FILE_NAME_MAX, "%s.%s", -+ cft->ss->name, cft->name); -+ else -+ strncpy(buf, cft->name, CGROUP_FILE_NAME_MAX); -+ return buf; ++ WARN_ON(!list_empty(&worker->events)); ++ kfree(worker); +} + +/** -+ * cgroup_file_mode - deduce file mode of a control file -+ * @cft: the control file in question ++ * swork_queue - queue swork ++ * ++ * Returns %false if @work was already on a queue, %true otherwise. + * -+ * returns cft->mode if ->mode is not 0 -+ * returns S_IRUGO|S_IWUSR if it has both a read and a write handler -+ * returns S_IRUGO if it has only a read handler -+ * returns S_IWUSR if it has only a write hander ++ * The work is queued and processed on a random CPU + */ -+static umode_t cgroup_file_mode(const struct cftype *cft) ++bool swork_queue(struct swork_event *sev) +{ -+ umode_t mode = 0; -+ -+ if (cft->mode) -+ return cft->mode; -+ -+ if (cft->read_u64 || cft->read_s64 || cft->seq_show) -+ mode |= S_IRUGO; -+ -+ if (cft->write_u64 || cft->write_s64 || cft->write) -+ mode |= S_IWUSR; -+ -+ return mode; -+} ++ unsigned long flags; + -+static void cgroup_get(struct cgroup *cgrp) -+{ -+ WARN_ON_ONCE(cgroup_is_dead(cgrp)); -+ css_get(&cgrp->self); -+} ++ if (test_and_set_bit(SWORK_EVENT_PENDING, &sev->flags)) ++ return false; + -+static bool cgroup_tryget(struct cgroup *cgrp) -+{ -+ return css_tryget(&cgrp->self); -+} ++ raw_spin_lock_irqsave(&glob_worker->lock, flags); ++ list_add_tail(&sev->item, &glob_worker->events); ++ raw_spin_unlock_irqrestore(&glob_worker->lock, flags); + -+static void cgroup_put(struct cgroup *cgrp) -+{ -+ css_put(&cgrp->self); ++ swait_wake(&glob_worker->wq); ++ return true; +} ++EXPORT_SYMBOL_GPL(swork_queue); + +/** -+ * cgroup_calc_child_subsys_mask - calculate child_subsys_mask -+ * @cgrp: the target cgroup -+ * @subtree_control: the new subtree_control mask to consider ++ * swork_get - get an instance of the sworker + * -+ * On the default hierarchy, a subsystem may request other subsystems to be -+ * enabled together through its ->depends_on mask. In such cases, more -+ * subsystems than specified in "cgroup.subtree_control" may be enabled. ++ * Returns an negative error code if the initialization if the worker did not ++ * work, %0 otherwise. + * -+ * This function calculates which subsystems need to be enabled if -+ * @subtree_control is to be applied to @cgrp. The returned mask is always -+ * a superset of @subtree_control and follows the usual hierarchy rules. + */ -+static unsigned int cgroup_calc_child_subsys_mask(struct cgroup *cgrp, -+ unsigned int subtree_control) ++int swork_get(void) +{ -+ struct cgroup *parent = cgroup_parent(cgrp); -+ unsigned int cur_ss_mask = subtree_control; -+ struct cgroup_subsys *ss; -+ int ssid; -+ -+ lockdep_assert_held(&cgroup_mutex); -+ -+ if (!cgroup_on_dfl(cgrp)) -+ return cur_ss_mask; -+ -+ while (true) { -+ unsigned int new_ss_mask = cur_ss_mask; -+ -+ for_each_subsys(ss, ssid) -+ if (cur_ss_mask & (1 << ssid)) -+ new_ss_mask |= ss->depends_on; ++ struct sworker *worker; + -+ /* -+ * Mask out subsystems which aren't available. This can -+ * happen only if some depended-upon subsystems were bound -+ * to non-default hierarchies. -+ */ -+ if (parent) -+ new_ss_mask &= parent->child_subsys_mask; -+ else -+ new_ss_mask &= cgrp->root->subsys_mask; ++ mutex_lock(&worker_mutex); ++ if (!glob_worker) { ++ worker = swork_create(); ++ if (IS_ERR(worker)) { ++ mutex_unlock(&worker_mutex); ++ return -ENOMEM; ++ } + -+ if (new_ss_mask == cur_ss_mask) -+ break; -+ cur_ss_mask = new_ss_mask; ++ glob_worker = worker; + } + -+ return cur_ss_mask; -+} ++ glob_worker->refs++; ++ mutex_unlock(&worker_mutex); + -+/** -+ * cgroup_refresh_child_subsys_mask - update child_subsys_mask -+ * @cgrp: the target cgroup -+ * -+ * Update @cgrp->child_subsys_mask according to the current -+ * @cgrp->subtree_control using cgroup_calc_child_subsys_mask(). -+ */ -+static void cgroup_refresh_child_subsys_mask(struct cgroup *cgrp) -+{ -+ cgrp->child_subsys_mask = -+ cgroup_calc_child_subsys_mask(cgrp, cgrp->subtree_control); ++ return 0; +} ++EXPORT_SYMBOL_GPL(swork_get); + +/** -+ * cgroup_kn_unlock - unlocking helper for cgroup kernfs methods -+ * @kn: the kernfs_node being serviced ++ * swork_put - puts an instance of the sworker + * -+ * This helper undoes cgroup_kn_lock_live() and should be invoked before -+ * the method finishes if locking succeeded. Note that once this function -+ * returns the cgroup returned by cgroup_kn_lock_live() may become -+ * inaccessible any time. If the caller intends to continue to access the -+ * cgroup, it should pin it before invoking this function. ++ * Will destroy the sworker thread. This function must not be called until all ++ * queued events have been completed. + */ -+static void cgroup_kn_unlock(struct kernfs_node *kn) ++void swork_put(void) +{ -+ struct cgroup *cgrp; -+ -+ if (kernfs_type(kn) == KERNFS_DIR) -+ cgrp = kn->priv; -+ else -+ cgrp = kn->parent->priv; ++ mutex_lock(&worker_mutex); + -+ mutex_unlock(&cgroup_mutex); ++ glob_worker->refs--; ++ if (glob_worker->refs > 0) ++ goto out; + -+ kernfs_unbreak_active_protection(kn); -+ cgroup_put(cgrp); ++ swork_destroy(glob_worker); ++ glob_worker = NULL; ++out: ++ mutex_unlock(&worker_mutex); +} -+ -+/** -+ * cgroup_kn_lock_live - locking helper for cgroup kernfs methods -+ * @kn: the kernfs_node being serviced -+ * -+ * This helper is to be used by a cgroup kernfs method currently servicing -+ * @kn. It breaks the active protection, performs cgroup locking and -+ * verifies that the associated cgroup is alive. Returns the cgroup if -+ * alive; otherwise, %NULL. A successful return should be undone by a -+ * matching cgroup_kn_unlock() invocation. -+ * -+ * Any cgroup kernfs method implementation which requires locking the -+ * associated cgroup should use this helper. It avoids nesting cgroup -+ * locking under kernfs active protection and allows all kernfs operations -+ * including self-removal. -+ */ -+static struct cgroup *cgroup_kn_lock_live(struct kernfs_node *kn) ++EXPORT_SYMBOL_GPL(swork_put); +diff -Nur linux-4.1.10.orig/kernel/signal.c linux-4.1.10/kernel/signal.c +--- linux-4.1.10.orig/kernel/signal.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/signal.c 2015-10-12 22:33:32.312674375 +0200 +@@ -14,6 +14,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -352,13 +353,45 @@ + return false; + } + ++#ifdef __HAVE_ARCH_CMPXCHG ++static inline struct sigqueue *get_task_cache(struct task_struct *t) +{ -+ struct cgroup *cgrp; -+ -+ if (kernfs_type(kn) == KERNFS_DIR) -+ cgrp = kn->priv; -+ else -+ cgrp = kn->parent->priv; ++ struct sigqueue *q = t->sigqueue_cache; + -+ /* -+ * We're gonna grab cgroup_mutex which nests outside kernfs -+ * active_ref. cgroup liveliness check alone provides enough -+ * protection against removal. Ensure @cgrp stays accessible and -+ * break the active_ref protection. -+ */ -+ if (!cgroup_tryget(cgrp)) ++ if (cmpxchg(&t->sigqueue_cache, q, NULL) != q) + return NULL; -+ kernfs_break_active_protection(kn); ++ return q; ++} + -+ mutex_lock(&cgroup_mutex); ++static inline int put_task_cache(struct task_struct *t, struct sigqueue *q) ++{ ++ if (cmpxchg(&t->sigqueue_cache, NULL, q) == NULL) ++ return 0; ++ return 1; ++} + -+ if (!cgroup_is_dead(cgrp)) -+ return cgrp; ++#else + -+ cgroup_kn_unlock(kn); ++static inline struct sigqueue *get_task_cache(struct task_struct *t) ++{ + return NULL; +} + -+static void cgroup_rm_file(struct cgroup *cgrp, const struct cftype *cft) ++static inline int put_task_cache(struct task_struct *t, struct sigqueue *q) +{ -+ char name[CGROUP_FILE_NAME_MAX]; -+ -+ lockdep_assert_held(&cgroup_mutex); -+ kernfs_remove_by_name(cgrp->kn, cgroup_file_name(cgrp, cft, name)); ++ return 1; +} + -+/** -+ * cgroup_clear_dir - remove subsys files in a cgroup directory -+ * @cgrp: target cgroup -+ * @subsys_mask: mask of the subsystem ids whose files should be removed -+ */ -+static void cgroup_clear_dir(struct cgroup *cgrp, unsigned int subsys_mask) -+{ -+ struct cgroup_subsys *ss; -+ int i; -+ -+ for_each_subsys(ss, i) { -+ struct cftype *cfts; ++#endif + -+ if (!(subsys_mask & (1 << i))) -+ continue; -+ list_for_each_entry(cfts, &ss->cfts, node) -+ cgroup_addrm_files(cgrp, cfts, false); -+ } + /* + * allocate a new signal queue record + * - this may be called without locks if and only if t == current, otherwise an + * appropriate lock must be held to stop the target task from exiting + */ + static struct sigqueue * +-__sigqueue_alloc(int sig, struct task_struct *t, gfp_t flags, int override_rlimit) ++__sigqueue_do_alloc(int sig, struct task_struct *t, gfp_t flags, ++ int override_rlimit, int fromslab) + { + struct sigqueue *q = NULL; + struct user_struct *user; +@@ -375,7 +408,10 @@ + if (override_rlimit || + atomic_read(&user->sigpending) <= + task_rlimit(t, RLIMIT_SIGPENDING)) { +- q = kmem_cache_alloc(sigqueue_cachep, flags); ++ if (!fromslab) ++ q = get_task_cache(t); ++ if (!q) ++ q = kmem_cache_alloc(sigqueue_cachep, flags); + } else { + print_dropped_signal(sig); + } +@@ -392,6 +428,13 @@ + return q; + } + ++static struct sigqueue * ++__sigqueue_alloc(int sig, struct task_struct *t, gfp_t flags, ++ int override_rlimit) ++{ ++ return __sigqueue_do_alloc(sig, t, flags, override_rlimit, 0); +} + -+static int rebind_subsystems(struct cgroup_root *dst_root, unsigned int ss_mask) + static void __sigqueue_free(struct sigqueue *q) + { + if (q->flags & SIGQUEUE_PREALLOC) +@@ -401,6 +444,21 @@ + kmem_cache_free(sigqueue_cachep, q); + } + ++static void sigqueue_free_current(struct sigqueue *q) +{ -+ struct cgroup_subsys *ss; -+ unsigned int tmp_ss_mask; -+ int ssid, i, ret; -+ -+ lockdep_assert_held(&cgroup_mutex); -+ -+ for_each_subsys(ss, ssid) { -+ if (!(ss_mask & (1 << ssid))) -+ continue; -+ -+ /* if @ss has non-root csses attached to it, can't move */ -+ if (css_next_child(NULL, cgroup_css(&ss->root->cgrp, ss))) -+ return -EBUSY; -+ -+ /* can't move between two non-dummy roots either */ -+ if (ss->root != &cgrp_dfl_root && dst_root != &cgrp_dfl_root) -+ return -EBUSY; -+ } -+ -+ /* skip creating root files on dfl_root for inhibited subsystems */ -+ tmp_ss_mask = ss_mask; -+ if (dst_root == &cgrp_dfl_root) -+ tmp_ss_mask &= ~cgrp_dfl_root_inhibit_ss_mask; -+ -+ ret = cgroup_populate_dir(&dst_root->cgrp, tmp_ss_mask); -+ if (ret) { -+ if (dst_root != &cgrp_dfl_root) -+ return ret; -+ -+ /* -+ * Rebinding back to the default root is not allowed to -+ * fail. Using both default and non-default roots should -+ * be rare. Moving subsystems back and forth even more so. -+ * Just warn about it and continue. -+ */ -+ if (cgrp_dfl_root_visible) { -+ pr_warn("failed to create files (%d) while rebinding 0x%x to default root\n", -+ ret, ss_mask); -+ pr_warn("you may retry by moving them to a different hierarchy and unbinding\n"); -+ } -+ } -+ -+ /* -+ * Nothing can fail from this point on. Remove files for the -+ * removed subsystems and rebind each subsystem. -+ */ -+ for_each_subsys(ss, ssid) -+ if (ss_mask & (1 << ssid)) -+ cgroup_clear_dir(&ss->root->cgrp, 1 << ssid); ++ struct user_struct *up; + -+ for_each_subsys(ss, ssid) { -+ struct cgroup_root *src_root; -+ struct cgroup_subsys_state *css; -+ struct css_set *cset; ++ if (q->flags & SIGQUEUE_PREALLOC) ++ return; + -+ if (!(ss_mask & (1 << ssid))) -+ continue; ++ up = q->user; ++ if (rt_prio(current->normal_prio) && !put_task_cache(current, q)) { ++ atomic_dec(&up->sigpending); ++ free_uid(up); ++ } else ++ __sigqueue_free(q); ++} + -+ src_root = ss->root; -+ css = cgroup_css(&src_root->cgrp, ss); + void flush_sigqueue(struct sigpending *queue) + { + struct sigqueue *q; +@@ -414,6 +472,21 @@ + } + + /* ++ * Called from __exit_signal. Flush tsk->pending and ++ * tsk->sigqueue_cache ++ */ ++void flush_task_sigqueue(struct task_struct *tsk) ++{ ++ struct sigqueue *q; + -+ WARN_ON(!css || cgroup_css(&dst_root->cgrp, ss)); ++ flush_sigqueue(&tsk->pending); + -+ RCU_INIT_POINTER(src_root->cgrp.subsys[ssid], NULL); -+ rcu_assign_pointer(dst_root->cgrp.subsys[ssid], css); -+ ss->root = dst_root; -+ css->cgroup = &dst_root->cgrp; ++ q = get_task_cache(tsk); ++ if (q) ++ kmem_cache_free(sigqueue_cachep, q); ++} + -+ down_write(&css_set_rwsem); -+ hash_for_each(css_set_table, i, cset, hlist) -+ list_move_tail(&cset->e_cset_node[ss->id], -+ &dst_root->cgrp.e_csets[ss->id]); -+ up_write(&css_set_rwsem); ++/* + * Flush all pending signals for a task. + */ + void __flush_signals(struct task_struct *t) +@@ -565,7 +638,7 @@ + still_pending: + list_del_init(&first->list); + copy_siginfo(info, &first->info); +- __sigqueue_free(first); ++ sigqueue_free_current(first); + } else { + /* + * Ok, it wasn't in the queue. This must be +@@ -611,6 +684,8 @@ + { + int signr; + ++ WARN_ON_ONCE(tsk != current); + -+ src_root->subsys_mask &= ~(1 << ssid); -+ src_root->cgrp.subtree_control &= ~(1 << ssid); -+ cgroup_refresh_child_subsys_mask(&src_root->cgrp); + /* We only dequeue private signals from ourselves, we don't let + * signalfd steal them + */ +@@ -1207,8 +1282,8 @@ + * We don't want to have recursive SIGSEGV's etc, for example, + * that is why we also clear SIGNAL_UNKILLABLE. + */ +-int +-force_sig_info(int sig, struct siginfo *info, struct task_struct *t) ++static int ++do_force_sig_info(int sig, struct siginfo *info, struct task_struct *t) + { + unsigned long int flags; + int ret, blocked, ignored; +@@ -1233,6 +1308,39 @@ + return ret; + } + ++int force_sig_info(int sig, struct siginfo *info, struct task_struct *t) ++{ ++/* ++ * On some archs, PREEMPT_RT has to delay sending a signal from a trap ++ * since it can not enable preemption, and the signal code's spin_locks ++ * turn into mutexes. Instead, it must set TIF_NOTIFY_RESUME which will ++ * send the signal on exit of the trap. ++ */ ++#ifdef ARCH_RT_DELAYS_SIGNAL_SEND ++ if (in_atomic()) { ++ if (WARN_ON_ONCE(t != current)) ++ return 0; ++ if (WARN_ON_ONCE(t->forced_info.si_signo)) ++ return 0; + -+ /* default hierarchy doesn't enable controllers by default */ -+ dst_root->subsys_mask |= 1 << ssid; -+ if (dst_root != &cgrp_dfl_root) { -+ dst_root->cgrp.subtree_control |= 1 << ssid; -+ cgroup_refresh_child_subsys_mask(&dst_root->cgrp); ++ if (is_si_special(info)) { ++ WARN_ON_ONCE(info != SEND_SIG_PRIV); ++ t->forced_info.si_signo = sig; ++ t->forced_info.si_errno = 0; ++ t->forced_info.si_code = SI_KERNEL; ++ t->forced_info.si_pid = 0; ++ t->forced_info.si_uid = 0; ++ } else { ++ t->forced_info = *info; + } + -+ if (ss->bind) -+ ss->bind(css); ++ set_tsk_thread_flag(t, TIF_NOTIFY_RESUME); ++ return 0; + } -+ -+ kernfs_activate(dst_root->cgrp.kn); -+ return 0; ++#endif ++ return do_force_sig_info(sig, info, t); +} + -+static int cgroup_show_options(struct seq_file *seq, -+ struct kernfs_root *kf_root) -+{ -+ struct cgroup_root *root = cgroup_root_from_kf(kf_root); -+ struct cgroup_subsys *ss; -+ int ssid; -+ -+ for_each_subsys(ss, ssid) -+ if (root->subsys_mask & (1 << ssid)) -+ seq_show_option(seq, ss->name, NULL); -+ if (root->flags & CGRP_ROOT_NOPREFIX) -+ seq_puts(seq, ",noprefix"); -+ if (root->flags & CGRP_ROOT_XATTR) -+ seq_puts(seq, ",xattr"); -+ -+ spin_lock(&release_agent_path_lock); -+ if (strlen(root->release_agent_path)) -+ seq_show_option(seq, "release_agent", -+ root->release_agent_path); -+ spin_unlock(&release_agent_path_lock); -+ -+ if (test_bit(CGRP_CPUSET_CLONE_CHILDREN, &root->cgrp.flags)) -+ seq_puts(seq, ",clone_children"); -+ if (strlen(root->name)) -+ seq_show_option(seq, "name", root->name); -+ return 0; -+} + /* + * Nuke all other threads in the group. + */ +@@ -1267,12 +1375,12 @@ + * Disable interrupts early to avoid deadlocks. + * See rcu_read_unlock() comment header for details. + */ +- local_irq_save(*flags); ++ local_irq_save_nort(*flags); + rcu_read_lock(); + sighand = rcu_dereference(tsk->sighand); + if (unlikely(sighand == NULL)) { + rcu_read_unlock(); +- local_irq_restore(*flags); ++ local_irq_restore_nort(*flags); + break; + } + /* +@@ -1293,7 +1401,7 @@ + } + spin_unlock(&sighand->siglock); + rcu_read_unlock(); +- local_irq_restore(*flags); ++ local_irq_restore_nort(*flags); + } + + return sighand; +@@ -1536,7 +1644,8 @@ + */ + struct sigqueue *sigqueue_alloc(void) + { +- struct sigqueue *q = __sigqueue_alloc(-1, current, GFP_KERNEL, 0); ++ /* Preallocated sigqueue objects always from the slabcache ! */ ++ struct sigqueue *q = __sigqueue_do_alloc(-1, current, GFP_KERNEL, 0, 1); + + if (q) + q->flags |= SIGQUEUE_PREALLOC; +@@ -1897,15 +2006,7 @@ + if (gstop_done && ptrace_reparented(current)) + do_notify_parent_cldstop(current, false, why); + +- /* +- * Don't want to allow preemption here, because +- * sys_ptrace() needs this task to be inactive. +- * +- * XXX: implement read_unlock_no_resched(). +- */ +- preempt_disable(); + read_unlock(&tasklist_lock); +- preempt_enable_no_resched(); + freezable_schedule(); + } else { + /* +diff -Nur linux-4.1.10.orig/kernel/softirq.c linux-4.1.10/kernel/softirq.c +--- linux-4.1.10.orig/kernel/softirq.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/softirq.c 2015-10-12 22:33:32.312674375 +0200 +@@ -21,10 +21,12 @@ + #include + #include + #include ++#include + #include + #include + #include + #include ++#include + #include + + #define CREATE_TRACE_POINTS +@@ -62,6 +64,98 @@ + "TASKLET", "SCHED", "HRTIMER", "RCU" + }; + ++#ifdef CONFIG_NO_HZ_COMMON ++# ifdef CONFIG_PREEMPT_RT_FULL + -+struct cgroup_sb_opts { -+ unsigned int subsys_mask; -+ unsigned int flags; -+ char *release_agent; -+ bool cpuset_clone_children; -+ char *name; -+ /* User explicitly requested empty subsystem */ -+ bool none; ++struct softirq_runner { ++ struct task_struct *runner[NR_SOFTIRQS]; +}; + -+static int parse_cgroupfs_options(char *data, struct cgroup_sb_opts *opts) ++static DEFINE_PER_CPU(struct softirq_runner, softirq_runners); ++ ++static inline void softirq_set_runner(unsigned int sirq) +{ -+ char *token, *o = data; -+ bool all_ss = false, one_ss = false; -+ unsigned int mask = -1U; -+ struct cgroup_subsys *ss; -+ int nr_opts = 0; -+ int i; ++ struct softirq_runner *sr = this_cpu_ptr(&softirq_runners); + -+#ifdef CONFIG_CPUSETS -+ mask = ~(1U << cpuset_cgrp_id); -+#endif ++ sr->runner[sirq] = current; ++} + -+ memset(opts, 0, sizeof(*opts)); ++static inline void softirq_clr_runner(unsigned int sirq) ++{ ++ struct softirq_runner *sr = this_cpu_ptr(&softirq_runners); + -+ while ((token = strsep(&o, ",")) != NULL) { -+ nr_opts++; ++ sr->runner[sirq] = NULL; ++} + -+ if (!*token) -+ return -EINVAL; -+ if (!strcmp(token, "none")) { -+ /* Explicitly have no subsystems */ -+ opts->none = true; -+ continue; -+ } -+ if (!strcmp(token, "all")) { -+ /* Mutually exclusive option 'all' + subsystem name */ -+ if (one_ss) -+ return -EINVAL; -+ all_ss = true; -+ continue; -+ } -+ if (!strcmp(token, "__DEVEL__sane_behavior")) { -+ opts->flags |= CGRP_ROOT_SANE_BEHAVIOR; -+ continue; -+ } -+ if (!strcmp(token, "noprefix")) { -+ opts->flags |= CGRP_ROOT_NOPREFIX; -+ continue; -+ } -+ if (!strcmp(token, "clone_children")) { -+ opts->cpuset_clone_children = true; -+ continue; -+ } -+ if (!strcmp(token, "xattr")) { -+ opts->flags |= CGRP_ROOT_XATTR; -+ continue; -+ } -+ if (!strncmp(token, "release_agent=", 14)) { -+ /* Specifying two release agents is forbidden */ -+ if (opts->release_agent) -+ return -EINVAL; -+ opts->release_agent = -+ kstrndup(token + 14, PATH_MAX - 1, GFP_KERNEL); -+ if (!opts->release_agent) -+ return -ENOMEM; -+ continue; -+ } -+ if (!strncmp(token, "name=", 5)) { -+ const char *name = token + 5; -+ /* Can't specify an empty name */ -+ if (!strlen(name)) -+ return -EINVAL; -+ /* Must match [\w.-]+ */ -+ for (i = 0; i < strlen(name); i++) { -+ char c = name[i]; -+ if (isalnum(c)) -+ continue; -+ if ((c == '.') || (c == '-') || (c == '_')) -+ continue; -+ return -EINVAL; -+ } -+ /* Specifying two names is forbidden */ -+ if (opts->name) -+ return -EINVAL; -+ opts->name = kstrndup(name, -+ MAX_CGROUP_ROOT_NAMELEN - 1, -+ GFP_KERNEL); -+ if (!opts->name) -+ return -ENOMEM; -+ -+ continue; -+ } ++/* ++ * On preempt-rt a softirq running context might be blocked on a ++ * lock. There might be no other runnable task on this CPU because the ++ * lock owner runs on some other CPU. So we have to go into idle with ++ * the pending bit set. Therefor we need to check this otherwise we ++ * warn about false positives which confuses users and defeats the ++ * whole purpose of this test. ++ * ++ * This code is called with interrupts disabled. ++ */ ++void softirq_check_pending_idle(void) ++{ ++ static int rate_limit; ++ struct softirq_runner *sr = this_cpu_ptr(&softirq_runners); ++ u32 warnpending; ++ int i; + -+ for_each_subsys(ss, i) { -+ if (strcmp(token, ss->name)) -+ continue; -+ if (ss->disabled) -+ continue; ++ if (rate_limit >= 10) ++ return; + -+ /* Mutually exclusive option 'all' + subsystem name */ -+ if (all_ss) -+ return -EINVAL; -+ opts->subsys_mask |= (1 << i); -+ one_ss = true; ++ warnpending = local_softirq_pending() & SOFTIRQ_STOP_IDLE_MASK; ++ for (i = 0; i < NR_SOFTIRQS; i++) { ++ struct task_struct *tsk = sr->runner[i]; + -+ break; ++ /* ++ * The wakeup code in rtmutex.c wakes up the task ++ * _before_ it sets pi_blocked_on to NULL under ++ * tsk->pi_lock. So we need to check for both: state ++ * and pi_blocked_on. ++ */ ++ if (tsk) { ++ raw_spin_lock(&tsk->pi_lock); ++ if (tsk->pi_blocked_on || tsk->state == TASK_RUNNING) { ++ /* Clear all bits pending in that task */ ++ warnpending &= ~(tsk->softirqs_raised); ++ warnpending &= ~(1 << i); ++ } ++ raw_spin_unlock(&tsk->pi_lock); + } -+ if (i == CGROUP_SUBSYS_COUNT) -+ return -ENOENT; + } + -+ if (opts->flags & CGRP_ROOT_SANE_BEHAVIOR) { -+ pr_warn("sane_behavior: this is still under development and its behaviors will change, proceed at your own risk\n"); -+ if (nr_opts != 1) { -+ pr_err("sane_behavior: no other mount options allowed\n"); -+ return -EINVAL; -+ } -+ return 0; ++ if (warnpending) { ++ printk(KERN_ERR "NOHZ: local_softirq_pending %02x\n", ++ warnpending); ++ rate_limit++; + } -+ -+ /* -+ * If the 'all' option was specified select all the subsystems, -+ * otherwise if 'none', 'name=' and a subsystem name options were -+ * not specified, let's default to 'all' -+ */ -+ if (all_ss || (!one_ss && !opts->none && !opts->name)) -+ for_each_subsys(ss, i) -+ if (!ss->disabled) -+ opts->subsys_mask |= (1 << i); -+ -+ /* -+ * We either have to specify by name or by subsystems. (So all -+ * empty hierarchies must have a name). -+ */ -+ if (!opts->subsys_mask && !opts->name) -+ return -EINVAL; -+ -+ /* -+ * Option noprefix was introduced just for backward compatibility -+ * with the old cpuset, so we allow noprefix only if mounting just -+ * the cpuset subsystem. -+ */ -+ if ((opts->flags & CGRP_ROOT_NOPREFIX) && (opts->subsys_mask & mask)) -+ return -EINVAL; -+ -+ /* Can't specify "none" and some subsystems */ -+ if (opts->subsys_mask && opts->none) -+ return -EINVAL; -+ -+ return 0; +} -+ -+static int cgroup_remount(struct kernfs_root *kf_root, int *flags, char *data) ++# else ++/* ++ * On !PREEMPT_RT we just printk rate limited: ++ */ ++void softirq_check_pending_idle(void) +{ -+ int ret = 0; -+ struct cgroup_root *root = cgroup_root_from_kf(kf_root); -+ struct cgroup_sb_opts opts; -+ unsigned int added_mask, removed_mask; -+ -+ if (root == &cgrp_dfl_root) { -+ pr_err("remount is not allowed\n"); -+ return -EINVAL; -+ } -+ -+ mutex_lock(&cgroup_mutex); -+ -+ /* See what subsystems are wanted */ -+ ret = parse_cgroupfs_options(data, &opts); -+ if (ret) -+ goto out_unlock; -+ -+ if (opts.subsys_mask != root->subsys_mask || opts.release_agent) -+ pr_warn("option changes via remount are deprecated (pid=%d comm=%s)\n", -+ task_tgid_nr(current), current->comm); -+ -+ added_mask = opts.subsys_mask & ~root->subsys_mask; -+ removed_mask = root->subsys_mask & ~opts.subsys_mask; -+ -+ /* Don't allow flags or name to change at remount */ -+ if ((opts.flags ^ root->flags) || -+ (opts.name && strcmp(opts.name, root->name))) { -+ pr_err("option or name mismatch, new: 0x%x \"%s\", old: 0x%x \"%s\"\n", -+ opts.flags, opts.name ?: "", root->flags, root->name); -+ ret = -EINVAL; -+ goto out_unlock; -+ } -+ -+ /* remounting is not allowed for populated hierarchies */ -+ if (!list_empty(&root->cgrp.self.children)) { -+ ret = -EBUSY; -+ goto out_unlock; -+ } -+ -+ ret = rebind_subsystems(root, added_mask); -+ if (ret) -+ goto out_unlock; -+ -+ rebind_subsystems(&cgrp_dfl_root, removed_mask); ++ static int rate_limit; + -+ if (opts.release_agent) { -+ spin_lock(&release_agent_path_lock); -+ strcpy(root->release_agent_path, opts.release_agent); -+ spin_unlock(&release_agent_path_lock); ++ if (rate_limit < 10 && ++ (local_softirq_pending() & SOFTIRQ_STOP_IDLE_MASK)) { ++ printk(KERN_ERR "NOHZ: local_softirq_pending %02x\n", ++ local_softirq_pending()); ++ rate_limit++; + } -+ out_unlock: -+ kfree(opts.release_agent); -+ kfree(opts.name); -+ mutex_unlock(&cgroup_mutex); -+ return ret; +} ++# endif + -+/* -+ * To reduce the fork() overhead for systems that are not actually using -+ * their cgroups capability, we don't maintain the lists running through -+ * each css_set to its tasks until we see the list actually used - in other -+ * words after the first mount. -+ */ -+static bool use_task_css_set_links __read_mostly; ++#else /* !CONFIG_NO_HZ_COMMON */ ++static inline void softirq_set_runner(unsigned int sirq) { } ++static inline void softirq_clr_runner(unsigned int sirq) { } ++#endif + -+static void cgroup_enable_task_cg_lists(void) + /* + * we cannot loop indefinitely here to avoid userspace starvation, + * but we also don't want to introduce a worst case 1/HZ latency +@@ -77,6 +171,68 @@ + wake_up_process(tsk); + } + ++static void handle_softirq(unsigned int vec_nr) +{ -+ struct task_struct *p, *g; -+ -+ down_write(&css_set_rwsem); -+ -+ if (use_task_css_set_links) -+ goto out_unlock; -+ -+ use_task_css_set_links = true; -+ -+ /* -+ * We need tasklist_lock because RCU is not safe against -+ * while_each_thread(). Besides, a forking task that has passed -+ * cgroup_post_fork() without seeing use_task_css_set_links = 1 -+ * is not guaranteed to have its child immediately visible in the -+ * tasklist if we walk through it with RCU. -+ */ -+ read_lock(&tasklist_lock); -+ do_each_thread(g, p) { -+ WARN_ON_ONCE(!list_empty(&p->cg_list) || -+ task_css_set(p) != &init_css_set); ++ struct softirq_action *h = softirq_vec + vec_nr; ++ int prev_count; + -+ /* -+ * We should check if the process is exiting, otherwise -+ * it will race with cgroup_exit() in that the list -+ * entry won't be deleted though the process has exited. -+ * Do it while holding siglock so that we don't end up -+ * racing against cgroup_exit(). -+ */ -+ spin_lock_irq(&p->sighand->siglock); -+ if (!(p->flags & PF_EXITING)) { -+ struct css_set *cset = task_css_set(p); ++ prev_count = preempt_count(); + -+ list_add(&p->cg_list, &cset->tasks); -+ get_css_set(cset); -+ } -+ spin_unlock_irq(&p->sighand->siglock); -+ } while_each_thread(g, p); -+ read_unlock(&tasklist_lock); -+out_unlock: -+ up_write(&css_set_rwsem); -+} ++ kstat_incr_softirqs_this_cpu(vec_nr); + -+static void init_cgroup_housekeeping(struct cgroup *cgrp) -+{ -+ struct cgroup_subsys *ss; -+ int ssid; -+ -+ INIT_LIST_HEAD(&cgrp->self.sibling); -+ INIT_LIST_HEAD(&cgrp->self.children); -+ INIT_LIST_HEAD(&cgrp->cset_links); -+ INIT_LIST_HEAD(&cgrp->pidlists); -+ mutex_init(&cgrp->pidlist_mutex); -+ cgrp->self.cgroup = cgrp; -+ cgrp->self.flags |= CSS_ONLINE; -+ -+ for_each_subsys(ss, ssid) -+ INIT_LIST_HEAD(&cgrp->e_csets[ssid]); -+ -+ init_waitqueue_head(&cgrp->offline_waitq); -+ INIT_WORK(&cgrp->release_agent_work, cgroup_release_agent); ++ trace_softirq_entry(vec_nr); ++ h->action(h); ++ trace_softirq_exit(vec_nr); ++ if (unlikely(prev_count != preempt_count())) { ++ pr_err("huh, entered softirq %u %s %p with preempt_count %08x, exited with %08x?\n", ++ vec_nr, softirq_to_name[vec_nr], h->action, ++ prev_count, preempt_count()); ++ preempt_count_set(prev_count); ++ } +} + -+static void init_cgroup_root(struct cgroup_root *root, -+ struct cgroup_sb_opts *opts) ++#ifndef CONFIG_PREEMPT_RT_FULL ++static inline int ksoftirqd_softirq_pending(void) +{ -+ struct cgroup *cgrp = &root->cgrp; -+ -+ INIT_LIST_HEAD(&root->root_list); -+ atomic_set(&root->nr_cgrps, 1); -+ cgrp->root = root; -+ init_cgroup_housekeeping(cgrp); -+ idr_init(&root->cgroup_idr); -+ -+ root->flags = opts->flags; -+ if (opts->release_agent) -+ strcpy(root->release_agent_path, opts->release_agent); -+ if (opts->name) -+ strcpy(root->name, opts->name); -+ if (opts->cpuset_clone_children) -+ set_bit(CGRP_CPUSET_CLONE_CHILDREN, &root->cgrp.flags); ++ return local_softirq_pending(); +} + -+static int cgroup_setup_root(struct cgroup_root *root, unsigned int ss_mask) ++static void handle_pending_softirqs(u32 pending) +{ -+ LIST_HEAD(tmp_links); -+ struct cgroup *root_cgrp = &root->cgrp; -+ struct cftype *base_files; -+ struct css_set *cset; -+ int i, ret; ++ struct softirq_action *h = softirq_vec; ++ int softirq_bit; + -+ lockdep_assert_held(&cgroup_mutex); ++ local_irq_enable(); + -+ ret = cgroup_idr_alloc(&root->cgroup_idr, root_cgrp, 1, 2, GFP_NOWAIT); -+ if (ret < 0) -+ goto out; -+ root_cgrp->id = ret; ++ h = softirq_vec; + -+ ret = percpu_ref_init(&root_cgrp->self.refcnt, css_release, 0, -+ GFP_KERNEL); -+ if (ret) -+ goto out; ++ while ((softirq_bit = ffs(pending))) { ++ unsigned int vec_nr; + -+ /* -+ * We're accessing css_set_count without locking css_set_rwsem here, -+ * but that's OK - it can only be increased by someone holding -+ * cgroup_lock, and that's us. The worst that can happen is that we -+ * have some link structures left over -+ */ -+ ret = allocate_cgrp_cset_links(css_set_count, &tmp_links); -+ if (ret) -+ goto cancel_ref; ++ h += softirq_bit - 1; ++ vec_nr = h - softirq_vec; ++ handle_softirq(vec_nr); + -+ ret = cgroup_init_root_id(root); -+ if (ret) -+ goto cancel_ref; -+ -+ root->kf_root = kernfs_create_root(&cgroup_kf_syscall_ops, -+ KERNFS_ROOT_CREATE_DEACTIVATED, -+ root_cgrp); -+ if (IS_ERR(root->kf_root)) { -+ ret = PTR_ERR(root->kf_root); -+ goto exit_root_id; ++ h++; ++ pending >>= softirq_bit; + } -+ root_cgrp->kn = root->kf_root->kn; + -+ if (root == &cgrp_dfl_root) -+ base_files = cgroup_dfl_base_files; -+ else -+ base_files = cgroup_legacy_base_files; ++ rcu_bh_qs(); ++ local_irq_disable(); ++} + -+ ret = cgroup_addrm_files(root_cgrp, base_files, true); -+ if (ret) -+ goto destroy_root; ++static void run_ksoftirqd(unsigned int cpu) ++{ ++ local_irq_disable(); ++ if (ksoftirqd_softirq_pending()) { ++ __do_softirq(); ++ local_irq_enable(); ++ cond_resched_rcu_qs(); ++ return; ++ } ++ local_irq_enable(); ++} + -+ ret = rebind_subsystems(root, ss_mask); -+ if (ret) -+ goto destroy_root; + /* + * preempt_count and SOFTIRQ_OFFSET usage: + * - preempt_count is changed by SOFTIRQ_OFFSET on entering or leaving +@@ -232,10 +388,8 @@ + unsigned long end = jiffies + MAX_SOFTIRQ_TIME; + unsigned long old_flags = current->flags; + int max_restart = MAX_SOFTIRQ_RESTART; +- struct softirq_action *h; + bool in_hardirq; + __u32 pending; +- int softirq_bit; + + /* + * Mask out PF_MEMALLOC s current task context is borrowed for the +@@ -254,36 +408,7 @@ + /* Reset the pending bitmask before enabling irqs */ + set_softirq_pending(0); + +- local_irq_enable(); +- +- h = softirq_vec; +- +- while ((softirq_bit = ffs(pending))) { +- unsigned int vec_nr; +- int prev_count; +- +- h += softirq_bit - 1; +- +- vec_nr = h - softirq_vec; +- prev_count = preempt_count(); +- +- kstat_incr_softirqs_this_cpu(vec_nr); +- +- trace_softirq_entry(vec_nr); +- h->action(h); +- trace_softirq_exit(vec_nr); +- if (unlikely(prev_count != preempt_count())) { +- pr_err("huh, entered softirq %u %s %p with preempt_count %08x, exited with %08x?\n", +- vec_nr, softirq_to_name[vec_nr], h->action, +- prev_count, preempt_count()); +- preempt_count_set(prev_count); +- } +- h++; +- pending >>= softirq_bit; +- } +- +- rcu_bh_qs(); +- local_irq_disable(); ++ handle_pending_softirqs(pending); + + pending = local_softirq_pending(); + if (pending) { +@@ -320,6 +445,285 @@ + } + + /* ++ * This function must run with irqs disabled! ++ */ ++void raise_softirq_irqoff(unsigned int nr) ++{ ++ __raise_softirq_irqoff(nr); + + /* -+ * There must be no failure case after here, since rebinding takes -+ * care of subsystems' refcounts, which are explicitly dropped in -+ * the failure exit path. ++ * If we're in an interrupt or softirq, we're done ++ * (this also catches softirq-disabled code). We will ++ * actually run the softirq once we return from ++ * the irq or softirq. ++ * ++ * Otherwise we wake up ksoftirqd to make sure we ++ * schedule the softirq soon. + */ -+ list_add(&root->root_list, &cgroup_roots); -+ cgroup_root_count++; ++ if (!in_interrupt()) ++ wakeup_softirqd(); ++} + -+ /* -+ * Link the root cgroup in this hierarchy into all the css_set -+ * objects. -+ */ -+ down_write(&css_set_rwsem); -+ hash_for_each(css_set_table, i, cset, hlist) -+ link_css_set(&tmp_links, cset, root_cgrp); -+ up_write(&css_set_rwsem); ++void __raise_softirq_irqoff(unsigned int nr) ++{ ++ trace_softirq_raise(nr); ++ or_softirq_pending(1UL << nr); ++} + -+ BUG_ON(!list_empty(&root_cgrp->self.children)); -+ BUG_ON(atomic_read(&root->nr_cgrps) != 1); ++static inline void local_bh_disable_nort(void) { local_bh_disable(); } ++static inline void _local_bh_enable_nort(void) { _local_bh_enable(); } ++static void ksoftirqd_set_sched_params(unsigned int cpu) { } ++static void ksoftirqd_clr_sched_params(unsigned int cpu, bool online) { } + -+ kernfs_activate(root_cgrp->kn); -+ ret = 0; -+ goto out; ++#else /* !PREEMPT_RT_FULL */ + -+destroy_root: -+ kernfs_destroy_root(root->kf_root); -+ root->kf_root = NULL; -+exit_root_id: -+ cgroup_exit_root_id(root); -+cancel_ref: -+ percpu_ref_exit(&root_cgrp->self.refcnt); -+out: -+ free_cgrp_cset_links(&tmp_links); -+ return ret; -+} ++/* ++ * On RT we serialize softirq execution with a cpu local lock per softirq ++ */ ++static DEFINE_PER_CPU(struct local_irq_lock [NR_SOFTIRQS], local_softirq_locks); + -+static struct dentry *cgroup_mount(struct file_system_type *fs_type, -+ int flags, const char *unused_dev_name, -+ void *data) ++void __init softirq_early_init(void) +{ -+ struct super_block *pinned_sb = NULL; -+ struct cgroup_subsys *ss; -+ struct cgroup_root *root; -+ struct cgroup_sb_opts opts; -+ struct dentry *dentry; -+ int ret; + int i; -+ bool new_sb; -+ -+ /* -+ * The first time anyone tries to mount a cgroup, enable the list -+ * linking each css_set to its tasks and fix up all existing tasks. -+ */ -+ if (!use_task_css_set_links) -+ cgroup_enable_task_cg_lists(); + -+ mutex_lock(&cgroup_mutex); -+ -+ /* First find the desired set of subsystems */ -+ ret = parse_cgroupfs_options(data, &opts); -+ if (ret) -+ goto out_unlock; ++ for (i = 0; i < NR_SOFTIRQS; i++) ++ local_irq_lock_init(local_softirq_locks[i]); ++} + -+ /* look for a matching existing root */ -+ if (opts.flags & CGRP_ROOT_SANE_BEHAVIOR) { -+ cgrp_dfl_root_visible = true; -+ root = &cgrp_dfl_root; -+ cgroup_get(&root->cgrp); -+ ret = 0; -+ goto out_unlock; -+ } ++static void lock_softirq(int which) ++{ ++ local_lock(local_softirq_locks[which]); ++} + -+ /* -+ * Destruction of cgroup root is asynchronous, so subsystems may -+ * still be dying after the previous unmount. Let's drain the -+ * dying subsystems. We just need to ensure that the ones -+ * unmounted previously finish dying and don't care about new ones -+ * starting. Testing ref liveliness is good enough. -+ */ -+ for_each_subsys(ss, i) { -+ if (!(opts.subsys_mask & (1 << i)) || -+ ss->root == &cgrp_dfl_root) -+ continue; ++static void unlock_softirq(int which) ++{ ++ local_unlock(local_softirq_locks[which]); ++} + -+ if (!percpu_ref_tryget_live(&ss->root->cgrp.self.refcnt)) { -+ mutex_unlock(&cgroup_mutex); -+ msleep(10); -+ ret = restart_syscall(); -+ goto out_free; -+ } -+ cgroup_put(&ss->root->cgrp); -+ } ++static void do_single_softirq(int which) ++{ ++ unsigned long old_flags = current->flags; + -+ for_each_root(root) { -+ bool name_match = false; ++ current->flags &= ~PF_MEMALLOC; ++ vtime_account_irq_enter(current); ++ current->flags |= PF_IN_SOFTIRQ; ++ lockdep_softirq_enter(); ++ local_irq_enable(); ++ handle_softirq(which); ++ local_irq_disable(); ++ lockdep_softirq_exit(); ++ current->flags &= ~PF_IN_SOFTIRQ; ++ vtime_account_irq_enter(current); ++ tsk_restore_flags(current, old_flags, PF_MEMALLOC); ++} + -+ if (root == &cgrp_dfl_root) -+ continue; ++/* ++ * Called with interrupts disabled. Process softirqs which were raised ++ * in current context (or on behalf of ksoftirqd). ++ */ ++static void do_current_softirqs(void) ++{ ++ while (current->softirqs_raised) { ++ int i = __ffs(current->softirqs_raised); ++ unsigned int pending, mask = (1U << i); + -+ /* -+ * If we asked for a name then it must match. Also, if -+ * name matches but sybsys_mask doesn't, we should fail. -+ * Remember whether name matched. -+ */ -+ if (opts.name) { -+ if (strcmp(opts.name, root->name)) -+ continue; -+ name_match = true; -+ } ++ current->softirqs_raised &= ~mask; ++ local_irq_enable(); + + /* -+ * If we asked for subsystems (or explicitly for no -+ * subsystems) then they must match. ++ * If the lock is contended, we boost the owner to ++ * process the softirq or leave the critical section ++ * now. + */ -+ if ((opts.subsys_mask || opts.none) && -+ (opts.subsys_mask != root->subsys_mask)) { -+ if (!name_match) -+ continue; -+ ret = -EBUSY; -+ goto out_unlock; -+ } -+ -+ if (root->flags ^ opts.flags) -+ pr_warn("new mount options do not match the existing superblock, will be ignored\n"); -+ ++ lock_softirq(i); ++ local_irq_disable(); ++ softirq_set_runner(i); + /* -+ * We want to reuse @root whose lifetime is governed by its -+ * ->cgrp. Let's check whether @root is alive and keep it -+ * that way. As cgroup_kill_sb() can happen anytime, we -+ * want to block it by pinning the sb so that @root doesn't -+ * get killed before mount is complete. -+ * -+ * With the sb pinned, tryget_live can reliably indicate -+ * whether @root can be reused. If it's being killed, -+ * drain it. We can use wait_queue for the wait but this -+ * path is super cold. Let's just sleep a bit and retry. ++ * Check with the local_softirq_pending() bits, ++ * whether we need to process this still or if someone ++ * else took care of it. + */ -+ pinned_sb = kernfs_pin_sb(root->kf_root, NULL); -+ if (IS_ERR(pinned_sb) || -+ !percpu_ref_tryget_live(&root->cgrp.self.refcnt)) { -+ mutex_unlock(&cgroup_mutex); -+ if (!IS_ERR_OR_NULL(pinned_sb)) -+ deactivate_super(pinned_sb); -+ msleep(10); -+ ret = restart_syscall(); -+ goto out_free; ++ pending = local_softirq_pending(); ++ if (pending & mask) { ++ set_softirq_pending(pending & ~mask); ++ do_single_softirq(i); + } -+ -+ ret = 0; -+ goto out_unlock; -+ } -+ -+ /* -+ * No such thing, create a new one. name= matching without subsys -+ * specification is allowed for already existing hierarchies but we -+ * can't create new one without subsys specification. -+ */ -+ if (!opts.subsys_mask && !opts.none) { -+ ret = -EINVAL; -+ goto out_unlock; -+ } -+ -+ root = kzalloc(sizeof(*root), GFP_KERNEL); -+ if (!root) { -+ ret = -ENOMEM; -+ goto out_unlock; ++ softirq_clr_runner(i); ++ unlock_softirq(i); ++ WARN_ON(current->softirq_nestcnt != 1); + } ++} + -+ init_cgroup_root(root, &opts); -+ -+ ret = cgroup_setup_root(root, opts.subsys_mask); -+ if (ret) -+ cgroup_free_root(root); -+ -+out_unlock: -+ mutex_unlock(&cgroup_mutex); -+out_free: -+ kfree(opts.release_agent); -+ kfree(opts.name); -+ -+ if (ret) -+ return ERR_PTR(ret); -+ -+ dentry = kernfs_mount(fs_type, flags, root->kf_root, -+ CGROUP_SUPER_MAGIC, &new_sb); -+ if (IS_ERR(dentry) || !new_sb) -+ cgroup_put(&root->cgrp); ++static void __local_bh_disable(void) ++{ ++ if (++current->softirq_nestcnt == 1) ++ migrate_disable(); ++} + -+ /* -+ * If @pinned_sb, we're reusing an existing root and holding an -+ * extra ref on its sb. Mount is complete. Put the extra ref. -+ */ -+ if (pinned_sb) { -+ WARN_ON(new_sb); -+ deactivate_super(pinned_sb); -+ } ++void local_bh_disable(void) ++{ ++ __local_bh_disable(); ++} ++EXPORT_SYMBOL(local_bh_disable); + -+ return dentry; ++void __local_bh_disable_ip(unsigned long ip, unsigned int cnt) ++{ ++ __local_bh_disable(); ++ if (cnt & PREEMPT_CHECK_OFFSET) ++ preempt_disable(); +} + -+static void cgroup_kill_sb(struct super_block *sb) ++static void __local_bh_enable(void) +{ -+ struct kernfs_root *kf_root = kernfs_root_from_sb(sb); -+ struct cgroup_root *root = cgroup_root_from_kf(kf_root); ++ if (WARN_ON(current->softirq_nestcnt == 0)) ++ return; + -+ /* -+ * If @root doesn't have any mounts or children, start killing it. -+ * This prevents new mounts by disabling percpu_ref_tryget_live(). -+ * cgroup_mount() may wait for @root's release. -+ * -+ * And don't kill the default root. -+ */ -+ if (!list_empty(&root->cgrp.self.children) || -+ root == &cgrp_dfl_root) -+ cgroup_put(&root->cgrp); -+ else -+ percpu_ref_kill(&root->cgrp.self.refcnt); ++ local_irq_disable(); ++ if (current->softirq_nestcnt == 1 && current->softirqs_raised) ++ do_current_softirqs(); ++ local_irq_enable(); + -+ kernfs_kill_sb(sb); ++ if (--current->softirq_nestcnt == 0) ++ migrate_enable(); +} + -+static struct file_system_type cgroup_fs_type = { -+ .name = "cgroup", -+ .mount = cgroup_mount, -+ .kill_sb = cgroup_kill_sb, -+}; -+ -+/** -+ * task_cgroup_path - cgroup path of a task in the first cgroup hierarchy -+ * @task: target task -+ * @buf: the buffer to write the path into -+ * @buflen: the length of the buffer -+ * -+ * Determine @task's cgroup on the first (the one with the lowest non-zero -+ * hierarchy_id) cgroup hierarchy and copy its path into @buf. This -+ * function grabs cgroup_mutex and shouldn't be used inside locks used by -+ * cgroup controller callbacks. -+ * -+ * Return value is the same as kernfs_path(). -+ */ -+char *task_cgroup_path(struct task_struct *task, char *buf, size_t buflen) ++void local_bh_enable(void) +{ -+ struct cgroup_root *root; -+ struct cgroup *cgrp; -+ int hierarchy_id = 1; -+ char *path = NULL; -+ -+ mutex_lock(&cgroup_mutex); -+ down_read(&css_set_rwsem); -+ -+ root = idr_get_next(&cgroup_hierarchy_idr, &hierarchy_id); ++ __local_bh_enable(); ++} ++EXPORT_SYMBOL(local_bh_enable); + -+ if (root) { -+ cgrp = task_cgroup_from_root(task, root); -+ path = cgroup_path(cgrp, buf, buflen); -+ } else { -+ /* if no hierarchy exists, everyone is in "/" */ -+ if (strlcpy(buf, "/", buflen) < buflen) -+ path = buf; -+ } ++extern void __local_bh_enable_ip(unsigned long ip, unsigned int cnt) ++{ ++ __local_bh_enable(); ++ if (cnt & PREEMPT_CHECK_OFFSET) ++ preempt_enable(); ++} + -+ up_read(&css_set_rwsem); -+ mutex_unlock(&cgroup_mutex); -+ return path; ++void local_bh_enable_ip(unsigned long ip) ++{ ++ local_bh_enable(); +} -+EXPORT_SYMBOL_GPL(task_cgroup_path); ++EXPORT_SYMBOL(local_bh_enable_ip); + -+/* used to track tasks and other necessary states during migration */ -+struct cgroup_taskset { -+ /* the src and dst cset list running through cset->mg_node */ -+ struct list_head src_csets; -+ struct list_head dst_csets; ++void _local_bh_enable(void) ++{ ++ if (WARN_ON(current->softirq_nestcnt == 0)) ++ return; ++ if (--current->softirq_nestcnt == 0) ++ migrate_enable(); ++} ++EXPORT_SYMBOL(_local_bh_enable); + -+ /* -+ * Fields for cgroup_taskset_*() iteration. -+ * -+ * Before migration is committed, the target migration tasks are on -+ * ->mg_tasks of the csets on ->src_csets. After, on ->mg_tasks of -+ * the csets on ->dst_csets. ->csets point to either ->src_csets -+ * or ->dst_csets depending on whether migration is committed. -+ * -+ * ->cur_csets and ->cur_task point to the current task position -+ * during iteration. -+ */ -+ struct list_head *csets; -+ struct css_set *cur_cset; -+ struct task_struct *cur_task; -+}; ++int in_serving_softirq(void) ++{ ++ return current->flags & PF_IN_SOFTIRQ; ++} ++EXPORT_SYMBOL(in_serving_softirq); + -+/** -+ * cgroup_taskset_first - reset taskset and return the first task -+ * @tset: taskset of interest -+ * -+ * @tset iteration is initialized and the first task is returned. -+ */ -+struct task_struct *cgroup_taskset_first(struct cgroup_taskset *tset) ++/* Called with preemption disabled */ ++static void run_ksoftirqd(unsigned int cpu) +{ -+ tset->cur_cset = list_first_entry(tset->csets, struct css_set, mg_node); -+ tset->cur_task = NULL; ++ local_irq_disable(); ++ current->softirq_nestcnt++; + -+ return cgroup_taskset_next(tset); ++ do_current_softirqs(); ++ current->softirq_nestcnt--; ++ rcu_note_context_switch(); ++ local_irq_enable(); +} + -+/** -+ * cgroup_taskset_next - iterate to the next task in taskset -+ * @tset: taskset of interest -+ * -+ * Return the next task in @tset. Iteration must have been initialized -+ * with cgroup_taskset_first(). ++/* ++ * Called from netif_rx_ni(). Preemption enabled, but migration ++ * disabled. So the cpu can't go away under us. + */ -+struct task_struct *cgroup_taskset_next(struct cgroup_taskset *tset) ++void thread_do_softirq(void) +{ -+ struct css_set *cset = tset->cur_cset; -+ struct task_struct *task = tset->cur_task; -+ -+ while (&cset->mg_node != tset->csets) { -+ if (!task) -+ task = list_first_entry(&cset->mg_tasks, -+ struct task_struct, cg_list); -+ else -+ task = list_next_entry(task, cg_list); -+ -+ if (&task->cg_list != &cset->mg_tasks) { -+ tset->cur_cset = cset; -+ tset->cur_task = task; -+ return task; -+ } -+ -+ cset = list_next_entry(cset, mg_node); -+ task = NULL; ++ if (!in_serving_softirq() && current->softirqs_raised) { ++ current->softirq_nestcnt++; ++ do_current_softirqs(); ++ current->softirq_nestcnt--; + } -+ -+ return NULL; +} + -+/** -+ * cgroup_task_migrate - move a task from one cgroup to another. -+ * @old_cgrp: the cgroup @tsk is being migrated from -+ * @tsk: the task being migrated -+ * @new_cset: the new css_set @tsk is being attached to -+ * -+ * Must be called with cgroup_mutex, threadgroup and css_set_rwsem locked. -+ */ -+static void cgroup_task_migrate(struct cgroup *old_cgrp, -+ struct task_struct *tsk, -+ struct css_set *new_cset) ++static void do_raise_softirq_irqoff(unsigned int nr) +{ -+ struct css_set *old_cset; -+ -+ lockdep_assert_held(&cgroup_mutex); -+ lockdep_assert_held(&css_set_rwsem); ++ trace_softirq_raise(nr); ++ or_softirq_pending(1UL << nr); + + /* -+ * We are synchronized through threadgroup_lock() against PF_EXITING -+ * setting such that we can't race against cgroup_exit() changing the -+ * css_set to init_css_set and dropping the old one. ++ * If we are not in a hard interrupt and inside a bh disabled ++ * region, we simply raise the flag on current. local_bh_enable() ++ * will make sure that the softirq is executed. Otherwise we ++ * delegate it to ksoftirqd. + */ -+ WARN_ON_ONCE(tsk->flags & PF_EXITING); -+ old_cset = task_css_set(tsk); ++ if (!in_irq() && current->softirq_nestcnt) ++ current->softirqs_raised |= (1U << nr); ++ else if (__this_cpu_read(ksoftirqd)) ++ __this_cpu_read(ksoftirqd)->softirqs_raised |= (1U << nr); ++} ++ ++void __raise_softirq_irqoff(unsigned int nr) ++{ ++ do_raise_softirq_irqoff(nr); ++ if (!in_irq() && !current->softirq_nestcnt) ++ wakeup_softirqd(); ++} + -+ get_css_set(new_cset); -+ rcu_assign_pointer(tsk->cgroups, new_cset); ++/* ++ * This function must run with irqs disabled! ++ */ ++void raise_softirq_irqoff(unsigned int nr) ++{ ++ do_raise_softirq_irqoff(nr); + + /* -+ * Use move_tail so that cgroup_taskset_first() still returns the -+ * leader after migration. This works because cgroup_migrate() -+ * ensures that the dst_cset of the leader is the first on the -+ * tset's dst_csets list. ++ * If we're in an hard interrupt we let irq return code deal ++ * with the wakeup of ksoftirqd. + */ -+ list_move_tail(&tsk->cg_list, &new_cset->mg_tasks); -+ ++ if (in_irq()) ++ return; + /* -+ * We just gained a reference on old_cset by taking it from the -+ * task. As trading it for new_cset is protected by cgroup_mutex, -+ * we're safe to drop it here; it will be freed under RCU. ++ * If we are in thread context but outside of a bh disabled ++ * region, we need to wake ksoftirqd as well. ++ * ++ * CHECKME: Some of the places which do that could be wrapped ++ * into local_bh_disable/enable pairs. Though it's unclear ++ * whether this is worth the effort. To find those places just ++ * raise a WARN() if the condition is met. + */ -+ put_css_set_locked(old_cset); ++ if (!current->softirq_nestcnt) ++ wakeup_softirqd(); +} + -+/** -+ * cgroup_migrate_finish - cleanup after attach -+ * @preloaded_csets: list of preloaded css_sets -+ * -+ * Undo cgroup_migrate_add_src() and cgroup_migrate_prepare_dst(). See -+ * those functions for details. -+ */ -+static void cgroup_migrate_finish(struct list_head *preloaded_csets) ++static inline int ksoftirqd_softirq_pending(void) +{ -+ struct css_set *cset, *tmp_cset; ++ return current->softirqs_raised; ++} + -+ lockdep_assert_held(&cgroup_mutex); -+ -+ down_write(&css_set_rwsem); -+ list_for_each_entry_safe(cset, tmp_cset, preloaded_csets, mg_preload_node) { -+ cset->mg_src_cgrp = NULL; -+ cset->mg_dst_cset = NULL; -+ list_del_init(&cset->mg_preload_node); -+ put_css_set_locked(cset); -+ } -+ up_write(&css_set_rwsem); -+} ++static inline void local_bh_disable_nort(void) { } ++static inline void _local_bh_enable_nort(void) { } + -+/** -+ * cgroup_migrate_add_src - add a migration source css_set -+ * @src_cset: the source css_set to add -+ * @dst_cgrp: the destination cgroup -+ * @preloaded_csets: list of preloaded css_sets -+ * -+ * Tasks belonging to @src_cset are about to be migrated to @dst_cgrp. Pin -+ * @src_cset and add it to @preloaded_csets, which should later be cleaned -+ * up by cgroup_migrate_finish(). -+ * -+ * This function may be called without holding threadgroup_lock even if the -+ * target is a process. Threads may be created and destroyed but as long -+ * as cgroup_mutex is not dropped, no new css_set can be put into play and -+ * the preloaded css_sets are guaranteed to cover all migrations. -+ */ -+static void cgroup_migrate_add_src(struct css_set *src_cset, -+ struct cgroup *dst_cgrp, -+ struct list_head *preloaded_csets) ++static inline void ksoftirqd_set_sched_params(unsigned int cpu) +{ -+ struct cgroup *src_cgrp; -+ -+ lockdep_assert_held(&cgroup_mutex); -+ lockdep_assert_held(&css_set_rwsem); -+ -+ src_cgrp = cset_cgroup_from_root(src_cset, dst_cgrp->root); -+ -+ if (!list_empty(&src_cset->mg_preload_node)) -+ return; -+ -+ WARN_ON(src_cset->mg_src_cgrp); -+ WARN_ON(!list_empty(&src_cset->mg_tasks)); -+ WARN_ON(!list_empty(&src_cset->mg_node)); ++ struct sched_param param = { .sched_priority = 1 }; + -+ src_cset->mg_src_cgrp = src_cgrp; -+ get_css_set(src_cset); -+ list_add(&src_cset->mg_preload_node, preloaded_csets); ++ sched_setscheduler(current, SCHED_FIFO, ¶m); ++ /* Take over all pending softirqs when starting */ ++ local_irq_disable(); ++ current->softirqs_raised = local_softirq_pending(); ++ local_irq_enable(); +} + -+/** -+ * cgroup_migrate_prepare_dst - prepare destination css_sets for migration -+ * @dst_cgrp: the destination cgroup (may be %NULL) -+ * @preloaded_csets: list of preloaded source css_sets -+ * -+ * Tasks are about to be moved to @dst_cgrp and all the source css_sets -+ * have been preloaded to @preloaded_csets. This function looks up and -+ * pins all destination css_sets, links each to its source, and append them -+ * to @preloaded_csets. If @dst_cgrp is %NULL, the destination of each -+ * source css_set is assumed to be its cgroup on the default hierarchy. -+ * -+ * This function must be called after cgroup_migrate_add_src() has been -+ * called on each migration source css_set. After migration is performed -+ * using cgroup_migrate(), cgroup_migrate_finish() must be called on -+ * @preloaded_csets. -+ */ -+static int cgroup_migrate_prepare_dst(struct cgroup *dst_cgrp, -+ struct list_head *preloaded_csets) ++static inline void ksoftirqd_clr_sched_params(unsigned int cpu, bool online) +{ -+ LIST_HEAD(csets); -+ struct css_set *src_cset, *tmp_cset; -+ -+ lockdep_assert_held(&cgroup_mutex); -+ -+ /* -+ * Except for the root, child_subsys_mask must be zero for a cgroup -+ * with tasks so that child cgroups don't compete against tasks. -+ */ -+ if (dst_cgrp && cgroup_on_dfl(dst_cgrp) && cgroup_parent(dst_cgrp) && -+ dst_cgrp->child_subsys_mask) -+ return -EBUSY; -+ -+ /* look up the dst cset for each src cset and link it to src */ -+ list_for_each_entry_safe(src_cset, tmp_cset, preloaded_csets, mg_preload_node) { -+ struct css_set *dst_cset; -+ -+ dst_cset = find_css_set(src_cset, -+ dst_cgrp ?: src_cset->dfl_cgrp); -+ if (!dst_cset) -+ goto err; -+ -+ WARN_ON_ONCE(src_cset->mg_dst_cset || dst_cset->mg_dst_cset); -+ -+ /* -+ * If src cset equals dst, it's noop. Drop the src. -+ * cgroup_migrate() will skip the cset too. Note that we -+ * can't handle src == dst as some nodes are used by both. -+ */ -+ if (src_cset == dst_cset) { -+ src_cset->mg_src_cgrp = NULL; -+ list_del_init(&src_cset->mg_preload_node); -+ put_css_set(src_cset); -+ put_css_set(dst_cset); -+ continue; -+ } -+ -+ src_cset->mg_dst_cset = dst_cset; -+ -+ if (list_empty(&dst_cset->mg_preload_node)) -+ list_add(&dst_cset->mg_preload_node, &csets); -+ else -+ put_css_set(dst_cset); -+ } ++ struct sched_param param = { .sched_priority = 0 }; + -+ list_splice_tail(&csets, preloaded_csets); -+ return 0; -+err: -+ cgroup_migrate_finish(&csets); -+ return -ENOMEM; ++ sched_setscheduler(current, SCHED_NORMAL, ¶m); +} + -+/** -+ * cgroup_migrate - migrate a process or task to a cgroup -+ * @cgrp: the destination cgroup -+ * @leader: the leader of the process or the task to migrate -+ * @threadgroup: whether @leader points to the whole process or a single task -+ * -+ * Migrate a process or task denoted by @leader to @cgrp. If migrating a -+ * process, the caller must be holding threadgroup_lock of @leader. The -+ * caller is also responsible for invoking cgroup_migrate_add_src() and -+ * cgroup_migrate_prepare_dst() on the targets before invoking this -+ * function and following up with cgroup_migrate_finish(). -+ * -+ * As long as a controller's ->can_attach() doesn't fail, this function is -+ * guaranteed to succeed. This means that, excluding ->can_attach() -+ * failure, when migrating multiple targets, the success or failure can be -+ * decided for all targets by invoking group_migrate_prepare_dst() before -+ * actually starting migrating. -+ */ -+static int cgroup_migrate(struct cgroup *cgrp, struct task_struct *leader, -+ bool threadgroup) -+{ -+ struct cgroup_taskset tset = { -+ .src_csets = LIST_HEAD_INIT(tset.src_csets), -+ .dst_csets = LIST_HEAD_INIT(tset.dst_csets), -+ .csets = &tset.src_csets, -+ }; -+ struct cgroup_subsys_state *css, *failed_css = NULL; -+ struct css_set *cset, *tmp_cset; -+ struct task_struct *task, *tmp_task; -+ int i, ret; -+ -+ /* -+ * Prevent freeing of tasks while we take a snapshot. Tasks that are -+ * already PF_EXITING could be freed from underneath us unless we -+ * take an rcu_read_lock. -+ */ -+ down_write(&css_set_rwsem); -+ rcu_read_lock(); -+ task = leader; -+ do { -+ /* @task either already exited or can't exit until the end */ -+ if (task->flags & PF_EXITING) -+ goto next; -+ -+ /* leave @task alone if post_fork() hasn't linked it yet */ -+ if (list_empty(&task->cg_list)) -+ goto next; -+ -+ cset = task_css_set(task); -+ if (!cset->mg_src_cgrp) -+ goto next; ++#endif /* PREEMPT_RT_FULL */ ++/* + * Enter an interrupt context. + */ + void irq_enter(void) +@@ -330,9 +734,9 @@ + * Prevent raise_softirq from needlessly waking up ksoftirqd + * here, as softirq will be serviced on return from interrupt. + */ +- local_bh_disable(); ++ local_bh_disable_nort(); + tick_irq_enter(); +- _local_bh_enable(); ++ _local_bh_enable_nort(); + } + + __irq_enter(); +@@ -340,6 +744,7 @@ + + static inline void invoke_softirq(void) + { ++#ifndef CONFIG_PREEMPT_RT_FULL + if (!force_irqthreads) { + #ifdef CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK + /* +@@ -359,6 +764,15 @@ + } else { + wakeup_softirqd(); + } ++#else /* PREEMPT_RT_FULL */ ++ unsigned long flags; + -+ /* -+ * cgroup_taskset_first() must always return the leader. -+ * Take care to avoid disturbing the ordering. ++ local_irq_save(flags); ++ if (__this_cpu_read(ksoftirqd) && ++ __this_cpu_read(ksoftirqd)->softirqs_raised) ++ wakeup_softirqd(); ++ local_irq_restore(flags); ++#endif + } + + static inline void tick_irq_exit(void) +@@ -395,26 +809,6 @@ + trace_hardirq_exit(); /* must be last! */ + } + +-/* +- * This function must run with irqs disabled! +- */ +-inline void raise_softirq_irqoff(unsigned int nr) +-{ +- __raise_softirq_irqoff(nr); +- +- /* +- * If we're in an interrupt or softirq, we're done +- * (this also catches softirq-disabled code). We will +- * actually run the softirq once we return from +- * the irq or softirq. +- * +- * Otherwise we wake up ksoftirqd to make sure we +- * schedule the softirq soon. +- */ +- if (!in_interrupt()) +- wakeup_softirqd(); +-} +- + void raise_softirq(unsigned int nr) + { + unsigned long flags; +@@ -424,12 +818,6 @@ + local_irq_restore(flags); + } + +-void __raise_softirq_irqoff(unsigned int nr) +-{ +- trace_softirq_raise(nr); +- or_softirq_pending(1UL << nr); +-} +- + void open_softirq(int nr, void (*action)(struct softirq_action *)) + { + softirq_vec[nr].action = action; +@@ -446,15 +834,45 @@ + static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec); + static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec); + ++static void inline ++__tasklet_common_schedule(struct tasklet_struct *t, struct tasklet_head *head, unsigned int nr) ++{ ++ if (tasklet_trylock(t)) { ++again: ++ /* We may have been preempted before tasklet_trylock ++ * and __tasklet_action may have already run. ++ * So double check the sched bit while the takslet ++ * is locked before adding it to the list. + */ -+ list_move_tail(&task->cg_list, &cset->mg_tasks); -+ if (list_empty(&cset->mg_node)) -+ list_add_tail(&cset->mg_node, &tset.src_csets); -+ if (list_empty(&cset->mg_dst_cset->mg_node)) -+ list_move_tail(&cset->mg_dst_cset->mg_node, -+ &tset.dst_csets); -+ next: -+ if (!threadgroup) -+ break; -+ } while_each_thread(leader, task); -+ rcu_read_unlock(); -+ up_write(&css_set_rwsem); -+ -+ /* methods shouldn't be called if no task is actually migrating */ -+ if (list_empty(&tset.src_csets)) -+ return 0; -+ -+ /* check that we can legitimately attach to the cgroup */ -+ for_each_e_css(css, i, cgrp) { -+ if (css->ss->can_attach) { -+ ret = css->ss->can_attach(css, &tset); -+ if (ret) { -+ failed_css = css; -+ goto out_cancel_attach; -+ } ++ if (test_bit(TASKLET_STATE_SCHED, &t->state)) { ++ t->next = NULL; ++ *head->tail = t; ++ head->tail = &(t->next); ++ raise_softirq_irqoff(nr); ++ tasklet_unlock(t); ++ } else { ++ /* This is subtle. If we hit the corner case above ++ * It is possible that we get preempted right here, ++ * and another task has successfully called ++ * tasklet_schedule(), then this function, and ++ * failed on the trylock. Thus we must be sure ++ * before releasing the tasklet lock, that the ++ * SCHED_BIT is clear. Otherwise the tasklet ++ * may get its SCHED_BIT set, but not added to the ++ * list ++ */ ++ if (!tasklet_tryunlock(t)) ++ goto again; + } + } -+ -+ /* -+ * Now that we're guaranteed success, proceed to move all tasks to -+ * the new cgroup. There are no failure cases after here, so this -+ * is the commit point. -+ */ -+ down_write(&css_set_rwsem); -+ list_for_each_entry(cset, &tset.src_csets, mg_node) { -+ list_for_each_entry_safe(task, tmp_task, &cset->mg_tasks, cg_list) -+ cgroup_task_migrate(cset->mg_src_cgrp, task, -+ cset->mg_dst_cset); -+ } -+ up_write(&css_set_rwsem); -+ -+ /* -+ * Migration is committed, all target tasks are now on dst_csets. -+ * Nothing is sensitive to fork() after this point. Notify -+ * controllers that migration is complete. -+ */ -+ tset.csets = &tset.dst_csets; -+ -+ for_each_e_css(css, i, cgrp) -+ if (css->ss->attach) -+ css->ss->attach(css, &tset); -+ -+ ret = 0; -+ goto out_release_tset; -+ -+out_cancel_attach: -+ for_each_e_css(css, i, cgrp) { -+ if (css == failed_css) -+ break; -+ if (css->ss->cancel_attach) -+ css->ss->cancel_attach(css, &tset); -+ } -+out_release_tset: -+ down_write(&css_set_rwsem); -+ list_splice_init(&tset.dst_csets, &tset.src_csets); -+ list_for_each_entry_safe(cset, tmp_cset, &tset.src_csets, mg_node) { -+ list_splice_tail_init(&cset->mg_tasks, &cset->tasks); -+ list_del_init(&cset->mg_node); -+ } -+ up_write(&css_set_rwsem); -+ return ret; +} + -+/** -+ * cgroup_attach_task - attach a task or a whole threadgroup to a cgroup -+ * @dst_cgrp: the cgroup to attach to -+ * @leader: the task or the leader of the threadgroup to be attached -+ * @threadgroup: attach the whole threadgroup? -+ * -+ * Call holding cgroup_mutex and threadgroup_lock of @leader. -+ */ -+static int cgroup_attach_task(struct cgroup *dst_cgrp, -+ struct task_struct *leader, bool threadgroup) -+{ -+ LIST_HEAD(preloaded_csets); -+ struct task_struct *task; -+ int ret; -+ -+ /* look up all src csets */ -+ down_read(&css_set_rwsem); -+ rcu_read_lock(); -+ task = leader; -+ do { -+ cgroup_migrate_add_src(task_css_set(task), dst_cgrp, -+ &preloaded_csets); -+ if (!threadgroup) -+ break; -+ } while_each_thread(leader, task); -+ rcu_read_unlock(); -+ up_read(&css_set_rwsem); -+ -+ /* prepare dst csets and commit */ -+ ret = cgroup_migrate_prepare_dst(dst_cgrp, &preloaded_csets); -+ if (!ret) -+ ret = cgroup_migrate(dst_cgrp, leader, threadgroup); -+ -+ cgroup_migrate_finish(&preloaded_csets); -+ return ret; + void __tasklet_schedule(struct tasklet_struct *t) + { + unsigned long flags; + + local_irq_save(flags); +- t->next = NULL; +- *__this_cpu_read(tasklet_vec.tail) = t; +- __this_cpu_write(tasklet_vec.tail, &(t->next)); +- raise_softirq_irqoff(TASKLET_SOFTIRQ); ++ __tasklet_common_schedule(t, this_cpu_ptr(&tasklet_vec), TASKLET_SOFTIRQ); + local_irq_restore(flags); + } + EXPORT_SYMBOL(__tasklet_schedule); +@@ -464,10 +882,7 @@ + unsigned long flags; + + local_irq_save(flags); +- t->next = NULL; +- *__this_cpu_read(tasklet_hi_vec.tail) = t; +- __this_cpu_write(tasklet_hi_vec.tail, &(t->next)); +- raise_softirq_irqoff(HI_SOFTIRQ); ++ __tasklet_common_schedule(t, this_cpu_ptr(&tasklet_hi_vec), HI_SOFTIRQ); + local_irq_restore(flags); + } + EXPORT_SYMBOL(__tasklet_hi_schedule); +@@ -476,82 +891,122 @@ + { + BUG_ON(!irqs_disabled()); + +- t->next = __this_cpu_read(tasklet_hi_vec.head); +- __this_cpu_write(tasklet_hi_vec.head, t); +- __raise_softirq_irqoff(HI_SOFTIRQ); ++ __tasklet_hi_schedule(t); + } + EXPORT_SYMBOL(__tasklet_hi_schedule_first); + +-static void tasklet_action(struct softirq_action *a) ++void tasklet_enable(struct tasklet_struct *t) + { +- struct tasklet_struct *list; ++ if (!atomic_dec_and_test(&t->count)) ++ return; ++ if (test_and_clear_bit(TASKLET_STATE_PENDING, &t->state)) ++ tasklet_schedule(t); +} -+ -+/* -+ * Find the task_struct of the task to attach by vpid and pass it along to the -+ * function to attach either it or all tasks in its threadgroup. Will lock -+ * cgroup_mutex and threadgroup. -+ */ -+static ssize_t __cgroup_procs_write(struct kernfs_open_file *of, char *buf, -+ size_t nbytes, loff_t off, bool threadgroup) ++EXPORT_SYMBOL(tasklet_enable); + +- local_irq_disable(); +- list = __this_cpu_read(tasklet_vec.head); +- __this_cpu_write(tasklet_vec.head, NULL); +- __this_cpu_write(tasklet_vec.tail, this_cpu_ptr(&tasklet_vec.head)); +- local_irq_enable(); ++static void __tasklet_action(struct softirq_action *a, ++ struct tasklet_struct *list) +{ -+ struct task_struct *tsk; -+ const struct cred *cred = current_cred(), *tcred; -+ struct cgroup *cgrp; -+ pid_t pid; -+ int ret; -+ -+ if (kstrtoint(strstrip(buf), 0, &pid) || pid < 0) -+ return -EINVAL; -+ -+ cgrp = cgroup_kn_lock_live(of->kn); -+ if (!cgrp) -+ return -ENODEV; ++ int loops = 1000000; + + while (list) { + struct tasklet_struct *t = list; + + list = list->next; + +- if (tasklet_trylock(t)) { +- if (!atomic_read(&t->count)) { +- if (!test_and_clear_bit(TASKLET_STATE_SCHED, +- &t->state)) +- BUG(); +- t->func(t->data); +- tasklet_unlock(t); +- continue; +- } +- tasklet_unlock(t); ++ /* ++ * Should always succeed - after a tasklist got on the ++ * list (after getting the SCHED bit set from 0 to 1), ++ * nothing but the tasklet softirq it got queued to can ++ * lock it: ++ */ ++ if (!tasklet_trylock(t)) { ++ WARN_ON(1); ++ continue; + } + +- local_irq_disable(); + t->next = NULL; +- *__this_cpu_read(tasklet_vec.tail) = t; +- __this_cpu_write(tasklet_vec.tail, &(t->next)); +- __raise_softirq_irqoff(TASKLET_SOFTIRQ); +- local_irq_enable(); + -+retry_find_task: -+ rcu_read_lock(); -+ if (pid) { -+ tsk = find_task_by_vpid(pid); -+ if (!tsk) { -+ rcu_read_unlock(); -+ ret = -ESRCH; -+ goto out_unlock_cgroup; -+ } + /* -+ * even if we're attaching all tasks in the thread group, we -+ * only need to check permissions on one of them. ++ * If we cannot handle the tasklet because it's disabled, ++ * mark it as pending. tasklet_enable() will later ++ * re-schedule the tasklet. + */ -+ tcred = __task_cred(tsk); -+ if (!uid_eq(cred->euid, GLOBAL_ROOT_UID) && -+ !uid_eq(cred->euid, tcred->uid) && -+ !uid_eq(cred->euid, tcred->suid)) { -+ rcu_read_unlock(); -+ ret = -EACCES; -+ goto out_unlock_cgroup; ++ if (unlikely(atomic_read(&t->count))) { ++out_disabled: ++ /* implicit unlock: */ ++ wmb(); ++ t->state = TASKLET_STATEF_PENDING; ++ continue; + } -+ } else -+ tsk = current; + -+ if (threadgroup) -+ tsk = tsk->group_leader; -+ -+ /* -+ * Workqueue threads may acquire PF_NO_SETAFFINITY and become -+ * trapped in a cpuset, or RT worker may be born in a cgroup -+ * with no rt_runtime allocated. Just say no. -+ */ -+ if (tsk == kthreadd_task || (tsk->flags & PF_NO_SETAFFINITY)) { -+ ret = -EINVAL; -+ rcu_read_unlock(); -+ goto out_unlock_cgroup; -+ } ++ /* ++ * After this point on the tasklet might be rescheduled ++ * on another CPU, but it can only be added to another ++ * CPU's tasklet list if we unlock the tasklet (which we ++ * dont do yet). ++ */ ++ if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state)) ++ WARN_ON(1); + -+ get_task_struct(tsk); -+ rcu_read_unlock(); ++again: ++ t->func(t->data); + -+ threadgroup_lock(tsk); -+ if (threadgroup) { -+ if (!thread_group_leader(tsk)) { ++ /* ++ * Try to unlock the tasklet. We must use cmpxchg, because ++ * another CPU might have scheduled or disabled the tasklet. ++ * We only allow the STATE_RUN -> 0 transition here. ++ */ ++ while (!tasklet_tryunlock(t)) { + /* -+ * a race with de_thread from another thread's exec() -+ * may strip us of our leadership, if this happens, -+ * there is no choice but to throw this task away and -+ * try again; this is -+ * "double-double-toil-and-trouble-check locking". ++ * If it got disabled meanwhile, bail out: + */ -+ threadgroup_unlock(tsk); -+ put_task_struct(tsk); -+ goto retry_find_task; ++ if (atomic_read(&t->count)) ++ goto out_disabled; ++ /* ++ * If it got scheduled meanwhile, re-execute ++ * the tasklet function: ++ */ ++ if (test_and_clear_bit(TASKLET_STATE_SCHED, &t->state)) ++ goto again; ++ if (!--loops) { ++ printk("hm, tasklet state: %08lx\n", t->state); ++ WARN_ON(1); ++ tasklet_unlock(t); ++ break; ++ } + } -+ } -+ -+ ret = cgroup_attach_task(cgrp, tsk, threadgroup); -+ -+ threadgroup_unlock(tsk); -+ -+ put_task_struct(tsk); -+out_unlock_cgroup: -+ cgroup_kn_unlock(of->kn); -+ return ret ?: nbytes; -+} -+ -+/** -+ * cgroup_attach_task_all - attach task 'tsk' to all cgroups of task 'from' -+ * @from: attach to all cgroups of a given task -+ * @tsk: the task to be attached -+ */ -+int cgroup_attach_task_all(struct task_struct *from, struct task_struct *tsk) -+{ -+ struct cgroup_root *root; -+ int retval = 0; -+ -+ mutex_lock(&cgroup_mutex); -+ for_each_root(root) { -+ struct cgroup *from_cgrp; -+ -+ if (root == &cgrp_dfl_root) -+ continue; -+ -+ down_read(&css_set_rwsem); -+ from_cgrp = task_cgroup_from_root(from, root); -+ up_read(&css_set_rwsem); -+ -+ retval = cgroup_attach_task(from_cgrp, tsk, false); -+ if (retval) -+ break; -+ } -+ mutex_unlock(&cgroup_mutex); -+ -+ return retval; -+} -+EXPORT_SYMBOL_GPL(cgroup_attach_task_all); -+ -+static ssize_t cgroup_tasks_write(struct kernfs_open_file *of, -+ char *buf, size_t nbytes, loff_t off) -+{ -+ return __cgroup_procs_write(of, buf, nbytes, off, false); -+} -+ -+static ssize_t cgroup_procs_write(struct kernfs_open_file *of, -+ char *buf, size_t nbytes, loff_t off) -+{ -+ return __cgroup_procs_write(of, buf, nbytes, off, true); -+} -+ -+static ssize_t cgroup_release_agent_write(struct kernfs_open_file *of, -+ char *buf, size_t nbytes, loff_t off) -+{ -+ struct cgroup *cgrp; -+ -+ BUILD_BUG_ON(sizeof(cgrp->root->release_agent_path) < PATH_MAX); -+ -+ cgrp = cgroup_kn_lock_live(of->kn); -+ if (!cgrp) -+ return -ENODEV; -+ spin_lock(&release_agent_path_lock); -+ strlcpy(cgrp->root->release_agent_path, strstrip(buf), -+ sizeof(cgrp->root->release_agent_path)); -+ spin_unlock(&release_agent_path_lock); -+ cgroup_kn_unlock(of->kn); -+ return nbytes; -+} -+ -+static int cgroup_release_agent_show(struct seq_file *seq, void *v) -+{ -+ struct cgroup *cgrp = seq_css(seq)->cgroup; -+ -+ spin_lock(&release_agent_path_lock); -+ seq_puts(seq, cgrp->root->release_agent_path); -+ spin_unlock(&release_agent_path_lock); -+ seq_putc(seq, '\n'); -+ return 0; -+} -+ -+static int cgroup_sane_behavior_show(struct seq_file *seq, void *v) -+{ -+ seq_puts(seq, "0\n"); -+ return 0; -+} -+ -+static void cgroup_print_ss_mask(struct seq_file *seq, unsigned int ss_mask) + } + } + ++static void tasklet_action(struct softirq_action *a) +{ -+ struct cgroup_subsys *ss; -+ bool printed = false; -+ int ssid; -+ -+ for_each_subsys(ss, ssid) { -+ if (ss_mask & (1 << ssid)) { -+ if (printed) -+ seq_putc(seq, ' '); -+ seq_printf(seq, "%s", ss->name); -+ printed = true; -+ } -+ } -+ if (printed) -+ seq_putc(seq, '\n'); -+} ++ struct tasklet_struct *list; + -+/* show controllers which are currently attached to the default hierarchy */ -+static int cgroup_root_controllers_show(struct seq_file *seq, void *v) -+{ -+ struct cgroup *cgrp = seq_css(seq)->cgroup; ++ local_irq_disable(); + -+ cgroup_print_ss_mask(seq, cgrp->root->subsys_mask & -+ ~cgrp_dfl_root_inhibit_ss_mask); -+ return 0; -+} ++ list = __this_cpu_read(tasklet_vec.head); ++ __this_cpu_write(tasklet_vec.head, NULL); ++ __this_cpu_write(tasklet_vec.tail, this_cpu_ptr(&tasklet_vec.head)); + -+/* show controllers which are enabled from the parent */ -+static int cgroup_controllers_show(struct seq_file *seq, void *v) -+{ -+ struct cgroup *cgrp = seq_css(seq)->cgroup; ++ local_irq_enable(); + -+ cgroup_print_ss_mask(seq, cgroup_parent(cgrp)->subtree_control); -+ return 0; ++ __tasklet_action(a, list); +} + -+/* show controllers which are enabled for a given cgroup's children */ -+static int cgroup_subtree_control_show(struct seq_file *seq, void *v) -+{ -+ struct cgroup *cgrp = seq_css(seq)->cgroup; + static void tasklet_hi_action(struct softirq_action *a) + { + struct tasklet_struct *list; + + local_irq_disable(); + -+ cgroup_print_ss_mask(seq, cgrp->subtree_control); -+ return 0; + list = __this_cpu_read(tasklet_hi_vec.head); + __this_cpu_write(tasklet_hi_vec.head, NULL); + __this_cpu_write(tasklet_hi_vec.tail, this_cpu_ptr(&tasklet_hi_vec.head)); +- local_irq_enable(); +- +- while (list) { +- struct tasklet_struct *t = list; +- +- list = list->next; + +- if (tasklet_trylock(t)) { +- if (!atomic_read(&t->count)) { +- if (!test_and_clear_bit(TASKLET_STATE_SCHED, +- &t->state)) +- BUG(); +- t->func(t->data); +- tasklet_unlock(t); +- continue; +- } +- tasklet_unlock(t); +- } ++ local_irq_enable(); + +- local_irq_disable(); +- t->next = NULL; +- *__this_cpu_read(tasklet_hi_vec.tail) = t; +- __this_cpu_write(tasklet_hi_vec.tail, &(t->next)); +- __raise_softirq_irqoff(HI_SOFTIRQ); +- local_irq_enable(); +- } ++ __tasklet_action(a, list); + } + + void tasklet_init(struct tasklet_struct *t, +@@ -572,7 +1027,7 @@ + + while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) { + do { +- yield(); ++ msleep(1); + } while (test_bit(TASKLET_STATE_SCHED, &t->state)); + } + tasklet_unlock_wait(t); +@@ -646,25 +1101,26 @@ + open_softirq(HI_SOFTIRQ, tasklet_hi_action); + } + +-static int ksoftirqd_should_run(unsigned int cpu) +-{ +- return local_softirq_pending(); +-} +- +-static void run_ksoftirqd(unsigned int cpu) ++#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT_FULL) ++void tasklet_unlock_wait(struct tasklet_struct *t) + { +- local_irq_disable(); +- if (local_softirq_pending()) { ++ while (test_bit(TASKLET_STATE_RUN, &(t)->state)) { + /* +- * We can safely run softirq on inline stack, as we are not deep +- * in the task stack here. ++ * Hack for now to avoid this busy-loop: + */ +- __do_softirq(); +- local_irq_enable(); +- cond_resched_rcu_qs(); +- return; ++#ifdef CONFIG_PREEMPT_RT_FULL ++ msleep(1); ++#else ++ barrier(); ++#endif + } +- local_irq_enable(); +} ++EXPORT_SYMBOL(tasklet_unlock_wait); ++#endif + -+/** -+ * cgroup_update_dfl_csses - update css assoc of a subtree in default hierarchy -+ * @cgrp: root of the subtree to update csses for -+ * -+ * @cgrp's child_subsys_mask has changed and its subtree's (self excluded) -+ * css associations need to be updated accordingly. This function looks up -+ * all css_sets which are attached to the subtree, creates the matching -+ * updated css_sets and migrates the tasks to the new ones. -+ */ -+static int cgroup_update_dfl_csses(struct cgroup *cgrp) ++static int ksoftirqd_should_run(unsigned int cpu) +{ -+ LIST_HEAD(preloaded_csets); -+ struct cgroup_subsys_state *css; -+ struct css_set *src_cset; -+ int ret; -+ -+ lockdep_assert_held(&cgroup_mutex); -+ -+ /* look up all csses currently attached to @cgrp's subtree */ -+ down_read(&css_set_rwsem); -+ css_for_each_descendant_pre(css, cgroup_css(cgrp, NULL)) { -+ struct cgrp_cset_link *link; -+ -+ /* self is not affected by child_subsys_mask change */ -+ if (css->cgroup == cgrp) -+ continue; -+ -+ list_for_each_entry(link, &css->cgroup->cset_links, cset_link) -+ cgroup_migrate_add_src(link->cset, cgrp, -+ &preloaded_csets); -+ } -+ up_read(&css_set_rwsem); -+ -+ /* NULL dst indicates self on default hierarchy */ -+ ret = cgroup_migrate_prepare_dst(NULL, &preloaded_csets); -+ if (ret) -+ goto out_finish; -+ -+ list_for_each_entry(src_cset, &preloaded_csets, mg_preload_node) { -+ struct task_struct *last_task = NULL, *task; -+ -+ /* src_csets precede dst_csets, break on the first dst_cset */ -+ if (!src_cset->mg_src_cgrp) -+ break; -+ -+ /* -+ * All tasks in src_cset need to be migrated to the -+ * matching dst_cset. Empty it process by process. We -+ * walk tasks but migrate processes. The leader might even -+ * belong to a different cset but such src_cset would also -+ * be among the target src_csets because the default -+ * hierarchy enforces per-process membership. -+ */ -+ while (true) { -+ down_read(&css_set_rwsem); -+ task = list_first_entry_or_null(&src_cset->tasks, -+ struct task_struct, cg_list); -+ if (task) { -+ task = task->group_leader; -+ WARN_ON_ONCE(!task_css_set(task)->mg_src_cgrp); -+ get_task_struct(task); -+ } -+ up_read(&css_set_rwsem); -+ -+ if (!task) -+ break; -+ -+ /* guard against possible infinite loop */ -+ if (WARN(last_task == task, -+ "cgroup: update_dfl_csses failed to make progress, aborting in inconsistent state\n")) -+ goto out_finish; -+ last_task = task; -+ -+ threadgroup_lock(task); -+ /* raced against de_thread() from another thread? */ -+ if (!thread_group_leader(task)) { -+ threadgroup_unlock(task); -+ put_task_struct(task); -+ continue; -+ } -+ -+ ret = cgroup_migrate(src_cset->dfl_cgrp, task, true); -+ -+ threadgroup_unlock(task); -+ put_task_struct(task); -+ -+ if (WARN(ret, "cgroup: failed to update controllers for the default hierarchy (%d), further operations may crash or hang\n", ret)) -+ goto out_finish; ++ return ksoftirqd_softirq_pending(); + } + + #ifdef CONFIG_HOTPLUG_CPU +@@ -746,6 +1202,8 @@ + + static struct smp_hotplug_thread softirq_threads = { + .store = &ksoftirqd, ++ .setup = ksoftirqd_set_sched_params, ++ .cleanup = ksoftirqd_clr_sched_params, + .thread_should_run = ksoftirqd_should_run, + .thread_fn = run_ksoftirqd, + .thread_comm = "ksoftirqd/%u", +diff -Nur linux-4.1.10.orig/kernel/stop_machine.c linux-4.1.10/kernel/stop_machine.c +--- linux-4.1.10.orig/kernel/stop_machine.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/stop_machine.c 2015-10-12 22:33:32.312674375 +0200 +@@ -30,12 +30,12 @@ + atomic_t nr_todo; /* nr left to execute */ + bool executed; /* actually executed? */ + int ret; /* collected return value */ +- struct completion completion; /* fired if nr_todo reaches 0 */ ++ struct task_struct *waiter; /* woken when nr_todo reaches 0 */ + }; + + /* the actual stopper, one per every possible cpu, enabled on online cpus */ + struct cpu_stopper { +- spinlock_t lock; ++ raw_spinlock_t lock; + bool enabled; /* is this stopper enabled? */ + struct list_head works; /* list of pending works */ + }; +@@ -56,7 +56,7 @@ + { + memset(done, 0, sizeof(*done)); + atomic_set(&done->nr_todo, nr_todo); +- init_completion(&done->completion); ++ done->waiter = current; + } + + /* signal completion unless @done is NULL */ +@@ -65,8 +65,10 @@ + if (done) { + if (executed) + done->executed = true; +- if (atomic_dec_and_test(&done->nr_todo)) +- complete(&done->completion); ++ if (atomic_dec_and_test(&done->nr_todo)) { ++ wake_up_process(done->waiter); ++ done->waiter = NULL; + } -+ } -+ -+out_finish: -+ cgroup_migrate_finish(&preloaded_csets); -+ return ret; + } + } + +@@ -78,7 +80,7 @@ + + unsigned long flags; + +- spin_lock_irqsave(&stopper->lock, flags); ++ raw_spin_lock_irqsave(&stopper->lock, flags); + + if (stopper->enabled) { + list_add_tail(&work->list, &stopper->works); +@@ -86,7 +88,23 @@ + } else + cpu_stop_signal_done(work->done, false); + +- spin_unlock_irqrestore(&stopper->lock, flags); ++ raw_spin_unlock_irqrestore(&stopper->lock, flags); +} + -+/* change the enabled child controllers for a cgroup in the default hierarchy */ -+static ssize_t cgroup_subtree_control_write(struct kernfs_open_file *of, -+ char *buf, size_t nbytes, -+ loff_t off) ++static void wait_for_stop_done(struct cpu_stop_done *done) +{ -+ unsigned int enable = 0, disable = 0; -+ unsigned int css_enable, css_disable, old_sc, new_sc, old_ss, new_ss; -+ struct cgroup *cgrp, *child; -+ struct cgroup_subsys *ss; -+ char *tok; -+ int ssid, ret; -+ -+ /* -+ * Parse input - space separated list of subsystem names prefixed -+ * with either + or -. -+ */ -+ buf = strstrip(buf); -+ while ((tok = strsep(&buf, " "))) { -+ if (tok[0] == '\0') -+ continue; -+ for_each_subsys(ss, ssid) { -+ if (ss->disabled || strcmp(tok + 1, ss->name) || -+ ((1 << ss->id) & cgrp_dfl_root_inhibit_ss_mask)) -+ continue; -+ -+ if (*tok == '+') { -+ enable |= 1 << ssid; -+ disable &= ~(1 << ssid); -+ } else if (*tok == '-') { -+ disable |= 1 << ssid; -+ enable &= ~(1 << ssid); -+ } else { -+ return -EINVAL; -+ } -+ break; -+ } -+ if (ssid == CGROUP_SUBSYS_COUNT) -+ return -EINVAL; -+ } -+ -+ cgrp = cgroup_kn_lock_live(of->kn); -+ if (!cgrp) -+ return -ENODEV; -+ -+ for_each_subsys(ss, ssid) { -+ if (enable & (1 << ssid)) { -+ if (cgrp->subtree_control & (1 << ssid)) { -+ enable &= ~(1 << ssid); -+ continue; -+ } -+ -+ /* unavailable or not enabled on the parent? */ -+ if (!(cgrp_dfl_root.subsys_mask & (1 << ssid)) || -+ (cgroup_parent(cgrp) && -+ !(cgroup_parent(cgrp)->subtree_control & (1 << ssid)))) { -+ ret = -ENOENT; -+ goto out_unlock; -+ } -+ } else if (disable & (1 << ssid)) { -+ if (!(cgrp->subtree_control & (1 << ssid))) { -+ disable &= ~(1 << ssid); -+ continue; -+ } -+ -+ /* a child has it enabled? */ -+ cgroup_for_each_live_child(child, cgrp) { -+ if (child->subtree_control & (1 << ssid)) { -+ ret = -EBUSY; -+ goto out_unlock; -+ } -+ } -+ } -+ } -+ -+ if (!enable && !disable) { -+ ret = 0; -+ goto out_unlock; -+ } -+ -+ /* -+ * Except for the root, subtree_control must be zero for a cgroup -+ * with tasks so that child cgroups don't compete against tasks. -+ */ -+ if (enable && cgroup_parent(cgrp) && !list_empty(&cgrp->cset_links)) { -+ ret = -EBUSY; -+ goto out_unlock; -+ } -+ -+ /* -+ * Update subsys masks and calculate what needs to be done. More -+ * subsystems than specified may need to be enabled or disabled -+ * depending on subsystem dependencies. -+ */ -+ old_sc = cgrp->subtree_control; -+ old_ss = cgrp->child_subsys_mask; -+ new_sc = (old_sc | enable) & ~disable; -+ new_ss = cgroup_calc_child_subsys_mask(cgrp, new_sc); -+ -+ css_enable = ~old_ss & new_ss; -+ css_disable = old_ss & ~new_ss; -+ enable |= css_enable; -+ disable |= css_disable; -+ -+ /* -+ * Because css offlining is asynchronous, userland might try to -+ * re-enable the same controller while the previous instance is -+ * still around. In such cases, wait till it's gone using -+ * offline_waitq. -+ */ -+ for_each_subsys(ss, ssid) { -+ if (!(css_enable & (1 << ssid))) -+ continue; -+ -+ cgroup_for_each_live_child(child, cgrp) { -+ DEFINE_WAIT(wait); -+ -+ if (!cgroup_css(child, ss)) -+ continue; -+ -+ cgroup_get(child); -+ prepare_to_wait(&child->offline_waitq, &wait, -+ TASK_UNINTERRUPTIBLE); -+ cgroup_kn_unlock(of->kn); -+ schedule(); -+ finish_wait(&child->offline_waitq, &wait); -+ cgroup_put(child); -+ -+ return restart_syscall(); -+ } -+ } -+ -+ cgrp->subtree_control = new_sc; -+ cgrp->child_subsys_mask = new_ss; -+ -+ /* -+ * Create new csses or make the existing ones visible. A css is -+ * created invisible if it's being implicitly enabled through -+ * dependency. An invisible css is made visible when the userland -+ * explicitly enables it. -+ */ -+ for_each_subsys(ss, ssid) { -+ if (!(enable & (1 << ssid))) -+ continue; -+ -+ cgroup_for_each_live_child(child, cgrp) { -+ if (css_enable & (1 << ssid)) -+ ret = create_css(child, ss, -+ cgrp->subtree_control & (1 << ssid)); -+ else -+ ret = cgroup_populate_dir(child, 1 << ssid); -+ if (ret) -+ goto err_undo_css; -+ } -+ } -+ -+ /* -+ * At this point, cgroup_e_css() results reflect the new csses -+ * making the following cgroup_update_dfl_csses() properly update -+ * css associations of all tasks in the subtree. -+ */ -+ ret = cgroup_update_dfl_csses(cgrp); -+ if (ret) -+ goto err_undo_css; -+ -+ /* -+ * All tasks are migrated out of disabled csses. Kill or hide -+ * them. A css is hidden when the userland requests it to be -+ * disabled while other subsystems are still depending on it. The -+ * css must not actively control resources and be in the vanilla -+ * state if it's made visible again later. Controllers which may -+ * be depended upon should provide ->css_reset() for this purpose. -+ */ -+ for_each_subsys(ss, ssid) { -+ if (!(disable & (1 << ssid))) -+ continue; -+ -+ cgroup_for_each_live_child(child, cgrp) { -+ struct cgroup_subsys_state *css = cgroup_css(child, ss); -+ -+ if (css_disable & (1 << ssid)) { -+ kill_css(css); -+ } else { -+ cgroup_clear_dir(child, 1 << ssid); -+ if (ss->css_reset) -+ ss->css_reset(css); -+ } -+ } ++ set_current_state(TASK_UNINTERRUPTIBLE); ++ while (atomic_read(&done->nr_todo)) { ++ schedule(); ++ set_current_state(TASK_UNINTERRUPTIBLE); + } -+ + /* -+ * The effective csses of all the descendants (excluding @cgrp) may -+ * have changed. Subsystems can optionally subscribe to this event -+ * by implementing ->css_e_css_changed() which is invoked if any of -+ * the effective csses seen from the css's cgroup may have changed. ++ * We need to wait until cpu_stop_signal_done() has cleared ++ * done->waiter. + */ -+ for_each_subsys(ss, ssid) { -+ struct cgroup_subsys_state *this_css = cgroup_css(cgrp, ss); -+ struct cgroup_subsys_state *css; -+ -+ if (!ss->css_e_css_changed || !this_css) -+ continue; -+ -+ css_for_each_descendant_pre(css, this_css) -+ if (css != this_css) -+ ss->css_e_css_changed(css); -+ } -+ -+ kernfs_activate(cgrp->kn); -+ ret = 0; -+out_unlock: -+ cgroup_kn_unlock(of->kn); -+ return ret ?: nbytes; -+ -+err_undo_css: -+ cgrp->subtree_control = old_sc; -+ cgrp->child_subsys_mask = old_ss; -+ -+ for_each_subsys(ss, ssid) { -+ if (!(enable & (1 << ssid))) -+ continue; -+ -+ cgroup_for_each_live_child(child, cgrp) { -+ struct cgroup_subsys_state *css = cgroup_css(child, ss); ++ while (done->waiter) ++ cpu_relax(); ++ set_current_state(TASK_RUNNING); + } + + /** +@@ -120,7 +138,7 @@ + + cpu_stop_init_done(&done, 1); + cpu_stop_queue_work(cpu, &work); +- wait_for_completion(&done.completion); ++ wait_for_stop_done(&done); + return done.executed ? done.ret : -ENOENT; + } + +@@ -248,7 +266,7 @@ + struct irq_cpu_stop_queue_work_info call_args; + struct multi_stop_data msdata; + +- preempt_disable(); ++ preempt_disable_nort(); + msdata = (struct multi_stop_data){ + .fn = fn, + .data = arg, +@@ -281,7 +299,7 @@ + * This relies on the stopper workqueues to be FIFO. + */ + if (!cpu_active(cpu1) || !cpu_active(cpu2)) { +- preempt_enable(); ++ preempt_enable_nort(); + return -ENOENT; + } + +@@ -295,9 +313,9 @@ + &irq_cpu_stop_queue_work, + &call_args, 1); + lg_local_unlock(&stop_cpus_lock); +- preempt_enable(); ++ preempt_enable_nort(); + +- wait_for_completion(&done.completion); ++ wait_for_stop_done(&done); + + return done.executed ? done.ret : -ENOENT; + } +@@ -329,7 +347,7 @@ + + static void queue_stop_cpus_work(const struct cpumask *cpumask, + cpu_stop_fn_t fn, void *arg, +- struct cpu_stop_done *done) ++ struct cpu_stop_done *done, bool inactive) + { + struct cpu_stop_work *work; + unsigned int cpu; +@@ -343,11 +361,13 @@ + } + + /* +- * Disable preemption while queueing to avoid getting +- * preempted by a stopper which might wait for other stoppers +- * to enter @fn which can lead to deadlock. ++ * Make sure that all work is queued on all cpus before ++ * any of the cpus can execute it. + */ +- lg_global_lock(&stop_cpus_lock); ++ if (!inactive) ++ lg_global_lock(&stop_cpus_lock); ++ else ++ lg_global_trylock_relax(&stop_cpus_lock); + for_each_cpu(cpu, cpumask) + cpu_stop_queue_work(cpu, &per_cpu(stop_cpus_work, cpu)); + lg_global_unlock(&stop_cpus_lock); +@@ -359,8 +379,8 @@ + struct cpu_stop_done done; + + cpu_stop_init_done(&done, cpumask_weight(cpumask)); +- queue_stop_cpus_work(cpumask, fn, arg, &done); +- wait_for_completion(&done.completion); ++ queue_stop_cpus_work(cpumask, fn, arg, &done, false); ++ wait_for_stop_done(&done); + return done.executed ? done.ret : -ENOENT; + } + +@@ -439,9 +459,9 @@ + unsigned long flags; + int run; + +- spin_lock_irqsave(&stopper->lock, flags); ++ raw_spin_lock_irqsave(&stopper->lock, flags); + run = !list_empty(&stopper->works); +- spin_unlock_irqrestore(&stopper->lock, flags); ++ raw_spin_unlock_irqrestore(&stopper->lock, flags); + return run; + } + +@@ -453,13 +473,13 @@ + + repeat: + work = NULL; +- spin_lock_irq(&stopper->lock); ++ raw_spin_lock_irq(&stopper->lock); + if (!list_empty(&stopper->works)) { + work = list_first_entry(&stopper->works, + struct cpu_stop_work, list); + list_del_init(&work->list); + } +- spin_unlock_irq(&stopper->lock); ++ raw_spin_unlock_irq(&stopper->lock); + + if (work) { + cpu_stop_fn_t fn = work->fn; +@@ -467,6 +487,16 @@ + struct cpu_stop_done *done = work->done; + char ksym_buf[KSYM_NAME_LEN] __maybe_unused; + ++ /* ++ * Wait until the stopper finished scheduling on all ++ * cpus ++ */ ++ lg_global_lock(&stop_cpus_lock); ++ /* ++ * Let other cpu threads continue as well ++ */ ++ lg_global_unlock(&stop_cpus_lock); + -+ if (!css) -+ continue; + /* cpu stop callbacks are not allowed to sleep */ + preempt_disable(); + +@@ -481,7 +511,13 @@ + kallsyms_lookup((unsigned long)fn, NULL, NULL, NULL, + ksym_buf), arg); + ++ /* ++ * Make sure that the wakeup and setting done->waiter ++ * to NULL is atomic. ++ */ ++ local_irq_disable(); + cpu_stop_signal_done(done, true); ++ local_irq_enable(); + goto repeat; + } + } +@@ -500,20 +536,20 @@ + unsigned long flags; + + /* drain remaining works */ +- spin_lock_irqsave(&stopper->lock, flags); ++ raw_spin_lock_irqsave(&stopper->lock, flags); + list_for_each_entry(work, &stopper->works, list) + cpu_stop_signal_done(work->done, false); + stopper->enabled = false; +- spin_unlock_irqrestore(&stopper->lock, flags); ++ raw_spin_unlock_irqrestore(&stopper->lock, flags); + } + + static void cpu_stop_unpark(unsigned int cpu) + { + struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu); + +- spin_lock_irq(&stopper->lock); ++ raw_spin_lock_irq(&stopper->lock); + stopper->enabled = true; +- spin_unlock_irq(&stopper->lock); ++ raw_spin_unlock_irq(&stopper->lock); + } + + static struct smp_hotplug_thread cpu_stop_threads = { +@@ -535,10 +571,12 @@ + for_each_possible_cpu(cpu) { + struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu); + +- spin_lock_init(&stopper->lock); ++ raw_spin_lock_init(&stopper->lock); + INIT_LIST_HEAD(&stopper->works); + } + ++ lg_lock_init(&stop_cpus_lock, "stop_cpus_lock"); + -+ if (css_enable & (1 << ssid)) -+ kill_css(css); -+ else -+ cgroup_clear_dir(child, 1 << ssid); -+ } -+ } -+ goto out_unlock; -+} + BUG_ON(smpboot_register_percpu_thread(&cpu_stop_threads)); + stop_machine_initialized = true; + return 0; +@@ -634,11 +672,11 @@ + set_state(&msdata, MULTI_STOP_PREPARE); + cpu_stop_init_done(&done, num_active_cpus()); + queue_stop_cpus_work(cpu_active_mask, multi_cpu_stop, &msdata, +- &done); ++ &done, true); + ret = multi_cpu_stop(&msdata); + + /* Busy wait for completion. */ +- while (!completion_done(&done.completion)) ++ while (atomic_read(&done.nr_todo)) + cpu_relax(); + + mutex_unlock(&stop_cpus_mutex); +diff -Nur linux-4.1.10.orig/kernel/time/hrtimer.c linux-4.1.10/kernel/time/hrtimer.c +--- linux-4.1.10.orig/kernel/time/hrtimer.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/time/hrtimer.c 2015-10-12 22:33:32.312674375 +0200 +@@ -48,11 +48,13 @@ + #include + #include + #include ++#include + #include + + #include + + #include ++#include + + #include "tick-internal.h" + +@@ -576,8 +578,7 @@ + * When the callback is running, we do not reprogram the clock event + * device. The timer callback is either running on a different CPU or + * the callback is executed in the hrtimer_interrupt context. The +- * reprogramming is handled either by the softirq, which called the +- * callback or at the end of the hrtimer_interrupt. ++ * reprogramming is handled at the end of the hrtimer_interrupt. + */ + if (hrtimer_callback_running(timer)) + return 0; +@@ -621,6 +622,9 @@ + return res; + } + ++static void __run_hrtimer(struct hrtimer *timer, ktime_t *now); ++static int hrtimer_rt_defer(struct hrtimer *timer); + -+static int cgroup_populated_show(struct seq_file *seq, void *v) + /* + * Initialize the high resolution related parts of cpu_base + */ +@@ -630,6 +634,21 @@ + base->hres_active = 0; + } + ++static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer, ++ struct hrtimer_clock_base *base, ++ int wakeup) +{ -+ seq_printf(seq, "%d\n", (bool)seq_css(seq)->cgroup->populated_cnt); -+ return 0; ++ if (!hrtimer_reprogram(timer, base)) ++ return 0; ++ if (!wakeup) ++ return -ETIME; ++#ifdef CONFIG_PREEMPT_RT_BASE ++ if (!hrtimer_rt_defer(timer)) ++ return -ETIME; ++#endif ++ return 1; +} + -+static ssize_t cgroup_file_write(struct kernfs_open_file *of, char *buf, -+ size_t nbytes, loff_t off) -+{ -+ struct cgroup *cgrp = of->kn->parent->priv; -+ struct cftype *cft = of->kn->priv; -+ struct cgroup_subsys_state *css; -+ int ret; -+ -+ if (cft->write) -+ return cft->write(of, buf, nbytes, off); -+ -+ /* -+ * kernfs guarantees that a file isn't deleted with operations in -+ * flight, which means that the matching css is and stays alive and -+ * doesn't need to be pinned. The RCU locking is not necessary -+ * either. It's just for the convenience of using cgroup_css(). -+ */ -+ rcu_read_lock(); -+ css = cgroup_css(cgrp, cft->ss); -+ rcu_read_unlock(); -+ -+ if (cft->write_u64) { -+ unsigned long long v; -+ ret = kstrtoull(buf, 0, &v); -+ if (!ret) -+ ret = cft->write_u64(css, cft, v); -+ } else if (cft->write_s64) { -+ long long v; -+ ret = kstrtoll(buf, 0, &v); -+ if (!ret) -+ ret = cft->write_s64(css, cft, v); -+ } else { -+ ret = -EINVAL; -+ } -+ -+ return ret ?: nbytes; -+} + static inline ktime_t hrtimer_update_base(struct hrtimer_cpu_base *base) + { + ktime_t *offs_real = &base->clock_base[HRTIMER_BASE_REALTIME].offset; +@@ -695,6 +714,44 @@ + + static DECLARE_WORK(hrtimer_work, clock_was_set_work); + ++#ifdef CONFIG_PREEMPT_RT_FULL ++/* ++ * RT can not call schedule_work from real interrupt context. ++ * Need to make a thread to do the real work. ++ */ ++static struct task_struct *clock_set_delay_thread; ++static bool do_clock_set_delay; + -+static void *cgroup_seqfile_start(struct seq_file *seq, loff_t *ppos) ++static int run_clock_set_delay(void *ignore) +{ -+ return seq_cft(seq)->seq_start(seq, ppos); ++ while (!kthread_should_stop()) { ++ set_current_state(TASK_INTERRUPTIBLE); ++ if (do_clock_set_delay) { ++ do_clock_set_delay = false; ++ schedule_work(&hrtimer_work); ++ } ++ schedule(); ++ } ++ __set_current_state(TASK_RUNNING); ++ return 0; +} + -+static void *cgroup_seqfile_next(struct seq_file *seq, void *v, loff_t *ppos) ++void clock_was_set_delayed(void) +{ -+ return seq_cft(seq)->seq_next(seq, v, ppos); ++ do_clock_set_delay = true; ++ /* Make visible before waking up process */ ++ smp_wmb(); ++ wake_up_process(clock_set_delay_thread); +} + -+static void cgroup_seqfile_stop(struct seq_file *seq, void *v) ++static __init int create_clock_set_delay_thread(void) +{ -+ seq_cft(seq)->seq_stop(seq, v); ++ clock_set_delay_thread = kthread_run(run_clock_set_delay, NULL, "kclksetdelayd"); ++ BUG_ON(!clock_set_delay_thread); ++ return 0; +} -+ -+static int cgroup_seqfile_show(struct seq_file *m, void *arg) ++early_initcall(create_clock_set_delay_thread); ++#else /* PREEMPT_RT_FULL */ + /* + * Called from timekeeping and resume code to reprogramm the hrtimer + * interrupt device on all cpus. +@@ -703,6 +760,7 @@ + { + schedule_work(&hrtimer_work); + } ++#endif + + #else + +@@ -711,6 +769,13 @@ + static inline int hrtimer_switch_to_hres(void) { return 0; } + static inline void + hrtimer_force_reprogram(struct hrtimer_cpu_base *base, int skip_equal) { } ++static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer, ++ struct hrtimer_clock_base *base, ++ int wakeup) +{ -+ struct cftype *cft = seq_cft(m); -+ struct cgroup_subsys_state *css = seq_css(m); -+ -+ if (cft->seq_show) -+ return cft->seq_show(m, arg); -+ -+ if (cft->read_u64) -+ seq_printf(m, "%llu\n", cft->read_u64(css, cft)); -+ else if (cft->read_s64) -+ seq_printf(m, "%lld\n", cft->read_s64(css, cft)); -+ else -+ return -EINVAL; + return 0; +} + -+static struct kernfs_ops cgroup_kf_single_ops = { -+ .atomic_write_len = PAGE_SIZE, -+ .write = cgroup_file_write, -+ .seq_show = cgroup_seqfile_show, -+}; -+ -+static struct kernfs_ops cgroup_kf_ops = { -+ .atomic_write_len = PAGE_SIZE, -+ .write = cgroup_file_write, -+ .seq_start = cgroup_seqfile_start, -+ .seq_next = cgroup_seqfile_next, -+ .seq_stop = cgroup_seqfile_stop, -+ .seq_show = cgroup_seqfile_show, -+}; -+ -+/* -+ * cgroup_rename - Only allow simple rename of directories in place. -+ */ -+static int cgroup_rename(struct kernfs_node *kn, struct kernfs_node *new_parent, -+ const char *new_name_str) -+{ -+ struct cgroup *cgrp = kn->priv; -+ int ret; -+ -+ if (kernfs_type(kn) != KERNFS_DIR) -+ return -ENOTDIR; -+ if (kn->parent != new_parent) -+ return -EIO; -+ -+ /* -+ * This isn't a proper migration and its usefulness is very -+ * limited. Disallow on the default hierarchy. -+ */ -+ if (cgroup_on_dfl(cgrp)) -+ return -EPERM; -+ -+ /* -+ * We're gonna grab cgroup_mutex which nests outside kernfs -+ * active_ref. kernfs_rename() doesn't require active_ref -+ * protection. Break them before grabbing cgroup_mutex. -+ */ -+ kernfs_break_active_protection(new_parent); -+ kernfs_break_active_protection(kn); -+ -+ mutex_lock(&cgroup_mutex); -+ -+ ret = kernfs_rename(kn, new_parent, new_name_str); -+ -+ mutex_unlock(&cgroup_mutex); -+ -+ kernfs_unbreak_active_protection(kn); -+ kernfs_unbreak_active_protection(new_parent); -+ return ret; -+} -+ -+/* set uid and gid of cgroup dirs and files to that of the creator */ -+static int cgroup_kn_set_ugid(struct kernfs_node *kn) -+{ -+ struct iattr iattr = { .ia_valid = ATTR_UID | ATTR_GID, -+ .ia_uid = current_fsuid(), -+ .ia_gid = current_fsgid(), }; -+ -+ if (uid_eq(iattr.ia_uid, GLOBAL_ROOT_UID) && -+ gid_eq(iattr.ia_gid, GLOBAL_ROOT_GID)) -+ return 0; -+ -+ return kernfs_setattr(kn, &iattr); -+} -+ -+static int cgroup_add_file(struct cgroup *cgrp, struct cftype *cft) -+{ -+ char name[CGROUP_FILE_NAME_MAX]; -+ struct kernfs_node *kn; -+ struct lock_class_key *key = NULL; -+ int ret; -+ -+#ifdef CONFIG_DEBUG_LOCK_ALLOC -+ key = &cft->lockdep_key; -+#endif -+ kn = __kernfs_create_file(cgrp->kn, cgroup_file_name(cgrp, cft, name), -+ cgroup_file_mode(cft), 0, cft->kf_ops, cft, -+ NULL, key); -+ if (IS_ERR(kn)) -+ return PTR_ERR(kn); -+ -+ ret = cgroup_kn_set_ugid(kn); -+ if (ret) { -+ kernfs_remove(kn); -+ return ret; -+ } -+ -+ if (cft->seq_show == cgroup_populated_show) -+ cgrp->populated_kn = kn; -+ return 0; -+} + static inline int hrtimer_reprogram(struct hrtimer *timer, + struct hrtimer_clock_base *base) + { +@@ -718,7 +783,6 @@ + } + static inline void hrtimer_init_hres(struct hrtimer_cpu_base *base) { } + static inline void retrigger_next_event(void *arg) { } +- + #endif /* CONFIG_HIGH_RES_TIMERS */ + + /* +@@ -836,6 +900,32 @@ + } + EXPORT_SYMBOL_GPL(hrtimer_forward); + ++#ifdef CONFIG_PREEMPT_RT_BASE ++# define wake_up_timer_waiters(b) wake_up(&(b)->wait) + +/** -+ * cgroup_addrm_files - add or remove files to a cgroup directory -+ * @cgrp: the target cgroup -+ * @cfts: array of cftypes to be added -+ * @is_add: whether to add or remove ++ * hrtimer_wait_for_timer - Wait for a running timer ++ * ++ * @timer: timer to wait for + * -+ * Depending on @is_add, add or remove files defined by @cfts on @cgrp. -+ * For removals, this function never fails. If addition fails, this -+ * function doesn't remove files already added. The caller is responsible -+ * for cleaning up. ++ * The function waits in case the timers callback function is ++ * currently executed on the waitqueue of the timer base. The ++ * waitqueue is woken up after the timer callback function has ++ * finished execution. + */ -+static int cgroup_addrm_files(struct cgroup *cgrp, struct cftype cfts[], -+ bool is_add) -+{ -+ struct cftype *cft; -+ int ret; -+ -+ lockdep_assert_held(&cgroup_mutex); -+ -+ for (cft = cfts; cft->name[0] != '\0'; cft++) { -+ /* does cft->flags tell us to skip this file on @cgrp? */ -+ if ((cft->flags & __CFTYPE_ONLY_ON_DFL) && !cgroup_on_dfl(cgrp)) -+ continue; -+ if ((cft->flags & __CFTYPE_NOT_ON_DFL) && cgroup_on_dfl(cgrp)) -+ continue; -+ if ((cft->flags & CFTYPE_NOT_ON_ROOT) && !cgroup_parent(cgrp)) -+ continue; -+ if ((cft->flags & CFTYPE_ONLY_ON_ROOT) && cgroup_parent(cgrp)) -+ continue; -+ -+ if (is_add) { -+ ret = cgroup_add_file(cgrp, cft); -+ if (ret) { -+ pr_warn("%s: failed to add %s, err=%d\n", -+ __func__, cft->name, ret); -+ return ret; -+ } -+ } else { -+ cgroup_rm_file(cgrp, cft); -+ } -+ } -+ return 0; -+} -+ -+static int cgroup_apply_cftypes(struct cftype *cfts, bool is_add) ++void hrtimer_wait_for_timer(const struct hrtimer *timer) +{ -+ LIST_HEAD(pending); -+ struct cgroup_subsys *ss = cfts[0].ss; -+ struct cgroup *root = &ss->root->cgrp; -+ struct cgroup_subsys_state *css; -+ int ret = 0; -+ -+ lockdep_assert_held(&cgroup_mutex); -+ -+ /* add/rm files for all cgroups created before */ -+ css_for_each_descendant_pre(css, cgroup_css(root, ss)) { -+ struct cgroup *cgrp = css->cgroup; -+ -+ if (cgroup_is_dead(cgrp)) -+ continue; -+ -+ ret = cgroup_addrm_files(cgrp, cfts, is_add); -+ if (ret) -+ break; -+ } ++ struct hrtimer_clock_base *base = timer->base; + -+ if (is_add && !ret) -+ kernfs_activate(root->kn); -+ return ret; ++ if (base && base->cpu_base && !timer->irqsafe) ++ wait_event(base->cpu_base->wait, ++ !(timer->state & HRTIMER_STATE_CALLBACK)); +} + -+static void cgroup_exit_cftypes(struct cftype *cfts) -+{ -+ struct cftype *cft; -+ -+ for (cft = cfts; cft->name[0] != '\0'; cft++) { -+ /* free copy for custom atomic_write_len, see init_cftypes() */ -+ if (cft->max_write_len && cft->max_write_len != PAGE_SIZE) -+ kfree(cft->kf_ops); -+ cft->kf_ops = NULL; -+ cft->ss = NULL; ++#else ++# define wake_up_timer_waiters(b) do { } while (0) ++#endif + -+ /* revert flags set by cgroup core while adding @cfts */ -+ cft->flags &= ~(__CFTYPE_ONLY_ON_DFL | __CFTYPE_NOT_ON_DFL); + /* + * enqueue_hrtimer - internal function to (re)start a timer + * +@@ -879,6 +969,11 @@ + if (!(timer->state & HRTIMER_STATE_ENQUEUED)) + goto out; + ++ if (unlikely(!list_empty(&timer->cb_entry))) { ++ list_del_init(&timer->cb_entry); ++ goto out; + } -+} -+ -+static int cgroup_init_cftypes(struct cgroup_subsys *ss, struct cftype *cfts) -+{ -+ struct cftype *cft; -+ -+ for (cft = cfts; cft->name[0] != '\0'; cft++) { -+ struct kernfs_ops *kf_ops; + -+ WARN_ON(cft->ss || cft->kf_ops); -+ -+ if (cft->seq_start) -+ kf_ops = &cgroup_kf_ops; + next_timer = timerqueue_getnext(&base->active); + timerqueue_del(&base->active, &timer->node); + if (&timer->node == next_timer) { +@@ -966,7 +1061,16 @@ + new_base = switch_hrtimer_base(timer, base, mode & HRTIMER_MODE_PINNED); + + timer_stats_hrtimer_set_start_info(timer); ++#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST ++ { ++ ktime_t now = new_base->get_time(); + ++ if (ktime_to_ns(tim) < ktime_to_ns(now)) ++ timer->praecox = now; + else -+ kf_ops = &cgroup_kf_single_ops; -+ -+ /* -+ * Ugh... if @cft wants a custom max_write_len, we need to -+ * make a copy of kf_ops to set its atomic_write_len. -+ */ -+ if (cft->max_write_len && cft->max_write_len != PAGE_SIZE) { -+ kf_ops = kmemdup(kf_ops, sizeof(*kf_ops), GFP_KERNEL); -+ if (!kf_ops) { -+ cgroup_exit_cftypes(cfts); -+ return -ENOMEM; -+ } -+ kf_ops->atomic_write_len = cft->max_write_len; -+ } -+ -+ cft->kf_ops = kf_ops; -+ cft->ss = ss; ++ timer->praecox = ktime_set(0, 0); + } ++#endif + leftmost = enqueue_hrtimer(timer, new_base); + + if (!leftmost) { +@@ -980,15 +1084,26 @@ + * on dynticks target. + */ + wake_up_nohz_cpu(new_base->cpu_base->cpu); +- } else if (new_base->cpu_base == this_cpu_ptr(&hrtimer_bases) && +- hrtimer_reprogram(timer, new_base)) { ++ } else if (new_base->cpu_base == this_cpu_ptr(&hrtimer_bases)) { + -+ return 0; -+} -+ -+static int cgroup_rm_cftypes_locked(struct cftype *cfts) -+{ -+ lockdep_assert_held(&cgroup_mutex); -+ -+ if (!cfts || !cfts[0].ss) -+ return -ENOENT; -+ -+ list_del(&cfts->node); -+ cgroup_apply_cftypes(cfts, false); -+ cgroup_exit_cftypes(cfts); -+ return 0; -+} -+ -+/** -+ * cgroup_rm_cftypes - remove an array of cftypes from a subsystem -+ * @cfts: zero-length name terminated array of cftypes -+ * -+ * Unregister @cfts. Files described by @cfts are removed from all -+ * existing cgroups and all future cgroups won't have them either. This -+ * function can be called anytime whether @cfts' subsys is attached or not. -+ * -+ * Returns 0 on successful unregistration, -ENOENT if @cfts is not -+ * registered. -+ */ -+int cgroup_rm_cftypes(struct cftype *cfts) -+{ -+ int ret; -+ -+ mutex_lock(&cgroup_mutex); -+ ret = cgroup_rm_cftypes_locked(cfts); -+ mutex_unlock(&cgroup_mutex); -+ return ret; -+} -+ -+/** -+ * cgroup_add_cftypes - add an array of cftypes to a subsystem -+ * @ss: target cgroup subsystem -+ * @cfts: zero-length name terminated array of cftypes -+ * -+ * Register @cfts to @ss. Files described by @cfts are created for all -+ * existing cgroups to which @ss is attached and all future cgroups will -+ * have them too. This function can be called anytime whether @ss is -+ * attached or not. -+ * -+ * Returns 0 on successful registration, -errno on failure. Note that this -+ * function currently returns 0 as long as @cfts registration is successful -+ * even if some file creation attempts on existing cgroups fail. -+ */ -+static int cgroup_add_cftypes(struct cgroup_subsys *ss, struct cftype *cfts) -+{ -+ int ret; -+ -+ if (ss->disabled) -+ return 0; -+ -+ if (!cfts || cfts[0].name[0] == '\0') -+ return 0; -+ -+ ret = cgroup_init_cftypes(ss, cfts); -+ if (ret) -+ return ret; -+ -+ mutex_lock(&cgroup_mutex); -+ -+ list_add_tail(&cfts->node, &ss->cfts); -+ ret = cgroup_apply_cftypes(cfts, true); -+ if (ret) -+ cgroup_rm_cftypes_locked(cfts); -+ -+ mutex_unlock(&cgroup_mutex); -+ return ret; -+} -+ -+/** -+ * cgroup_add_dfl_cftypes - add an array of cftypes for default hierarchy -+ * @ss: target cgroup subsystem -+ * @cfts: zero-length name terminated array of cftypes -+ * -+ * Similar to cgroup_add_cftypes() but the added files are only used for -+ * the default hierarchy. -+ */ -+int cgroup_add_dfl_cftypes(struct cgroup_subsys *ss, struct cftype *cfts) -+{ -+ struct cftype *cft; -+ -+ for (cft = cfts; cft && cft->name[0] != '\0'; cft++) -+ cft->flags |= __CFTYPE_ONLY_ON_DFL; -+ return cgroup_add_cftypes(ss, cfts); -+} ++ ret = hrtimer_enqueue_reprogram(timer, new_base, wakeup); ++ if (ret < 0) { ++ /* ++ * In case we failed to reprogram the timer (mostly ++ * because out current timer is already elapsed), ++ * remove it again and report a failure. This avoids ++ * stale base->first entries. ++ */ ++ debug_deactivate(timer); ++ __remove_hrtimer(timer, new_base, ++ timer->state & HRTIMER_STATE_CALLBACK, 0); ++ } else if (ret > 0) { + /* + * Only allow reprogramming if the new base is on this CPU. + * (it might still be on another CPU if the timer was pending) + * + * XXX send_remote_softirq() ? + */ +- if (wakeup) { + /* + * We need to drop cpu_base->lock to avoid a + * lock ordering issue vs. rq->lock. +@@ -996,9 +1111,7 @@ + raw_spin_unlock(&new_base->cpu_base->lock); + raise_softirq_irqoff(HRTIMER_SOFTIRQ); + local_irq_restore(flags); +- return ret; +- } else { +- __raise_softirq_irqoff(HRTIMER_SOFTIRQ); ++ return 0; + } + } + +@@ -1089,7 +1202,7 @@ + + if (ret >= 0) + return ret; +- cpu_relax(); ++ hrtimer_wait_for_timer(timer); + } + } + EXPORT_SYMBOL_GPL(hrtimer_cancel); +@@ -1153,6 +1266,7 @@ + + base = hrtimer_clockid_to_base(clock_id); + timer->base = &cpu_base->clock_base[base]; ++ INIT_LIST_HEAD(&timer->cb_entry); + timerqueue_init(&timer->node); + + #ifdef CONFIG_TIMER_STATS +@@ -1236,6 +1350,126 @@ + timer->state &= ~HRTIMER_STATE_CALLBACK; + } + ++static enum hrtimer_restart hrtimer_wakeup(struct hrtimer *timer); + -+/** -+ * cgroup_add_legacy_cftypes - add an array of cftypes for legacy hierarchies -+ * @ss: target cgroup subsystem -+ * @cfts: zero-length name terminated array of cftypes -+ * -+ * Similar to cgroup_add_cftypes() but the added files are only used for -+ * the legacy hierarchies. -+ */ -+int cgroup_add_legacy_cftypes(struct cgroup_subsys *ss, struct cftype *cfts) ++#ifdef CONFIG_PREEMPT_RT_BASE ++static void hrtimer_rt_reprogram(int restart, struct hrtimer *timer, ++ struct hrtimer_clock_base *base) +{ -+ struct cftype *cft; -+ + /* -+ * If legacy_flies_on_dfl, we want to show the legacy files on the -+ * dfl hierarchy but iff the target subsystem hasn't been updated -+ * for the dfl hierarchy yet. ++ * Note, we clear the callback flag before we requeue the ++ * timer otherwise we trigger the callback_running() check ++ * in hrtimer_reprogram(). + */ -+ if (!cgroup_legacy_files_on_dfl || -+ ss->dfl_cftypes != ss->legacy_cftypes) { -+ for (cft = cfts; cft && cft->name[0] != '\0'; cft++) -+ cft->flags |= __CFTYPE_NOT_ON_DFL; -+ } -+ -+ return cgroup_add_cftypes(ss, cfts); -+} -+ -+/** -+ * cgroup_task_count - count the number of tasks in a cgroup. -+ * @cgrp: the cgroup in question -+ * -+ * Return the number of tasks in the cgroup. -+ */ -+static int cgroup_task_count(const struct cgroup *cgrp) -+{ -+ int count = 0; -+ struct cgrp_cset_link *link; -+ -+ down_read(&css_set_rwsem); -+ list_for_each_entry(link, &cgrp->cset_links, cset_link) -+ count += atomic_read(&link->cset->refcount); -+ up_read(&css_set_rwsem); -+ return count; -+} ++ timer->state &= ~HRTIMER_STATE_CALLBACK; + -+/** -+ * css_next_child - find the next child of a given css -+ * @pos: the current position (%NULL to initiate traversal) -+ * @parent: css whose children to walk -+ * -+ * This function returns the next child of @parent and should be called -+ * under either cgroup_mutex or RCU read lock. The only requirement is -+ * that @parent and @pos are accessible. The next sibling is guaranteed to -+ * be returned regardless of their states. -+ * -+ * If a subsystem synchronizes ->css_online() and the start of iteration, a -+ * css which finished ->css_online() is guaranteed to be visible in the -+ * future iterations and will stay visible until the last reference is put. -+ * A css which hasn't finished ->css_online() or already finished -+ * ->css_offline() may show up during traversal. It's each subsystem's -+ * responsibility to synchronize against on/offlining. -+ */ -+struct cgroup_subsys_state *css_next_child(struct cgroup_subsys_state *pos, -+ struct cgroup_subsys_state *parent) -+{ -+ struct cgroup_subsys_state *next; ++ if (restart != HRTIMER_NORESTART) { ++ BUG_ON(hrtimer_active(timer)); ++ /* ++ * Enqueue the timer, if it's the leftmost timer then ++ * we need to reprogram it. ++ */ ++ if (!enqueue_hrtimer(timer, base)) ++ return; + -+ cgroup_assert_mutex_or_rcu_locked(); ++#ifndef CONFIG_HIGH_RES_TIMERS ++ } ++#else ++ if (base->cpu_base->hres_active && ++ hrtimer_reprogram(timer, base)) ++ goto requeue; + -+ /* -+ * @pos could already have been unlinked from the sibling list. -+ * Once a cgroup is removed, its ->sibling.next is no longer -+ * updated when its next sibling changes. CSS_RELEASED is set when -+ * @pos is taken off list, at which time its next pointer is valid, -+ * and, as releases are serialized, the one pointed to by the next -+ * pointer is guaranteed to not have started release yet. This -+ * implies that if we observe !CSS_RELEASED on @pos in this RCU -+ * critical section, the one pointed to by its next pointer is -+ * guaranteed to not have finished its RCU grace period even if we -+ * have dropped rcu_read_lock() inbetween iterations. -+ * -+ * If @pos has CSS_RELEASED set, its next pointer can't be -+ * dereferenced; however, as each css is given a monotonically -+ * increasing unique serial number and always appended to the -+ * sibling list, the next one can be found by walking the parent's -+ * children until the first css with higher serial number than -+ * @pos's. While this path can be slower, it happens iff iteration -+ * races against release and the race window is very small. -+ */ -+ if (!pos) { -+ next = list_entry_rcu(parent->children.next, struct cgroup_subsys_state, sibling); -+ } else if (likely(!(pos->flags & CSS_RELEASED))) { -+ next = list_entry_rcu(pos->sibling.next, struct cgroup_subsys_state, sibling); -+ } else { -+ list_for_each_entry_rcu(next, &parent->children, sibling) -+ if (next->serial_nr > pos->serial_nr) -+ break; ++ } else if (hrtimer_active(timer)) { ++ /* ++ * If the timer was rearmed on another CPU, reprogram ++ * the event device. ++ */ ++ if (&timer->node == base->active.next && ++ base->cpu_base->hres_active && ++ hrtimer_reprogram(timer, base)) ++ goto requeue; + } ++ return; + ++requeue: + /* -+ * @next, if not pointing to the head, can be dereferenced and is -+ * the next sibling. ++ * Timer is expired. Thus move it from tree to pending list ++ * again. + */ -+ if (&next->sibling != &parent->children) -+ return next; -+ return NULL; -+} -+ -+/** -+ * css_next_descendant_pre - find the next descendant for pre-order walk -+ * @pos: the current position (%NULL to initiate traversal) -+ * @root: css whose descendants to walk -+ * -+ * To be used by css_for_each_descendant_pre(). Find the next descendant -+ * to visit for pre-order traversal of @root's descendants. @root is -+ * included in the iteration and the first node to be visited. -+ * -+ * While this function requires cgroup_mutex or RCU read locking, it -+ * doesn't require the whole traversal to be contained in a single critical -+ * section. This function will return the correct next descendant as long -+ * as both @pos and @root are accessible and @pos is a descendant of @root. -+ * -+ * If a subsystem synchronizes ->css_online() and the start of iteration, a -+ * css which finished ->css_online() is guaranteed to be visible in the -+ * future iterations and will stay visible until the last reference is put. -+ * A css which hasn't finished ->css_online() or already finished -+ * ->css_offline() may show up during traversal. It's each subsystem's -+ * responsibility to synchronize against on/offlining. -+ */ -+struct cgroup_subsys_state * -+css_next_descendant_pre(struct cgroup_subsys_state *pos, -+ struct cgroup_subsys_state *root) -+{ -+ struct cgroup_subsys_state *next; -+ -+ cgroup_assert_mutex_or_rcu_locked(); -+ -+ /* if first iteration, visit @root */ -+ if (!pos) -+ return root; -+ -+ /* visit the first child if exists */ -+ next = css_next_child(NULL, pos); -+ if (next) -+ return next; -+ -+ /* no child, visit my or the closest ancestor's next sibling */ -+ while (pos != root) { -+ next = css_next_child(pos, pos->parent); -+ if (next) -+ return next; -+ pos = pos->parent; -+ } -+ -+ return NULL; -+} -+ -+/** -+ * css_rightmost_descendant - return the rightmost descendant of a css -+ * @pos: css of interest -+ * -+ * Return the rightmost descendant of @pos. If there's no descendant, @pos -+ * is returned. This can be used during pre-order traversal to skip -+ * subtree of @pos. -+ * -+ * While this function requires cgroup_mutex or RCU read locking, it -+ * doesn't require the whole traversal to be contained in a single critical -+ * section. This function will return the correct rightmost descendant as -+ * long as @pos is accessible. -+ */ -+struct cgroup_subsys_state * -+css_rightmost_descendant(struct cgroup_subsys_state *pos) -+{ -+ struct cgroup_subsys_state *last, *tmp; -+ -+ cgroup_assert_mutex_or_rcu_locked(); -+ -+ do { -+ last = pos; -+ /* ->prev isn't RCU safe, walk ->next till the end */ -+ pos = NULL; -+ css_for_each_child(tmp, last) -+ pos = tmp; -+ } while (pos); -+ -+ return last; -+} -+ -+static struct cgroup_subsys_state * -+css_leftmost_descendant(struct cgroup_subsys_state *pos) -+{ -+ struct cgroup_subsys_state *last; -+ -+ do { -+ last = pos; -+ pos = css_next_child(NULL, pos); -+ } while (pos); -+ -+ return last; ++ __remove_hrtimer(timer, base, timer->state, 0); ++ list_add_tail(&timer->cb_entry, &base->expired); ++#endif +} + -+/** -+ * css_next_descendant_post - find the next descendant for post-order walk -+ * @pos: the current position (%NULL to initiate traversal) -+ * @root: css whose descendants to walk -+ * -+ * To be used by css_for_each_descendant_post(). Find the next descendant -+ * to visit for post-order traversal of @root's descendants. @root is -+ * included in the iteration and the last node to be visited. -+ * -+ * While this function requires cgroup_mutex or RCU read locking, it -+ * doesn't require the whole traversal to be contained in a single critical -+ * section. This function will return the correct next descendant as long -+ * as both @pos and @cgroup are accessible and @pos is a descendant of -+ * @cgroup. -+ * -+ * If a subsystem synchronizes ->css_online() and the start of iteration, a -+ * css which finished ->css_online() is guaranteed to be visible in the -+ * future iterations and will stay visible until the last reference is put. -+ * A css which hasn't finished ->css_online() or already finished -+ * ->css_offline() may show up during traversal. It's each subsystem's -+ * responsibility to synchronize against on/offlining. ++/* ++ * The changes in mainline which removed the callback modes from ++ * hrtimer are not yet working with -rt. The non wakeup_process() ++ * based callbacks which involve sleeping locks need to be treated ++ * seperately. + */ -+struct cgroup_subsys_state * -+css_next_descendant_post(struct cgroup_subsys_state *pos, -+ struct cgroup_subsys_state *root) ++static void hrtimer_rt_run_pending(void) +{ -+ struct cgroup_subsys_state *next; ++ enum hrtimer_restart (*fn)(struct hrtimer *); ++ struct hrtimer_cpu_base *cpu_base; ++ struct hrtimer_clock_base *base; ++ struct hrtimer *timer; ++ int index, restart; + -+ cgroup_assert_mutex_or_rcu_locked(); ++ local_irq_disable(); ++ cpu_base = &per_cpu(hrtimer_bases, smp_processor_id()); + -+ /* if first iteration, visit leftmost descendant which may be @root */ -+ if (!pos) -+ return css_leftmost_descendant(root); ++ raw_spin_lock(&cpu_base->lock); + -+ /* if we visited @root, we're done */ -+ if (pos == root) -+ return NULL; ++ for (index = 0; index < HRTIMER_MAX_CLOCK_BASES; index++) { ++ base = &cpu_base->clock_base[index]; + -+ /* if there's an unvisited sibling, visit its leftmost descendant */ -+ next = css_next_child(pos, pos->parent); -+ if (next) -+ return css_leftmost_descendant(next); ++ while (!list_empty(&base->expired)) { ++ timer = list_first_entry(&base->expired, ++ struct hrtimer, cb_entry); + -+ /* no sibling left, visit parent */ -+ return pos->parent; -+} ++ /* ++ * Same as the above __run_hrtimer function ++ * just we run with interrupts enabled. ++ */ ++ debug_hrtimer_deactivate(timer); ++ __remove_hrtimer(timer, base, HRTIMER_STATE_CALLBACK, 0); ++ timer_stats_account_hrtimer(timer); ++ fn = timer->function; + -+/** -+ * css_has_online_children - does a css have online children -+ * @css: the target css -+ * -+ * Returns %true if @css has any online children; otherwise, %false. This -+ * function can be called from any context but the caller is responsible -+ * for synchronizing against on/offlining as necessary. -+ */ -+bool css_has_online_children(struct cgroup_subsys_state *css) -+{ -+ struct cgroup_subsys_state *child; -+ bool ret = false; ++ raw_spin_unlock_irq(&cpu_base->lock); ++ restart = fn(timer); ++ raw_spin_lock_irq(&cpu_base->lock); + -+ rcu_read_lock(); -+ css_for_each_child(child, css) { -+ if (child->flags & CSS_ONLINE) { -+ ret = true; -+ break; ++ hrtimer_rt_reprogram(restart, timer, base); + } + } -+ rcu_read_unlock(); -+ return ret; -+} -+ -+/** -+ * css_advance_task_iter - advance a task itererator to the next css_set -+ * @it: the iterator to advance -+ * -+ * Advance @it to the next css_set to walk. -+ */ -+static void css_advance_task_iter(struct css_task_iter *it) -+{ -+ struct list_head *l = it->cset_pos; -+ struct cgrp_cset_link *link; -+ struct css_set *cset; -+ -+ /* Advance to the next non-empty css_set */ -+ do { -+ l = l->next; -+ if (l == it->cset_head) { -+ it->cset_pos = NULL; -+ return; -+ } -+ -+ if (it->ss) { -+ cset = container_of(l, struct css_set, -+ e_cset_node[it->ss->id]); -+ } else { -+ link = list_entry(l, struct cgrp_cset_link, cset_link); -+ cset = link->cset; -+ } -+ } while (list_empty(&cset->tasks) && list_empty(&cset->mg_tasks)); -+ -+ it->cset_pos = l; + -+ if (!list_empty(&cset->tasks)) -+ it->task_pos = cset->tasks.next; -+ else -+ it->task_pos = cset->mg_tasks.next; ++ raw_spin_unlock_irq(&cpu_base->lock); + -+ it->tasks_head = &cset->tasks; -+ it->mg_tasks_head = &cset->mg_tasks; ++ wake_up_timer_waiters(cpu_base); +} + -+/** -+ * css_task_iter_start - initiate task iteration -+ * @css: the css to walk tasks of -+ * @it: the task iterator to use -+ * -+ * Initiate iteration through the tasks of @css. The caller can call -+ * css_task_iter_next() to walk through the tasks until the function -+ * returns NULL. On completion of iteration, css_task_iter_end() must be -+ * called. -+ * -+ * Note that this function acquires a lock which is released when the -+ * iteration finishes. The caller can't sleep while iteration is in -+ * progress. -+ */ -+void css_task_iter_start(struct cgroup_subsys_state *css, -+ struct css_task_iter *it) -+ __acquires(css_set_rwsem) ++static int hrtimer_rt_defer(struct hrtimer *timer) +{ -+ /* no one should try to iterate before mounting cgroups */ -+ WARN_ON_ONCE(!use_task_css_set_links); -+ -+ down_read(&css_set_rwsem); -+ -+ it->ss = css->ss; -+ -+ if (it->ss) -+ it->cset_pos = &css->cgroup->e_csets[css->ss->id]; -+ else -+ it->cset_pos = &css->cgroup->cset_links; -+ -+ it->cset_head = it->cset_pos; ++ if (timer->irqsafe) ++ return 0; + -+ css_advance_task_iter(it); ++ __remove_hrtimer(timer, timer->base, timer->state, 0); ++ list_add_tail(&timer->cb_entry, &timer->base->expired); ++ return 1; +} + -+/** -+ * css_task_iter_next - return the next task for the iterator -+ * @it: the task iterator being iterated -+ * -+ * The "next" function for task iteration. @it should have been -+ * initialized via css_task_iter_start(). Returns NULL when the iteration -+ * reaches the end. -+ */ -+struct task_struct *css_task_iter_next(struct css_task_iter *it) -+{ -+ struct task_struct *res; -+ struct list_head *l = it->task_pos; -+ -+ /* If the iterator cg is NULL, we have no tasks */ -+ if (!it->cset_pos) -+ return NULL; -+ res = list_entry(l, struct task_struct, cg_list); -+ -+ /* -+ * Advance iterator to find next entry. cset->tasks is consumed -+ * first and then ->mg_tasks. After ->mg_tasks, we move onto the -+ * next cset. -+ */ -+ l = l->next; -+ -+ if (l == it->tasks_head) -+ l = it->mg_tasks_head->next; -+ -+ if (l == it->mg_tasks_head) -+ css_advance_task_iter(it); -+ else -+ it->task_pos = l; -+ -+ return res; -+} ++#else + -+/** -+ * css_task_iter_end - finish task iteration -+ * @it: the task iterator to finish -+ * -+ * Finish task iteration started by css_task_iter_start(). -+ */ -+void css_task_iter_end(struct css_task_iter *it) -+ __releases(css_set_rwsem) ++static inline void hrtimer_rt_run_pending(void) +{ -+ up_read(&css_set_rwsem); ++ hrtimer_peek_ahead_timers(); +} + -+/** -+ * cgroup_trasnsfer_tasks - move tasks from one cgroup to another -+ * @to: cgroup to which the tasks will be moved -+ * @from: cgroup in which the tasks currently reside -+ * -+ * Locking rules between cgroup_post_fork() and the migration path -+ * guarantee that, if a task is forking while being migrated, the new child -+ * is guaranteed to be either visible in the source cgroup after the -+ * parent's migration is complete or put into the target cgroup. No task -+ * can slip out of migration through forking. -+ */ -+int cgroup_transfer_tasks(struct cgroup *to, struct cgroup *from) -+{ -+ LIST_HEAD(preloaded_csets); -+ struct cgrp_cset_link *link; -+ struct css_task_iter it; -+ struct task_struct *task; -+ int ret; ++static inline int hrtimer_rt_defer(struct hrtimer *timer) { return 0; } + -+ mutex_lock(&cgroup_mutex); ++#endif + -+ /* all tasks in @from are being moved, all csets are source */ -+ down_read(&css_set_rwsem); -+ list_for_each_entry(link, &from->cset_links, cset_link) -+ cgroup_migrate_add_src(link->cset, to, &preloaded_csets); -+ up_read(&css_set_rwsem); + #ifdef CONFIG_HIGH_RES_TIMERS + + /* +@@ -1246,7 +1480,7 @@ + { + struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases); + ktime_t expires_next, now, entry_time, delta; +- int i, retries = 0; ++ int i, retries = 0, raise = 0; + + BUG_ON(!cpu_base->hres_active); + cpu_base->nr_events++; +@@ -1281,6 +1515,15 @@ + + timer = container_of(node, struct hrtimer, node); + ++ trace_hrtimer_interrupt(raw_smp_processor_id(), ++ ktime_to_ns(ktime_sub(ktime_to_ns(timer->praecox) ? ++ timer->praecox : hrtimer_get_expires(timer), ++ basenow)), ++ current, ++ timer->function == hrtimer_wakeup ? ++ container_of(timer, struct hrtimer_sleeper, ++ timer)->task : NULL); + -+ ret = cgroup_migrate_prepare_dst(to, &preloaded_csets); -+ if (ret) -+ goto out_err; + /* + * The immediate goal for using the softexpires is + * minimizing wakeups, not running timers at the +@@ -1296,7 +1539,10 @@ + if (basenow.tv64 < hrtimer_get_softexpires_tv64(timer)) + break; + +- __run_hrtimer(timer, &basenow); ++ if (!hrtimer_rt_defer(timer)) ++ __run_hrtimer(timer, &basenow); ++ else ++ raise = 1; + } + } + /* Reevaluate the clock bases for the next expiry */ +@@ -1313,7 +1559,7 @@ + if (expires_next.tv64 == KTIME_MAX || + !tick_program_event(expires_next, 0)) { + cpu_base->hang_detected = 0; +- return; ++ goto out; + } + + /* +@@ -1357,6 +1603,9 @@ + tick_program_event(expires_next, 1); + printk_once(KERN_WARNING "hrtimer: interrupt took %llu ns\n", + ktime_to_ns(delta)); ++out: ++ if (raise) ++ raise_softirq_irqoff(HRTIMER_SOFTIRQ); + } + + /* +@@ -1392,18 +1641,18 @@ + __hrtimer_peek_ahead_timers(); + local_irq_restore(flags); + } +- +-static void run_hrtimer_softirq(struct softirq_action *h) +-{ +- hrtimer_peek_ahead_timers(); +-} +- + #else /* CONFIG_HIGH_RES_TIMERS */ + + static inline void __hrtimer_peek_ahead_timers(void) { } + + #endif /* !CONFIG_HIGH_RES_TIMERS */ + + -+ /* -+ * Migrate tasks one-by-one until @form is empty. This fails iff -+ * ->can_attach() fails. -+ */ -+ do { -+ css_task_iter_start(&from->self, &it); -+ task = css_task_iter_next(&it); -+ if (task) -+ get_task_struct(task); -+ css_task_iter_end(&it); -+ -+ if (task) { -+ ret = cgroup_migrate(to, task, false); -+ put_task_struct(task); -+ } -+ } while (task && !ret); -+out_err: -+ cgroup_migrate_finish(&preloaded_csets); -+ mutex_unlock(&cgroup_mutex); -+ return ret; ++static void run_hrtimer_softirq(struct softirq_action *h) ++{ ++ hrtimer_rt_run_pending(); +} + -+/* -+ * Stuff for reading the 'tasks'/'procs' files. -+ * -+ * Reading this file can return large amounts of data if a cgroup has -+ * *lots* of attached tasks. So it may need several calls to read(), -+ * but we cannot guarantee that the information we produce is correct -+ * unless we produce it entirely atomically. -+ * -+ */ -+ -+/* which pidlist file are we talking about? */ -+enum cgroup_filetype { -+ CGROUP_FILE_PROCS, -+ CGROUP_FILE_TASKS, -+}; -+ -+/* -+ * A pidlist is a list of pids that virtually represents the contents of one -+ * of the cgroup files ("procs" or "tasks"). We keep a list of such pidlists, -+ * a pair (one each for procs, tasks) for each pid namespace that's relevant -+ * to the cgroup. -+ */ -+struct cgroup_pidlist { -+ /* -+ * used to find which pidlist is wanted. doesn't change as long as -+ * this particular list stays in the list. -+ */ -+ struct { enum cgroup_filetype type; struct pid_namespace *ns; } key; -+ /* array of xids */ -+ pid_t *list; -+ /* how many elements the above list has */ -+ int length; -+ /* each of these stored in a list by its cgroup */ -+ struct list_head links; -+ /* pointer to the cgroup we belong to, for list removal purposes */ -+ struct cgroup *owner; -+ /* for delayed destruction */ -+ struct delayed_work destroy_dwork; -+}; -+ -+/* -+ * The following two functions "fix" the issue where there are more pids -+ * than kmalloc will give memory for; in such cases, we use vmalloc/vfree. -+ * TODO: replace with a kernel-wide solution to this problem -+ */ -+#define PIDLIST_TOO_LARGE(c) ((c) * sizeof(pid_t) > (PAGE_SIZE * 2)) -+static void *pidlist_allocate(int count) -+{ -+ if (PIDLIST_TOO_LARGE(count)) -+ return vmalloc(count * sizeof(pid_t)); -+ else -+ return kmalloc(count * sizeof(pid_t), GFP_KERNEL); -+} + /* + * Called from timer softirq every jiffy, expire hrtimers: + * +@@ -1436,7 +1685,7 @@ + struct timerqueue_node *node; + struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases); + struct hrtimer_clock_base *base; +- int index, gettime = 1; ++ int index, gettime = 1, raise = 0; + + if (hrtimer_hres_active()) + return; +@@ -1461,10 +1710,16 @@ + hrtimer_get_expires_tv64(timer)) + break; + +- __run_hrtimer(timer, &base->softirq_time); ++ if (!hrtimer_rt_defer(timer)) ++ __run_hrtimer(timer, &base->softirq_time); ++ else ++ raise = 1; + } + raw_spin_unlock(&cpu_base->lock); + } + -+static void pidlist_free(void *p) ++ if (raise) ++ raise_softirq_irqoff(HRTIMER_SOFTIRQ); + } + + /* +@@ -1486,16 +1741,18 @@ + void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, struct task_struct *task) + { + sl->timer.function = hrtimer_wakeup; ++ sl->timer.irqsafe = 1; + sl->task = task; + } + EXPORT_SYMBOL_GPL(hrtimer_init_sleeper); + +-static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mode) ++static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mode, ++ unsigned long state) + { + hrtimer_init_sleeper(t, current); + + do { +- set_current_state(TASK_INTERRUPTIBLE); ++ set_current_state(state); + hrtimer_start_expires(&t->timer, mode); + if (!hrtimer_active(&t->timer)) + t->task = NULL; +@@ -1539,7 +1796,8 @@ + HRTIMER_MODE_ABS); + hrtimer_set_expires_tv64(&t.timer, restart->nanosleep.expires); + +- if (do_nanosleep(&t, HRTIMER_MODE_ABS)) ++ /* cpu_chill() does not care about restart state. */ ++ if (do_nanosleep(&t, HRTIMER_MODE_ABS, TASK_INTERRUPTIBLE)) + goto out; + + rmtp = restart->nanosleep.rmtp; +@@ -1556,8 +1814,10 @@ + return ret; + } + +-long hrtimer_nanosleep(struct timespec *rqtp, struct timespec __user *rmtp, +- const enum hrtimer_mode mode, const clockid_t clockid) ++static long ++__hrtimer_nanosleep(struct timespec *rqtp, struct timespec __user *rmtp, ++ const enum hrtimer_mode mode, const clockid_t clockid, ++ unsigned long state) + { + struct restart_block *restart; + struct hrtimer_sleeper t; +@@ -1570,7 +1830,7 @@ + + hrtimer_init_on_stack(&t.timer, clockid, mode); + hrtimer_set_expires_range_ns(&t.timer, timespec_to_ktime(*rqtp), slack); +- if (do_nanosleep(&t, mode)) ++ if (do_nanosleep(&t, mode, state)) + goto out; + + /* Absolute timers do not update the rmtp value and restart: */ +@@ -1597,6 +1857,12 @@ + return ret; + } + ++long hrtimer_nanosleep(struct timespec *rqtp, struct timespec __user *rmtp, ++ const enum hrtimer_mode mode, const clockid_t clockid) +{ -+ kvfree(p); ++ return __hrtimer_nanosleep(rqtp, rmtp, mode, clockid, TASK_INTERRUPTIBLE); +} + + SYSCALL_DEFINE2(nanosleep, struct timespec __user *, rqtp, + struct timespec __user *, rmtp) + { +@@ -1611,6 +1877,26 @@ + return hrtimer_nanosleep(&tu, rmtp, HRTIMER_MODE_REL, CLOCK_MONOTONIC); + } + ++#ifdef CONFIG_PREEMPT_RT_FULL +/* -+ * Used to destroy all pidlists lingering waiting for destroy timer. None -+ * should be left afterwards. ++ * Sleep for 1 ms in hope whoever holds what we want will let it go. + */ -+static void cgroup_pidlist_destroy_all(struct cgroup *cgrp) -+{ -+ struct cgroup_pidlist *l, *tmp_l; -+ -+ mutex_lock(&cgrp->pidlist_mutex); -+ list_for_each_entry_safe(l, tmp_l, &cgrp->pidlists, links) -+ mod_delayed_work(cgroup_pidlist_destroy_wq, &l->destroy_dwork, 0); -+ mutex_unlock(&cgrp->pidlist_mutex); -+ -+ flush_workqueue(cgroup_pidlist_destroy_wq); -+ BUG_ON(!list_empty(&cgrp->pidlists)); -+} -+ -+static void cgroup_pidlist_destroy_work_fn(struct work_struct *work) ++void cpu_chill(void) +{ -+ struct delayed_work *dwork = to_delayed_work(work); -+ struct cgroup_pidlist *l = container_of(dwork, struct cgroup_pidlist, -+ destroy_dwork); -+ struct cgroup_pidlist *tofree = NULL; -+ -+ mutex_lock(&l->owner->pidlist_mutex); -+ -+ /* -+ * Destroy iff we didn't get queued again. The state won't change -+ * as destroy_dwork can only be queued while locked. -+ */ -+ if (!delayed_work_pending(dwork)) { -+ list_del(&l->links); -+ pidlist_free(l->list); -+ put_pid_ns(l->key.ns); -+ tofree = l; -+ } ++ struct timespec tu = { ++ .tv_nsec = NSEC_PER_MSEC, ++ }; ++ unsigned int freeze_flag = current->flags & PF_NOFREEZE; + -+ mutex_unlock(&l->owner->pidlist_mutex); -+ kfree(tofree); ++ current->flags |= PF_NOFREEZE; ++ __hrtimer_nanosleep(&tu, NULL, HRTIMER_MODE_REL, CLOCK_MONOTONIC, ++ TASK_UNINTERRUPTIBLE); ++ if (!freeze_flag) ++ current->flags &= ~PF_NOFREEZE; +} ++EXPORT_SYMBOL(cpu_chill); ++#endif + + /* + * Functions related to boot-time initialization: + */ +@@ -1622,10 +1908,14 @@ + for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) { + cpu_base->clock_base[i].cpu_base = cpu_base; + timerqueue_init_head(&cpu_base->clock_base[i].active); ++ INIT_LIST_HEAD(&cpu_base->clock_base[i].expired); + } + + cpu_base->cpu = cpu; + hrtimer_init_hres(cpu_base); ++#ifdef CONFIG_PREEMPT_RT_BASE ++ init_waitqueue_head(&cpu_base->wait); ++#endif + } + + #ifdef CONFIG_HOTPLUG_CPU +@@ -1731,9 +2021,7 @@ + hrtimer_cpu_notify(&hrtimers_nb, (unsigned long)CPU_UP_PREPARE, + (void *)(long)smp_processor_id()); + register_cpu_notifier(&hrtimers_nb); +-#ifdef CONFIG_HIGH_RES_TIMERS + open_softirq(HRTIMER_SOFTIRQ, run_hrtimer_softirq); +-#endif + } + + /** +diff -Nur linux-4.1.10.orig/kernel/time/itimer.c linux-4.1.10/kernel/time/itimer.c +--- linux-4.1.10.orig/kernel/time/itimer.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/time/itimer.c 2015-10-12 22:33:32.312674375 +0200 +@@ -213,6 +213,7 @@ + /* We are sharing ->siglock with it_real_fn() */ + if (hrtimer_try_to_cancel(timer) < 0) { + spin_unlock_irq(&tsk->sighand->siglock); ++ hrtimer_wait_for_timer(&tsk->signal->real_timer); + goto again; + } + expires = timeval_to_ktime(value->it_value); +diff -Nur linux-4.1.10.orig/kernel/time/jiffies.c linux-4.1.10/kernel/time/jiffies.c +--- linux-4.1.10.orig/kernel/time/jiffies.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/time/jiffies.c 2015-10-12 22:33:32.312674375 +0200 +@@ -74,7 +74,8 @@ + .max_cycles = 10, + }; + +-__cacheline_aligned_in_smp DEFINE_SEQLOCK(jiffies_lock); ++__cacheline_aligned_in_smp DEFINE_RAW_SPINLOCK(jiffies_lock); ++__cacheline_aligned_in_smp seqcount_t jiffies_seq; + + #if (BITS_PER_LONG < 64) + u64 get_jiffies_64(void) +@@ -83,9 +84,9 @@ + u64 ret; + + do { +- seq = read_seqbegin(&jiffies_lock); ++ seq = read_seqcount_begin(&jiffies_seq); + ret = jiffies_64; +- } while (read_seqretry(&jiffies_lock, seq)); ++ } while (read_seqcount_retry(&jiffies_seq, seq)); + return ret; + } + EXPORT_SYMBOL(get_jiffies_64); +diff -Nur linux-4.1.10.orig/kernel/time/ntp.c linux-4.1.10/kernel/time/ntp.c +--- linux-4.1.10.orig/kernel/time/ntp.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/time/ntp.c 2015-10-12 22:33:32.312674375 +0200 +@@ -10,6 +10,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -529,10 +530,52 @@ + &sync_cmos_work, timespec_to_jiffies(&next)); + } + ++#ifdef CONFIG_PREEMPT_RT_FULL +/* -+ * pidlist_uniq - given a kmalloc()ed list, strip out all duplicate entries -+ * Returns the number of unique elements. ++ * RT can not call schedule_delayed_work from real interrupt context. ++ * Need to make a thread to do the real work. + */ -+static int pidlist_uniq(pid_t *list, int length) -+{ -+ int src, dest = 1; ++static struct task_struct *cmos_delay_thread; ++static bool do_cmos_delay; + -+ /* -+ * we presume the 0th element is unique, so i starts at 1. trivial -+ * edge cases first; no work needs to be done for either -+ */ -+ if (length == 0 || length == 1) -+ return length; -+ /* src and dest walk down the list; dest counts unique elements */ -+ for (src = 1; src < length; src++) { -+ /* find next unique element */ -+ while (list[src] == list[src-1]) { -+ src++; -+ if (src == length) -+ goto after; ++static int run_cmos_delay(void *ignore) ++{ ++ while (!kthread_should_stop()) { ++ set_current_state(TASK_INTERRUPTIBLE); ++ if (do_cmos_delay) { ++ do_cmos_delay = false; ++ queue_delayed_work(system_power_efficient_wq, ++ &sync_cmos_work, 0); + } -+ /* dest always points to where the next unique element goes */ -+ list[dest] = list[src]; -+ dest++; ++ schedule(); + } -+after: -+ return dest; ++ __set_current_state(TASK_RUNNING); ++ return 0; +} + -+/* -+ * The two pid files - task and cgroup.procs - guaranteed that the result -+ * is sorted, which forced this whole pidlist fiasco. As pid order is -+ * different per namespace, each namespace needs differently sorted list, -+ * making it impossible to use, for example, single rbtree of member tasks -+ * sorted by task pointer. As pidlists can be fairly large, allocating one -+ * per open file is dangerous, so cgroup had to implement shared pool of -+ * pidlists keyed by cgroup and namespace. -+ * -+ * All this extra complexity was caused by the original implementation -+ * committing to an entirely unnecessary property. In the long term, we -+ * want to do away with it. Explicitly scramble sort order if on the -+ * default hierarchy so that no such expectation exists in the new -+ * interface. -+ * -+ * Scrambling is done by swapping every two consecutive bits, which is -+ * non-identity one-to-one mapping which disturbs sort order sufficiently. -+ */ -+static pid_t pid_fry(pid_t pid) ++void ntp_notify_cmos_timer(void) +{ -+ unsigned a = pid & 0x55555555; -+ unsigned b = pid & 0xAAAAAAAA; -+ -+ return (a << 1) | (b >> 1); ++ do_cmos_delay = true; ++ /* Make visible before waking up process */ ++ smp_wmb(); ++ wake_up_process(cmos_delay_thread); +} + -+static pid_t cgroup_pid_fry(struct cgroup *cgrp, pid_t pid) ++static __init int create_cmos_delay_thread(void) +{ -+ if (cgroup_on_dfl(cgrp)) -+ return pid_fry(pid); -+ else -+ return pid; ++ cmos_delay_thread = kthread_run(run_cmos_delay, NULL, "kcmosdelayd"); ++ BUG_ON(!cmos_delay_thread); ++ return 0; +} ++early_initcall(create_cmos_delay_thread); + -+static int cmppid(const void *a, const void *b) -+{ -+ return *(pid_t *)a - *(pid_t *)b; -+} ++#else + -+static int fried_cmppid(const void *a, const void *b) -+{ -+ return pid_fry(*(pid_t *)a) - pid_fry(*(pid_t *)b); -+} + void ntp_notify_cmos_timer(void) + { + queue_delayed_work(system_power_efficient_wq, &sync_cmos_work, 0); + } ++#endif /* CONFIG_PREEMPT_RT_FULL */ + + #else + void ntp_notify_cmos_timer(void) { } +diff -Nur linux-4.1.10.orig/kernel/time/posix-cpu-timers.c linux-4.1.10/kernel/time/posix-cpu-timers.c +--- linux-4.1.10.orig/kernel/time/posix-cpu-timers.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/time/posix-cpu-timers.c 2015-10-12 22:33:32.316674111 +0200 +@@ -3,6 +3,7 @@ + */ + + #include ++#include + #include + #include + #include +@@ -626,7 +627,7 @@ + /* + * Disarm any old timer after extracting its expiry time. + */ +- WARN_ON_ONCE(!irqs_disabled()); ++ WARN_ON_ONCE_NONRT(!irqs_disabled()); + + ret = 0; + old_incr = timer->it.cpu.incr; +@@ -1047,7 +1048,7 @@ + /* + * Now re-arm for the new expiry time. + */ +- WARN_ON_ONCE(!irqs_disabled()); ++ WARN_ON_ONCE_NONRT(!irqs_disabled()); + arm_timer(timer); + unlock_task_sighand(p, &flags); + +@@ -1113,10 +1114,11 @@ + sig = tsk->signal; + if (sig->cputimer.running) { + struct task_cputime group_sample; ++ unsigned long flags; + +- raw_spin_lock(&sig->cputimer.lock); ++ raw_spin_lock_irqsave(&sig->cputimer.lock, flags); + group_sample = sig->cputimer.cputime; +- raw_spin_unlock(&sig->cputimer.lock); ++ raw_spin_unlock_irqrestore(&sig->cputimer.lock, flags); + + if (task_cputime_expired(&group_sample, &sig->cputime_expires)) + return 1; +@@ -1130,13 +1132,13 @@ + * already updated our counts. We need to check if any timers fire now. + * Interrupts are disabled. + */ +-void run_posix_cpu_timers(struct task_struct *tsk) ++static void __run_posix_cpu_timers(struct task_struct *tsk) + { + LIST_HEAD(firing); + struct k_itimer *timer, *next; + unsigned long flags; + +- WARN_ON_ONCE(!irqs_disabled()); ++ WARN_ON_ONCE_NONRT(!irqs_disabled()); + + /* + * The fast path checks that there are no expired thread or thread +@@ -1194,6 +1196,190 @@ + } + } + ++#ifdef CONFIG_PREEMPT_RT_BASE ++#include ++#include ++DEFINE_PER_CPU(struct task_struct *, posix_timer_task); ++DEFINE_PER_CPU(struct task_struct *, posix_timer_tasklist); + -+static struct cgroup_pidlist *cgroup_pidlist_find(struct cgroup *cgrp, -+ enum cgroup_filetype type) ++static int posix_cpu_timers_thread(void *data) +{ -+ struct cgroup_pidlist *l; -+ /* don't need task_nsproxy() if we're looking at ourself */ -+ struct pid_namespace *ns = task_active_pid_ns(current); ++ int cpu = (long)data; + -+ lockdep_assert_held(&cgrp->pidlist_mutex); ++ BUG_ON(per_cpu(posix_timer_task,cpu) != current); + -+ list_for_each_entry(l, &cgrp->pidlists, links) -+ if (l->key.type == type && l->key.ns == ns) -+ return l; -+ return NULL; -+} ++ while (!kthread_should_stop()) { ++ struct task_struct *tsk = NULL; ++ struct task_struct *next = NULL; + -+/* -+ * find the appropriate pidlist for our purpose (given procs vs tasks) -+ * returns with the lock on that pidlist already held, and takes care -+ * of the use count, or returns NULL with no locks held if we're out of -+ * memory. -+ */ -+static struct cgroup_pidlist *cgroup_pidlist_find_create(struct cgroup *cgrp, -+ enum cgroup_filetype type) -+{ -+ struct cgroup_pidlist *l; -+ -+ lockdep_assert_held(&cgrp->pidlist_mutex); -+ -+ l = cgroup_pidlist_find(cgrp, type); -+ if (l) -+ return l; -+ -+ /* entry not found; create a new one */ -+ l = kzalloc(sizeof(struct cgroup_pidlist), GFP_KERNEL); -+ if (!l) -+ return l; -+ -+ INIT_DELAYED_WORK(&l->destroy_dwork, cgroup_pidlist_destroy_work_fn); -+ l->key.type = type; -+ /* don't need task_nsproxy() if we're looking at ourself */ -+ l->key.ns = get_pid_ns(task_active_pid_ns(current)); -+ l->owner = cgrp; -+ list_add(&l->links, &cgrp->pidlists); -+ return l; -+} ++ if (cpu_is_offline(cpu)) ++ goto wait_to_die; + -+/* -+ * Load a cgroup's pidarray with either procs' tgids or tasks' pids -+ */ -+static int pidlist_array_load(struct cgroup *cgrp, enum cgroup_filetype type, -+ struct cgroup_pidlist **lp) -+{ -+ pid_t *array; -+ int length; -+ int pid, n = 0; /* used for populating the array */ -+ struct css_task_iter it; -+ struct task_struct *tsk; -+ struct cgroup_pidlist *l; ++ /* grab task list */ ++ raw_local_irq_disable(); ++ tsk = per_cpu(posix_timer_tasklist, cpu); ++ per_cpu(posix_timer_tasklist, cpu) = NULL; ++ raw_local_irq_enable(); ++ ++ /* its possible the list is empty, just return */ ++ if (!tsk) { ++ set_current_state(TASK_INTERRUPTIBLE); ++ schedule(); ++ __set_current_state(TASK_RUNNING); ++ continue; ++ } + -+ lockdep_assert_held(&cgrp->pidlist_mutex); ++ /* Process task list */ ++ while (1) { ++ /* save next */ ++ next = tsk->posix_timer_list; + -+ /* -+ * If cgroup gets more users after we read count, we won't have -+ * enough space - tough. This race is indistinguishable to the -+ * caller from the case that the additional cgroup users didn't -+ * show up until sometime later on. -+ */ -+ length = cgroup_task_count(cgrp); -+ array = pidlist_allocate(length); -+ if (!array) -+ return -ENOMEM; -+ /* now, populate the array */ -+ css_task_iter_start(&cgrp->self, &it); -+ while ((tsk = css_task_iter_next(&it))) { -+ if (unlikely(n == length)) -+ break; -+ /* get tgid or pid for procs or tasks file respectively */ -+ if (type == CGROUP_FILE_PROCS) -+ pid = task_tgid_vnr(tsk); -+ else -+ pid = task_pid_vnr(tsk); -+ if (pid > 0) /* make sure to only use valid results */ -+ array[n++] = pid; -+ } -+ css_task_iter_end(&it); -+ length = n; -+ /* now sort & (if procs) strip out duplicates */ -+ if (cgroup_on_dfl(cgrp)) -+ sort(array, length, sizeof(pid_t), fried_cmppid, NULL); -+ else -+ sort(array, length, sizeof(pid_t), cmppid, NULL); -+ if (type == CGROUP_FILE_PROCS) -+ length = pidlist_uniq(array, length); ++ /* run the task timers, clear its ptr and ++ * unreference it ++ */ ++ __run_posix_cpu_timers(tsk); ++ tsk->posix_timer_list = NULL; ++ put_task_struct(tsk); + -+ l = cgroup_pidlist_find_create(cgrp, type); -+ if (!l) { -+ pidlist_free(array); -+ return -ENOMEM; ++ /* check if this is the last on the list */ ++ if (next == tsk) ++ break; ++ tsk = next; ++ } + } ++ return 0; + -+ /* store array, freeing old if necessary */ -+ pidlist_free(l->list); -+ l->list = array; -+ l->length = length; -+ *lp = l; ++wait_to_die: ++ /* Wait for kthread_stop */ ++ set_current_state(TASK_INTERRUPTIBLE); ++ while (!kthread_should_stop()) { ++ schedule(); ++ set_current_state(TASK_INTERRUPTIBLE); ++ } ++ __set_current_state(TASK_RUNNING); + return 0; +} + -+/** -+ * cgroupstats_build - build and fill cgroupstats -+ * @stats: cgroupstats to fill information into -+ * @dentry: A dentry entry belonging to the cgroup for which stats have -+ * been requested. -+ * -+ * Build and fill cgroupstats so that taskstats can export it to user -+ * space. -+ */ -+int cgroupstats_build(struct cgroupstats *stats, struct dentry *dentry) ++static inline int __fastpath_timer_check(struct task_struct *tsk) +{ -+ struct kernfs_node *kn = kernfs_node_from_dentry(dentry); -+ struct cgroup *cgrp; -+ struct css_task_iter it; -+ struct task_struct *tsk; -+ -+ /* it should be kernfs_node belonging to cgroupfs and is a directory */ -+ if (dentry->d_sb->s_type != &cgroup_fs_type || !kn || -+ kernfs_type(kn) != KERNFS_DIR) -+ return -EINVAL; -+ -+ mutex_lock(&cgroup_mutex); ++ /* tsk == current, ensure it is safe to use ->signal/sighand */ ++ if (unlikely(tsk->exit_state)) ++ return 0; + -+ /* -+ * We aren't being called from kernfs and there's no guarantee on -+ * @kn->priv's validity. For this and css_tryget_online_from_dir(), -+ * @kn->priv is RCU safe. Let's do the RCU dancing. -+ */ -+ rcu_read_lock(); -+ cgrp = rcu_dereference(kn->priv); -+ if (!cgrp || cgroup_is_dead(cgrp)) { -+ rcu_read_unlock(); -+ mutex_unlock(&cgroup_mutex); -+ return -ENOENT; -+ } -+ rcu_read_unlock(); ++ if (!task_cputime_zero(&tsk->cputime_expires)) ++ return 1; + -+ css_task_iter_start(&cgrp->self, &it); -+ while ((tsk = css_task_iter_next(&it))) { -+ switch (tsk->state) { -+ case TASK_RUNNING: -+ stats->nr_running++; -+ break; -+ case TASK_INTERRUPTIBLE: -+ stats->nr_sleeping++; -+ break; -+ case TASK_UNINTERRUPTIBLE: -+ stats->nr_uninterruptible++; -+ break; -+ case TASK_STOPPED: -+ stats->nr_stopped++; -+ break; -+ default: -+ if (delayacct_is_task_waiting_on_io(tsk)) -+ stats->nr_io_wait++; -+ break; -+ } -+ } -+ css_task_iter_end(&it); ++ if (!task_cputime_zero(&tsk->signal->cputime_expires)) ++ return 1; + -+ mutex_unlock(&cgroup_mutex); + return 0; +} + -+ -+/* -+ * seq_file methods for the tasks/procs files. The seq_file position is the -+ * next pid to display; the seq_file iterator is a pointer to the pid -+ * in the cgroup->l->list array. -+ */ -+ -+static void *cgroup_pidlist_start(struct seq_file *s, loff_t *pos) ++void run_posix_cpu_timers(struct task_struct *tsk) +{ -+ /* -+ * Initially we receive a position value that corresponds to -+ * one more than the last pid shown (or 0 on the first call or -+ * after a seek to the start). Use a binary-search to find the -+ * next pid to display, if any -+ */ -+ struct kernfs_open_file *of = s->private; -+ struct cgroup *cgrp = seq_css(s)->cgroup; -+ struct cgroup_pidlist *l; -+ enum cgroup_filetype type = seq_cft(s)->private; -+ int index = 0, pid = *pos; -+ int *iter, ret; -+ -+ mutex_lock(&cgrp->pidlist_mutex); -+ -+ /* -+ * !NULL @of->priv indicates that this isn't the first start() -+ * after open. If the matching pidlist is around, we can use that. -+ * Look for it. Note that @of->priv can't be used directly. It -+ * could already have been destroyed. -+ */ -+ if (of->priv) -+ of->priv = cgroup_pidlist_find(cgrp, type); -+ -+ /* -+ * Either this is the first start() after open or the matching -+ * pidlist has been destroyed inbetween. Create a new one. -+ */ -+ if (!of->priv) { -+ ret = pidlist_array_load(cgrp, type, -+ (struct cgroup_pidlist **)&of->priv); -+ if (ret) -+ return ERR_PTR(ret); -+ } -+ l = of->priv; ++ unsigned long cpu = smp_processor_id(); ++ struct task_struct *tasklist; + -+ if (pid) { -+ int end = l->length; ++ BUG_ON(!irqs_disabled()); ++ if(!per_cpu(posix_timer_task, cpu)) ++ return; ++ /* get per-cpu references */ ++ tasklist = per_cpu(posix_timer_tasklist, cpu); + -+ while (index < end) { -+ int mid = (index + end) / 2; -+ if (cgroup_pid_fry(cgrp, l->list[mid]) == pid) { -+ index = mid; -+ break; -+ } else if (cgroup_pid_fry(cgrp, l->list[mid]) <= pid) -+ index = mid + 1; -+ else -+ end = mid; ++ /* check to see if we're already queued */ ++ if (!tsk->posix_timer_list && __fastpath_timer_check(tsk)) { ++ get_task_struct(tsk); ++ if (tasklist) { ++ tsk->posix_timer_list = tasklist; ++ } else { ++ /* ++ * The list is terminated by a self-pointing ++ * task_struct ++ */ ++ tsk->posix_timer_list = tsk; + } ++ per_cpu(posix_timer_tasklist, cpu) = tsk; ++ ++ wake_up_process(per_cpu(posix_timer_task, cpu)); + } -+ /* If we're off the end of the array, we're done */ -+ if (index >= l->length) -+ return NULL; -+ /* Update the abstract position to be the actual pid that we found */ -+ iter = l->list + index; -+ *pos = cgroup_pid_fry(cgrp, *iter); -+ return iter; +} + -+static void cgroup_pidlist_stop(struct seq_file *s, void *v) ++/* ++ * posix_cpu_thread_call - callback that gets triggered when a CPU is added. ++ * Here we can start up the necessary migration thread for the new CPU. ++ */ ++static int posix_cpu_thread_call(struct notifier_block *nfb, ++ unsigned long action, void *hcpu) +{ -+ struct kernfs_open_file *of = s->private; -+ struct cgroup_pidlist *l = of->priv; -+ -+ if (l) -+ mod_delayed_work(cgroup_pidlist_destroy_wq, &l->destroy_dwork, -+ CGROUP_PIDLIST_DESTROY_DELAY); -+ mutex_unlock(&seq_css(s)->cgroup->pidlist_mutex); -+} ++ int cpu = (long)hcpu; ++ struct task_struct *p; ++ struct sched_param param; + -+static void *cgroup_pidlist_next(struct seq_file *s, void *v, loff_t *pos) -+{ -+ struct kernfs_open_file *of = s->private; -+ struct cgroup_pidlist *l = of->priv; -+ pid_t *p = v; -+ pid_t *end = l->list + l->length; -+ /* -+ * Advance to the next pid in the array. If this goes off the -+ * end, we're done -+ */ -+ p++; -+ if (p >= end) { -+ return NULL; -+ } else { -+ *pos = cgroup_pid_fry(seq_css(s)->cgroup, *p); -+ return p; ++ switch (action) { ++ case CPU_UP_PREPARE: ++ p = kthread_create(posix_cpu_timers_thread, hcpu, ++ "posixcputmr/%d",cpu); ++ if (IS_ERR(p)) ++ return NOTIFY_BAD; ++ p->flags |= PF_NOFREEZE; ++ kthread_bind(p, cpu); ++ /* Must be high prio to avoid getting starved */ ++ param.sched_priority = MAX_RT_PRIO-1; ++ sched_setscheduler(p, SCHED_FIFO, ¶m); ++ per_cpu(posix_timer_task,cpu) = p; ++ break; ++ case CPU_ONLINE: ++ /* Strictly unneccessary, as first user will wake it. */ ++ wake_up_process(per_cpu(posix_timer_task,cpu)); ++ break; ++#ifdef CONFIG_HOTPLUG_CPU ++ case CPU_UP_CANCELED: ++ /* Unbind it from offline cpu so it can run. Fall thru. */ ++ kthread_bind(per_cpu(posix_timer_task, cpu), ++ cpumask_any(cpu_online_mask)); ++ kthread_stop(per_cpu(posix_timer_task,cpu)); ++ per_cpu(posix_timer_task,cpu) = NULL; ++ break; ++ case CPU_DEAD: ++ kthread_stop(per_cpu(posix_timer_task,cpu)); ++ per_cpu(posix_timer_task,cpu) = NULL; ++ break; ++#endif + } ++ return NOTIFY_OK; +} + -+static int cgroup_pidlist_show(struct seq_file *s, void *v) -+{ -+ seq_printf(s, "%d\n", *(int *)v); -+ -+ return 0; -+} ++/* Register at highest priority so that task migration (migrate_all_tasks) ++ * happens before everything else. ++ */ ++static struct notifier_block posix_cpu_thread_notifier = { ++ .notifier_call = posix_cpu_thread_call, ++ .priority = 10 ++}; + -+static u64 cgroup_read_notify_on_release(struct cgroup_subsys_state *css, -+ struct cftype *cft) -+{ -+ return notify_on_release(css->cgroup); -+} -+ -+static int cgroup_write_notify_on_release(struct cgroup_subsys_state *css, -+ struct cftype *cft, u64 val) ++static int __init posix_cpu_thread_init(void) +{ -+ if (val) -+ set_bit(CGRP_NOTIFY_ON_RELEASE, &css->cgroup->flags); -+ else -+ clear_bit(CGRP_NOTIFY_ON_RELEASE, &css->cgroup->flags); -+ return 0; -+} ++ void *hcpu = (void *)(long)smp_processor_id(); ++ /* Start one for boot CPU. */ ++ unsigned long cpu; + -+static u64 cgroup_clone_children_read(struct cgroup_subsys_state *css, -+ struct cftype *cft) -+{ -+ return test_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags); -+} ++ /* init the per-cpu posix_timer_tasklets */ ++ for_each_possible_cpu(cpu) ++ per_cpu(posix_timer_tasklist, cpu) = NULL; + -+static int cgroup_clone_children_write(struct cgroup_subsys_state *css, -+ struct cftype *cft, u64 val) -+{ -+ if (val) -+ set_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags); -+ else -+ clear_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags); ++ posix_cpu_thread_call(&posix_cpu_thread_notifier, CPU_UP_PREPARE, hcpu); ++ posix_cpu_thread_call(&posix_cpu_thread_notifier, CPU_ONLINE, hcpu); ++ register_cpu_notifier(&posix_cpu_thread_notifier); + return 0; +} -+ -+/* cgroup core interface files for the default hierarchy */ -+static struct cftype cgroup_dfl_base_files[] = { -+ { -+ .name = "cgroup.procs", -+ .seq_start = cgroup_pidlist_start, -+ .seq_next = cgroup_pidlist_next, -+ .seq_stop = cgroup_pidlist_stop, -+ .seq_show = cgroup_pidlist_show, -+ .private = CGROUP_FILE_PROCS, -+ .write = cgroup_procs_write, -+ .mode = S_IRUGO | S_IWUSR, -+ }, -+ { -+ .name = "cgroup.controllers", -+ .flags = CFTYPE_ONLY_ON_ROOT, -+ .seq_show = cgroup_root_controllers_show, -+ }, -+ { -+ .name = "cgroup.controllers", -+ .flags = CFTYPE_NOT_ON_ROOT, -+ .seq_show = cgroup_controllers_show, -+ }, -+ { -+ .name = "cgroup.subtree_control", -+ .seq_show = cgroup_subtree_control_show, -+ .write = cgroup_subtree_control_write, -+ }, -+ { -+ .name = "cgroup.populated", -+ .flags = CFTYPE_NOT_ON_ROOT, -+ .seq_show = cgroup_populated_show, -+ }, -+ { } /* terminate */ -+}; -+ -+/* cgroup core interface files for the legacy hierarchies */ -+static struct cftype cgroup_legacy_base_files[] = { -+ { -+ .name = "cgroup.procs", -+ .seq_start = cgroup_pidlist_start, -+ .seq_next = cgroup_pidlist_next, -+ .seq_stop = cgroup_pidlist_stop, -+ .seq_show = cgroup_pidlist_show, -+ .private = CGROUP_FILE_PROCS, -+ .write = cgroup_procs_write, -+ .mode = S_IRUGO | S_IWUSR, -+ }, -+ { -+ .name = "cgroup.clone_children", -+ .read_u64 = cgroup_clone_children_read, -+ .write_u64 = cgroup_clone_children_write, -+ }, -+ { -+ .name = "cgroup.sane_behavior", -+ .flags = CFTYPE_ONLY_ON_ROOT, -+ .seq_show = cgroup_sane_behavior_show, -+ }, -+ { -+ .name = "tasks", -+ .seq_start = cgroup_pidlist_start, -+ .seq_next = cgroup_pidlist_next, -+ .seq_stop = cgroup_pidlist_stop, -+ .seq_show = cgroup_pidlist_show, -+ .private = CGROUP_FILE_TASKS, -+ .write = cgroup_tasks_write, -+ .mode = S_IRUGO | S_IWUSR, -+ }, -+ { -+ .name = "notify_on_release", -+ .read_u64 = cgroup_read_notify_on_release, -+ .write_u64 = cgroup_write_notify_on_release, -+ }, -+ { -+ .name = "release_agent", -+ .flags = CFTYPE_ONLY_ON_ROOT, -+ .seq_show = cgroup_release_agent_show, -+ .write = cgroup_release_agent_write, -+ .max_write_len = PATH_MAX - 1, -+ }, -+ { } /* terminate */ -+}; -+ -+/** -+ * cgroup_populate_dir - create subsys files in a cgroup directory -+ * @cgrp: target cgroup -+ * @subsys_mask: mask of the subsystem ids whose files should be added -+ * -+ * On failure, no file is added. -+ */ -+static int cgroup_populate_dir(struct cgroup *cgrp, unsigned int subsys_mask) ++early_initcall(posix_cpu_thread_init); ++#else /* CONFIG_PREEMPT_RT_BASE */ ++void run_posix_cpu_timers(struct task_struct *tsk) +{ -+ struct cgroup_subsys *ss; -+ int i, ret = 0; -+ -+ /* process cftsets of each subsystem */ -+ for_each_subsys(ss, i) { -+ struct cftype *cfts; -+ -+ if (!(subsys_mask & (1 << i))) -+ continue; -+ -+ list_for_each_entry(cfts, &ss->cfts, node) { -+ ret = cgroup_addrm_files(cgrp, cfts, true); -+ if (ret < 0) -+ goto err; -+ } -+ } -+ return 0; -+err: -+ cgroup_clear_dir(cgrp, subsys_mask); -+ return ret; ++ __run_posix_cpu_timers(tsk); +} ++#endif /* CONFIG_PREEMPT_RT_BASE */ + + /* + * Set one of the process-wide special case CPU timers or RLIMIT_CPU. + * The tsk->sighand->siglock must be held by the caller. +diff -Nur linux-4.1.10.orig/kernel/time/posix-timers.c linux-4.1.10/kernel/time/posix-timers.c +--- linux-4.1.10.orig/kernel/time/posix-timers.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/time/posix-timers.c 2015-10-12 22:33:32.316674111 +0200 +@@ -499,6 +499,7 @@ + static struct pid *good_sigevent(sigevent_t * event) + { + struct task_struct *rtn = current->group_leader; ++ int sig = event->sigev_signo; + + if ((event->sigev_notify & SIGEV_THREAD_ID ) && + (!(rtn = find_task_by_vpid(event->sigev_notify_thread_id)) || +@@ -507,7 +508,8 @@ + return NULL; + + if (((event->sigev_notify & ~SIGEV_THREAD_ID) != SIGEV_NONE) && +- ((event->sigev_signo <= 0) || (event->sigev_signo > SIGRTMAX))) ++ (sig <= 0 || sig > SIGRTMAX || sig_kernel_only(sig) || ++ sig_kernel_coredump(sig))) + return NULL; + + return task_pid(rtn); +@@ -819,6 +821,20 @@ + return overrun; + } + +/* -+ * css destruction is four-stage process. -+ * -+ * 1. Destruction starts. Killing of the percpu_ref is initiated. -+ * Implemented in kill_css(). -+ * -+ * 2. When the percpu_ref is confirmed to be visible as killed on all CPUs -+ * and thus css_tryget_online() is guaranteed to fail, the css can be -+ * offlined by invoking offline_css(). After offlining, the base ref is -+ * put. Implemented in css_killed_work_fn(). -+ * -+ * 3. When the percpu_ref reaches zero, the only possible remaining -+ * accessors are inside RCU read sections. css_release() schedules the -+ * RCU callback. -+ * -+ * 4. After the grace period, the css can be freed. Implemented in -+ * css_free_work_fn(). -+ * -+ * It is actually hairier because both step 2 and 4 require process context -+ * and thus involve punting to css->destroy_work adding two additional -+ * steps to the already complex sequence. ++ * Protected by RCU! + */ -+static void css_free_work_fn(struct work_struct *work) -+{ -+ struct cgroup_subsys_state *css = -+ container_of(work, struct cgroup_subsys_state, destroy_work); -+ struct cgroup_subsys *ss = css->ss; -+ struct cgroup *cgrp = css->cgroup; -+ -+ percpu_ref_exit(&css->refcnt); -+ -+ if (ss) { -+ /* css free path */ -+ int id = css->id; -+ -+ if (css->parent) -+ css_put(css->parent); -+ -+ ss->css_free(css); -+ cgroup_idr_remove(&ss->css_idr, id); -+ cgroup_put(cgrp); -+ } else { -+ /* cgroup free path */ -+ atomic_dec(&cgrp->root->nr_cgrps); -+ cgroup_pidlist_destroy_all(cgrp); -+ cancel_work_sync(&cgrp->release_agent_work); -+ -+ if (cgroup_parent(cgrp)) { -+ /* -+ * We get a ref to the parent, and put the ref when -+ * this cgroup is being freed, so it's guaranteed -+ * that the parent won't be destroyed before its -+ * children. -+ */ -+ cgroup_put(cgroup_parent(cgrp)); -+ kernfs_put(cgrp->kn); -+ kfree(cgrp); -+ } else { -+ /* -+ * This is root cgroup's refcnt reaching zero, -+ * which indicates that the root should be -+ * released. -+ */ -+ cgroup_destroy_root(cgrp->root); -+ } -+ } -+} -+ -+static void css_free_rcu_fn(struct rcu_head *rcu_head) -+{ -+ struct cgroup_subsys_state *css = -+ container_of(rcu_head, struct cgroup_subsys_state, rcu_head); -+ -+ INIT_WORK(&css->destroy_work, css_free_work_fn); -+ queue_work(cgroup_destroy_wq, &css->destroy_work); -+} -+ -+static void css_release_work_fn(struct work_struct *work) -+{ -+ struct cgroup_subsys_state *css = -+ container_of(work, struct cgroup_subsys_state, destroy_work); -+ struct cgroup_subsys *ss = css->ss; -+ struct cgroup *cgrp = css->cgroup; -+ -+ mutex_lock(&cgroup_mutex); -+ -+ css->flags |= CSS_RELEASED; -+ list_del_rcu(&css->sibling); -+ -+ if (ss) { -+ /* css release path */ -+ cgroup_idr_replace(&ss->css_idr, NULL, css->id); -+ if (ss->css_released) -+ ss->css_released(css); -+ } else { -+ /* cgroup release path */ -+ cgroup_idr_remove(&cgrp->root->cgroup_idr, cgrp->id); -+ cgrp->id = -1; -+ -+ /* -+ * There are two control paths which try to determine -+ * cgroup from dentry without going through kernfs - -+ * cgroupstats_build() and css_tryget_online_from_dir(). -+ * Those are supported by RCU protecting clearing of -+ * cgrp->kn->priv backpointer. -+ */ -+ RCU_INIT_POINTER(*(void __rcu __force **)&cgrp->kn->priv, NULL); -+ } -+ -+ mutex_unlock(&cgroup_mutex); -+ -+ call_rcu(&css->rcu_head, css_free_rcu_fn); -+} -+ -+static void css_release(struct percpu_ref *ref) -+{ -+ struct cgroup_subsys_state *css = -+ container_of(ref, struct cgroup_subsys_state, refcnt); -+ -+ INIT_WORK(&css->destroy_work, css_release_work_fn); -+ queue_work(cgroup_destroy_wq, &css->destroy_work); -+} -+ -+static void init_and_link_css(struct cgroup_subsys_state *css, -+ struct cgroup_subsys *ss, struct cgroup *cgrp) -+{ -+ lockdep_assert_held(&cgroup_mutex); -+ -+ cgroup_get(cgrp); -+ -+ memset(css, 0, sizeof(*css)); -+ css->cgroup = cgrp; -+ css->ss = ss; -+ INIT_LIST_HEAD(&css->sibling); -+ INIT_LIST_HEAD(&css->children); -+ css->serial_nr = css_serial_nr_next++; -+ -+ if (cgroup_parent(cgrp)) { -+ css->parent = cgroup_css(cgroup_parent(cgrp), ss); -+ css_get(css->parent); -+ } -+ -+ BUG_ON(cgroup_css(cgrp, ss)); -+} -+ -+/* invoke ->css_online() on a new CSS and mark it online if successful */ -+static int online_css(struct cgroup_subsys_state *css) ++static void timer_wait_for_callback(struct k_clock *kc, struct k_itimer *timr) +{ -+ struct cgroup_subsys *ss = css->ss; -+ int ret = 0; -+ -+ lockdep_assert_held(&cgroup_mutex); -+ -+ if (ss->css_online) -+ ret = ss->css_online(css); -+ if (!ret) { -+ css->flags |= CSS_ONLINE; -+ rcu_assign_pointer(css->cgroup->subsys[ss->id], css); -+ } -+ return ret; ++#ifdef CONFIG_PREEMPT_RT_FULL ++ if (kc->timer_set == common_timer_set) ++ hrtimer_wait_for_timer(&timr->it.real.timer); ++ else ++ /* FIXME: Whacky hack for posix-cpu-timers */ ++ schedule_timeout(1); ++#endif +} + -+/* if the CSS is online, invoke ->css_offline() on it and mark it offline */ -+static void offline_css(struct cgroup_subsys_state *css) -+{ -+ struct cgroup_subsys *ss = css->ss; -+ -+ lockdep_assert_held(&cgroup_mutex); -+ -+ if (!(css->flags & CSS_ONLINE)) + /* Set a POSIX.1b interval timer. */ + /* timr->it_lock is taken. */ + static int +@@ -896,6 +912,7 @@ + if (!timr) + return -EINVAL; + ++ rcu_read_lock(); + kc = clockid_to_kclock(timr->it_clock); + if (WARN_ON_ONCE(!kc || !kc->timer_set)) + error = -EINVAL; +@@ -904,9 +921,12 @@ + + unlock_timer(timr, flag); + if (error == TIMER_RETRY) { ++ timer_wait_for_callback(kc, timr); + rtn = NULL; // We already got the old time... ++ rcu_read_unlock(); + goto retry; + } ++ rcu_read_unlock(); + + if (old_setting && !error && + copy_to_user(old_setting, &old_spec, sizeof (old_spec))) +@@ -944,10 +964,15 @@ + if (!timer) + return -EINVAL; + ++ rcu_read_lock(); + if (timer_delete_hook(timer) == TIMER_RETRY) { + unlock_timer(timer, flags); ++ timer_wait_for_callback(clockid_to_kclock(timer->it_clock), ++ timer); ++ rcu_read_unlock(); + goto retry_delete; + } ++ rcu_read_unlock(); + + spin_lock(¤t->sighand->siglock); + list_del(&timer->list); +@@ -973,8 +998,18 @@ + retry_delete: + spin_lock_irqsave(&timer->it_lock, flags); + ++ /* On RT we can race with a deletion */ ++ if (!timer->it_signal) { ++ unlock_timer(timer, flags); + return; -+ -+ if (ss->css_offline) -+ ss->css_offline(css); -+ -+ css->flags &= ~CSS_ONLINE; -+ RCU_INIT_POINTER(css->cgroup->subsys[ss->id], NULL); -+ -+ wake_up_all(&css->cgroup->offline_waitq); -+} -+ -+/** -+ * create_css - create a cgroup_subsys_state -+ * @cgrp: the cgroup new css will be associated with -+ * @ss: the subsys of new css -+ * @visible: whether to create control knobs for the new css or not -+ * -+ * Create a new css associated with @cgrp - @ss pair. On success, the new -+ * css is online and installed in @cgrp with all interface files created if -+ * @visible. Returns 0 on success, -errno on failure. -+ */ -+static int create_css(struct cgroup *cgrp, struct cgroup_subsys *ss, -+ bool visible) -+{ -+ struct cgroup *parent = cgroup_parent(cgrp); -+ struct cgroup_subsys_state *parent_css = cgroup_css(parent, ss); -+ struct cgroup_subsys_state *css; -+ int err; -+ -+ lockdep_assert_held(&cgroup_mutex); -+ -+ css = ss->css_alloc(parent_css); -+ if (IS_ERR(css)) -+ return PTR_ERR(css); -+ -+ init_and_link_css(css, ss, cgrp); -+ -+ err = percpu_ref_init(&css->refcnt, css_release, 0, GFP_KERNEL); -+ if (err) -+ goto err_free_css; -+ -+ err = cgroup_idr_alloc(&ss->css_idr, NULL, 2, 0, GFP_NOWAIT); -+ if (err < 0) -+ goto err_free_percpu_ref; -+ css->id = err; -+ -+ if (visible) { -+ err = cgroup_populate_dir(cgrp, 1 << ss->id); -+ if (err) -+ goto err_free_id; + } + -+ /* @css is ready to be brought online now, make it visible */ -+ list_add_tail_rcu(&css->sibling, &parent_css->children); -+ cgroup_idr_replace(&ss->css_idr, css, css->id); -+ -+ err = online_css(css); -+ if (err) -+ goto err_list_del; -+ -+ if (ss->broken_hierarchy && !ss->warned_broken_hierarchy && -+ cgroup_parent(parent)) { -+ pr_warn("%s (%d) created nested cgroup for controller \"%s\" which has incomplete hierarchy support. Nested cgroups may change behavior in the future.\n", -+ current->comm, current->pid, ss->name); -+ if (!strcmp(ss->name, "memory")) -+ pr_warn("\"memory\" requires setting use_hierarchy to 1 on the root\n"); -+ ss->warned_broken_hierarchy = true; + if (timer_delete_hook(timer) == TIMER_RETRY) { ++ rcu_read_lock(); + unlock_timer(timer, flags); ++ timer_wait_for_callback(clockid_to_kclock(timer->it_clock), ++ timer); ++ rcu_read_unlock(); + goto retry_delete; + } + list_del(&timer->list); +diff -Nur linux-4.1.10.orig/kernel/time/tick-common.c linux-4.1.10/kernel/time/tick-common.c +--- linux-4.1.10.orig/kernel/time/tick-common.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/time/tick-common.c 2015-10-12 22:33:32.316674111 +0200 +@@ -78,13 +78,15 @@ + static void tick_periodic(int cpu) + { + if (tick_do_timer_cpu == cpu) { +- write_seqlock(&jiffies_lock); ++ raw_spin_lock(&jiffies_lock); ++ write_seqcount_begin(&jiffies_seq); + + /* Keep track of the next tick event */ + tick_next_period = ktime_add(tick_next_period, tick_period); + + do_timer(1); +- write_sequnlock(&jiffies_lock); ++ write_seqcount_end(&jiffies_seq); ++ raw_spin_unlock(&jiffies_lock); + update_wall_time(); + } + +@@ -146,9 +148,9 @@ + ktime_t next; + + do { +- seq = read_seqbegin(&jiffies_lock); ++ seq = read_seqcount_begin(&jiffies_seq); + next = tick_next_period; +- } while (read_seqretry(&jiffies_lock, seq)); ++ } while (read_seqcount_retry(&jiffies_seq, seq)); + + clockevents_set_state(dev, CLOCK_EVT_STATE_ONESHOT); + +diff -Nur linux-4.1.10.orig/kernel/time/tick-sched.c linux-4.1.10/kernel/time/tick-sched.c +--- linux-4.1.10.orig/kernel/time/tick-sched.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/time/tick-sched.c 2015-10-12 22:33:32.316674111 +0200 +@@ -62,7 +62,8 @@ + return; + + /* Reevalute with jiffies_lock held */ +- write_seqlock(&jiffies_lock); ++ raw_spin_lock(&jiffies_lock); ++ write_seqcount_begin(&jiffies_seq); + + delta = ktime_sub(now, last_jiffies_update); + if (delta.tv64 >= tick_period.tv64) { +@@ -85,10 +86,12 @@ + /* Keep the tick_next_period variable up to date */ + tick_next_period = ktime_add(last_jiffies_update, tick_period); + } else { +- write_sequnlock(&jiffies_lock); ++ write_seqcount_end(&jiffies_seq); ++ raw_spin_unlock(&jiffies_lock); + return; + } +- write_sequnlock(&jiffies_lock); ++ write_seqcount_end(&jiffies_seq); ++ raw_spin_unlock(&jiffies_lock); + update_wall_time(); + } + +@@ -99,12 +102,14 @@ + { + ktime_t period; + +- write_seqlock(&jiffies_lock); ++ raw_spin_lock(&jiffies_lock); ++ write_seqcount_begin(&jiffies_seq); + /* Did we start the jiffies update yet ? */ + if (last_jiffies_update.tv64 == 0) + last_jiffies_update = tick_next_period; + period = last_jiffies_update; +- write_sequnlock(&jiffies_lock); ++ write_seqcount_end(&jiffies_seq); ++ raw_spin_unlock(&jiffies_lock); + return period; + } + +@@ -176,6 +181,11 @@ + return false; + } + ++ if (!arch_irq_work_has_interrupt()) { ++ trace_tick_stop(0, "missing irq work interrupt\n"); ++ return false; + } + -+ return 0; -+ -+err_list_del: -+ list_del_rcu(&css->sibling); -+ cgroup_clear_dir(css->cgroup, 1 << css->ss->id); -+err_free_id: -+ cgroup_idr_remove(&ss->css_idr, css->id); -+err_free_percpu_ref: -+ percpu_ref_exit(&css->refcnt); -+err_free_css: -+ call_rcu(&css->rcu_head, css_free_rcu_fn); -+ return err; + /* sched_clock_tick() needs us? */ + #ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK + /* +@@ -222,6 +232,7 @@ + + static DEFINE_PER_CPU(struct irq_work, nohz_full_kick_work) = { + .func = nohz_full_kick_work_func, ++ .flags = IRQ_WORK_HARD_IRQ, + }; + + /* +@@ -578,10 +589,10 @@ + + /* Read jiffies and the time when jiffies were updated last */ + do { +- seq = read_seqbegin(&jiffies_lock); ++ seq = read_seqcount_begin(&jiffies_seq); + last_update = last_jiffies_update; + last_jiffies = jiffies; +- } while (read_seqretry(&jiffies_lock, seq)); ++ } while (read_seqcount_retry(&jiffies_seq, seq)); + + if (rcu_needs_cpu(&rcu_delta_jiffies) || + arch_needs_cpu() || irq_work_needs_cpu()) { +@@ -759,14 +770,7 @@ + return false; + + if (unlikely(local_softirq_pending() && cpu_online(cpu))) { +- static int ratelimit; +- +- if (ratelimit < 10 && +- (local_softirq_pending() & SOFTIRQ_STOP_IDLE_MASK)) { +- pr_warn("NOHZ: local_softirq_pending %02x\n", +- (unsigned int) local_softirq_pending()); +- ratelimit++; +- } ++ softirq_check_pending_idle(); + return false; + } + +@@ -1154,6 +1158,7 @@ + * Emulate tick processing via per-CPU hrtimers: + */ + hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); ++ ts->sched_timer.irqsafe = 1; + ts->sched_timer.function = tick_sched_timer; + + /* Get the next period (per cpu) */ +diff -Nur linux-4.1.10.orig/kernel/time/timekeeping.c linux-4.1.10/kernel/time/timekeeping.c +--- linux-4.1.10.orig/kernel/time/timekeeping.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/time/timekeeping.c 2015-10-12 22:33:32.316674111 +0200 +@@ -2065,8 +2065,10 @@ + */ + void xtime_update(unsigned long ticks) + { +- write_seqlock(&jiffies_lock); ++ raw_spin_lock(&jiffies_lock); ++ write_seqcount_begin(&jiffies_seq); + do_timer(ticks); +- write_sequnlock(&jiffies_lock); ++ write_seqcount_end(&jiffies_seq); ++ raw_spin_unlock(&jiffies_lock); + update_wall_time(); + } +diff -Nur linux-4.1.10.orig/kernel/time/timekeeping.h linux-4.1.10/kernel/time/timekeeping.h +--- linux-4.1.10.orig/kernel/time/timekeeping.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/time/timekeeping.h 2015-10-12 22:33:32.316674111 +0200 +@@ -22,7 +22,8 @@ + extern void do_timer(unsigned long ticks); + extern void update_wall_time(void); + +-extern seqlock_t jiffies_lock; ++extern raw_spinlock_t jiffies_lock; ++extern seqcount_t jiffies_seq; + + #define CS_NAME_LEN 32 + +diff -Nur linux-4.1.10.orig/kernel/time/timer.c linux-4.1.10/kernel/time/timer.c +--- linux-4.1.10.orig/kernel/time/timer.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/time/timer.c 2015-10-12 22:33:32.316674111 +0200 +@@ -78,6 +78,9 @@ + struct tvec_base { + spinlock_t lock; + struct timer_list *running_timer; ++#ifdef CONFIG_PREEMPT_RT_FULL ++ wait_queue_head_t wait_for_running_timer; ++#endif + unsigned long timer_jiffies; + unsigned long next_timer; + unsigned long active_timers; +@@ -768,6 +771,36 @@ + } + } + ++#ifndef CONFIG_PREEMPT_RT_FULL ++static inline struct tvec_base *switch_timer_base(struct timer_list *timer, ++ struct tvec_base *old, ++ struct tvec_base *new) ++{ ++ /* See the comment in lock_timer_base() */ ++ timer_set_base(timer, NULL); ++ spin_unlock(&old->lock); ++ spin_lock(&new->lock); ++ timer_set_base(timer, new); ++ return new; +} -+ -+static int cgroup_mkdir(struct kernfs_node *parent_kn, const char *name, -+ umode_t mode) ++#else ++static inline struct tvec_base *switch_timer_base(struct timer_list *timer, ++ struct tvec_base *old, ++ struct tvec_base *new) +{ -+ struct cgroup *parent, *cgrp; -+ struct cgroup_root *root; -+ struct cgroup_subsys *ss; -+ struct kernfs_node *kn; -+ struct cftype *base_files; -+ int ssid, ret; -+ -+ /* Do not accept '\n' to prevent making /proc//cgroup unparsable. -+ */ -+ if (strchr(name, '\n')) -+ return -EINVAL; -+ -+ parent = cgroup_kn_lock_live(parent_kn); -+ if (!parent) -+ return -ENODEV; -+ root = parent->root; -+ -+ /* allocate the cgroup and its ID, 0 is reserved for the root */ -+ cgrp = kzalloc(sizeof(*cgrp), GFP_KERNEL); -+ if (!cgrp) { -+ ret = -ENOMEM; -+ goto out_unlock; -+ } -+ -+ ret = percpu_ref_init(&cgrp->self.refcnt, css_release, 0, GFP_KERNEL); -+ if (ret) -+ goto out_free_cgrp; -+ -+ /* -+ * Temporarily set the pointer to NULL, so idr_find() won't return -+ * a half-baked cgroup. -+ */ -+ cgrp->id = cgroup_idr_alloc(&root->cgroup_idr, NULL, 2, 0, GFP_NOWAIT); -+ if (cgrp->id < 0) { -+ ret = -ENOMEM; -+ goto out_cancel_ref; -+ } -+ -+ init_cgroup_housekeeping(cgrp); -+ -+ cgrp->self.parent = &parent->self; -+ cgrp->root = root; -+ -+ if (notify_on_release(parent)) -+ set_bit(CGRP_NOTIFY_ON_RELEASE, &cgrp->flags); -+ -+ if (test_bit(CGRP_CPUSET_CLONE_CHILDREN, &parent->flags)) -+ set_bit(CGRP_CPUSET_CLONE_CHILDREN, &cgrp->flags); -+ -+ /* create the directory */ -+ kn = kernfs_create_dir(parent->kn, name, mode, cgrp); -+ if (IS_ERR(kn)) { -+ ret = PTR_ERR(kn); -+ goto out_free_id; -+ } -+ cgrp->kn = kn; -+ -+ /* -+ * This extra ref will be put in cgroup_free_fn() and guarantees -+ * that @cgrp->kn is always accessible. -+ */ -+ kernfs_get(kn); -+ -+ cgrp->self.serial_nr = css_serial_nr_next++; -+ -+ /* allocation complete, commit to creation */ -+ list_add_tail_rcu(&cgrp->self.sibling, &cgroup_parent(cgrp)->self.children); -+ atomic_inc(&root->nr_cgrps); -+ cgroup_get(parent); -+ -+ /* -+ * @cgrp is now fully operational. If something fails after this -+ * point, it'll be released via the normal destruction path. -+ */ -+ cgroup_idr_replace(&root->cgroup_idr, cgrp, cgrp->id); -+ -+ ret = cgroup_kn_set_ugid(kn); -+ if (ret) -+ goto out_destroy; -+ -+ if (cgroup_on_dfl(cgrp)) -+ base_files = cgroup_dfl_base_files; -+ else -+ base_files = cgroup_legacy_base_files; -+ -+ ret = cgroup_addrm_files(cgrp, base_files, true); -+ if (ret) -+ goto out_destroy; -+ -+ /* let's create and online css's */ -+ for_each_subsys(ss, ssid) { -+ if (parent->child_subsys_mask & (1 << ssid)) { -+ ret = create_css(cgrp, ss, -+ parent->subtree_control & (1 << ssid)); -+ if (ret) -+ goto out_destroy; -+ } -+ } -+ + /* -+ * On the default hierarchy, a child doesn't automatically inherit -+ * subtree_control from the parent. Each is configured manually. ++ * We cannot do the above because we might be preempted and ++ * then the preempter would see NULL and loop forever. + */ -+ if (!cgroup_on_dfl(cgrp)) { -+ cgrp->subtree_control = parent->subtree_control; -+ cgroup_refresh_child_subsys_mask(cgrp); ++ if (spin_trylock(&new->lock)) { ++ timer_set_base(timer, new); ++ spin_unlock(&old->lock); ++ return new; + } -+ -+ kernfs_activate(kn); -+ -+ ret = 0; -+ goto out_unlock; -+ -+out_free_id: -+ cgroup_idr_remove(&root->cgroup_idr, cgrp->id); -+out_cancel_ref: -+ percpu_ref_exit(&cgrp->self.refcnt); -+out_free_cgrp: -+ kfree(cgrp); -+out_unlock: -+ cgroup_kn_unlock(parent_kn); -+ return ret; -+ -+out_destroy: -+ cgroup_destroy_locked(cgrp); -+ goto out_unlock; ++ return old; +} ++#endif + + static inline int + __mod_timer(struct timer_list *timer, unsigned long expires, + bool pending_only, int pinned) +@@ -798,14 +831,8 @@ + * handler yet has not finished. This also guarantees that + * the timer is serialized wrt itself. + */ +- if (likely(base->running_timer != timer)) { +- /* See the comment in lock_timer_base() */ +- timer_set_base(timer, NULL); +- spin_unlock(&base->lock); +- base = new_base; +- spin_lock(&base->lock); +- timer_set_base(timer, base); +- } ++ if (likely(base->running_timer != timer)) ++ base = switch_timer_base(timer, base, new_base); + } + + timer->expires = expires; +@@ -979,6 +1006,29 @@ + } + EXPORT_SYMBOL_GPL(add_timer_on); + ++#ifdef CONFIG_PREEMPT_RT_FULL +/* -+ * This is called when the refcnt of a css is confirmed to be killed. -+ * css_tryget_online() is now guaranteed to fail. Tell the subsystem to -+ * initate destruction and put the css ref from kill_css(). ++ * Wait for a running timer + */ -+static void css_killed_work_fn(struct work_struct *work) ++static void wait_for_running_timer(struct timer_list *timer) +{ -+ struct cgroup_subsys_state *css = -+ container_of(work, struct cgroup_subsys_state, destroy_work); -+ -+ mutex_lock(&cgroup_mutex); -+ offline_css(css); -+ mutex_unlock(&cgroup_mutex); ++ struct tvec_base *base = timer->base; + -+ css_put(css); ++ if (base->running_timer == timer) ++ wait_event(base->wait_for_running_timer, ++ base->running_timer != timer); +} + -+/* css kill confirmation processing requires process context, bounce */ -+static void css_killed_ref_fn(struct percpu_ref *ref) ++# define wakeup_timer_waiters(b) wake_up(&(b)->wait_for_running_timer) ++#else ++static inline void wait_for_running_timer(struct timer_list *timer) +{ -+ struct cgroup_subsys_state *css = -+ container_of(ref, struct cgroup_subsys_state, refcnt); -+ -+ INIT_WORK(&css->destroy_work, css_killed_work_fn); -+ queue_work(cgroup_destroy_wq, &css->destroy_work); ++ cpu_relax(); +} + -+/** -+ * kill_css - destroy a css -+ * @css: css to destroy -+ * -+ * This function initiates destruction of @css by removing cgroup interface -+ * files and putting its base reference. ->css_offline() will be invoked -+ * asynchronously once css_tryget_online() is guaranteed to fail and when -+ * the reference count reaches zero, @css will be released. -+ */ -+static void kill_css(struct cgroup_subsys_state *css) -+{ -+ lockdep_assert_held(&cgroup_mutex); -+ -+ /* -+ * This must happen before css is disassociated with its cgroup. -+ * See seq_css() for details. -+ */ -+ cgroup_clear_dir(css->cgroup, 1 << css->ss->id); -+ -+ /* -+ * Killing would put the base ref, but we need to keep it alive -+ * until after ->css_offline(). -+ */ -+ css_get(css); ++# define wakeup_timer_waiters(b) do { } while (0) ++#endif + + /** + * del_timer - deactive a timer. + * @timer: the timer to be deactivated +@@ -1036,7 +1086,7 @@ + } + EXPORT_SYMBOL(try_to_del_timer_sync); + +-#ifdef CONFIG_SMP ++#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT_FULL) + static DEFINE_PER_CPU(struct tvec_base, __tvec_bases); + + /** +@@ -1098,7 +1148,7 @@ + int ret = try_to_del_timer_sync(timer); + if (ret >= 0) + return ret; +- cpu_relax(); ++ wait_for_running_timer(timer); + } + } + EXPORT_SYMBOL(del_timer_sync); +@@ -1219,15 +1269,17 @@ + if (irqsafe) { + spin_unlock(&base->lock); + call_timer_fn(timer, fn, data); ++ base->running_timer = NULL; + spin_lock(&base->lock); + } else { + spin_unlock_irq(&base->lock); + call_timer_fn(timer, fn, data); ++ base->running_timer = NULL; + spin_lock_irq(&base->lock); + } + } + } +- base->running_timer = NULL; ++ wakeup_timer_waiters(base); + spin_unlock_irq(&base->lock); + } + +@@ -1367,6 +1419,14 @@ + if (cpu_is_offline(smp_processor_id())) + return expires; + ++#ifdef CONFIG_PREEMPT_RT_FULL + /* -+ * cgroup core guarantees that, by the time ->css_offline() is -+ * invoked, no new css reference will be given out via -+ * css_tryget_online(). We can't simply call percpu_ref_kill() and -+ * proceed to offlining css's because percpu_ref_kill() doesn't -+ * guarantee that the ref is seen as killed on all CPUs on return. -+ * -+ * Use percpu_ref_kill_and_confirm() to get notifications as each -+ * css is confirmed to be seen as killed on all CPUs. ++ * On PREEMPT_RT we cannot sleep here. As a result we can't take ++ * the base lock to check when the next timer is pending and so ++ * we assume the next jiffy. + */ -+ percpu_ref_kill_and_confirm(&css->refcnt, css_killed_ref_fn); -+} -+ -+/** -+ * cgroup_destroy_locked - the first stage of cgroup destruction -+ * @cgrp: cgroup to be destroyed -+ * -+ * css's make use of percpu refcnts whose killing latency shouldn't be -+ * exposed to userland and are RCU protected. Also, cgroup core needs to -+ * guarantee that css_tryget_online() won't succeed by the time -+ * ->css_offline() is invoked. To satisfy all the requirements, -+ * destruction is implemented in the following two steps. -+ * -+ * s1. Verify @cgrp can be destroyed and mark it dying. Remove all -+ * userland visible parts and start killing the percpu refcnts of -+ * css's. Set up so that the next stage will be kicked off once all -+ * the percpu refcnts are confirmed to be killed. -+ * -+ * s2. Invoke ->css_offline(), mark the cgroup dead and proceed with the -+ * rest of destruction. Once all cgroup references are gone, the -+ * cgroup is RCU-freed. -+ * -+ * This function implements s1. After this step, @cgrp is gone as far as -+ * the userland is concerned and a new cgroup with the same name may be -+ * created. As cgroup doesn't care about the names internally, this -+ * doesn't cause any problem. -+ */ -+static int cgroup_destroy_locked(struct cgroup *cgrp) -+ __releases(&cgroup_mutex) __acquires(&cgroup_mutex) -+{ -+ struct cgroup_subsys_state *css; -+ bool empty; -+ int ssid; ++ return now + 1; ++#endif + spin_lock(&base->lock); + if (base->active_timers) { + if (time_before_eq(base->next_timer, base->timer_jiffies)) +@@ -1392,13 +1452,13 @@ + + /* Note: this timer irq context must be accounted for as well. */ + account_process_tick(p, user_tick); ++ scheduler_tick(); + run_local_timers(); + rcu_check_callbacks(user_tick); +-#ifdef CONFIG_IRQ_WORK ++#if defined(CONFIG_IRQ_WORK) && !defined(CONFIG_PREEMPT_RT_FULL) + if (in_irq()) + irq_work_tick(); + #endif +- scheduler_tick(); + run_posix_cpu_timers(p); + } + +@@ -1411,6 +1471,10 @@ + + hrtimer_run_pending(); + ++#if defined(CONFIG_IRQ_WORK) && defined(CONFIG_PREEMPT_RT_FULL) ++ irq_work_tick(); ++#endif + -+ lockdep_assert_held(&cgroup_mutex); + if (time_after_eq(jiffies, base->timer_jiffies)) + __run_timers(base); + } +@@ -1566,7 +1630,7 @@ + + BUG_ON(cpu_online(cpu)); + old_base = per_cpu(tvec_bases, cpu); +- new_base = get_cpu_var(tvec_bases); ++ new_base = get_local_var(tvec_bases); + /* + * The caller is globally serialized and nobody else + * takes two locks at once, deadlock is not possible. +@@ -1590,7 +1654,7 @@ + + spin_unlock(&old_base->lock); + spin_unlock_irq(&new_base->lock); +- put_cpu_var(tvec_bases); ++ put_local_var(tvec_bases); + } + + static int timer_cpu_notify(struct notifier_block *self, +@@ -1625,6 +1689,9 @@ + base->cpu = cpu; + per_cpu(tvec_bases, cpu) = base; + spin_lock_init(&base->lock); ++#ifdef CONFIG_PREEMPT_RT_FULL ++ init_waitqueue_head(&base->wait_for_running_timer); ++#endif + + for (j = 0; j < TVN_SIZE; j++) { + INIT_LIST_HEAD(base->tv5.vec + j); +diff -Nur linux-4.1.10.orig/kernel/trace/Kconfig linux-4.1.10/kernel/trace/Kconfig +--- linux-4.1.10.orig/kernel/trace/Kconfig 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/trace/Kconfig 2015-10-12 22:33:32.316674111 +0200 +@@ -187,6 +187,24 @@ + enabled. This option and the preempt-off timing option can be + used together or separately.) + ++config INTERRUPT_OFF_HIST ++ bool "Interrupts-off Latency Histogram" ++ depends on IRQSOFF_TRACER ++ help ++ This option generates continuously updated histograms (one per cpu) ++ of the duration of time periods with interrupts disabled. The ++ histograms are disabled by default. To enable them, write a non-zero ++ number to + -+ /* -+ * css_set_rwsem synchronizes access to ->cset_links and prevents -+ * @cgrp from being removed while put_css_set() is in progress. -+ */ -+ down_read(&css_set_rwsem); -+ empty = list_empty(&cgrp->cset_links); -+ up_read(&css_set_rwsem); -+ if (!empty) -+ return -EBUSY; ++ /sys/kernel/debug/tracing/latency_hist/enable/preemptirqsoff + -+ /* -+ * Make sure there's no live children. We can't test emptiness of -+ * ->self.children as dead children linger on it while being -+ * drained; otherwise, "rmdir parent/child parent" may fail. -+ */ -+ if (css_has_online_children(&cgrp->self)) -+ return -EBUSY; ++ If PREEMPT_OFF_HIST is also selected, additional histograms (one ++ per cpu) are generated that accumulate the duration of time periods ++ when both interrupts and preemption are disabled. The histogram data ++ will be located in the debug file system at + -+ /* -+ * Mark @cgrp dead. This prevents further task migration and child -+ * creation by disabling cgroup_lock_live_group(). -+ */ -+ cgrp->self.flags &= ~CSS_ONLINE; ++ /sys/kernel/debug/tracing/latency_hist/irqsoff + -+ /* initiate massacre of all css's */ -+ for_each_css(css, ssid, cgrp) -+ kill_css(css); + config PREEMPT_TRACER + bool "Preemption-off Latency Tracer" + default n +@@ -211,6 +229,24 @@ + enabled. This option and the irqs-off timing option can be + used together or separately.) + ++config PREEMPT_OFF_HIST ++ bool "Preemption-off Latency Histogram" ++ depends on PREEMPT_TRACER ++ help ++ This option generates continuously updated histograms (one per cpu) ++ of the duration of time periods with preemption disabled. The ++ histograms are disabled by default. To enable them, write a non-zero ++ number to + -+ /* -+ * Remove @cgrp directory along with the base files. @cgrp has an -+ * extra ref on its kn. -+ */ -+ kernfs_remove(cgrp->kn); ++ /sys/kernel/debug/tracing/latency_hist/enable/preemptirqsoff + -+ check_for_release(cgroup_parent(cgrp)); ++ If INTERRUPT_OFF_HIST is also selected, additional histograms (one ++ per cpu) are generated that accumulate the duration of time periods ++ when both interrupts and preemption are disabled. The histogram data ++ will be located in the debug file system at + -+ /* put the base reference */ -+ percpu_ref_kill(&cgrp->self.refcnt); ++ /sys/kernel/debug/tracing/latency_hist/preemptoff + -+ return 0; -+}; + config SCHED_TRACER + bool "Scheduling Latency Tracer" + select GENERIC_TRACER +@@ -221,6 +257,74 @@ + This tracer tracks the latency of the highest priority task + to be scheduled in, starting from the point it has woken up. + ++config WAKEUP_LATENCY_HIST ++ bool "Scheduling Latency Histogram" ++ depends on SCHED_TRACER ++ help ++ This option generates continuously updated histograms (one per cpu) ++ of the scheduling latency of the highest priority task. ++ The histograms are disabled by default. To enable them, write a ++ non-zero number to + -+static int cgroup_rmdir(struct kernfs_node *kn) -+{ -+ struct cgroup *cgrp; -+ int ret = 0; ++ /sys/kernel/debug/tracing/latency_hist/enable/wakeup + -+ cgrp = cgroup_kn_lock_live(kn); -+ if (!cgrp) -+ return 0; ++ Two different algorithms are used, one to determine the latency of ++ processes that exclusively use the highest priority of the system and ++ another one to determine the latency of processes that share the ++ highest system priority with other processes. The former is used to ++ improve hardware and system software, the latter to optimize the ++ priority design of a given system. The histogram data will be ++ located in the debug file system at + -+ ret = cgroup_destroy_locked(cgrp); ++ /sys/kernel/debug/tracing/latency_hist/wakeup + -+ cgroup_kn_unlock(kn); -+ return ret; -+} ++ and + -+static struct kernfs_syscall_ops cgroup_kf_syscall_ops = { -+ .remount_fs = cgroup_remount, -+ .show_options = cgroup_show_options, -+ .mkdir = cgroup_mkdir, -+ .rmdir = cgroup_rmdir, -+ .rename = cgroup_rename, -+}; ++ /sys/kernel/debug/tracing/latency_hist/wakeup/sharedprio + -+static void __init cgroup_init_subsys(struct cgroup_subsys *ss, bool early) -+{ -+ struct cgroup_subsys_state *css; ++ If both Scheduling Latency Histogram and Missed Timer Offsets ++ Histogram are selected, additional histogram data will be collected ++ that contain, in addition to the wakeup latency, the timer latency, in ++ case the wakeup was triggered by an expired timer. These histograms ++ are available in the + -+ printk(KERN_INFO "Initializing cgroup subsys %s\n", ss->name); ++ /sys/kernel/debug/tracing/latency_hist/timerandwakeup + -+ mutex_lock(&cgroup_mutex); ++ directory. They reflect the apparent interrupt and scheduling latency ++ and are best suitable to determine the worst-case latency of a given ++ system. To enable these histograms, write a non-zero number to + -+ idr_init(&ss->css_idr); -+ INIT_LIST_HEAD(&ss->cfts); ++ /sys/kernel/debug/tracing/latency_hist/enable/timerandwakeup + -+ /* Create the root cgroup state for this subsystem */ -+ ss->root = &cgrp_dfl_root; -+ css = ss->css_alloc(cgroup_css(&cgrp_dfl_root.cgrp, ss)); -+ /* We don't handle early failures gracefully */ -+ BUG_ON(IS_ERR(css)); -+ init_and_link_css(css, ss, &cgrp_dfl_root.cgrp); ++config MISSED_TIMER_OFFSETS_HIST ++ depends on HIGH_RES_TIMERS ++ select GENERIC_TRACER ++ bool "Missed Timer Offsets Histogram" ++ help ++ Generate a histogram of missed timer offsets in microseconds. The ++ histograms are disabled by default. To enable them, write a non-zero ++ number to + -+ /* -+ * Root csses are never destroyed and we can't initialize -+ * percpu_ref during early init. Disable refcnting. -+ */ -+ css->flags |= CSS_NO_REF; ++ /sys/kernel/debug/tracing/latency_hist/enable/missed_timer_offsets + -+ if (early) { -+ /* allocation can't be done safely during early init */ -+ css->id = 1; -+ } else { -+ css->id = cgroup_idr_alloc(&ss->css_idr, css, 1, 2, GFP_KERNEL); -+ BUG_ON(css->id < 0); -+ } ++ The histogram data will be located in the debug file system at + -+ /* Update the init_css_set to contain a subsys -+ * pointer to this state - since the subsystem is -+ * newly registered, all tasks and hence the -+ * init_css_set is in the subsystem's root cgroup. */ -+ init_css_set.subsys[ss->id] = css; ++ /sys/kernel/debug/tracing/latency_hist/missed_timer_offsets + -+ need_forkexit_callback |= ss->fork || ss->exit; ++ If both Scheduling Latency Histogram and Missed Timer Offsets ++ Histogram are selected, additional histogram data will be collected ++ that contain, in addition to the wakeup latency, the timer latency, in ++ case the wakeup was triggered by an expired timer. These histograms ++ are available in the + -+ /* At system boot, before all subsystems have been -+ * registered, no tasks have been forked, so we don't -+ * need to invoke fork callbacks here. */ -+ BUG_ON(!list_empty(&init_task.tasks)); ++ /sys/kernel/debug/tracing/latency_hist/timerandwakeup + -+ BUG_ON(online_css(css)); ++ directory. They reflect the apparent interrupt and scheduling latency ++ and are best suitable to determine the worst-case latency of a given ++ system. To enable these histograms, write a non-zero number to + -+ mutex_unlock(&cgroup_mutex); -+} ++ /sys/kernel/debug/tracing/latency_hist/enable/timerandwakeup + -+/** -+ * cgroup_init_early - cgroup initialization at system boot + config ENABLE_DEFAULT_TRACERS + bool "Trace process context switches and events" + depends on !GENERIC_TRACER +diff -Nur linux-4.1.10.orig/kernel/trace/latency_hist.c linux-4.1.10/kernel/trace/latency_hist.c +--- linux-4.1.10.orig/kernel/trace/latency_hist.c 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/kernel/trace/latency_hist.c 2015-10-12 22:33:32.316674111 +0200 +@@ -0,0 +1,1178 @@ ++/* ++ * kernel/trace/latency_hist.c + * -+ * Initialize cgroups at system boot, and initialize any -+ * subsystems that request early init. -+ */ -+int __init cgroup_init_early(void) -+{ -+ static struct cgroup_sb_opts __initdata opts; -+ struct cgroup_subsys *ss; -+ int i; -+ -+ init_cgroup_root(&cgrp_dfl_root, &opts); -+ cgrp_dfl_root.cgrp.self.flags |= CSS_NO_REF; -+ -+ RCU_INIT_POINTER(init_task.cgroups, &init_css_set); -+ -+ for_each_subsys(ss, i) { -+ WARN(!ss->css_alloc || !ss->css_free || ss->name || ss->id, -+ "invalid cgroup_subsys %d:%s css_alloc=%p css_free=%p name:id=%d:%s\n", -+ i, cgroup_subsys_name[i], ss->css_alloc, ss->css_free, -+ ss->id, ss->name); -+ WARN(strlen(cgroup_subsys_name[i]) > MAX_CGROUP_TYPE_NAMELEN, -+ "cgroup_subsys_name %s too long\n", cgroup_subsys_name[i]); -+ -+ ss->id = i; -+ ss->name = cgroup_subsys_name[i]; -+ -+ if (ss->early_init) -+ cgroup_init_subsys(ss, true); -+ } -+ return 0; -+} -+ -+/** -+ * cgroup_init - cgroup initialization ++ * Add support for histograms of preemption-off latency and ++ * interrupt-off latency and wakeup latency, it depends on ++ * Real-Time Preemption Support. ++ * ++ * Copyright (C) 2005 MontaVista Software, Inc. ++ * Yi Yang ++ * ++ * Converted to work with the new latency tracer. ++ * Copyright (C) 2008 Red Hat, Inc. ++ * Steven Rostedt + * -+ * Register cgroup filesystem and /proc file, and initialize -+ * any subsystems that didn't request early init. + */ -+int __init cgroup_init(void) -+{ -+ struct cgroup_subsys *ss; -+ unsigned long key; -+ int ssid, err; -+ -+ BUG_ON(cgroup_init_cftypes(NULL, cgroup_dfl_base_files)); -+ BUG_ON(cgroup_init_cftypes(NULL, cgroup_legacy_base_files)); -+ -+ mutex_lock(&cgroup_mutex); -+ -+ /* Add init_css_set to the hash table */ -+ key = css_set_hash(init_css_set.subsys); -+ hash_add(css_set_table, &init_css_set.hlist, key); -+ -+ BUG_ON(cgroup_setup_root(&cgrp_dfl_root, 0)); ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include + -+ mutex_unlock(&cgroup_mutex); ++#include "trace.h" ++#include + -+ for_each_subsys(ss, ssid) { -+ if (ss->early_init) { -+ struct cgroup_subsys_state *css = -+ init_css_set.subsys[ss->id]; ++#define NSECS_PER_USECS 1000L + -+ css->id = cgroup_idr_alloc(&ss->css_idr, css, 1, 2, -+ GFP_KERNEL); -+ BUG_ON(css->id < 0); -+ } else { -+ cgroup_init_subsys(ss, false); -+ } ++#define CREATE_TRACE_POINTS ++#include + -+ list_add_tail(&init_css_set.e_cset_node[ssid], -+ &cgrp_dfl_root.cgrp.e_csets[ssid]); ++enum { ++ IRQSOFF_LATENCY = 0, ++ PREEMPTOFF_LATENCY, ++ PREEMPTIRQSOFF_LATENCY, ++ WAKEUP_LATENCY, ++ WAKEUP_LATENCY_SHAREDPRIO, ++ MISSED_TIMER_OFFSETS, ++ TIMERANDWAKEUP_LATENCY, ++ MAX_LATENCY_TYPE, ++}; + -+ /* -+ * Setting dfl_root subsys_mask needs to consider the -+ * disabled flag and cftype registration needs kmalloc, -+ * both of which aren't available during early_init. -+ */ -+ if (ss->disabled) -+ continue; ++#define MAX_ENTRY_NUM 10240 + -+ cgrp_dfl_root.subsys_mask |= 1 << ss->id; ++struct hist_data { ++ atomic_t hist_mode; /* 0 log, 1 don't log */ ++ long offset; /* set it to MAX_ENTRY_NUM/2 for a bipolar scale */ ++ long min_lat; ++ long max_lat; ++ unsigned long long below_hist_bound_samples; ++ unsigned long long above_hist_bound_samples; ++ long long accumulate_lat; ++ unsigned long long total_samples; ++ unsigned long long hist_array[MAX_ENTRY_NUM]; ++}; + -+ if (cgroup_legacy_files_on_dfl && !ss->dfl_cftypes) -+ ss->dfl_cftypes = ss->legacy_cftypes; ++struct enable_data { ++ int latency_type; ++ int enabled; ++}; + -+ if (!ss->dfl_cftypes) -+ cgrp_dfl_root_inhibit_ss_mask |= 1 << ss->id; ++static char *latency_hist_dir_root = "latency_hist"; + -+ if (ss->dfl_cftypes == ss->legacy_cftypes) { -+ WARN_ON(cgroup_add_cftypes(ss, ss->dfl_cftypes)); -+ } else { -+ WARN_ON(cgroup_add_dfl_cftypes(ss, ss->dfl_cftypes)); -+ WARN_ON(cgroup_add_legacy_cftypes(ss, ss->legacy_cftypes)); -+ } ++#ifdef CONFIG_INTERRUPT_OFF_HIST ++static DEFINE_PER_CPU(struct hist_data, irqsoff_hist); ++static char *irqsoff_hist_dir = "irqsoff"; ++static DEFINE_PER_CPU(cycles_t, hist_irqsoff_start); ++static DEFINE_PER_CPU(int, hist_irqsoff_counting); ++#endif + -+ if (ss->bind) -+ ss->bind(init_css_set.subsys[ssid]); -+ } ++#ifdef CONFIG_PREEMPT_OFF_HIST ++static DEFINE_PER_CPU(struct hist_data, preemptoff_hist); ++static char *preemptoff_hist_dir = "preemptoff"; ++static DEFINE_PER_CPU(cycles_t, hist_preemptoff_start); ++static DEFINE_PER_CPU(int, hist_preemptoff_counting); ++#endif + -+ err = sysfs_create_mount_point(fs_kobj, "cgroup"); -+ if (err) -+ return err; ++#if defined(CONFIG_PREEMPT_OFF_HIST) && defined(CONFIG_INTERRUPT_OFF_HIST) ++static DEFINE_PER_CPU(struct hist_data, preemptirqsoff_hist); ++static char *preemptirqsoff_hist_dir = "preemptirqsoff"; ++static DEFINE_PER_CPU(cycles_t, hist_preemptirqsoff_start); ++static DEFINE_PER_CPU(int, hist_preemptirqsoff_counting); ++#endif + -+ err = register_filesystem(&cgroup_fs_type); -+ if (err < 0) { -+ sysfs_remove_mount_point(fs_kobj, "cgroup"); -+ return err; -+ } ++#if defined(CONFIG_PREEMPT_OFF_HIST) || defined(CONFIG_INTERRUPT_OFF_HIST) ++static notrace void probe_preemptirqsoff_hist(void *v, int reason, int start); ++static struct enable_data preemptirqsoff_enabled_data = { ++ .latency_type = PREEMPTIRQSOFF_LATENCY, ++ .enabled = 0, ++}; ++#endif + -+ proc_create("cgroups", 0, NULL, &proc_cgroupstats_operations); -+ return 0; -+} ++#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \ ++ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) ++struct maxlatproc_data { ++ char comm[FIELD_SIZEOF(struct task_struct, comm)]; ++ char current_comm[FIELD_SIZEOF(struct task_struct, comm)]; ++ int pid; ++ int current_pid; ++ int prio; ++ int current_prio; ++ long latency; ++ long timeroffset; ++ cycle_t timestamp; ++}; ++#endif + -+static int __init cgroup_wq_init(void) -+{ -+ /* -+ * There isn't much point in executing destruction path in -+ * parallel. Good chunk is serialized with cgroup_mutex anyway. -+ * Use 1 for @max_active. -+ * -+ * We would prefer to do this in cgroup_init() above, but that -+ * is called before init_workqueues(): so leave this until after. -+ */ -+ cgroup_destroy_wq = alloc_workqueue("cgroup_destroy", 0, 1); -+ BUG_ON(!cgroup_destroy_wq); ++#ifdef CONFIG_WAKEUP_LATENCY_HIST ++static DEFINE_PER_CPU(struct hist_data, wakeup_latency_hist); ++static DEFINE_PER_CPU(struct hist_data, wakeup_latency_hist_sharedprio); ++static char *wakeup_latency_hist_dir = "wakeup"; ++static char *wakeup_latency_hist_dir_sharedprio = "sharedprio"; ++static notrace void probe_wakeup_latency_hist_start(void *v, ++ struct task_struct *p, int success); ++static notrace void probe_wakeup_latency_hist_stop(void *v, ++ struct task_struct *prev, struct task_struct *next); ++static notrace void probe_sched_migrate_task(void *, ++ struct task_struct *task, int cpu); ++static struct enable_data wakeup_latency_enabled_data = { ++ .latency_type = WAKEUP_LATENCY, ++ .enabled = 0, ++}; ++static DEFINE_PER_CPU(struct maxlatproc_data, wakeup_maxlatproc); ++static DEFINE_PER_CPU(struct maxlatproc_data, wakeup_maxlatproc_sharedprio); ++static DEFINE_PER_CPU(struct task_struct *, wakeup_task); ++static DEFINE_PER_CPU(int, wakeup_sharedprio); ++static unsigned long wakeup_pid; ++#endif + -+ /* -+ * Used to destroy pidlists and separate to serve as flush domain. -+ * Cap @max_active to 1 too. -+ */ -+ cgroup_pidlist_destroy_wq = alloc_workqueue("cgroup_pidlist_destroy", -+ 0, 1); -+ BUG_ON(!cgroup_pidlist_destroy_wq); ++#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST ++static DEFINE_PER_CPU(struct hist_data, missed_timer_offsets); ++static char *missed_timer_offsets_dir = "missed_timer_offsets"; ++static notrace void probe_hrtimer_interrupt(void *v, int cpu, ++ long long offset, struct task_struct *curr, struct task_struct *task); ++static struct enable_data missed_timer_offsets_enabled_data = { ++ .latency_type = MISSED_TIMER_OFFSETS, ++ .enabled = 0, ++}; ++static DEFINE_PER_CPU(struct maxlatproc_data, missed_timer_offsets_maxlatproc); ++static unsigned long missed_timer_offsets_pid; ++#endif + -+ return 0; -+} -+core_initcall(cgroup_wq_init); ++#if defined(CONFIG_WAKEUP_LATENCY_HIST) && \ ++ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) ++static DEFINE_PER_CPU(struct hist_data, timerandwakeup_latency_hist); ++static char *timerandwakeup_latency_hist_dir = "timerandwakeup"; ++static struct enable_data timerandwakeup_enabled_data = { ++ .latency_type = TIMERANDWAKEUP_LATENCY, ++ .enabled = 0, ++}; ++static DEFINE_PER_CPU(struct maxlatproc_data, timerandwakeup_maxlatproc); ++#endif + -+/* -+ * proc_cgroup_show() -+ * - Print task's cgroup paths into seq_file, one line for each hierarchy -+ * - Used for /proc//cgroup. -+ */ -+int proc_cgroup_show(struct seq_file *m, struct pid_namespace *ns, -+ struct pid *pid, struct task_struct *tsk) ++void notrace latency_hist(int latency_type, int cpu, long latency, ++ long timeroffset, cycle_t stop, ++ struct task_struct *p) +{ -+ char *buf, *path; -+ int retval; -+ struct cgroup_root *root; -+ -+ retval = -ENOMEM; -+ buf = kmalloc(PATH_MAX, GFP_KERNEL); -+ if (!buf) -+ goto out; -+ -+ mutex_lock(&cgroup_mutex); -+ down_read(&css_set_rwsem); ++ struct hist_data *my_hist; ++#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \ ++ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) ++ struct maxlatproc_data *mp = NULL; ++#endif + -+ for_each_root(root) { -+ struct cgroup_subsys *ss; -+ struct cgroup *cgrp; -+ int ssid, count = 0; ++ if (!cpu_possible(cpu) || latency_type < 0 || ++ latency_type >= MAX_LATENCY_TYPE) ++ return; + -+ if (root == &cgrp_dfl_root && !cgrp_dfl_root_visible) -+ continue; ++ switch (latency_type) { ++#ifdef CONFIG_INTERRUPT_OFF_HIST ++ case IRQSOFF_LATENCY: ++ my_hist = &per_cpu(irqsoff_hist, cpu); ++ break; ++#endif ++#ifdef CONFIG_PREEMPT_OFF_HIST ++ case PREEMPTOFF_LATENCY: ++ my_hist = &per_cpu(preemptoff_hist, cpu); ++ break; ++#endif ++#if defined(CONFIG_PREEMPT_OFF_HIST) && defined(CONFIG_INTERRUPT_OFF_HIST) ++ case PREEMPTIRQSOFF_LATENCY: ++ my_hist = &per_cpu(preemptirqsoff_hist, cpu); ++ break; ++#endif ++#ifdef CONFIG_WAKEUP_LATENCY_HIST ++ case WAKEUP_LATENCY: ++ my_hist = &per_cpu(wakeup_latency_hist, cpu); ++ mp = &per_cpu(wakeup_maxlatproc, cpu); ++ break; ++ case WAKEUP_LATENCY_SHAREDPRIO: ++ my_hist = &per_cpu(wakeup_latency_hist_sharedprio, cpu); ++ mp = &per_cpu(wakeup_maxlatproc_sharedprio, cpu); ++ break; ++#endif ++#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST ++ case MISSED_TIMER_OFFSETS: ++ my_hist = &per_cpu(missed_timer_offsets, cpu); ++ mp = &per_cpu(missed_timer_offsets_maxlatproc, cpu); ++ break; ++#endif ++#if defined(CONFIG_WAKEUP_LATENCY_HIST) && \ ++ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) ++ case TIMERANDWAKEUP_LATENCY: ++ my_hist = &per_cpu(timerandwakeup_latency_hist, cpu); ++ mp = &per_cpu(timerandwakeup_maxlatproc, cpu); ++ break; ++#endif + -+ seq_printf(m, "%d:", root->hierarchy_id); -+ for_each_subsys(ss, ssid) -+ if (root->subsys_mask & (1 << ssid)) -+ seq_printf(m, "%s%s", count++ ? "," : "", ss->name); -+ if (strlen(root->name)) -+ seq_printf(m, "%sname=%s", count ? "," : "", -+ root->name); -+ seq_putc(m, ':'); -+ cgrp = task_cgroup_from_root(tsk, root); -+ path = cgroup_path(cgrp, buf, PATH_MAX); -+ if (!path) { -+ retval = -ENAMETOOLONG; -+ goto out_unlock; -+ } -+ seq_puts(m, path); -+ seq_putc(m, '\n'); ++ default: ++ return; + } + -+ retval = 0; -+out_unlock: -+ up_read(&css_set_rwsem); -+ mutex_unlock(&cgroup_mutex); -+ kfree(buf); -+out: -+ return retval; -+} ++ latency += my_hist->offset; + -+/* Display information about each subsystem and each hierarchy */ -+static int proc_cgroupstats_show(struct seq_file *m, void *v) -+{ -+ struct cgroup_subsys *ss; -+ int i; ++ if (atomic_read(&my_hist->hist_mode) == 0) ++ return; + -+ seq_puts(m, "#subsys_name\thierarchy\tnum_cgroups\tenabled\n"); -+ /* -+ * ideally we don't want subsystems moving around while we do this. -+ * cgroup_mutex is also necessary to guarantee an atomic snapshot of -+ * subsys/hierarchy state. -+ */ -+ mutex_lock(&cgroup_mutex); ++ if (latency < 0 || latency >= MAX_ENTRY_NUM) { ++ if (latency < 0) ++ my_hist->below_hist_bound_samples++; ++ else ++ my_hist->above_hist_bound_samples++; ++ } else ++ my_hist->hist_array[latency]++; + -+ for_each_subsys(ss, i) -+ seq_printf(m, "%s\t%d\t%d\t%d\n", -+ ss->name, ss->root->hierarchy_id, -+ atomic_read(&ss->root->nr_cgrps), !ss->disabled); -+ -+ mutex_unlock(&cgroup_mutex); -+ return 0; ++ if (unlikely(latency > my_hist->max_lat || ++ my_hist->min_lat == LONG_MAX)) { ++#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \ ++ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) ++ if (latency_type == WAKEUP_LATENCY || ++ latency_type == WAKEUP_LATENCY_SHAREDPRIO || ++ latency_type == MISSED_TIMER_OFFSETS || ++ latency_type == TIMERANDWAKEUP_LATENCY) { ++ strncpy(mp->comm, p->comm, sizeof(mp->comm)); ++ strncpy(mp->current_comm, current->comm, ++ sizeof(mp->current_comm)); ++ mp->pid = task_pid_nr(p); ++ mp->current_pid = task_pid_nr(current); ++ mp->prio = p->prio; ++ mp->current_prio = current->prio; ++ mp->latency = latency; ++ mp->timeroffset = timeroffset; ++ mp->timestamp = stop; ++ } ++#endif ++ my_hist->max_lat = latency; ++ } ++ if (unlikely(latency < my_hist->min_lat)) ++ my_hist->min_lat = latency; ++ my_hist->total_samples++; ++ my_hist->accumulate_lat += latency; +} + -+static int cgroupstats_open(struct inode *inode, struct file *file) ++static void *l_start(struct seq_file *m, loff_t *pos) +{ -+ return single_open(file, proc_cgroupstats_show, NULL); -+} -+ -+static const struct file_operations proc_cgroupstats_operations = { -+ .open = cgroupstats_open, -+ .read = seq_read, -+ .llseek = seq_lseek, -+ .release = single_release, -+}; ++ loff_t *index_ptr = NULL; ++ loff_t index = *pos; ++ struct hist_data *my_hist = m->private; + -+/** -+ * cgroup_fork - initialize cgroup related fields during copy_process() -+ * @child: pointer to task_struct of forking parent process. -+ * -+ * A task is associated with the init_css_set until cgroup_post_fork() -+ * attaches it to the parent's css_set. Empty cg_list indicates that -+ * @child isn't holding reference to its css_set. -+ */ -+void cgroup_fork(struct task_struct *child) -+{ -+ RCU_INIT_POINTER(child->cgroups, &init_css_set); -+ INIT_LIST_HEAD(&child->cg_list); -+} ++ if (index == 0) { ++ char minstr[32], avgstr[32], maxstr[32]; + -+/** -+ * cgroup_post_fork - called on a new task after adding it to the task list -+ * @child: the task in question -+ * -+ * Adds the task to the list running through its css_set if necessary and -+ * call the subsystem fork() callbacks. Has to be after the task is -+ * visible on the task list in case we race with the first call to -+ * cgroup_task_iter_start() - to guarantee that the new task ends up on its -+ * list. -+ */ -+void cgroup_post_fork(struct task_struct *child) -+{ -+ struct cgroup_subsys *ss; -+ int i; ++ atomic_dec(&my_hist->hist_mode); + -+ /* -+ * This may race against cgroup_enable_task_cg_lists(). As that -+ * function sets use_task_css_set_links before grabbing -+ * tasklist_lock and we just went through tasklist_lock to add -+ * @child, it's guaranteed that either we see the set -+ * use_task_css_set_links or cgroup_enable_task_cg_lists() sees -+ * @child during its iteration. -+ * -+ * If we won the race, @child is associated with %current's -+ * css_set. Grabbing css_set_rwsem guarantees both that the -+ * association is stable, and, on completion of the parent's -+ * migration, @child is visible in the source of migration or -+ * already in the destination cgroup. This guarantee is necessary -+ * when implementing operations which need to migrate all tasks of -+ * a cgroup to another. -+ * -+ * Note that if we lose to cgroup_enable_task_cg_lists(), @child -+ * will remain in init_css_set. This is safe because all tasks are -+ * in the init_css_set before cg_links is enabled and there's no -+ * operation which transfers all tasks out of init_css_set. -+ */ -+ if (use_task_css_set_links) { -+ struct css_set *cset; -+ -+ down_write(&css_set_rwsem); -+ cset = task_css_set(current); -+ if (list_empty(&child->cg_list)) { -+ rcu_assign_pointer(child->cgroups, cset); -+ list_add(&child->cg_list, &cset->tasks); -+ get_css_set(cset); ++ if (likely(my_hist->total_samples)) { ++ long avg = (long) div64_s64(my_hist->accumulate_lat, ++ my_hist->total_samples); ++ snprintf(minstr, sizeof(minstr), "%ld", ++ my_hist->min_lat - my_hist->offset); ++ snprintf(avgstr, sizeof(avgstr), "%ld", ++ avg - my_hist->offset); ++ snprintf(maxstr, sizeof(maxstr), "%ld", ++ my_hist->max_lat - my_hist->offset); ++ } else { ++ strcpy(minstr, ""); ++ strcpy(avgstr, minstr); ++ strcpy(maxstr, minstr); + } -+ up_write(&css_set_rwsem); -+ } + -+ /* -+ * Call ss->fork(). This must happen after @child is linked on -+ * css_set; otherwise, @child might change state between ->fork() -+ * and addition to css_set. -+ */ -+ if (need_forkexit_callback) { -+ for_each_subsys(ss, i) -+ if (ss->fork) -+ ss->fork(child); ++ seq_printf(m, "#Minimum latency: %s microseconds\n" ++ "#Average latency: %s microseconds\n" ++ "#Maximum latency: %s microseconds\n" ++ "#Total samples: %llu\n" ++ "#There are %llu samples lower than %ld" ++ " microseconds.\n" ++ "#There are %llu samples greater or equal" ++ " than %ld microseconds.\n" ++ "#usecs\t%16s\n", ++ minstr, avgstr, maxstr, ++ my_hist->total_samples, ++ my_hist->below_hist_bound_samples, ++ -my_hist->offset, ++ my_hist->above_hist_bound_samples, ++ MAX_ENTRY_NUM - my_hist->offset, ++ "samples"); + } -+} -+ -+/** -+ * cgroup_exit - detach cgroup from exiting task -+ * @tsk: pointer to task_struct of exiting process -+ * -+ * Description: Detach cgroup from @tsk and release it. -+ * -+ * Note that cgroups marked notify_on_release force every task in -+ * them to take the global cgroup_mutex mutex when exiting. -+ * This could impact scaling on very large systems. Be reluctant to -+ * use notify_on_release cgroups where very high task exit scaling -+ * is required on large systems. -+ * -+ * We set the exiting tasks cgroup to the root cgroup (top_cgroup). We -+ * call cgroup_exit() while the task is still competent to handle -+ * notify_on_release(), then leave the task attached to the root cgroup in -+ * each hierarchy for the remainder of its exit. No need to bother with -+ * init_css_set refcnting. init_css_set never goes away and we can't race -+ * with migration path - PF_EXITING is visible to migration path. -+ */ -+void cgroup_exit(struct task_struct *tsk) -+{ -+ struct cgroup_subsys *ss; -+ struct css_set *cset; -+ bool put_cset = false; -+ int i; -+ -+ /* -+ * Unlink from @tsk from its css_set. As migration path can't race -+ * with us, we can check cg_list without grabbing css_set_rwsem. -+ */ -+ if (!list_empty(&tsk->cg_list)) { -+ down_write(&css_set_rwsem); -+ list_del_init(&tsk->cg_list); -+ up_write(&css_set_rwsem); -+ put_cset = true; ++ if (index < MAX_ENTRY_NUM) { ++ index_ptr = kmalloc(sizeof(loff_t), GFP_KERNEL); ++ if (index_ptr) ++ *index_ptr = index; + } + -+ /* Reassign the task to the init_css_set. */ -+ cset = task_css_set(tsk); -+ RCU_INIT_POINTER(tsk->cgroups, &init_css_set); ++ return index_ptr; ++} + -+ if (need_forkexit_callback) { -+ /* see cgroup_post_fork() for details */ -+ for_each_subsys(ss, i) { -+ if (ss->exit) { -+ struct cgroup_subsys_state *old_css = cset->subsys[i]; -+ struct cgroup_subsys_state *css = task_css(tsk, i); ++static void *l_next(struct seq_file *m, void *p, loff_t *pos) ++{ ++ loff_t *index_ptr = p; ++ struct hist_data *my_hist = m->private; + -+ ss->exit(css, old_css, tsk); -+ } -+ } ++ if (++*pos >= MAX_ENTRY_NUM) { ++ atomic_inc(&my_hist->hist_mode); ++ return NULL; + } ++ *index_ptr = *pos; ++ return index_ptr; ++} + -+ if (put_cset) -+ put_css_set(cset); ++static void l_stop(struct seq_file *m, void *p) ++{ ++ kfree(p); +} + -+static void check_for_release(struct cgroup *cgrp) ++static int l_show(struct seq_file *m, void *p) +{ -+ if (notify_on_release(cgrp) && !cgroup_has_tasks(cgrp) && -+ !css_has_online_children(&cgrp->self) && !cgroup_is_dead(cgrp)) -+ schedule_work(&cgrp->release_agent_work); ++ int index = *(loff_t *) p; ++ struct hist_data *my_hist = m->private; ++ ++ seq_printf(m, "%6ld\t%16llu\n", index - my_hist->offset, ++ my_hist->hist_array[index]); ++ return 0; +} + -+/* -+ * Notify userspace when a cgroup is released, by running the -+ * configured release agent with the name of the cgroup (path -+ * relative to the root of cgroup file system) as the argument. -+ * -+ * Most likely, this user command will try to rmdir this cgroup. -+ * -+ * This races with the possibility that some other task will be -+ * attached to this cgroup before it is removed, or that some other -+ * user task will 'mkdir' a child cgroup of this cgroup. That's ok. -+ * The presumed 'rmdir' will fail quietly if this cgroup is no longer -+ * unused, and this cgroup will be reprieved from its death sentence, -+ * to continue to serve a useful existence. Next time it's released, -+ * we will get notified again, if it still has 'notify_on_release' set. -+ * -+ * The final arg to call_usermodehelper() is UMH_WAIT_EXEC, which -+ * means only wait until the task is successfully execve()'d. The -+ * separate release agent task is forked by call_usermodehelper(), -+ * then control in this thread returns here, without waiting for the -+ * release agent task. We don't bother to wait because the caller of -+ * this routine has no use for the exit status of the release agent -+ * task, so no sense holding our caller up for that. -+ */ -+static void cgroup_release_agent(struct work_struct *work) ++static const struct seq_operations latency_hist_seq_op = { ++ .start = l_start, ++ .next = l_next, ++ .stop = l_stop, ++ .show = l_show ++}; ++ ++static int latency_hist_open(struct inode *inode, struct file *file) +{ -+ struct cgroup *cgrp = -+ container_of(work, struct cgroup, release_agent_work); -+ char *pathbuf = NULL, *agentbuf = NULL, *path; -+ char *argv[3], *envp[3]; ++ int ret; + -+ mutex_lock(&cgroup_mutex); ++ ret = seq_open(file, &latency_hist_seq_op); ++ if (!ret) { ++ struct seq_file *seq = file->private_data; ++ seq->private = inode->i_private; ++ } ++ return ret; ++} + -+ pathbuf = kmalloc(PATH_MAX, GFP_KERNEL); -+ agentbuf = kstrdup(cgrp->root->release_agent_path, GFP_KERNEL); -+ if (!pathbuf || !agentbuf) -+ goto out; ++static const struct file_operations latency_hist_fops = { ++ .open = latency_hist_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = seq_release, ++}; + -+ path = cgroup_path(cgrp, pathbuf, PATH_MAX); -+ if (!path) -+ goto out; ++#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \ ++ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) ++static void clear_maxlatprocdata(struct maxlatproc_data *mp) ++{ ++ mp->comm[0] = mp->current_comm[0] = '\0'; ++ mp->prio = mp->current_prio = mp->pid = mp->current_pid = ++ mp->latency = mp->timeroffset = -1; ++ mp->timestamp = 0; ++} ++#endif + -+ argv[0] = agentbuf; -+ argv[1] = path; -+ argv[2] = NULL; ++static void hist_reset(struct hist_data *hist) ++{ ++ atomic_dec(&hist->hist_mode); + -+ /* minimal command environment */ -+ envp[0] = "HOME=/"; -+ envp[1] = "PATH=/sbin:/bin:/usr/sbin:/usr/bin"; -+ envp[2] = NULL; ++ memset(hist->hist_array, 0, sizeof(hist->hist_array)); ++ hist->below_hist_bound_samples = 0ULL; ++ hist->above_hist_bound_samples = 0ULL; ++ hist->min_lat = LONG_MAX; ++ hist->max_lat = LONG_MIN; ++ hist->total_samples = 0ULL; ++ hist->accumulate_lat = 0LL; + -+ mutex_unlock(&cgroup_mutex); -+ call_usermodehelper(argv[0], argv, envp, UMH_WAIT_EXEC); -+ goto out_free; -+out: -+ mutex_unlock(&cgroup_mutex); -+out_free: -+ kfree(agentbuf); -+ kfree(pathbuf); ++ atomic_inc(&hist->hist_mode); +} + -+static int __init cgroup_disable(char *str) ++static ssize_t ++latency_hist_reset(struct file *file, const char __user *a, ++ size_t size, loff_t *off) +{ -+ struct cgroup_subsys *ss; -+ char *token; -+ int i; ++ int cpu; ++ struct hist_data *hist = NULL; ++#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \ ++ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) ++ struct maxlatproc_data *mp = NULL; ++#endif ++ off_t latency_type = (off_t) file->private_data; + -+ while ((token = strsep(&str, ",")) != NULL) { -+ if (!*token) -+ continue; ++ for_each_online_cpu(cpu) { + -+ for_each_subsys(ss, i) { -+ if (!strcmp(token, ss->name)) { -+ ss->disabled = 1; -+ printk(KERN_INFO "Disabling %s control group" -+ " subsystem\n", ss->name); -+ break; -+ } ++ switch (latency_type) { ++#ifdef CONFIG_PREEMPT_OFF_HIST ++ case PREEMPTOFF_LATENCY: ++ hist = &per_cpu(preemptoff_hist, cpu); ++ break; ++#endif ++#ifdef CONFIG_INTERRUPT_OFF_HIST ++ case IRQSOFF_LATENCY: ++ hist = &per_cpu(irqsoff_hist, cpu); ++ break; ++#endif ++#if defined(CONFIG_INTERRUPT_OFF_HIST) && defined(CONFIG_PREEMPT_OFF_HIST) ++ case PREEMPTIRQSOFF_LATENCY: ++ hist = &per_cpu(preemptirqsoff_hist, cpu); ++ break; ++#endif ++#ifdef CONFIG_WAKEUP_LATENCY_HIST ++ case WAKEUP_LATENCY: ++ hist = &per_cpu(wakeup_latency_hist, cpu); ++ mp = &per_cpu(wakeup_maxlatproc, cpu); ++ break; ++ case WAKEUP_LATENCY_SHAREDPRIO: ++ hist = &per_cpu(wakeup_latency_hist_sharedprio, cpu); ++ mp = &per_cpu(wakeup_maxlatproc_sharedprio, cpu); ++ break; ++#endif ++#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST ++ case MISSED_TIMER_OFFSETS: ++ hist = &per_cpu(missed_timer_offsets, cpu); ++ mp = &per_cpu(missed_timer_offsets_maxlatproc, cpu); ++ break; ++#endif ++#if defined(CONFIG_WAKEUP_LATENCY_HIST) && \ ++ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) ++ case TIMERANDWAKEUP_LATENCY: ++ hist = &per_cpu(timerandwakeup_latency_hist, cpu); ++ mp = &per_cpu(timerandwakeup_maxlatproc, cpu); ++ break; ++#endif + } ++ ++ hist_reset(hist); ++#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \ ++ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) ++ if (latency_type == WAKEUP_LATENCY || ++ latency_type == WAKEUP_LATENCY_SHAREDPRIO || ++ latency_type == MISSED_TIMER_OFFSETS || ++ latency_type == TIMERANDWAKEUP_LATENCY) ++ clear_maxlatprocdata(mp); ++#endif + } -+ return 1; ++ ++ return size; +} -+__setup("cgroup_disable=", cgroup_disable); + -+static int __init cgroup_set_legacy_files_on_dfl(char *str) ++#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \ ++ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) ++static ssize_t ++show_pid(struct file *file, char __user *ubuf, size_t cnt, loff_t *ppos) +{ -+ printk("cgroup: using legacy files on the default hierarchy\n"); -+ cgroup_legacy_files_on_dfl = true; -+ return 0; ++ char buf[64]; ++ int r; ++ unsigned long *this_pid = file->private_data; ++ ++ r = snprintf(buf, sizeof(buf), "%lu\n", *this_pid); ++ return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); +} -+__setup("cgroup__DEVEL__legacy_files_on_dfl", cgroup_set_legacy_files_on_dfl); + -+/** -+ * css_tryget_online_from_dir - get corresponding css from a cgroup dentry -+ * @dentry: directory dentry of interest -+ * @ss: subsystem of interest -+ * -+ * If @dentry is a directory for a cgroup which has @ss enabled on it, try -+ * to get the corresponding css and return it. If such css doesn't exist -+ * or can't be pinned, an ERR_PTR value is returned. -+ */ -+struct cgroup_subsys_state *css_tryget_online_from_dir(struct dentry *dentry, -+ struct cgroup_subsys *ss) ++static ssize_t do_pid(struct file *file, const char __user *ubuf, ++ size_t cnt, loff_t *ppos) +{ -+ struct kernfs_node *kn = kernfs_node_from_dentry(dentry); -+ struct cgroup_subsys_state *css = NULL; -+ struct cgroup *cgrp; ++ char buf[64]; ++ unsigned long pid; ++ unsigned long *this_pid = file->private_data; + -+ /* is @dentry a cgroup dir? */ -+ if (dentry->d_sb->s_type != &cgroup_fs_type || !kn || -+ kernfs_type(kn) != KERNFS_DIR) -+ return ERR_PTR(-EBADF); ++ if (cnt >= sizeof(buf)) ++ return -EINVAL; + -+ rcu_read_lock(); ++ if (copy_from_user(&buf, ubuf, cnt)) ++ return -EFAULT; + -+ /* -+ * This path doesn't originate from kernfs and @kn could already -+ * have been or be removed at any point. @kn->priv is RCU -+ * protected for this access. See css_release_work_fn() for details. -+ */ -+ cgrp = rcu_dereference(kn->priv); -+ if (cgrp) -+ css = cgroup_css(cgrp, ss); ++ buf[cnt] = '\0'; + -+ if (!css || !css_tryget_online(css)) -+ css = ERR_PTR(-ENOENT); ++ if (kstrtoul(buf, 10, &pid)) ++ return -EINVAL; + -+ rcu_read_unlock(); -+ return css; -+} ++ *this_pid = pid; + -+/** -+ * css_from_id - lookup css by id -+ * @id: the cgroup id -+ * @ss: cgroup subsys to be looked into -+ * -+ * Returns the css if there's valid one with @id, otherwise returns NULL. -+ * Should be called under rcu_read_lock(). -+ */ -+struct cgroup_subsys_state *css_from_id(int id, struct cgroup_subsys *ss) -+{ -+ WARN_ON_ONCE(!rcu_read_lock_held()); -+ return id > 0 ? idr_find(&ss->css_idr, id) : NULL; ++ return cnt; +} ++#endif + -+#ifdef CONFIG_CGROUP_DEBUG -+static struct cgroup_subsys_state * -+debug_css_alloc(struct cgroup_subsys_state *parent_css) ++#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \ ++ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) ++static ssize_t ++show_maxlatproc(struct file *file, char __user *ubuf, size_t cnt, loff_t *ppos) +{ -+ struct cgroup_subsys_state *css = kzalloc(sizeof(*css), GFP_KERNEL); ++ int r; ++ struct maxlatproc_data *mp = file->private_data; ++ int strmaxlen = (TASK_COMM_LEN * 2) + (8 * 8); ++ unsigned long long t; ++ unsigned long usecs, secs; ++ char *buf; + -+ if (!css) -+ return ERR_PTR(-ENOMEM); ++ if (mp->pid == -1 || mp->current_pid == -1) { ++ buf = "(none)\n"; ++ return simple_read_from_buffer(ubuf, cnt, ppos, buf, ++ strlen(buf)); ++ } + -+ return css; -+} ++ buf = kmalloc(strmaxlen, GFP_KERNEL); ++ if (buf == NULL) ++ return -ENOMEM; + -+static void debug_css_free(struct cgroup_subsys_state *css) -+{ -+ kfree(css); ++ t = ns2usecs(mp->timestamp); ++ usecs = do_div(t, USEC_PER_SEC); ++ secs = (unsigned long) t; ++ r = snprintf(buf, strmaxlen, ++ "%d %d %ld (%ld) %s <- %d %d %s %lu.%06lu\n", mp->pid, ++ MAX_RT_PRIO-1 - mp->prio, mp->latency, mp->timeroffset, mp->comm, ++ mp->current_pid, MAX_RT_PRIO-1 - mp->current_prio, mp->current_comm, ++ secs, usecs); ++ r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r); ++ kfree(buf); ++ return r; +} ++#endif + -+static u64 debug_taskcount_read(struct cgroup_subsys_state *css, -+ struct cftype *cft) ++static ssize_t ++show_enable(struct file *file, char __user *ubuf, size_t cnt, loff_t *ppos) +{ -+ return cgroup_task_count(css->cgroup); -+} ++ char buf[64]; ++ struct enable_data *ed = file->private_data; ++ int r; + -+static u64 current_css_set_read(struct cgroup_subsys_state *css, -+ struct cftype *cft) -+{ -+ return (u64)(unsigned long)current->cgroups; ++ r = snprintf(buf, sizeof(buf), "%d\n", ed->enabled); ++ return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); +} + -+static u64 current_css_set_refcount_read(struct cgroup_subsys_state *css, -+ struct cftype *cft) ++static ssize_t ++do_enable(struct file *file, const char __user *ubuf, size_t cnt, loff_t *ppos) +{ -+ u64 count; ++ char buf[64]; ++ long enable; ++ struct enable_data *ed = file->private_data; + -+ rcu_read_lock(); -+ count = atomic_read(&task_css_set(current)->refcount); -+ rcu_read_unlock(); -+ return count; -+} ++ if (cnt >= sizeof(buf)) ++ return -EINVAL; + -+static int current_css_set_cg_links_read(struct seq_file *seq, void *v) -+{ -+ struct cgrp_cset_link *link; -+ struct css_set *cset; -+ char *name_buf; ++ if (copy_from_user(&buf, ubuf, cnt)) ++ return -EFAULT; + -+ name_buf = kmalloc(NAME_MAX + 1, GFP_KERNEL); -+ if (!name_buf) -+ return -ENOMEM; ++ buf[cnt] = 0; + -+ down_read(&css_set_rwsem); -+ rcu_read_lock(); -+ cset = rcu_dereference(current->cgroups); -+ list_for_each_entry(link, &cset->cgrp_links, cgrp_link) { -+ struct cgroup *c = link->cgrp; ++ if (kstrtoul(buf, 10, &enable)) ++ return -EINVAL; + -+ cgroup_name(c, name_buf, NAME_MAX + 1); -+ seq_printf(seq, "Root %d group %s\n", -+ c->root->hierarchy_id, name_buf); -+ } -+ rcu_read_unlock(); -+ up_read(&css_set_rwsem); -+ kfree(name_buf); -+ return 0; -+} ++ if ((enable && ed->enabled) || (!enable && !ed->enabled)) ++ return cnt; + -+#define MAX_TASKS_SHOWN_PER_CSS 25 -+static int cgroup_css_links_read(struct seq_file *seq, void *v) -+{ -+ struct cgroup_subsys_state *css = seq_css(seq); -+ struct cgrp_cset_link *link; ++ if (enable) { ++ int ret; + -+ down_read(&css_set_rwsem); -+ list_for_each_entry(link, &css->cgroup->cset_links, cset_link) { -+ struct css_set *cset = link->cset; -+ struct task_struct *task; -+ int count = 0; ++ switch (ed->latency_type) { ++#if defined(CONFIG_INTERRUPT_OFF_HIST) || defined(CONFIG_PREEMPT_OFF_HIST) ++ case PREEMPTIRQSOFF_LATENCY: ++ ret = register_trace_preemptirqsoff_hist( ++ probe_preemptirqsoff_hist, NULL); ++ if (ret) { ++ pr_info("wakeup trace: Couldn't assign " ++ "probe_preemptirqsoff_hist " ++ "to trace_preemptirqsoff_hist\n"); ++ return ret; ++ } ++ break; ++#endif ++#ifdef CONFIG_WAKEUP_LATENCY_HIST ++ case WAKEUP_LATENCY: ++ ret = register_trace_sched_wakeup( ++ probe_wakeup_latency_hist_start, NULL); ++ if (ret) { ++ pr_info("wakeup trace: Couldn't assign " ++ "probe_wakeup_latency_hist_start " ++ "to trace_sched_wakeup\n"); ++ return ret; ++ } ++ ret = register_trace_sched_wakeup_new( ++ probe_wakeup_latency_hist_start, NULL); ++ if (ret) { ++ pr_info("wakeup trace: Couldn't assign " ++ "probe_wakeup_latency_hist_start " ++ "to trace_sched_wakeup_new\n"); ++ unregister_trace_sched_wakeup( ++ probe_wakeup_latency_hist_start, NULL); ++ return ret; ++ } ++ ret = register_trace_sched_switch( ++ probe_wakeup_latency_hist_stop, NULL); ++ if (ret) { ++ pr_info("wakeup trace: Couldn't assign " ++ "probe_wakeup_latency_hist_stop " ++ "to trace_sched_switch\n"); ++ unregister_trace_sched_wakeup( ++ probe_wakeup_latency_hist_start, NULL); ++ unregister_trace_sched_wakeup_new( ++ probe_wakeup_latency_hist_start, NULL); ++ return ret; ++ } ++ ret = register_trace_sched_migrate_task( ++ probe_sched_migrate_task, NULL); ++ if (ret) { ++ pr_info("wakeup trace: Couldn't assign " ++ "probe_sched_migrate_task " ++ "to trace_sched_migrate_task\n"); ++ unregister_trace_sched_wakeup( ++ probe_wakeup_latency_hist_start, NULL); ++ unregister_trace_sched_wakeup_new( ++ probe_wakeup_latency_hist_start, NULL); ++ unregister_trace_sched_switch( ++ probe_wakeup_latency_hist_stop, NULL); ++ return ret; ++ } ++ break; ++#endif ++#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST ++ case MISSED_TIMER_OFFSETS: ++ ret = register_trace_hrtimer_interrupt( ++ probe_hrtimer_interrupt, NULL); ++ if (ret) { ++ pr_info("wakeup trace: Couldn't assign " ++ "probe_hrtimer_interrupt " ++ "to trace_hrtimer_interrupt\n"); ++ return ret; ++ } ++ break; ++#endif ++#if defined(CONFIG_WAKEUP_LATENCY_HIST) && \ ++ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) ++ case TIMERANDWAKEUP_LATENCY: ++ if (!wakeup_latency_enabled_data.enabled || ++ !missed_timer_offsets_enabled_data.enabled) ++ return -EINVAL; ++ break; ++#endif ++ default: ++ break; ++ } ++ } else { ++ switch (ed->latency_type) { ++#if defined(CONFIG_INTERRUPT_OFF_HIST) || defined(CONFIG_PREEMPT_OFF_HIST) ++ case PREEMPTIRQSOFF_LATENCY: ++ { ++ int cpu; + -+ seq_printf(seq, "css_set %p\n", cset); ++ unregister_trace_preemptirqsoff_hist( ++ probe_preemptirqsoff_hist, NULL); ++ for_each_online_cpu(cpu) { ++#ifdef CONFIG_INTERRUPT_OFF_HIST ++ per_cpu(hist_irqsoff_counting, ++ cpu) = 0; ++#endif ++#ifdef CONFIG_PREEMPT_OFF_HIST ++ per_cpu(hist_preemptoff_counting, ++ cpu) = 0; ++#endif ++#if defined(CONFIG_INTERRUPT_OFF_HIST) && defined(CONFIG_PREEMPT_OFF_HIST) ++ per_cpu(hist_preemptirqsoff_counting, ++ cpu) = 0; ++#endif ++ } ++ } ++ break; ++#endif ++#ifdef CONFIG_WAKEUP_LATENCY_HIST ++ case WAKEUP_LATENCY: ++ { ++ int cpu; + -+ list_for_each_entry(task, &cset->tasks, cg_list) { -+ if (count++ > MAX_TASKS_SHOWN_PER_CSS) -+ goto overflow; -+ seq_printf(seq, " task %d\n", task_pid_vnr(task)); -+ } ++ unregister_trace_sched_wakeup( ++ probe_wakeup_latency_hist_start, NULL); ++ unregister_trace_sched_wakeup_new( ++ probe_wakeup_latency_hist_start, NULL); ++ unregister_trace_sched_switch( ++ probe_wakeup_latency_hist_stop, NULL); ++ unregister_trace_sched_migrate_task( ++ probe_sched_migrate_task, NULL); + -+ list_for_each_entry(task, &cset->mg_tasks, cg_list) { -+ if (count++ > MAX_TASKS_SHOWN_PER_CSS) -+ goto overflow; -+ seq_printf(seq, " task %d\n", task_pid_vnr(task)); ++ for_each_online_cpu(cpu) { ++ per_cpu(wakeup_task, cpu) = NULL; ++ per_cpu(wakeup_sharedprio, cpu) = 0; ++ } ++ } ++#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST ++ timerandwakeup_enabled_data.enabled = 0; ++#endif ++ break; ++#endif ++#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST ++ case MISSED_TIMER_OFFSETS: ++ unregister_trace_hrtimer_interrupt( ++ probe_hrtimer_interrupt, NULL); ++#ifdef CONFIG_WAKEUP_LATENCY_HIST ++ timerandwakeup_enabled_data.enabled = 0; ++#endif ++ break; ++#endif ++ default: ++ break; + } -+ continue; -+ overflow: -+ seq_puts(seq, " ...\n"); + } -+ up_read(&css_set_rwsem); -+ return 0; ++ ed->enabled = enable; ++ return cnt; +} + -+static u64 releasable_read(struct cgroup_subsys_state *css, struct cftype *cft) -+{ -+ return (!cgroup_has_tasks(css->cgroup) && -+ !css_has_online_children(&css->cgroup->self)); -+} ++static const struct file_operations latency_hist_reset_fops = { ++ .open = tracing_open_generic, ++ .write = latency_hist_reset, ++}; + -+static struct cftype debug_files[] = { -+ { -+ .name = "taskcount", -+ .read_u64 = debug_taskcount_read, -+ }, ++static const struct file_operations enable_fops = { ++ .open = tracing_open_generic, ++ .read = show_enable, ++ .write = do_enable, ++}; + -+ { -+ .name = "current_css_set", -+ .read_u64 = current_css_set_read, -+ }, -+ -+ { -+ .name = "current_css_set_refcount", -+ .read_u64 = current_css_set_refcount_read, -+ }, ++#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \ ++ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) ++static const struct file_operations pid_fops = { ++ .open = tracing_open_generic, ++ .read = show_pid, ++ .write = do_pid, ++}; + -+ { -+ .name = "current_css_set_cg_links", -+ .seq_show = current_css_set_cg_links_read, -+ }, ++static const struct file_operations maxlatproc_fops = { ++ .open = tracing_open_generic, ++ .read = show_maxlatproc, ++}; ++#endif + -+ { -+ .name = "cgroup_css_links", -+ .seq_show = cgroup_css_links_read, -+ }, ++#if defined(CONFIG_INTERRUPT_OFF_HIST) || defined(CONFIG_PREEMPT_OFF_HIST) ++static notrace void probe_preemptirqsoff_hist(void *v, int reason, ++ int starthist) ++{ ++ int cpu = raw_smp_processor_id(); ++ int time_set = 0; + -+ { -+ .name = "releasable", -+ .read_u64 = releasable_read, -+ }, ++ if (starthist) { ++ cycle_t uninitialized_var(start); + -+ { } /* terminate */ -+}; ++ if (!preempt_count() && !irqs_disabled()) ++ return; + -+struct cgroup_subsys debug_cgrp_subsys = { -+ .css_alloc = debug_css_alloc, -+ .css_free = debug_css_free, -+ .legacy_cftypes = debug_files, -+}; -+#endif /* CONFIG_CGROUP_DEBUG */ -diff -Nur linux-4.1.10.orig/kernel/cpu.c linux-4.1.10/kernel/cpu.c ---- linux-4.1.10.orig/kernel/cpu.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/cpu.c 2015-10-07 18:00:08.000000000 +0200 -@@ -74,8 +74,8 @@ - #endif - } cpu_hotplug = { - .active_writer = NULL, -- .wq = __WAIT_QUEUE_HEAD_INITIALIZER(cpu_hotplug.wq), - .lock = __MUTEX_INITIALIZER(cpu_hotplug.lock), -+ .wq = __WAIT_QUEUE_HEAD_INITIALIZER(cpu_hotplug.wq), - #ifdef CONFIG_DEBUG_LOCK_ALLOC - .dep_map = {.name = "cpu_hotplug.lock" }, - #endif -@@ -88,6 +88,289 @@ - #define cpuhp_lock_acquire() lock_map_acquire(&cpu_hotplug.dep_map) - #define cpuhp_lock_release() lock_map_release(&cpu_hotplug.dep_map) - -+/** -+ * hotplug_pcp - per cpu hotplug descriptor -+ * @unplug: set when pin_current_cpu() needs to sync tasks -+ * @sync_tsk: the task that waits for tasks to finish pinned sections -+ * @refcount: counter of tasks in pinned sections -+ * @grab_lock: set when the tasks entering pinned sections should wait -+ * @synced: notifier for @sync_tsk to tell cpu_down it's finished -+ * @mutex: the mutex to make tasks wait (used when @grab_lock is true) -+ * @mutex_init: zero if the mutex hasn't been initialized yet. -+ * -+ * Although @unplug and @sync_tsk may point to the same task, the @unplug -+ * is used as a flag and still exists after @sync_tsk has exited and -+ * @sync_tsk set to NULL. -+ */ -+struct hotplug_pcp { -+ struct task_struct *unplug; -+ struct task_struct *sync_tsk; -+ int refcount; -+ int grab_lock; -+ struct completion synced; -+ struct completion unplug_wait; -+#ifdef CONFIG_PREEMPT_RT_FULL -+ /* -+ * Note, on PREEMPT_RT, the hotplug lock must save the state of -+ * the task, otherwise the mutex will cause the task to fail -+ * to sleep when required. (Because it's called from migrate_disable()) -+ * -+ * The spinlock_t on PREEMPT_RT is a mutex that saves the task's -+ * state. -+ */ -+ spinlock_t lock; -+#else -+ struct mutex mutex; ++#ifdef CONFIG_INTERRUPT_OFF_HIST ++ if ((reason == IRQS_OFF || reason == TRACE_START) && ++ !per_cpu(hist_irqsoff_counting, cpu)) { ++ per_cpu(hist_irqsoff_counting, cpu) = 1; ++ start = ftrace_now(cpu); ++ time_set++; ++ per_cpu(hist_irqsoff_start, cpu) = start; ++ } +#endif -+ int mutex_init; -+}; + -+#ifdef CONFIG_PREEMPT_RT_FULL -+# define hotplug_lock(hp) rt_spin_lock(&(hp)->lock) -+# define hotplug_unlock(hp) rt_spin_unlock(&(hp)->lock) -+#else -+# define hotplug_lock(hp) mutex_lock(&(hp)->mutex) -+# define hotplug_unlock(hp) mutex_unlock(&(hp)->mutex) ++#ifdef CONFIG_PREEMPT_OFF_HIST ++ if ((reason == PREEMPT_OFF || reason == TRACE_START) && ++ !per_cpu(hist_preemptoff_counting, cpu)) { ++ per_cpu(hist_preemptoff_counting, cpu) = 1; ++ if (!(time_set++)) ++ start = ftrace_now(cpu); ++ per_cpu(hist_preemptoff_start, cpu) = start; ++ } +#endif + -+static DEFINE_PER_CPU(struct hotplug_pcp, hotplug_pcp); ++#if defined(CONFIG_INTERRUPT_OFF_HIST) && defined(CONFIG_PREEMPT_OFF_HIST) ++ if (per_cpu(hist_irqsoff_counting, cpu) && ++ per_cpu(hist_preemptoff_counting, cpu) && ++ !per_cpu(hist_preemptirqsoff_counting, cpu)) { ++ per_cpu(hist_preemptirqsoff_counting, cpu) = 1; ++ if (!time_set) ++ start = ftrace_now(cpu); ++ per_cpu(hist_preemptirqsoff_start, cpu) = start; ++ } ++#endif ++ } else { ++ cycle_t uninitialized_var(stop); + -+/** -+ * pin_current_cpu - Prevent the current cpu from being unplugged -+ * -+ * Lightweight version of get_online_cpus() to prevent cpu from being -+ * unplugged when code runs in a migration disabled region. -+ * -+ * Must be called with preemption disabled (preempt_count = 1)! -+ */ -+void pin_current_cpu(void) -+{ -+ struct hotplug_pcp *hp; -+ int force = 0; ++#ifdef CONFIG_INTERRUPT_OFF_HIST ++ if ((reason == IRQS_ON || reason == TRACE_STOP) && ++ per_cpu(hist_irqsoff_counting, cpu)) { ++ cycle_t start = per_cpu(hist_irqsoff_start, cpu); + -+retry: -+ hp = this_cpu_ptr(&hotplug_pcp); ++ stop = ftrace_now(cpu); ++ time_set++; ++ if (start) { ++ long latency = ((long) (stop - start)) / ++ NSECS_PER_USECS; + -+ if (!hp->unplug || hp->refcount || force || preempt_count() > 1 || -+ hp->unplug == current) { -+ hp->refcount++; -+ return; -+ } -+ if (hp->grab_lock) { -+ preempt_enable(); -+ hotplug_lock(hp); -+ hotplug_unlock(hp); -+ } else { -+ preempt_enable(); -+ /* -+ * Try to push this task off of this CPU. -+ */ -+ if (!migrate_me()) { -+ preempt_disable(); -+ hp = this_cpu_ptr(&hotplug_pcp); -+ if (!hp->grab_lock) { -+ /* -+ * Just let it continue it's already pinned -+ * or about to sleep. -+ */ -+ force = 1; -+ goto retry; ++ latency_hist(IRQSOFF_LATENCY, cpu, latency, 0, ++ stop, NULL); + } -+ preempt_enable(); ++ per_cpu(hist_irqsoff_counting, cpu) = 0; + } -+ } -+ preempt_disable(); -+ goto retry; -+} ++#endif + -+/** -+ * unpin_current_cpu - Allow unplug of current cpu -+ * -+ * Must be called with preemption or interrupts disabled! -+ */ -+void unpin_current_cpu(void) -+{ -+ struct hotplug_pcp *hp = this_cpu_ptr(&hotplug_pcp); ++#ifdef CONFIG_PREEMPT_OFF_HIST ++ if ((reason == PREEMPT_ON || reason == TRACE_STOP) && ++ per_cpu(hist_preemptoff_counting, cpu)) { ++ cycle_t start = per_cpu(hist_preemptoff_start, cpu); + -+ WARN_ON(hp->refcount <= 0); ++ if (!(time_set++)) ++ stop = ftrace_now(cpu); ++ if (start) { ++ long latency = ((long) (stop - start)) / ++ NSECS_PER_USECS; + -+ /* This is safe. sync_unplug_thread is pinned to this cpu */ -+ if (!--hp->refcount && hp->unplug && hp->unplug != current) -+ wake_up_process(hp->unplug); -+} ++ latency_hist(PREEMPTOFF_LATENCY, cpu, latency, ++ 0, stop, NULL); ++ } ++ per_cpu(hist_preemptoff_counting, cpu) = 0; ++ } ++#endif + -+static void wait_for_pinned_cpus(struct hotplug_pcp *hp) -+{ -+ set_current_state(TASK_UNINTERRUPTIBLE); -+ while (hp->refcount) { -+ schedule_preempt_disabled(); -+ set_current_state(TASK_UNINTERRUPTIBLE); ++#if defined(CONFIG_INTERRUPT_OFF_HIST) && defined(CONFIG_PREEMPT_OFF_HIST) ++ if ((!per_cpu(hist_irqsoff_counting, cpu) || ++ !per_cpu(hist_preemptoff_counting, cpu)) && ++ per_cpu(hist_preemptirqsoff_counting, cpu)) { ++ cycle_t start = per_cpu(hist_preemptirqsoff_start, cpu); ++ ++ if (!time_set) ++ stop = ftrace_now(cpu); ++ if (start) { ++ long latency = ((long) (stop - start)) / ++ NSECS_PER_USECS; ++ ++ latency_hist(PREEMPTIRQSOFF_LATENCY, cpu, ++ latency, 0, stop, NULL); ++ } ++ per_cpu(hist_preemptirqsoff_counting, cpu) = 0; ++ } ++#endif + } +} ++#endif + -+static int sync_unplug_thread(void *data) ++#ifdef CONFIG_WAKEUP_LATENCY_HIST ++static DEFINE_RAW_SPINLOCK(wakeup_lock); ++static notrace void probe_sched_migrate_task(void *v, struct task_struct *task, ++ int cpu) +{ -+ struct hotplug_pcp *hp = data; ++ int old_cpu = task_cpu(task); + -+ wait_for_completion(&hp->unplug_wait); -+ preempt_disable(); -+ hp->unplug = current; -+ wait_for_pinned_cpus(hp); ++ if (cpu != old_cpu) { ++ unsigned long flags; ++ struct task_struct *cpu_wakeup_task; + -+ /* -+ * This thread will synchronize the cpu_down() with threads -+ * that have pinned the CPU. When the pinned CPU count reaches -+ * zero, we inform the cpu_down code to continue to the next step. -+ */ -+ set_current_state(TASK_UNINTERRUPTIBLE); -+ preempt_enable(); -+ complete(&hp->synced); ++ raw_spin_lock_irqsave(&wakeup_lock, flags); + -+ /* -+ * If all succeeds, the next step will need tasks to wait till -+ * the CPU is offline before continuing. To do this, the grab_lock -+ * is set and tasks going into pin_current_cpu() will block on the -+ * mutex. But we still need to wait for those that are already in -+ * pinned CPU sections. If the cpu_down() failed, the kthread_should_stop() -+ * will kick this thread out. -+ */ -+ while (!hp->grab_lock && !kthread_should_stop()) { -+ schedule(); -+ set_current_state(TASK_UNINTERRUPTIBLE); ++ cpu_wakeup_task = per_cpu(wakeup_task, old_cpu); ++ if (task == cpu_wakeup_task) { ++ put_task_struct(cpu_wakeup_task); ++ per_cpu(wakeup_task, old_cpu) = NULL; ++ cpu_wakeup_task = per_cpu(wakeup_task, cpu) = task; ++ get_task_struct(cpu_wakeup_task); ++ } ++ ++ raw_spin_unlock_irqrestore(&wakeup_lock, flags); + } ++} + -+ /* Make sure grab_lock is seen before we see a stale completion */ -+ smp_mb(); ++static notrace void probe_wakeup_latency_hist_start(void *v, ++ struct task_struct *p, int success) ++{ ++ unsigned long flags; ++ struct task_struct *curr = current; ++ int cpu = task_cpu(p); ++ struct task_struct *cpu_wakeup_task; + -+ /* -+ * Now just before cpu_down() enters stop machine, we need to make -+ * sure all tasks that are in pinned CPU sections are out, and new -+ * tasks will now grab the lock, keeping them from entering pinned -+ * CPU sections. -+ */ -+ if (!kthread_should_stop()) { -+ preempt_disable(); -+ wait_for_pinned_cpus(hp); -+ preempt_enable(); -+ complete(&hp->synced); -+ } ++ raw_spin_lock_irqsave(&wakeup_lock, flags); + -+ set_current_state(TASK_UNINTERRUPTIBLE); -+ while (!kthread_should_stop()) { -+ schedule(); -+ set_current_state(TASK_UNINTERRUPTIBLE); ++ cpu_wakeup_task = per_cpu(wakeup_task, cpu); ++ ++ if (wakeup_pid) { ++ if ((cpu_wakeup_task && p->prio == cpu_wakeup_task->prio) || ++ p->prio == curr->prio) ++ per_cpu(wakeup_sharedprio, cpu) = 1; ++ if (likely(wakeup_pid != task_pid_nr(p))) ++ goto out; ++ } else { ++ if (likely(!rt_task(p)) || ++ (cpu_wakeup_task && p->prio > cpu_wakeup_task->prio) || ++ p->prio > curr->prio) ++ goto out; ++ if ((cpu_wakeup_task && p->prio == cpu_wakeup_task->prio) || ++ p->prio == curr->prio) ++ per_cpu(wakeup_sharedprio, cpu) = 1; + } -+ set_current_state(TASK_RUNNING); + -+ /* -+ * Force this thread off this CPU as it's going down and -+ * we don't want any more work on this CPU. -+ */ -+ current->flags &= ~PF_NO_SETAFFINITY; -+ set_cpus_allowed_ptr(current, cpu_present_mask); -+ migrate_me(); -+ return 0; ++ if (cpu_wakeup_task) ++ put_task_struct(cpu_wakeup_task); ++ cpu_wakeup_task = per_cpu(wakeup_task, cpu) = p; ++ get_task_struct(cpu_wakeup_task); ++ cpu_wakeup_task->preempt_timestamp_hist = ++ ftrace_now(raw_smp_processor_id()); ++out: ++ raw_spin_unlock_irqrestore(&wakeup_lock, flags); +} + -+static void __cpu_unplug_sync(struct hotplug_pcp *hp) ++static notrace void probe_wakeup_latency_hist_stop(void *v, ++ struct task_struct *prev, struct task_struct *next) +{ -+ wake_up_process(hp->sync_tsk); -+ wait_for_completion(&hp->synced); -+} ++ unsigned long flags; ++ int cpu = task_cpu(next); ++ long latency; ++ cycle_t stop; ++ struct task_struct *cpu_wakeup_task; + -+static void __cpu_unplug_wait(unsigned int cpu) -+{ -+ struct hotplug_pcp *hp = &per_cpu(hotplug_pcp, cpu); ++ raw_spin_lock_irqsave(&wakeup_lock, flags); + -+ complete(&hp->unplug_wait); -+ wait_for_completion(&hp->synced); -+} ++ cpu_wakeup_task = per_cpu(wakeup_task, cpu); + -+/* -+ * Start the sync_unplug_thread on the target cpu and wait for it to -+ * complete. -+ */ -+static int cpu_unplug_begin(unsigned int cpu) -+{ -+ struct hotplug_pcp *hp = &per_cpu(hotplug_pcp, cpu); -+ int err; ++ if (cpu_wakeup_task == NULL) ++ goto out; + -+ /* Protected by cpu_hotplug.lock */ -+ if (!hp->mutex_init) { -+#ifdef CONFIG_PREEMPT_RT_FULL -+ spin_lock_init(&hp->lock); -+#else -+ mutex_init(&hp->mutex); -+#endif -+ hp->mutex_init = 1; -+ } ++ /* Already running? */ ++ if (unlikely(current == cpu_wakeup_task)) ++ goto out_reset; + -+ /* Inform the scheduler to migrate tasks off this CPU */ -+ tell_sched_cpu_down_begin(cpu); ++ if (next != cpu_wakeup_task) { ++ if (next->prio < cpu_wakeup_task->prio) ++ goto out_reset; + -+ init_completion(&hp->synced); -+ init_completion(&hp->unplug_wait); ++ if (next->prio == cpu_wakeup_task->prio) ++ per_cpu(wakeup_sharedprio, cpu) = 1; + -+ hp->sync_tsk = kthread_create(sync_unplug_thread, hp, "sync_unplug/%d", cpu); -+ if (IS_ERR(hp->sync_tsk)) { -+ err = PTR_ERR(hp->sync_tsk); -+ hp->sync_tsk = NULL; -+ return err; ++ goto out; + } -+ kthread_bind(hp->sync_tsk, cpu); ++ ++ if (current->prio == cpu_wakeup_task->prio) ++ per_cpu(wakeup_sharedprio, cpu) = 1; + + /* -+ * Wait for tasks to get out of the pinned sections, -+ * it's still OK if new tasks enter. Some CPU notifiers will -+ * wait for tasks that are going to enter these sections and -+ * we must not have them block. ++ * The task we are waiting for is about to be switched to. ++ * Calculate latency and store it in histogram. + */ -+ wake_up_process(hp->sync_tsk); -+ return 0; -+} ++ stop = ftrace_now(raw_smp_processor_id()); + -+static void cpu_unplug_sync(unsigned int cpu) -+{ -+ struct hotplug_pcp *hp = &per_cpu(hotplug_pcp, cpu); ++ latency = ((long) (stop - next->preempt_timestamp_hist)) / ++ NSECS_PER_USECS; + -+ init_completion(&hp->synced); -+ /* The completion needs to be initialzied before setting grab_lock */ -+ smp_wmb(); ++ if (per_cpu(wakeup_sharedprio, cpu)) { ++ latency_hist(WAKEUP_LATENCY_SHAREDPRIO, cpu, latency, 0, stop, ++ next); ++ per_cpu(wakeup_sharedprio, cpu) = 0; ++ } else { ++ latency_hist(WAKEUP_LATENCY, cpu, latency, 0, stop, next); ++#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST ++ if (timerandwakeup_enabled_data.enabled) { ++ latency_hist(TIMERANDWAKEUP_LATENCY, cpu, ++ next->timer_offset + latency, next->timer_offset, ++ stop, next); ++ } ++#endif ++ } + -+ /* Grab the mutex before setting grab_lock */ -+ hotplug_lock(hp); -+ hp->grab_lock = 1; ++out_reset: ++#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST ++ next->timer_offset = 0; ++#endif ++ put_task_struct(cpu_wakeup_task); ++ per_cpu(wakeup_task, cpu) = NULL; ++out: ++ raw_spin_unlock_irqrestore(&wakeup_lock, flags); ++} ++#endif + -+ /* -+ * The CPU notifiers have been completed. -+ * Wait for tasks to get out of pinned CPU sections and have new -+ * tasks block until the CPU is completely down. -+ */ -+ __cpu_unplug_sync(hp); ++#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST ++static notrace void probe_hrtimer_interrupt(void *v, int cpu, ++ long long latency_ns, struct task_struct *curr, ++ struct task_struct *task) ++{ ++ if (latency_ns <= 0 && task != NULL && rt_task(task) && ++ (task->prio < curr->prio || ++ (task->prio == curr->prio && ++ !cpumask_test_cpu(cpu, &task->cpus_allowed)))) { ++ long latency; ++ cycle_t now; + -+ /* All done with the sync thread */ -+ kthread_stop(hp->sync_tsk); -+ hp->sync_tsk = NULL; ++ if (missed_timer_offsets_pid) { ++ if (likely(missed_timer_offsets_pid != ++ task_pid_nr(task))) ++ return; ++ } ++ ++ now = ftrace_now(cpu); ++ latency = (long) div_s64(-latency_ns, NSECS_PER_USECS); ++ latency_hist(MISSED_TIMER_OFFSETS, cpu, latency, latency, now, ++ task); ++#ifdef CONFIG_WAKEUP_LATENCY_HIST ++ task->timer_offset = latency; ++#endif ++ } +} ++#endif + -+static void cpu_unplug_done(unsigned int cpu) ++static __init int latency_hist_init(void) +{ -+ struct hotplug_pcp *hp = &per_cpu(hotplug_pcp, cpu); ++ struct dentry *latency_hist_root = NULL; ++ struct dentry *dentry; ++#ifdef CONFIG_WAKEUP_LATENCY_HIST ++ struct dentry *dentry_sharedprio; ++#endif ++ struct dentry *entry; ++ struct dentry *enable_root; ++ int i = 0; ++ struct hist_data *my_hist; ++ char name[64]; ++ char *cpufmt = "CPU%d"; ++#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \ ++ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) ++ char *cpufmt_maxlatproc = "max_latency-CPU%d"; ++ struct maxlatproc_data *mp = NULL; ++#endif + -+ hp->unplug = NULL; -+ /* Let all tasks know cpu unplug is finished before cleaning up */ -+ smp_wmb(); ++ dentry = tracing_init_dentry(); ++ latency_hist_root = debugfs_create_dir(latency_hist_dir_root, dentry); ++ enable_root = debugfs_create_dir("enable", latency_hist_root); + -+ if (hp->sync_tsk) -+ kthread_stop(hp->sync_tsk); ++#ifdef CONFIG_INTERRUPT_OFF_HIST ++ dentry = debugfs_create_dir(irqsoff_hist_dir, latency_hist_root); ++ for_each_possible_cpu(i) { ++ sprintf(name, cpufmt, i); ++ entry = debugfs_create_file(name, 0444, dentry, ++ &per_cpu(irqsoff_hist, i), &latency_hist_fops); ++ my_hist = &per_cpu(irqsoff_hist, i); ++ atomic_set(&my_hist->hist_mode, 1); ++ my_hist->min_lat = LONG_MAX; ++ } ++ entry = debugfs_create_file("reset", 0644, dentry, ++ (void *)IRQSOFF_LATENCY, &latency_hist_reset_fops); ++#endif + -+ if (hp->grab_lock) { -+ hotplug_unlock(hp); -+ /* protected by cpu_hotplug.lock */ -+ hp->grab_lock = 0; ++#ifdef CONFIG_PREEMPT_OFF_HIST ++ dentry = debugfs_create_dir(preemptoff_hist_dir, ++ latency_hist_root); ++ for_each_possible_cpu(i) { ++ sprintf(name, cpufmt, i); ++ entry = debugfs_create_file(name, 0444, dentry, ++ &per_cpu(preemptoff_hist, i), &latency_hist_fops); ++ my_hist = &per_cpu(preemptoff_hist, i); ++ atomic_set(&my_hist->hist_mode, 1); ++ my_hist->min_lat = LONG_MAX; + } -+ tell_sched_cpu_down_done(cpu); -+} - - void get_online_cpus(void) - { -@@ -349,13 +632,15 @@ - /* Requires cpu_add_remove_lock to be held */ - static int __ref _cpu_down(unsigned int cpu, int tasks_frozen) - { -- int err, nr_calls = 0; -+ int mycpu, err, nr_calls = 0; - void *hcpu = (void *)(long)cpu; - unsigned long mod = tasks_frozen ? CPU_TASKS_FROZEN : 0; - struct take_cpu_down_param tcd_param = { - .mod = mod, - .hcpu = hcpu, - }; -+ cpumask_var_t cpumask; -+ cpumask_var_t cpumask_org; - - if (num_online_cpus() == 1) - return -EBUSY; -@@ -363,7 +648,34 @@ - if (!cpu_online(cpu)) - return -EINVAL; - -+ /* Move the downtaker off the unplug cpu */ -+ if (!alloc_cpumask_var(&cpumask, GFP_KERNEL)) -+ return -ENOMEM; -+ if (!alloc_cpumask_var(&cpumask_org, GFP_KERNEL)) { -+ free_cpumask_var(cpumask); -+ return -ENOMEM; ++ entry = debugfs_create_file("reset", 0644, dentry, ++ (void *)PREEMPTOFF_LATENCY, &latency_hist_reset_fops); ++#endif ++ ++#if defined(CONFIG_INTERRUPT_OFF_HIST) && defined(CONFIG_PREEMPT_OFF_HIST) ++ dentry = debugfs_create_dir(preemptirqsoff_hist_dir, ++ latency_hist_root); ++ for_each_possible_cpu(i) { ++ sprintf(name, cpufmt, i); ++ entry = debugfs_create_file(name, 0444, dentry, ++ &per_cpu(preemptirqsoff_hist, i), &latency_hist_fops); ++ my_hist = &per_cpu(preemptirqsoff_hist, i); ++ atomic_set(&my_hist->hist_mode, 1); ++ my_hist->min_lat = LONG_MAX; + } ++ entry = debugfs_create_file("reset", 0644, dentry, ++ (void *)PREEMPTIRQSOFF_LATENCY, &latency_hist_reset_fops); ++#endif + -+ cpumask_copy(cpumask_org, tsk_cpus_allowed(current)); -+ cpumask_andnot(cpumask, cpu_online_mask, cpumask_of(cpu)); -+ set_cpus_allowed_ptr(current, cpumask); -+ free_cpumask_var(cpumask); -+ migrate_disable(); -+ mycpu = smp_processor_id(); -+ if (mycpu == cpu) { -+ printk(KERN_ERR "Yuck! Still on unplug CPU\n!"); -+ migrate_enable(); -+ err = -EBUSY; -+ goto restore_cpus; ++#if defined(CONFIG_INTERRUPT_OFF_HIST) || defined(CONFIG_PREEMPT_OFF_HIST) ++ entry = debugfs_create_file("preemptirqsoff", 0644, ++ enable_root, (void *)&preemptirqsoff_enabled_data, ++ &enable_fops); ++#endif ++ ++#ifdef CONFIG_WAKEUP_LATENCY_HIST ++ dentry = debugfs_create_dir(wakeup_latency_hist_dir, ++ latency_hist_root); ++ dentry_sharedprio = debugfs_create_dir( ++ wakeup_latency_hist_dir_sharedprio, dentry); ++ for_each_possible_cpu(i) { ++ sprintf(name, cpufmt, i); ++ ++ entry = debugfs_create_file(name, 0444, dentry, ++ &per_cpu(wakeup_latency_hist, i), ++ &latency_hist_fops); ++ my_hist = &per_cpu(wakeup_latency_hist, i); ++ atomic_set(&my_hist->hist_mode, 1); ++ my_hist->min_lat = LONG_MAX; ++ ++ entry = debugfs_create_file(name, 0444, dentry_sharedprio, ++ &per_cpu(wakeup_latency_hist_sharedprio, i), ++ &latency_hist_fops); ++ my_hist = &per_cpu(wakeup_latency_hist_sharedprio, i); ++ atomic_set(&my_hist->hist_mode, 1); ++ my_hist->min_lat = LONG_MAX; ++ ++ sprintf(name, cpufmt_maxlatproc, i); ++ ++ mp = &per_cpu(wakeup_maxlatproc, i); ++ entry = debugfs_create_file(name, 0444, dentry, mp, ++ &maxlatproc_fops); ++ clear_maxlatprocdata(mp); ++ ++ mp = &per_cpu(wakeup_maxlatproc_sharedprio, i); ++ entry = debugfs_create_file(name, 0444, dentry_sharedprio, mp, ++ &maxlatproc_fops); ++ clear_maxlatprocdata(mp); + } -+ migrate_enable(); ++ entry = debugfs_create_file("pid", 0644, dentry, ++ (void *)&wakeup_pid, &pid_fops); ++ entry = debugfs_create_file("reset", 0644, dentry, ++ (void *)WAKEUP_LATENCY, &latency_hist_reset_fops); ++ entry = debugfs_create_file("reset", 0644, dentry_sharedprio, ++ (void *)WAKEUP_LATENCY_SHAREDPRIO, &latency_hist_reset_fops); ++ entry = debugfs_create_file("wakeup", 0644, ++ enable_root, (void *)&wakeup_latency_enabled_data, ++ &enable_fops); ++#endif + - cpu_hotplug_begin(); -+ err = cpu_unplug_begin(cpu); -+ if (err) { -+ printk("cpu_unplug_begin(%d) failed\n", cpu); -+ goto out_cancel; ++#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST ++ dentry = debugfs_create_dir(missed_timer_offsets_dir, ++ latency_hist_root); ++ for_each_possible_cpu(i) { ++ sprintf(name, cpufmt, i); ++ entry = debugfs_create_file(name, 0444, dentry, ++ &per_cpu(missed_timer_offsets, i), &latency_hist_fops); ++ my_hist = &per_cpu(missed_timer_offsets, i); ++ atomic_set(&my_hist->hist_mode, 1); ++ my_hist->min_lat = LONG_MAX; ++ ++ sprintf(name, cpufmt_maxlatproc, i); ++ mp = &per_cpu(missed_timer_offsets_maxlatproc, i); ++ entry = debugfs_create_file(name, 0444, dentry, mp, ++ &maxlatproc_fops); ++ clear_maxlatprocdata(mp); ++ } ++ entry = debugfs_create_file("pid", 0644, dentry, ++ (void *)&missed_timer_offsets_pid, &pid_fops); ++ entry = debugfs_create_file("reset", 0644, dentry, ++ (void *)MISSED_TIMER_OFFSETS, &latency_hist_reset_fops); ++ entry = debugfs_create_file("missed_timer_offsets", 0644, ++ enable_root, (void *)&missed_timer_offsets_enabled_data, ++ &enable_fops); ++#endif ++ ++#if defined(CONFIG_WAKEUP_LATENCY_HIST) && \ ++ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) ++ dentry = debugfs_create_dir(timerandwakeup_latency_hist_dir, ++ latency_hist_root); ++ for_each_possible_cpu(i) { ++ sprintf(name, cpufmt, i); ++ entry = debugfs_create_file(name, 0444, dentry, ++ &per_cpu(timerandwakeup_latency_hist, i), ++ &latency_hist_fops); ++ my_hist = &per_cpu(timerandwakeup_latency_hist, i); ++ atomic_set(&my_hist->hist_mode, 1); ++ my_hist->min_lat = LONG_MAX; ++ ++ sprintf(name, cpufmt_maxlatproc, i); ++ mp = &per_cpu(timerandwakeup_maxlatproc, i); ++ entry = debugfs_create_file(name, 0444, dentry, mp, ++ &maxlatproc_fops); ++ clear_maxlatprocdata(mp); + } ++ entry = debugfs_create_file("reset", 0644, dentry, ++ (void *)TIMERANDWAKEUP_LATENCY, &latency_hist_reset_fops); ++ entry = debugfs_create_file("timerandwakeup", 0644, ++ enable_root, (void *)&timerandwakeup_enabled_data, ++ &enable_fops); ++#endif ++ return 0; ++} ++ ++device_initcall(latency_hist_init); +diff -Nur linux-4.1.10.orig/kernel/trace/Makefile linux-4.1.10/kernel/trace/Makefile +--- linux-4.1.10.orig/kernel/trace/Makefile 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/trace/Makefile 2015-10-12 22:33:32.316674111 +0200 +@@ -36,6 +36,10 @@ + obj-$(CONFIG_IRQSOFF_TRACER) += trace_irqsoff.o + obj-$(CONFIG_PREEMPT_TRACER) += trace_irqsoff.o + obj-$(CONFIG_SCHED_TRACER) += trace_sched_wakeup.o ++obj-$(CONFIG_INTERRUPT_OFF_HIST) += latency_hist.o ++obj-$(CONFIG_PREEMPT_OFF_HIST) += latency_hist.o ++obj-$(CONFIG_WAKEUP_LATENCY_HIST) += latency_hist.o ++obj-$(CONFIG_MISSED_TIMER_OFFSETS_HIST) += latency_hist.o + obj-$(CONFIG_NOP_TRACER) += trace_nop.o + obj-$(CONFIG_STACK_TRACER) += trace_stack.o + obj-$(CONFIG_MMIOTRACE) += trace_mmiotrace.o +diff -Nur linux-4.1.10.orig/kernel/trace/trace.c linux-4.1.10/kernel/trace/trace.c +--- linux-4.1.10.orig/kernel/trace/trace.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/trace/trace.c 2015-10-12 22:33:32.316674111 +0200 +@@ -1630,6 +1630,7 @@ + struct task_struct *tsk = current; - err = __cpu_notify(CPU_DOWN_PREPARE | mod, hcpu, -1, &nr_calls); - if (err) { -@@ -389,8 +701,12 @@ + entry->preempt_count = pc & 0xff; ++ entry->preempt_lazy_count = preempt_lazy_count(); + entry->pid = (tsk) ? tsk->pid : 0; + entry->flags = + #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT +@@ -1639,8 +1640,11 @@ #endif - synchronize_rcu(); - -+ __cpu_unplug_wait(cpu); - smpboot_park_threads(cpu); - -+ /* Notifiers are done. Don't let any more tasks pin this CPU. */ -+ cpu_unplug_sync(cpu); + ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) | + ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) | +- (tif_need_resched() ? TRACE_FLAG_NEED_RESCHED : 0) | ++ (tif_need_resched_now() ? TRACE_FLAG_NEED_RESCHED : 0) | ++ (need_resched_lazy() ? TRACE_FLAG_NEED_RESCHED_LAZY : 0) | + (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED : 0); + - /* - * So now all preempt/rcu users must observe !cpu_active(). - */ -@@ -427,9 +743,14 @@ - check_for_tasks(cpu); - - out_release: -+ cpu_unplug_done(cpu); -+out_cancel: - cpu_hotplug_done(); - if (!err) - cpu_notify_nofail(CPU_POST_DEAD | mod, hcpu); -+restore_cpus: -+ set_cpus_allowed_ptr(current, cpumask_org); -+ free_cpumask_var(cpumask_org); - return err; ++ entry->migrate_disable = (tsk) ? __migrate_disabled(tsk) & 0xFF : 0; } + EXPORT_SYMBOL_GPL(tracing_generic_entry_update); -diff -Nur linux-4.1.10.orig/kernel/debug/kdb/kdb_io.c linux-4.1.10/kernel/debug/kdb/kdb_io.c ---- linux-4.1.10.orig/kernel/debug/kdb/kdb_io.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/debug/kdb/kdb_io.c 2015-10-07 18:00:08.000000000 +0200 -@@ -554,7 +554,6 @@ - int linecount; - int colcount; - int logging, saved_loglevel = 0; -- int saved_trap_printk; - int got_printf_lock = 0; - int retlen = 0; - int fnd, len; -@@ -565,8 +564,6 @@ - unsigned long uninitialized_var(flags); +@@ -2558,14 +2562,17 @@ - preempt_disable(); -- saved_trap_printk = kdb_trap_printk; -- kdb_trap_printk = 0; + static void print_lat_help_header(struct seq_file *m) + { +- seq_puts(m, "# _------=> CPU# \n" +- "# / _-----=> irqs-off \n" +- "# | / _----=> need-resched \n" +- "# || / _---=> hardirq/softirq \n" +- "# ||| / _--=> preempt-depth \n" +- "# |||| / delay \n" +- "# cmd pid ||||| time | caller \n" +- "# \\ / ||||| \\ | / \n"); ++ seq_puts(m, "# _--------=> CPU# \n" ++ "# / _-------=> irqs-off \n" ++ "# | / _------=> need-resched \n" ++ "# || / _-----=> need-resched_lazy \n" ++ "# ||| / _----=> hardirq/softirq \n" ++ "# |||| / _---=> preempt-depth \n" ++ "# ||||| / _--=> preempt-lazy-depth\n" ++ "# |||||| / _-=> migrate-disable \n" ++ "# ||||||| / delay \n" ++ "# cmd pid |||||||| time | caller \n" ++ "# \\ / |||||||| \\ | / \n"); + } - /* Serialize kdb_printf if multiple cpus try to write at once. - * But if any cpu goes recursive in kdb, just print the output, -@@ -855,7 +852,6 @@ - } else { - __release(kdb_printf_lock); - } -- kdb_trap_printk = saved_trap_printk; - preempt_enable(); - return retlen; + static void print_event_info(struct trace_buffer *buf, struct seq_file *m) +@@ -2591,11 +2598,14 @@ + print_event_info(buf, m); + seq_puts(m, "# _-----=> irqs-off\n" + "# / _----=> need-resched\n" +- "# | / _---=> hardirq/softirq\n" +- "# || / _--=> preempt-depth\n" +- "# ||| / delay\n" +- "# TASK-PID CPU# |||| TIMESTAMP FUNCTION\n" +- "# | | | |||| | |\n"); ++ "# |/ _-----=> need-resched_lazy\n" ++ "# || / _---=> hardirq/softirq\n" ++ "# ||| / _--=> preempt-depth\n" ++ "# |||| /_--=> preempt-lazy-depth\n" ++ "# ||||| _-=> migrate-disable \n" ++ "# ||||| / delay\n" ++ "# TASK-PID CPU# |||||| TIMESTAMP FUNCTION\n" ++ "# | | | |||||| | |\n"); } -@@ -865,9 +861,11 @@ - va_list ap; - int r; -+ kdb_trap_printk++; - va_start(ap, fmt); - r = vkdb_printf(KDB_MSGSRC_INTERNAL, fmt, ap); - va_end(ap); -+ kdb_trap_printk--; + void +diff -Nur linux-4.1.10.orig/kernel/trace/trace_events.c linux-4.1.10/kernel/trace/trace_events.c +--- linux-4.1.10.orig/kernel/trace/trace_events.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/trace/trace_events.c 2015-10-12 22:33:32.320673847 +0200 +@@ -162,6 +162,8 @@ + __common_field(unsigned char, flags); + __common_field(unsigned char, preempt_count); + __common_field(int, pid); ++ __common_field(unsigned short, migrate_disable); ++ __common_field(unsigned short, padding); - return r; + return ret; } -diff -Nur linux-4.1.10.orig/kernel/events/core.c linux-4.1.10/kernel/events/core.c ---- linux-4.1.10.orig/kernel/events/core.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/events/core.c 2015-10-07 18:00:08.000000000 +0200 -@@ -6933,6 +6933,7 @@ - - hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); - hwc->hrtimer.function = perf_swevent_hrtimer; -+ hwc->hrtimer.irqsafe = 1; +diff -Nur linux-4.1.10.orig/kernel/trace/trace.h linux-4.1.10/kernel/trace/trace.h +--- linux-4.1.10.orig/kernel/trace/trace.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/trace/trace.h 2015-10-12 22:33:32.320673847 +0200 +@@ -120,6 +120,7 @@ + * NEED_RESCHED - reschedule is requested + * HARDIRQ - inside an interrupt handler + * SOFTIRQ - inside a softirq handler ++ * NEED_RESCHED_LAZY - lazy reschedule is requested + */ + enum trace_flag_type { + TRACE_FLAG_IRQS_OFF = 0x01, +@@ -128,6 +129,7 @@ + TRACE_FLAG_HARDIRQ = 0x08, + TRACE_FLAG_SOFTIRQ = 0x10, + TRACE_FLAG_PREEMPT_RESCHED = 0x20, ++ TRACE_FLAG_NEED_RESCHED_LAZY = 0x40, + }; - /* - * Since hrtimers have a fixed rate, we can do a static freq->period -diff -Nur linux-4.1.10.orig/kernel/exit.c linux-4.1.10/kernel/exit.c ---- linux-4.1.10.orig/kernel/exit.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/exit.c 2015-10-07 18:00:08.000000000 +0200 -@@ -144,7 +144,7 @@ - * Do this under ->siglock, we can race with another thread - * doing sigqueue_free() if we have SIGQUEUE_PREALLOC signals. - */ -- flush_sigqueue(&tsk->pending); -+ flush_task_sigqueue(tsk); - tsk->sighand = NULL; - spin_unlock(&sighand->siglock); + #define TRACE_BUF_SIZE 1024 +diff -Nur linux-4.1.10.orig/kernel/trace/trace_irqsoff.c linux-4.1.10/kernel/trace/trace_irqsoff.c +--- linux-4.1.10.orig/kernel/trace/trace_irqsoff.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/trace/trace_irqsoff.c 2015-10-12 22:33:32.320673847 +0200 +@@ -13,6 +13,7 @@ + #include + #include + #include ++#include -diff -Nur linux-4.1.10.orig/kernel/fork.c linux-4.1.10/kernel/fork.c ---- linux-4.1.10.orig/kernel/fork.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/fork.c 2015-10-07 18:00:08.000000000 +0200 -@@ -108,7 +108,7 @@ + #include "trace.h" - DEFINE_PER_CPU(unsigned long, process_counts) = 0; +@@ -433,11 +434,13 @@ + { + if (preempt_trace() || irq_trace()) + start_critical_timing(CALLER_ADDR0, CALLER_ADDR1); ++ trace_preemptirqsoff_hist(TRACE_START, 1); + } + EXPORT_SYMBOL_GPL(start_critical_timings); --__cacheline_aligned DEFINE_RWLOCK(tasklist_lock); /* outer */ -+DEFINE_RWLOCK(tasklist_lock); /* outer */ + void stop_critical_timings(void) + { ++ trace_preemptirqsoff_hist(TRACE_STOP, 0); + if (preempt_trace() || irq_trace()) + stop_critical_timing(CALLER_ADDR0, CALLER_ADDR1); + } +@@ -447,6 +450,7 @@ + #ifdef CONFIG_PROVE_LOCKING + void time_hardirqs_on(unsigned long a0, unsigned long a1) + { ++ trace_preemptirqsoff_hist_rcuidle(IRQS_ON, 0); + if (!preempt_trace() && irq_trace()) + stop_critical_timing(a0, a1); + } +@@ -455,6 +459,7 @@ + { + if (!preempt_trace() && irq_trace()) + start_critical_timing(a0, a1); ++ trace_preemptirqsoff_hist_rcuidle(IRQS_OFF, 1); + } - #ifdef CONFIG_PROVE_RCU - int lockdep_tasklist_lock_is_held(void) -@@ -244,7 +244,9 @@ - if (atomic_dec_and_test(&sig->sigcnt)) - free_signal_struct(sig); + #else /* !CONFIG_PROVE_LOCKING */ +@@ -480,6 +485,7 @@ + */ + void trace_hardirqs_on(void) + { ++ trace_preemptirqsoff_hist(IRQS_ON, 0); + if (!preempt_trace() && irq_trace()) + stop_critical_timing(CALLER_ADDR0, CALLER_ADDR1); } -- -+#ifdef CONFIG_PREEMPT_RT_BASE -+static -+#endif - void __put_task_struct(struct task_struct *tsk) +@@ -489,11 +495,13 @@ { - WARN_ON(!tsk->exit_state); -@@ -260,7 +262,18 @@ - if (!profile_handoff_task(tsk)) - free_task(tsk); + if (!preempt_trace() && irq_trace()) + start_critical_timing(CALLER_ADDR0, CALLER_ADDR1); ++ trace_preemptirqsoff_hist(IRQS_OFF, 1); } -+#ifndef CONFIG_PREEMPT_RT_BASE - EXPORT_SYMBOL_GPL(__put_task_struct); -+#else -+void __put_task_struct_cb(struct rcu_head *rhp) -+{ -+ struct task_struct *tsk = container_of(rhp, struct task_struct, put_rcu); -+ -+ __put_task_struct(tsk); -+ -+} -+EXPORT_SYMBOL_GPL(__put_task_struct_cb); -+#endif + EXPORT_SYMBOL(trace_hardirqs_off); - void __init __weak arch_task_cache_init(void) { } + __visible void trace_hardirqs_on_caller(unsigned long caller_addr) + { ++ trace_preemptirqsoff_hist(IRQS_ON, 0); + if (!preempt_trace() && irq_trace()) + stop_critical_timing(CALLER_ADDR0, caller_addr); + } +@@ -503,6 +511,7 @@ + { + if (!preempt_trace() && irq_trace()) + start_critical_timing(CALLER_ADDR0, caller_addr); ++ trace_preemptirqsoff_hist(IRQS_OFF, 1); + } + EXPORT_SYMBOL(trace_hardirqs_off_caller); -@@ -680,6 +693,19 @@ +@@ -512,12 +521,14 @@ + #ifdef CONFIG_PREEMPT_TRACER + void trace_preempt_on(unsigned long a0, unsigned long a1) + { ++ trace_preemptirqsoff_hist(PREEMPT_ON, 0); + if (preempt_trace() && !irq_trace()) + stop_critical_timing(a0, a1); } - EXPORT_SYMBOL_GPL(__mmdrop); -+#ifdef CONFIG_PREEMPT_RT_BASE -+/* -+ * RCU callback for delayed mm drop. Not strictly rcu, but we don't -+ * want another facility to make this work. -+ */ -+void __mmdrop_delayed(struct rcu_head *rhp) -+{ -+ struct mm_struct *mm = container_of(rhp, struct mm_struct, delayed_drop); -+ -+ __mmdrop(mm); -+} -+#endif -+ - /* - * Decrement the use count and release all resources for an mm. - */ -@@ -1214,6 +1240,9 @@ - */ - static void posix_cpu_timers_init(struct task_struct *tsk) + void trace_preempt_off(unsigned long a0, unsigned long a1) { -+#ifdef CONFIG_PREEMPT_RT_BASE -+ tsk->posix_timer_list = NULL; -+#endif - tsk->cputime_expires.prof_exp = 0; - tsk->cputime_expires.virt_exp = 0; - tsk->cputime_expires.sched_exp = 0; -@@ -1338,6 +1367,7 @@ - spin_lock_init(&p->alloc_lock); ++ trace_preemptirqsoff_hist(PREEMPT_ON, 1); + if (preempt_trace() && !irq_trace()) + start_critical_timing(a0, a1); + } +diff -Nur linux-4.1.10.orig/kernel/trace/trace_output.c linux-4.1.10/kernel/trace/trace_output.c +--- linux-4.1.10.orig/kernel/trace/trace_output.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/trace/trace_output.c 2015-10-12 22:33:32.320673847 +0200 +@@ -430,6 +430,7 @@ + { + char hardsoft_irq; + char need_resched; ++ char need_resched_lazy; + char irqs_off; + int hardirq; + int softirq; +@@ -457,6 +458,8 @@ + need_resched = '.'; + break; + } ++ need_resched_lazy = ++ (entry->flags & TRACE_FLAG_NEED_RESCHED_LAZY) ? 'L' : '.'; - init_sigpending(&p->pending); -+ p->sigqueue_cache = NULL; + hardsoft_irq = + (hardirq && softirq) ? 'H' : +@@ -464,14 +467,25 @@ + softirq ? 's' : + '.'; - p->utime = p->stime = p->gtime = 0; - p->utimescaled = p->stimescaled = 0; -@@ -1345,7 +1375,8 @@ - p->prev_cputime.utime = p->prev_cputime.stime = 0; - #endif - #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN -- seqlock_init(&p->vtime_seqlock); -+ raw_spin_lock_init(&p->vtime_lock); -+ seqcount_init(&p->vtime_seq); - p->vtime_snap = 0; - p->vtime_snap_whence = VTIME_SLEEPING; - #endif -@@ -1396,6 +1427,9 @@ - p->hardirq_context = 0; - p->softirq_context = 0; - #endif -+ -+ p->pagefault_disabled = 0; -+ - #ifdef CONFIG_LOCKDEP - p->lockdep_depth = 0; /* no locks held yet */ - p->curr_chain_key = 0; -diff -Nur linux-4.1.10.orig/kernel/futex.c linux-4.1.10/kernel/futex.c ---- linux-4.1.10.orig/kernel/futex.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/futex.c 2015-10-07 18:00:08.000000000 +0200 -@@ -738,7 +738,9 @@ - * task still owns the PI-state: - */ - if (head->next != next) { -+ raw_spin_unlock_irq(&curr->pi_lock); - spin_unlock(&hb->lock); -+ raw_spin_lock_irq(&curr->pi_lock); - continue; - } +- trace_seq_printf(s, "%c%c%c", +- irqs_off, need_resched, hardsoft_irq); ++ trace_seq_printf(s, "%c%c%c%c", ++ irqs_off, need_resched, need_resched_lazy, ++ hardsoft_irq); -@@ -1090,9 +1092,11 @@ + if (entry->preempt_count) + trace_seq_printf(s, "%x", entry->preempt_count); + else + trace_seq_putc(s, '.'); - /* - * The hash bucket lock must be held when this is called. -- * Afterwards, the futex_q must not be accessed. -+ * Afterwards, the futex_q must not be accessed. Callers -+ * must ensure to later call wake_up_q() for the actual -+ * wakeups to occur. - */ --static void wake_futex(struct futex_q *q) -+static void mark_wake_futex(struct wake_q_head *wake_q, struct futex_q *q) - { - struct task_struct *p = q->task; ++ if (entry->preempt_lazy_count) ++ trace_seq_printf(s, "%x", entry->preempt_lazy_count); ++ else ++ trace_seq_putc(s, '.'); ++ ++ if (entry->migrate_disable) ++ trace_seq_printf(s, "%x", entry->migrate_disable); ++ else ++ trace_seq_putc(s, '.'); ++ + return !trace_seq_has_overflowed(s); + } -@@ -1100,14 +1104,10 @@ +diff -Nur linux-4.1.10.orig/kernel/user.c linux-4.1.10/kernel/user.c +--- linux-4.1.10.orig/kernel/user.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/user.c 2015-10-12 22:33:32.320673847 +0200 +@@ -161,11 +161,11 @@ + if (!up) return; - /* -- * We set q->lock_ptr = NULL _before_ we wake up the task. If -- * a non-futex wake up happens on another CPU then the task -- * might exit and p would dereference a non-existing task -- * struct. Prevent this by holding a reference on p across the -- * wake up. -+ * Queue the task for later wakeup for after we've released -+ * the hb->lock. wake_q_add() grabs reference to p. - */ -- get_task_struct(p); -- -+ wake_q_add(wake_q, p); - __unqueue_futex(q); - /* - * The waiting task can free the futex_q as soon as -@@ -1117,16 +1117,15 @@ - */ - smp_wmb(); - q->lock_ptr = NULL; -- -- wake_up_state(p, TASK_NORMAL); -- put_task_struct(p); +- local_irq_save(flags); ++ local_irq_save_nort(flags); + if (atomic_dec_and_lock(&up->__count, &uidhash_lock)) + free_user(up, flags); + else +- local_irq_restore(flags); ++ local_irq_restore_nort(flags); } --static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_q *this) -+static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_q *this, -+ struct futex_hash_bucket *hb) - { - struct task_struct *new_owner; - struct futex_pi_state *pi_state = this->pi_state; - u32 uninitialized_var(curval), newval; -+ bool deboost; - int ret = 0; + struct user_struct *alloc_uid(kuid_t uid) +diff -Nur linux-4.1.10.orig/kernel/watchdog.c linux-4.1.10/kernel/watchdog.c +--- linux-4.1.10.orig/kernel/watchdog.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/watchdog.c 2015-10-12 22:33:32.320673847 +0200 +@@ -262,6 +262,8 @@ - if (!pi_state) -@@ -1178,7 +1177,17 @@ - raw_spin_unlock_irq(&new_owner->pi_lock); + #ifdef CONFIG_HARDLOCKUP_DETECTOR - raw_spin_unlock(&pi_state->pi_mutex.wait_lock); -- rt_mutex_unlock(&pi_state->pi_mutex); -+ -+ deboost = rt_mutex_futex_unlock(&pi_state->pi_mutex); -+ -+ /* -+ * We deboost after dropping hb->lock. That prevents a double -+ * wakeup on RT. -+ */ -+ spin_unlock(&hb->lock); ++static DEFINE_RAW_SPINLOCK(watchdog_output_lock); + -+ if (deboost) -+ rt_mutex_adjust_prio(current); - - return 0; - } -@@ -1217,6 +1226,7 @@ - struct futex_q *this, *next; - union futex_key key = FUTEX_KEY_INIT; - int ret; -+ WAKE_Q(wake_q); - - if (!bitset) - return -EINVAL; -@@ -1244,13 +1254,14 @@ - if (!(this->bitset & bitset)) - continue; - -- wake_futex(this); -+ mark_wake_futex(&wake_q, this); - if (++ret >= nr_wake) - break; - } - } + static struct perf_event_attr wd_hw_attr = { + .type = PERF_TYPE_HARDWARE, + .config = PERF_COUNT_HW_CPU_CYCLES, +@@ -295,13 +297,21 @@ + /* only print hardlockups once */ + if (__this_cpu_read(hard_watchdog_warn) == true) + return; ++ /* ++ * If early-printk is enabled then make sure we do not ++ * lock up in printk() and kill console logging: ++ */ ++ printk_kill(); - spin_unlock(&hb->lock); -+ wake_up_q(&wake_q); - out_put_key: - put_futex_key(&key); - out: -@@ -1269,6 +1280,7 @@ - struct futex_hash_bucket *hb1, *hb2; - struct futex_q *this, *next; - int ret, op_ret; -+ WAKE_Q(wake_q); +- if (hardlockup_panic) ++ if (hardlockup_panic) { + panic("Watchdog detected hard LOCKUP on cpu %d", + this_cpu); +- else ++ } else { ++ raw_spin_lock(&watchdog_output_lock); + WARN(1, "Watchdog detected hard LOCKUP on cpu %d", + this_cpu); ++ raw_spin_unlock(&watchdog_output_lock); ++ } - retry: - ret = get_futex_key(uaddr1, flags & FLAGS_SHARED, &key1, VERIFY_READ); -@@ -1320,7 +1332,7 @@ - ret = -EINVAL; - goto out_unlock; - } -- wake_futex(this); -+ mark_wake_futex(&wake_q, this); - if (++ret >= nr_wake) - break; - } -@@ -1334,7 +1346,7 @@ - ret = -EINVAL; - goto out_unlock; - } -- wake_futex(this); -+ mark_wake_futex(&wake_q, this); - if (++op_ret >= nr_wake2) - break; - } -@@ -1344,6 +1356,7 @@ + __this_cpu_write(hard_watchdog_warn, true); + return; +@@ -444,6 +454,7 @@ + /* kick off the timer for the hardlockup detector */ + hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + hrtimer->function = watchdog_timer_fn; ++ hrtimer->irqsafe = 1; - out_unlock: - double_unlock_hb(hb1, hb2); -+ wake_up_q(&wake_q); - out_put_keys: - put_futex_key(&key2); - out_put_key1: -@@ -1503,6 +1516,7 @@ - struct futex_pi_state *pi_state = NULL; - struct futex_hash_bucket *hb1, *hb2; - struct futex_q *this, *next; -+ WAKE_Q(wake_q); + /* Enable the perf event */ + watchdog_nmi_enable(cpu); +diff -Nur linux-4.1.10.orig/kernel/workqueue.c linux-4.1.10/kernel/workqueue.c +--- linux-4.1.10.orig/kernel/workqueue.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/workqueue.c 2015-10-12 22:33:32.320673847 +0200 +@@ -48,6 +48,8 @@ + #include + #include + #include ++#include ++#include - if (requeue_pi) { - /* -@@ -1679,7 +1693,7 @@ - * woken by futex_unlock_pi(). - */ - if (++task_count <= nr_wake && !requeue_pi) { -- wake_futex(this); -+ mark_wake_futex(&wake_q, this); - continue; - } + #include "workqueue_internal.h" -@@ -1705,6 +1719,16 @@ - requeue_pi_wake_futex(this, &key2, hb2); - drop_count++; - continue; -+ } else if (ret == -EAGAIN) { -+ /* -+ * Waiter was woken by timeout or -+ * signal and has set pi_blocked_on to -+ * PI_WAKEUP_INPROGRESS before we -+ * tried to enqueue it on the rtmutex. -+ */ -+ this->pi_state = NULL; -+ free_pi_state(pi_state); -+ continue; - } else if (ret) { - /* -EDEADLK */ - this->pi_state = NULL; -@@ -1719,6 +1743,7 @@ - out_unlock: - free_pi_state(pi_state); - double_unlock_hb(hb1, hb2); -+ wake_up_q(&wake_q); - hb_waiters_dec(hb2); +@@ -121,15 +123,20 @@ + * cpu or grabbing pool->lock is enough for read access. If + * POOL_DISASSOCIATED is set, it's identical to L. + * ++ * On RT we need the extra protection via rt_lock_idle_list() for ++ * the list manipulations against read access from ++ * wq_worker_sleeping(). All other places are nicely serialized via ++ * pool->lock. ++ * + * A: pool->attach_mutex protected. + * + * PL: wq_pool_mutex protected. + * +- * PR: wq_pool_mutex protected for writes. Sched-RCU protected for reads. ++ * PR: wq_pool_mutex protected for writes. RCU protected for reads. + * + * WQ: wq->mutex protected. + * +- * WR: wq->mutex protected for writes. Sched-RCU protected for reads. ++ * WR: wq->mutex protected for writes. RCU protected for reads. + * + * MD: wq_mayday_lock protected. + */ +@@ -178,7 +185,7 @@ + atomic_t nr_running ____cacheline_aligned_in_smp; /* -@@ -2412,13 +2437,26 @@ +- * Destruction of pool is sched-RCU protected to allow dereferences ++ * Destruction of pool is RCU protected to allow dereferences + * from get_work_pool(). */ - match = futex_top_waiter(hb, &key); - if (match) { -- ret = wake_futex_pi(uaddr, uval, match); -+ ret = wake_futex_pi(uaddr, uval, match, hb); -+ -+ /* -+ * In case of success wake_futex_pi dropped the hash -+ * bucket lock. -+ */ -+ if (!ret) -+ goto out_putkey; -+ - /* - * The atomic access to the futex value generated a - * pagefault, so retry the user-access and the wakeup: - */ - if (ret == -EFAULT) - goto pi_faulted; + struct rcu_head rcu; +@@ -207,7 +214,7 @@ + /* + * Release of unbound pwq is punted to system_wq. See put_pwq() + * and pwq_unbound_release_workfn() for details. pool_workqueue +- * itself is also sched-RCU protected so that the first pwq can be ++ * itself is also RCU protected so that the first pwq can be + * determined without grabbing wq->mutex. + */ + struct work_struct unbound_release_work; +@@ -329,6 +336,8 @@ + struct workqueue_struct *system_freezable_power_efficient_wq __read_mostly; + EXPORT_SYMBOL_GPL(system_freezable_power_efficient_wq); + ++static DEFINE_LOCAL_IRQ_LOCK(pendingb_lock); + -+ /* -+ * wake_futex_pi has detected invalid state. Tell user -+ * space. -+ */ - goto out_unlock; - } + static int worker_thread(void *__worker); + static void copy_workqueue_attrs(struct workqueue_attrs *to, + const struct workqueue_attrs *from); +@@ -338,14 +347,14 @@ + #include -@@ -2439,6 +2477,7 @@ + #define assert_rcu_or_pool_mutex() \ +- rcu_lockdep_assert(rcu_read_lock_sched_held() || \ ++ rcu_lockdep_assert(rcu_read_lock_held() || \ + lockdep_is_held(&wq_pool_mutex), \ +- "sched RCU or wq_pool_mutex should be held") ++ "RCU or wq_pool_mutex should be held") - out_unlock: - spin_unlock(&hb->lock); -+out_putkey: - put_futex_key(&key); - return ret; + #define assert_rcu_or_wq_mutex(wq) \ +- rcu_lockdep_assert(rcu_read_lock_sched_held() || \ ++ rcu_lockdep_assert(rcu_read_lock_held() || \ + lockdep_is_held(&wq->mutex), \ +- "sched RCU or wq->mutex should be held") ++ "RCU or wq->mutex should be held") -@@ -2549,7 +2588,7 @@ - struct hrtimer_sleeper timeout, *to = NULL; - struct rt_mutex_waiter rt_waiter; - struct rt_mutex *pi_mutex = NULL; -- struct futex_hash_bucket *hb; -+ struct futex_hash_bucket *hb, *hb2; - union futex_key key2 = FUTEX_KEY_INIT; - struct futex_q q = futex_q_init; - int res, ret; -@@ -2574,10 +2613,7 @@ - * The waiter is allocated on our stack, manipulated by the requeue - * code while we sleep on uaddr. - */ -- debug_rt_mutex_init_waiter(&rt_waiter); -- RB_CLEAR_NODE(&rt_waiter.pi_tree_entry); -- RB_CLEAR_NODE(&rt_waiter.tree_entry); -- rt_waiter.task = NULL; -+ rt_mutex_init_waiter(&rt_waiter, false); + #define for_each_cpu_worker_pool(pool, cpu) \ + for ((pool) = &per_cpu(cpu_worker_pools, cpu)[0]; \ +@@ -357,7 +366,7 @@ + * @pool: iteration cursor + * @pi: integer used for iteration + * +- * This must be called either with wq_pool_mutex held or sched RCU read ++ * This must be called either with wq_pool_mutex held or RCU read + * locked. If the pool needs to be used beyond the locking in effect, the + * caller is responsible for guaranteeing that the pool stays online. + * +@@ -389,7 +398,7 @@ + * @pwq: iteration cursor + * @wq: the target workqueue + * +- * This must be called either with wq->mutex held or sched RCU read locked. ++ * This must be called either with wq->mutex held or RCU read locked. + * If the pwq needs to be used beyond the locking in effect, the caller is + * responsible for guaranteeing that the pwq stays online. + * +@@ -401,6 +410,31 @@ + if (({ assert_rcu_or_wq_mutex(wq); false; })) { } \ + else - ret = get_futex_key(uaddr2, flags & FLAGS_SHARED, &key2, VERIFY_WRITE); - if (unlikely(ret != 0)) -@@ -2608,20 +2644,55 @@ - /* Queue the futex_q, drop the hb lock, wait for wakeup. */ - futex_wait_queue_me(hb, &q, to); ++#ifdef CONFIG_PREEMPT_RT_BASE ++static inline void rt_lock_idle_list(struct worker_pool *pool) ++{ ++ preempt_disable(); ++} ++static inline void rt_unlock_idle_list(struct worker_pool *pool) ++{ ++ preempt_enable(); ++} ++static inline void sched_lock_idle_list(struct worker_pool *pool) { } ++static inline void sched_unlock_idle_list(struct worker_pool *pool) { } ++#else ++static inline void rt_lock_idle_list(struct worker_pool *pool) { } ++static inline void rt_unlock_idle_list(struct worker_pool *pool) { } ++static inline void sched_lock_idle_list(struct worker_pool *pool) ++{ ++ spin_lock_irq(&pool->lock); ++} ++static inline void sched_unlock_idle_list(struct worker_pool *pool) ++{ ++ spin_unlock_irq(&pool->lock); ++} ++#endif ++ ++ + #ifdef CONFIG_DEBUG_OBJECTS_WORK -- spin_lock(&hb->lock); -- ret = handle_early_requeue_pi_wakeup(hb, &q, &key2, to); -- spin_unlock(&hb->lock); -- if (ret) -- goto out_put_keys; -+ /* -+ * On RT we must avoid races with requeue and trying to block -+ * on two mutexes (hb->lock and uaddr2's rtmutex) by -+ * serializing access to pi_blocked_on with pi_lock. -+ */ -+ raw_spin_lock_irq(¤t->pi_lock); -+ if (current->pi_blocked_on) { -+ /* -+ * We have been requeued or are in the process of -+ * being requeued. -+ */ -+ raw_spin_unlock_irq(¤t->pi_lock); -+ } else { -+ /* -+ * Setting pi_blocked_on to PI_WAKEUP_INPROGRESS -+ * prevents a concurrent requeue from moving us to the -+ * uaddr2 rtmutex. After that we can safely acquire -+ * (and possibly block on) hb->lock. -+ */ -+ current->pi_blocked_on = PI_WAKEUP_INPROGRESS; -+ raw_spin_unlock_irq(¤t->pi_lock); + static struct debug_obj_descr work_debug_descr; +@@ -551,7 +585,7 @@ + * @wq: the target workqueue + * @node: the node ID + * +- * This must be called either with pwq_lock held or sched RCU read locked. ++ * This must be called either with pwq_lock held or RCU read locked. + * If the pwq needs to be used beyond the locking in effect, the caller is + * responsible for guaranteeing that the pwq stays online. + * +@@ -655,8 +689,8 @@ + * @work: the work item of interest + * + * Pools are created and destroyed under wq_pool_mutex, and allows read +- * access under sched-RCU read lock. As such, this function should be +- * called under wq_pool_mutex or with preemption disabled. ++ * access under RCU read lock. As such, this function should be ++ * called under wq_pool_mutex or inside of a rcu_read_lock() region. + * + * All fields of the returned pool are accessible as long as the above + * mentioned locking is in effect. If the returned pool needs to be used +@@ -793,51 +827,44 @@ + */ + static void wake_up_worker(struct worker_pool *pool) + { +- struct worker *worker = first_idle_worker(pool); ++ struct worker *worker; + -+ spin_lock(&hb->lock); ++ rt_lock_idle_list(pool); + -+ /* -+ * Clean up pi_blocked_on. We might leak it otherwise -+ * when we succeeded with the hb->lock in the fast -+ * path. -+ */ -+ raw_spin_lock_irq(¤t->pi_lock); -+ current->pi_blocked_on = NULL; -+ raw_spin_unlock_irq(¤t->pi_lock); ++ worker = first_idle_worker(pool); + + if (likely(worker)) + wake_up_process(worker->task); + -+ ret = handle_early_requeue_pi_wakeup(hb, &q, &key2, to); -+ spin_unlock(&hb->lock); -+ if (ret) -+ goto out_put_keys; -+ } ++ rt_unlock_idle_list(pool); + } - /* -- * In order for us to be here, we know our q.key == key2, and since -- * we took the hb->lock above, we also know that futex_requeue() has -- * completed and we no longer have to concern ourselves with a wakeup -- * race with the atomic proxy lock acquisition by the requeue code. The -- * futex_requeue dropped our key1 reference and incremented our key2 -- * reference count. -+ * In order to be here, we have either been requeued, are in -+ * the process of being requeued, or requeue successfully -+ * acquired uaddr2 on our behalf. If pi_blocked_on was -+ * non-null above, we may be racing with a requeue. Do not -+ * rely on q->lock_ptr to be hb2->lock until after blocking on -+ * hb->lock or hb2->lock. The futex_requeue dropped our key1 -+ * reference and incremented our key2 reference count. + /** +- * wq_worker_waking_up - a worker is waking up +- * @task: task waking up +- * @cpu: CPU @task is waking up to ++ * wq_worker_running - a worker is running again ++ * @task: task returning from sleep + * +- * This function is called during try_to_wake_up() when a worker is +- * being awoken. +- * +- * CONTEXT: +- * spin_lock_irq(rq->lock) ++ * This function is called when a worker returns from schedule() + */ +-void wq_worker_waking_up(struct task_struct *task, int cpu) ++void wq_worker_running(struct task_struct *task) + { + struct worker *worker = kthread_data(task); + +- if (!(worker->flags & WORKER_NOT_RUNNING)) { +- WARN_ON_ONCE(worker->pool->cpu != cpu); ++ if (!worker->sleeping) ++ return; ++ if (!(worker->flags & WORKER_NOT_RUNNING)) + atomic_inc(&worker->pool->nr_running); +- } ++ worker->sleeping = 0; + } + + /** + * wq_worker_sleeping - a worker is going to sleep + * @task: task going to sleep +- * @cpu: CPU in question, must be the current CPU number +- * +- * This function is called during schedule() when a busy worker is +- * going to sleep. Worker on the same cpu can be woken up by +- * returning pointer to its task. +- * +- * CONTEXT: +- * spin_lock_irq(rq->lock) +- * +- * Return: +- * Worker task on @cpu to wake up, %NULL if none. ++ * This function is called from schedule() when a busy worker is ++ * going to sleep. + */ +-struct task_struct *wq_worker_sleeping(struct task_struct *task, int cpu) ++void wq_worker_sleeping(struct task_struct *task) + { +- struct worker *worker = kthread_data(task), *to_wakeup = NULL; ++ struct worker *worker = kthread_data(task); + struct worker_pool *pool; + + /* +@@ -846,29 +873,26 @@ + * checking NOT_RUNNING. */ -+ hb2 = hash_futex(&key2); + if (worker->flags & WORKER_NOT_RUNNING) +- return NULL; ++ return; - /* Check if the requeue code acquired the second futex for us. */ - if (!q.rt_waiter) { -@@ -2630,9 +2701,10 @@ - * did a lock-steal - fix up the PI-state in that case. - */ - if (q.pi_state && (q.pi_state->owner != current)) { -- spin_lock(q.lock_ptr); -+ spin_lock(&hb2->lock); -+ BUG_ON(&hb2->lock != q.lock_ptr); - ret = fixup_pi_state_owner(uaddr2, &q, current); -- spin_unlock(q.lock_ptr); -+ spin_unlock(&hb2->lock); - } - } else { - /* -@@ -2645,7 +2717,8 @@ - ret = rt_mutex_finish_proxy_lock(pi_mutex, to, &rt_waiter); - debug_rt_mutex_free_waiter(&rt_waiter); + pool = worker->pool; -- spin_lock(q.lock_ptr); -+ spin_lock(&hb2->lock); -+ BUG_ON(&hb2->lock != q.lock_ptr); - /* - * Fixup the pi_state owner and possibly acquire the lock if we - * haven't already. -diff -Nur linux-4.1.10.orig/kernel/irq/handle.c linux-4.1.10/kernel/irq/handle.c ---- linux-4.1.10.orig/kernel/irq/handle.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/irq/handle.c 2015-10-07 18:00:08.000000000 +0200 -@@ -133,6 +133,8 @@ - irqreturn_t - handle_irq_event_percpu(struct irq_desc *desc, struct irqaction *action) +- /* this can only happen on the local cpu */ +- if (WARN_ON_ONCE(cpu != raw_smp_processor_id() || pool->cpu != cpu)) +- return NULL; ++ if (WARN_ON_ONCE(worker->sleeping)) ++ return; ++ ++ worker->sleeping = 1; + + /* + * The counterpart of the following dec_and_test, implied mb, + * worklist not empty test sequence is in insert_work(). + * Please read comment there. +- * +- * NOT_RUNNING is clear. This means that we're bound to and +- * running on the local cpu w/ rq lock held and preemption +- * disabled, which in turn means that none else could be +- * manipulating idle_list, so dereferencing idle_list without pool +- * lock is safe. + */ + if (atomic_dec_and_test(&pool->nr_running) && +- !list_empty(&pool->worklist)) +- to_wakeup = first_idle_worker(pool); +- return to_wakeup ? to_wakeup->task : NULL; ++ !list_empty(&pool->worklist)) { ++ sched_lock_idle_list(pool); ++ wake_up_worker(pool); ++ sched_unlock_idle_list(pool); ++ } + } + + /** +@@ -1062,12 +1086,12 @@ { -+ struct pt_regs *regs = get_irq_regs(); -+ u64 ip = regs ? instruction_pointer(regs) : 0; - irqreturn_t retval = IRQ_NONE; - unsigned int flags = 0, irq = desc->irq_data.irq; + if (pwq) { + /* +- * As both pwqs and pools are sched-RCU protected, the ++ * As both pwqs and pools are RCU protected, the + * following lock operations are safe. + */ +- spin_lock_irq(&pwq->pool->lock); ++ local_spin_lock_irq(pendingb_lock, &pwq->pool->lock); + put_pwq(pwq); +- spin_unlock_irq(&pwq->pool->lock); ++ local_spin_unlock_irq(pendingb_lock, &pwq->pool->lock); + } + } -@@ -173,7 +175,11 @@ - action = action->next; - } while (action); +@@ -1169,7 +1193,7 @@ + struct worker_pool *pool; + struct pool_workqueue *pwq; -- add_interrupt_randomness(irq, flags); -+#ifndef CONFIG_PREEMPT_RT_FULL -+ add_interrupt_randomness(irq, flags, ip); -+#else -+ desc->random_ip = ip; -+#endif +- local_irq_save(*flags); ++ local_lock_irqsave(pendingb_lock, *flags); - if (!noirqdebug) - note_interrupt(irq, desc, retval); -diff -Nur linux-4.1.10.orig/kernel/irq/manage.c linux-4.1.10/kernel/irq/manage.c ---- linux-4.1.10.orig/kernel/irq/manage.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/irq/manage.c 2015-10-07 18:00:08.000000000 +0200 -@@ -22,6 +22,7 @@ - #include "internals.h" + /* try to steal the timer if it exists */ + if (is_dwork) { +@@ -1188,6 +1212,7 @@ + if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) + return 0; - #ifdef CONFIG_IRQ_FORCED_THREADING -+# ifndef CONFIG_PREEMPT_RT_BASE - __read_mostly bool force_irqthreads; ++ rcu_read_lock(); + /* + * The queueing is in progress, or it is already queued. Try to + * steal it from ->worklist without clearing WORK_STRUCT_PENDING. +@@ -1226,14 +1251,16 @@ + set_work_pool_and_keep_pending(work, pool->id); - static int __init setup_forced_irqthreads(char *arg) -@@ -30,6 +31,7 @@ - return 0; + spin_unlock(&pool->lock); ++ rcu_read_unlock(); + return 1; + } + spin_unlock(&pool->lock); + fail: +- local_irq_restore(*flags); ++ rcu_read_unlock(); ++ local_unlock_irqrestore(pendingb_lock, *flags); + if (work_is_canceling(work)) + return -ENOENT; +- cpu_relax(); ++ cpu_chill(); + return -EAGAIN; } - early_param("threadirqs", setup_forced_irqthreads); -+# endif - #endif - static void __synchronize_hardirq(struct irq_desc *desc) -@@ -179,6 +181,62 @@ - irq_get_pending(struct cpumask *mask, struct irq_desc *desc) { } - #endif +@@ -1302,7 +1329,7 @@ + * queued or lose PENDING. Grabbing PENDING and queueing should + * happen with IRQ disabled. + */ +- WARN_ON_ONCE(!irqs_disabled()); ++ WARN_ON_ONCE_NONRT(!irqs_disabled()); -+#ifdef CONFIG_PREEMPT_RT_FULL -+static void _irq_affinity_notify(struct irq_affinity_notify *notify); -+static struct task_struct *set_affinity_helper; -+static LIST_HEAD(affinity_list); -+static DEFINE_RAW_SPINLOCK(affinity_list_lock); -+ -+static int set_affinity_thread(void *unused) -+{ -+ while (1) { -+ struct irq_affinity_notify *notify; -+ int empty; -+ -+ set_current_state(TASK_INTERRUPTIBLE); -+ -+ raw_spin_lock_irq(&affinity_list_lock); -+ empty = list_empty(&affinity_list); -+ raw_spin_unlock_irq(&affinity_list_lock); -+ -+ if (empty) -+ schedule(); -+ if (kthread_should_stop()) -+ break; -+ set_current_state(TASK_RUNNING); -+try_next: -+ notify = NULL; -+ -+ raw_spin_lock_irq(&affinity_list_lock); -+ if (!list_empty(&affinity_list)) { -+ notify = list_first_entry(&affinity_list, -+ struct irq_affinity_notify, list); -+ list_del_init(¬ify->list); -+ } -+ raw_spin_unlock_irq(&affinity_list_lock); -+ -+ if (!notify) -+ continue; -+ _irq_affinity_notify(notify); -+ goto try_next; -+ } -+ return 0; -+} -+ -+static void init_helper_thread(void) -+{ -+ if (set_affinity_helper) -+ return; -+ set_affinity_helper = kthread_run(set_affinity_thread, NULL, -+ "affinity-cb"); -+ WARN_ON(IS_ERR(set_affinity_helper)); -+} -+#else -+ -+static inline void init_helper_thread(void) { } -+ -+#endif -+ - int irq_do_set_affinity(struct irq_data *data, const struct cpumask *mask, - bool force) - { -@@ -218,7 +276,17 @@ + debug_work_activate(work); - if (desc->affinity_notify) { - kref_get(&desc->affinity_notify->kref); +@@ -1310,6 +1337,8 @@ + if (unlikely(wq->flags & __WQ_DRAINING) && + WARN_ON_ONCE(!is_chained_work(wq))) + return; + -+#ifdef CONFIG_PREEMPT_RT_FULL -+ raw_spin_lock(&affinity_list_lock); -+ if (list_empty(&desc->affinity_notify->list)) -+ list_add_tail(&affinity_list, -+ &desc->affinity_notify->list); -+ raw_spin_unlock(&affinity_list_lock); -+ wake_up_process(set_affinity_helper); -+#else - schedule_work(&desc->affinity_notify->work); -+#endif - } - irqd_set(data, IRQD_AFFINITY_SET); ++ rcu_read_lock(); + retry: + if (req_cpu == WORK_CPU_UNBOUND) + cpu = raw_smp_processor_id(); +@@ -1366,10 +1395,8 @@ + /* pwq determined, queue */ + trace_workqueue_queue_work(req_cpu, pwq, work); -@@ -256,10 +324,8 @@ +- if (WARN_ON(!list_empty(&work->entry))) { +- spin_unlock(&pwq->pool->lock); +- return; +- } ++ if (WARN_ON(!list_empty(&work->entry))) ++ goto out; + + pwq->nr_in_flight[pwq->work_color]++; + work_flags = work_color_to_flags(pwq->work_color); +@@ -1385,7 +1412,9 @@ + + insert_work(pwq, work, worklist, work_flags); + ++out: + spin_unlock(&pwq->pool->lock); ++ rcu_read_unlock(); } - EXPORT_SYMBOL_GPL(irq_set_affinity_hint); --static void irq_affinity_notify(struct work_struct *work) -+static void _irq_affinity_notify(struct irq_affinity_notify *notify) - { -- struct irq_affinity_notify *notify = -- container_of(work, struct irq_affinity_notify, work); - struct irq_desc *desc = irq_to_desc(notify->irq); - cpumask_var_t cpumask; + /** +@@ -1405,14 +1434,14 @@ + bool ret = false; unsigned long flags; -@@ -281,6 +347,13 @@ - kref_put(¬ify->kref, notify->release); + +- local_irq_save(flags); ++ local_lock_irqsave(pendingb_lock,flags); + + if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) { + __queue_work(cpu, wq, work); + ret = true; + } + +- local_irq_restore(flags); ++ local_unlock_irqrestore(pendingb_lock, flags); + return ret; } + EXPORT_SYMBOL(queue_work_on); +@@ -1479,14 +1508,14 @@ + unsigned long flags; -+static void irq_affinity_notify(struct work_struct *work) -+{ -+ struct irq_affinity_notify *notify = -+ container_of(work, struct irq_affinity_notify, work); -+ _irq_affinity_notify(notify); -+} -+ - /** - * irq_set_affinity_notifier - control notification of IRQ affinity changes - * @irq: Interrupt for which to enable/disable notification -@@ -310,6 +383,8 @@ - notify->irq = irq; - kref_init(¬ify->kref); - INIT_WORK(¬ify->work, irq_affinity_notify); -+ INIT_LIST_HEAD(¬ify->list); -+ init_helper_thread(); + /* read the comment in __queue_work() */ +- local_irq_save(flags); ++ local_lock_irqsave(pendingb_lock, flags); + + if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) { + __queue_delayed_work(cpu, wq, dwork, delay); + ret = true; } - raw_spin_lock_irqsave(&desc->lock, flags); -@@ -697,6 +772,12 @@ - return IRQ_NONE; +- local_irq_restore(flags); ++ local_unlock_irqrestore(pendingb_lock, flags); + return ret; } + EXPORT_SYMBOL(queue_delayed_work_on); +@@ -1521,7 +1550,7 @@ -+static irqreturn_t irq_forced_secondary_handler(int irq, void *dev_id) -+{ -+ WARN(1, "Secondary action handler called for irq %d\n", irq); -+ return IRQ_NONE; -+} -+ - static int irq_wait_for_interrupt(struct irqaction *action) - { - set_current_state(TASK_INTERRUPTIBLE); -@@ -723,7 +804,8 @@ - static void irq_finalize_oneshot(struct irq_desc *desc, - struct irqaction *action) - { -- if (!(desc->istate & IRQS_ONESHOT)) -+ if (!(desc->istate & IRQS_ONESHOT) || -+ action->handler == irq_forced_secondary_handler) + if (likely(ret >= 0)) { + __queue_delayed_work(cpu, wq, dwork, delay); +- local_irq_restore(flags); ++ local_unlock_irqrestore(pendingb_lock, flags); + } + + /* -ENOENT from try_to_grab_pending() becomes %true */ +@@ -1554,7 +1583,9 @@ + worker->last_active = jiffies; + + /* idle_list is LIFO */ ++ rt_lock_idle_list(pool); + list_add(&worker->entry, &pool->idle_list); ++ rt_unlock_idle_list(pool); + + if (too_many_workers(pool) && !timer_pending(&pool->idle_timer)) + mod_timer(&pool->idle_timer, jiffies + IDLE_WORKER_TIMEOUT); +@@ -1587,7 +1618,9 @@ return; - again: - chip_bus_lock(desc); -@@ -825,7 +907,15 @@ - local_bh_disable(); - ret = action->thread_fn(action->irq, action->dev_id); - irq_finalize_oneshot(desc, action); -- local_bh_enable(); -+ /* -+ * Interrupts which have real time requirements can be set up -+ * to avoid softirq processing in the thread handler. This is -+ * safe as these interrupts do not raise soft interrupts. -+ */ -+ if (irq_settings_no_softirq_call(desc)) -+ _local_bh_enable(); -+ else -+ local_bh_enable(); - return ret; + worker_clr_flags(worker, WORKER_IDLE); + pool->nr_idle--; ++ rt_lock_idle_list(pool); + list_del_init(&worker->entry); ++ rt_unlock_idle_list(pool); } -@@ -877,6 +967,18 @@ - irq_finalize_oneshot(desc, action); + static struct worker *alloc_worker(int node) +@@ -1755,7 +1788,9 @@ + pool->nr_workers--; + pool->nr_idle--; + ++ rt_lock_idle_list(pool); + list_del_init(&worker->entry); ++ rt_unlock_idle_list(pool); + worker->flags |= WORKER_DIE; + wake_up_process(worker->task); } +@@ -2672,14 +2707,14 @@ -+static void irq_wake_secondary(struct irq_desc *desc, struct irqaction *action) -+{ -+ struct irqaction *secondary = action->secondary; -+ -+ if (WARN_ON_ONCE(!secondary)) -+ return; -+ -+ raw_spin_lock_irq(&desc->lock); -+ __irq_wake_thread(desc, secondary); -+ raw_spin_unlock_irq(&desc->lock); -+} -+ - /* - * Interrupt handler thread - */ -@@ -907,7 +1009,15 @@ - action_ret = handler_fn(desc, action); - if (action_ret == IRQ_HANDLED) - atomic_inc(&desc->threads_handled); -+ if (action_ret == IRQ_WAKE_THREAD) -+ irq_wake_secondary(desc, action); + might_sleep(); -+#ifdef CONFIG_PREEMPT_RT_FULL -+ migrate_disable(); -+ add_interrupt_randomness(action->irq, 0, -+ desc->random_ip ^ (unsigned long) action); -+ migrate_enable(); -+#endif - wake_threads_waitq(desc); +- local_irq_disable(); ++ rcu_read_lock(); + pool = get_work_pool(work); + if (!pool) { +- local_irq_enable(); ++ rcu_read_unlock(); + return false; } -@@ -951,20 +1061,36 @@ +- spin_lock(&pool->lock); ++ spin_lock_irq(&pool->lock); + /* see the comment in try_to_grab_pending() with the same code */ + pwq = get_work_pwq(work); + if (pwq) { +@@ -2706,10 +2741,11 @@ + else + lock_map_acquire_read(&pwq->wq->lockdep_map); + lock_map_release(&pwq->wq->lockdep_map); +- ++ rcu_read_unlock(); + return true; + already_gone: + spin_unlock_irq(&pool->lock); ++ rcu_read_unlock(); + return false; } - EXPORT_SYMBOL_GPL(irq_wake_thread); - --static void irq_setup_forced_threading(struct irqaction *new) -+static int irq_setup_forced_threading(struct irqaction *new) - { - if (!force_irqthreads) -- return; -+ return 0; - if (new->flags & (IRQF_NO_THREAD | IRQF_PERCPU | IRQF_ONESHOT)) -- return; -+ return 0; - new->flags |= IRQF_ONESHOT; +@@ -2796,7 +2832,7 @@ -- if (!new->thread_fn) { -- set_bit(IRQTF_FORCED_THREAD, &new->thread_flags); -- new->thread_fn = new->handler; -- new->handler = irq_default_primary_handler; -- } -+ /* -+ * Handle the case where we have a real primary handler and a -+ * thread handler. We force thread them as well by creating a -+ * secondary action. -+ */ -+ if (new->handler != irq_default_primary_handler && new->thread_fn) { -+ /* Allocate the secondary action */ -+ new->secondary = kzalloc(sizeof(struct irqaction), GFP_KERNEL); -+ if (!new->secondary) -+ return -ENOMEM; -+ new->secondary->handler = irq_forced_secondary_handler; -+ new->secondary->thread_fn = new->thread_fn; -+ new->secondary->dev_id = new->dev_id; -+ new->secondary->irq = new->irq; -+ new->secondary->name = new->name; -+ } -+ /* Deal with the primary handler */ -+ set_bit(IRQTF_FORCED_THREAD, &new->thread_flags); -+ new->thread_fn = new->handler; -+ new->handler = irq_default_primary_handler; -+ return 0; + /* tell other tasks trying to grab @work to back off */ + mark_work_canceling(work); +- local_irq_restore(flags); ++ local_unlock_irqrestore(pendingb_lock, flags); + + flush_work(work); + clear_work_data(work); +@@ -2851,10 +2887,10 @@ + */ + bool flush_delayed_work(struct delayed_work *dwork) + { +- local_irq_disable(); ++ local_lock_irq(pendingb_lock); + if (del_timer_sync(&dwork->timer)) + __queue_work(dwork->cpu, dwork->wq, &dwork->work); +- local_irq_enable(); ++ local_unlock_irq(pendingb_lock); + return flush_work(&dwork->work); } + EXPORT_SYMBOL(flush_delayed_work); +@@ -2889,7 +2925,7 @@ - static int irq_request_resources(struct irq_desc *desc) -@@ -984,6 +1110,48 @@ - c->irq_release_resources(d); + set_work_pool_and_clear_pending(&dwork->work, + get_work_pool_id(&dwork->work)); +- local_irq_restore(flags); ++ local_unlock_irqrestore(pendingb_lock, flags); + return ret; + } + EXPORT_SYMBOL(cancel_delayed_work); +@@ -3147,7 +3183,7 @@ + * put_unbound_pool - put a worker_pool + * @pool: worker_pool to put + * +- * Put @pool. If its refcnt reaches zero, it gets destroyed in sched-RCU ++ * Put @pool. If its refcnt reaches zero, it gets destroyed in RCU + * safe manner. get_unbound_pool() calls this function on its failure path + * and this function should be able to release pools which went through, + * successfully or not, init_worker_pool(). +@@ -3201,8 +3237,8 @@ + del_timer_sync(&pool->idle_timer); + del_timer_sync(&pool->mayday_timer); + +- /* sched-RCU protected to allow dereferences from get_work_pool() */ +- call_rcu_sched(&pool->rcu, rcu_free_pool); ++ /* RCU protected to allow dereferences from get_work_pool() */ ++ call_rcu(&pool->rcu, rcu_free_pool); } -+static int -+setup_irq_thread(struct irqaction *new, unsigned int irq, bool secondary) -+{ -+ struct task_struct *t; -+ struct sched_param param = { -+ .sched_priority = MAX_USER_RT_PRIO/2, -+ }; -+ -+ if (!secondary) { -+ t = kthread_create(irq_thread, new, "irq/%d-%s", irq, -+ new->name); -+ } else { -+ t = kthread_create(irq_thread, new, "irq/%d-s-%s", irq, -+ new->name); -+ param.sched_priority += 1; -+ } -+ -+ if (IS_ERR(t)) -+ return PTR_ERR(t); -+ -+ sched_setscheduler_nocheck(t, SCHED_FIFO, ¶m); -+ -+ /* -+ * We keep the reference to the task struct even if -+ * the thread dies to avoid that the interrupt code -+ * references an already freed task_struct. -+ */ -+ get_task_struct(t); -+ new->thread = t; -+ /* -+ * Tell the thread to set its affinity. This is -+ * important for shared interrupt handlers as we do -+ * not invoke setup_affinity() for the secondary -+ * handlers as everything is already set up. Even for -+ * interrupts marked with IRQF_NO_BALANCE this is -+ * correct as we want the thread to move to the cpu(s) -+ * on which the requesting code placed the interrupt. -+ */ -+ set_bit(IRQTF_AFFINITY, &new->thread_flags); -+ return 0; -+} -+ - /* - * Internal function to register an irqaction - typically used to - * allocate special interrupts that are part of the architecture. -@@ -1004,6 +1172,8 @@ - if (!try_module_get(desc->owner)) - return -ENODEV; + /** +@@ -3307,14 +3343,14 @@ + put_unbound_pool(pool); + mutex_unlock(&wq_pool_mutex); + +- call_rcu_sched(&pwq->rcu, rcu_free_pwq); ++ call_rcu(&pwq->rcu, rcu_free_pwq); -+ new->irq = irq; -+ /* - * Check whether the interrupt nests into another interrupt - * thread. -@@ -1021,8 +1191,11 @@ + * If we're the last pwq going away, @wq is already dead and no one + * is gonna access it anymore. Schedule RCU free. + */ + if (is_last) +- call_rcu_sched(&wq->rcu, rcu_free_wq); ++ call_rcu(&wq->rcu, rcu_free_wq); + } + + /** +@@ -3920,7 +3956,7 @@ + * The base ref is never dropped on per-cpu pwqs. Directly + * schedule RCU free. */ - new->handler = irq_nested_primary_handler; +- call_rcu_sched(&wq->rcu, rcu_free_wq); ++ call_rcu(&wq->rcu, rcu_free_wq); } else { -- if (irq_settings_can_thread(desc)) -- irq_setup_forced_threading(new); -+ if (irq_settings_can_thread(desc)) { -+ ret = irq_setup_forced_threading(new); -+ if (ret) -+ goto out_mput; -+ } - } + /* + * We're the sole accessor of @wq at this point. Directly +@@ -4013,7 +4049,8 @@ + struct pool_workqueue *pwq; + bool ret; - /* -@@ -1031,37 +1204,14 @@ - * thread. - */ - if (new->thread_fn && !nested) { -- struct task_struct *t; -- static const struct sched_param param = { -- .sched_priority = MAX_USER_RT_PRIO/2, -- }; -- -- t = kthread_create(irq_thread, new, "irq/%d-%s", irq, -- new->name); -- if (IS_ERR(t)) { -- ret = PTR_ERR(t); -+ ret = setup_irq_thread(new, irq, false); -+ if (ret) - goto out_mput; -+ if (new->secondary) { -+ ret = setup_irq_thread(new->secondary, irq, true); -+ if (ret) -+ goto out_thread; - } -- -- sched_setscheduler_nocheck(t, SCHED_FIFO, ¶m); -- -- /* -- * We keep the reference to the task struct even if -- * the thread dies to avoid that the interrupt code -- * references an already freed task_struct. -- */ -- get_task_struct(t); -- new->thread = t; -- /* -- * Tell the thread to set its affinity. This is -- * important for shared interrupt handlers as we do -- * not invoke setup_affinity() for the secondary -- * handlers as everything is already set up. Even for -- * interrupts marked with IRQF_NO_BALANCE this is -- * correct as we want the thread to move to the cpu(s) -- * on which the requesting code placed the interrupt. -- */ -- set_bit(IRQTF_AFFINITY, &new->thread_flags); - } +- rcu_read_lock_sched(); ++ rcu_read_lock(); ++ preempt_disable(); - if (!alloc_cpumask_var(&mask, GFP_KERNEL)) { -@@ -1221,6 +1371,9 @@ - irqd_set(&desc->irq_data, IRQD_NO_BALANCING); - } + if (cpu == WORK_CPU_UNBOUND) + cpu = smp_processor_id(); +@@ -4024,7 +4061,8 @@ + pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu)); -+ if (new->flags & IRQF_NO_SOFTIRQ_CALL) -+ irq_settings_set_no_softirq_call(desc); -+ - /* Set default affinity mask once everything is setup */ - setup_affinity(irq, desc, mask); + ret = !list_empty(&pwq->delayed_works); +- rcu_read_unlock_sched(); ++ preempt_enable(); ++ rcu_read_unlock(); -@@ -1234,7 +1387,6 @@ - irq, nmsk, omsk); + return ret; + } +@@ -4050,15 +4088,15 @@ + if (work_pending(work)) + ret |= WORK_BUSY_PENDING; + +- local_irq_save(flags); ++ rcu_read_lock(); + pool = get_work_pool(work); + if (pool) { +- spin_lock(&pool->lock); ++ spin_lock_irqsave(&pool->lock, flags); + if (find_worker_executing_work(pool, work)) + ret |= WORK_BUSY_RUNNING; +- spin_unlock(&pool->lock); ++ spin_unlock_irqrestore(&pool->lock, flags); } +- local_irq_restore(flags); ++ rcu_read_unlock(); -- new->irq = irq; - *old_ptr = new; + return ret; + } +@@ -4247,7 +4285,7 @@ + unsigned long flags; + int pi; - irq_pm_install_action(desc, new); -@@ -1260,6 +1412,8 @@ - */ - if (new->thread) - wake_up_process(new->thread); -+ if (new->secondary) -+ wake_up_process(new->secondary->thread); +- rcu_read_lock_sched(); ++ rcu_read_lock(); - register_irq_proc(irq, desc); - new->dir = NULL; -@@ -1290,6 +1444,13 @@ - kthread_stop(t); - put_task_struct(t); + pr_info("Showing busy workqueues and worker pools:\n"); + +@@ -4298,7 +4336,7 @@ + spin_unlock_irqrestore(&pool->lock, flags); } -+ if (new->secondary && new->secondary->thread) { -+ struct task_struct *t = new->secondary->thread; -+ -+ new->secondary->thread = NULL; -+ kthread_stop(t); -+ put_task_struct(t); -+ } - out_mput: - module_put(desc->owner); - return ret; -@@ -1397,9 +1558,14 @@ - if (action->thread) { - kthread_stop(action->thread); - put_task_struct(action->thread); -+ if (action->secondary && action->secondary->thread) { -+ kthread_stop(action->secondary->thread); -+ put_task_struct(action->secondary->thread); -+ } + +- rcu_read_unlock_sched(); ++ rcu_read_unlock(); + } + + /* +@@ -4648,16 +4686,16 @@ + * nr_active is monotonically decreasing. It's safe + * to peek without lock. + */ +- rcu_read_lock_sched(); ++ rcu_read_lock(); + for_each_pwq(pwq, wq) { + WARN_ON_ONCE(pwq->nr_active < 0); + if (pwq->nr_active) { + busy = true; +- rcu_read_unlock_sched(); ++ rcu_read_unlock(); + goto out_unlock; + } + } +- rcu_read_unlock_sched(); ++ rcu_read_unlock(); } + out_unlock: + mutex_unlock(&wq_pool_mutex); +@@ -4771,7 +4809,8 @@ + const char *delim = ""; + int node, written = 0; - module_put(desc->owner); -+ kfree(action->secondary); - return action; +- rcu_read_lock_sched(); ++ get_online_cpus(); ++ rcu_read_lock(); + for_each_node(node) { + written += scnprintf(buf + written, PAGE_SIZE - written, + "%s%d:%d", delim, node, +@@ -4779,7 +4818,8 @@ + delim = " "; + } + written += scnprintf(buf + written, PAGE_SIZE - written, "\n"); +- rcu_read_unlock_sched(); ++ rcu_read_unlock(); ++ put_online_cpus(); + + return written; } +diff -Nur linux-4.1.10.orig/kernel/workqueue_internal.h linux-4.1.10/kernel/workqueue_internal.h +--- linux-4.1.10.orig/kernel/workqueue_internal.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/workqueue_internal.h 2015-10-12 22:33:32.320673847 +0200 +@@ -43,6 +43,7 @@ + unsigned long last_active; /* L: last active timestamp */ + unsigned int flags; /* X: flags */ + int id; /* I: worker id */ ++ int sleeping; /* None */ -@@ -1543,8 +1709,10 @@ - retval = __setup_irq(irq, desc, action); - chip_bus_sync_unlock(desc); + /* + * Opaque string set with work_set_desc(). Printed out with task +@@ -68,7 +69,7 @@ + * Scheduler hooks for concurrency managed workqueue. Only to be used from + * sched/core.c and workqueue.c. + */ +-void wq_worker_waking_up(struct task_struct *task, int cpu); +-struct task_struct *wq_worker_sleeping(struct task_struct *task, int cpu); ++void wq_worker_running(struct task_struct *task); ++void wq_worker_sleeping(struct task_struct *task); -- if (retval) -+ if (retval) { -+ kfree(action->secondary); - kfree(action); -+ } + #endif /* _KERNEL_WORKQUEUE_INTERNAL_H */ +diff -Nur linux-4.1.10.orig/lib/debugobjects.c linux-4.1.10/lib/debugobjects.c +--- linux-4.1.10.orig/lib/debugobjects.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/lib/debugobjects.c 2015-10-12 22:33:32.320673847 +0200 +@@ -309,7 +309,10 @@ + struct debug_obj *obj; + unsigned long flags; - #ifdef CONFIG_DEBUG_SHIRQ_FIXME - if (!retval && (irqflags & IRQF_SHARED)) { -diff -Nur linux-4.1.10.orig/kernel/irq/settings.h linux-4.1.10/kernel/irq/settings.h ---- linux-4.1.10.orig/kernel/irq/settings.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/irq/settings.h 2015-10-07 18:00:08.000000000 +0200 -@@ -15,6 +15,7 @@ - _IRQ_NESTED_THREAD = IRQ_NESTED_THREAD, - _IRQ_PER_CPU_DEVID = IRQ_PER_CPU_DEVID, - _IRQ_IS_POLLED = IRQ_IS_POLLED, -+ _IRQ_NO_SOFTIRQ_CALL = IRQ_NO_SOFTIRQ_CALL, - _IRQF_MODIFY_MASK = IRQF_MODIFY_MASK, - }; +- fill_pool(); ++#ifdef CONFIG_PREEMPT_RT_FULL ++ if (preempt_count() == 0 && !irqs_disabled()) ++#endif ++ fill_pool(); -@@ -28,6 +29,7 @@ - #define IRQ_NESTED_THREAD GOT_YOU_MORON - #define IRQ_PER_CPU_DEVID GOT_YOU_MORON - #define IRQ_IS_POLLED GOT_YOU_MORON -+#define IRQ_NO_SOFTIRQ_CALL GOT_YOU_MORON - #undef IRQF_MODIFY_MASK - #define IRQF_MODIFY_MASK GOT_YOU_MORON + db = get_bucket((unsigned long) addr); -@@ -38,6 +40,16 @@ - desc->status_use_accessors |= (set & _IRQF_MODIFY_MASK); +diff -Nur linux-4.1.10.orig/lib/dump_stack.c linux-4.1.10/lib/dump_stack.c +--- linux-4.1.10.orig/lib/dump_stack.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/lib/dump_stack.c 2015-10-12 22:33:32.320673847 +0200 +@@ -33,7 +33,7 @@ + * Permit this cpu to perform nested stack dumps while serialising + * against other CPUs + */ +- preempt_disable(); ++ migrate_disable(); + + retry: + cpu = smp_processor_id(); +@@ -52,7 +52,7 @@ + if (!was_locked) + atomic_set(&dump_lock, -1); + +- preempt_enable(); ++ migrate_enable(); + } + #else + asmlinkage __visible void dump_stack(void) +diff -Nur linux-4.1.10.orig/lib/idr.c linux-4.1.10/lib/idr.c +--- linux-4.1.10.orig/lib/idr.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/lib/idr.c 2015-10-12 22:33:32.324673582 +0200 +@@ -30,6 +30,7 @@ + #include + #include + #include ++#include + + #define MAX_IDR_SHIFT (sizeof(int) * 8 - 1) + #define MAX_IDR_BIT (1U << MAX_IDR_SHIFT) +@@ -366,6 +367,35 @@ + idr_mark_full(pa, id); } -+static inline bool irq_settings_no_softirq_call(struct irq_desc *desc) ++#ifdef CONFIG_PREEMPT_RT_FULL ++static DEFINE_LOCAL_IRQ_LOCK(idr_lock); ++ ++static inline void idr_preload_lock(void) +{ -+ return desc->status_use_accessors & _IRQ_NO_SOFTIRQ_CALL; ++ local_lock(idr_lock); +} + -+static inline void irq_settings_set_no_softirq_call(struct irq_desc *desc) ++static inline void idr_preload_unlock(void) +{ -+ desc->status_use_accessors |= _IRQ_NO_SOFTIRQ_CALL; ++ local_unlock(idr_lock); +} + - static inline bool irq_settings_is_per_cpu(struct irq_desc *desc) - { - return desc->status_use_accessors & _IRQ_PER_CPU; -diff -Nur linux-4.1.10.orig/kernel/irq/spurious.c linux-4.1.10/kernel/irq/spurious.c ---- linux-4.1.10.orig/kernel/irq/spurious.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/irq/spurious.c 2015-10-07 18:00:08.000000000 +0200 -@@ -444,6 +444,10 @@ - - static int __init irqfixup_setup(char *str) - { -+#ifdef CONFIG_PREEMPT_RT_BASE -+ pr_warn("irqfixup boot option not supported w/ CONFIG_PREEMPT_RT_BASE\n"); -+ return 1; ++void idr_preload_end(void) ++{ ++ idr_preload_unlock(); ++} ++EXPORT_SYMBOL(idr_preload_end); ++#else ++static inline void idr_preload_lock(void) ++{ ++ preempt_disable(); ++} ++ ++static inline void idr_preload_unlock(void) ++{ ++ preempt_enable(); ++} +#endif - irqfixup = 1; - printk(KERN_WARNING "Misrouted IRQ fixup support enabled.\n"); - printk(KERN_WARNING "This may impact system performance.\n"); -@@ -456,6 +460,10 @@ - static int __init irqpoll_setup(char *str) - { -+#ifdef CONFIG_PREEMPT_RT_BASE -+ pr_warn("irqpoll boot option not supported w/ CONFIG_PREEMPT_RT_BASE\n"); -+ return 1; -+#endif - irqfixup = 2; - printk(KERN_WARNING "Misrouted IRQ fixup and polling support " - "enabled\n"); -diff -Nur linux-4.1.10.orig/kernel/irq_work.c linux-4.1.10/kernel/irq_work.c ---- linux-4.1.10.orig/kernel/irq_work.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/irq_work.c 2015-10-07 18:00:08.000000000 +0200 -@@ -17,6 +17,7 @@ - #include - #include - #include -+#include - #include + /** + * idr_preload - preload for idr_alloc() +@@ -401,7 +431,7 @@ + WARN_ON_ONCE(in_interrupt()); + might_sleep_if(gfp_mask & __GFP_WAIT); +- preempt_disable(); ++ idr_preload_lock(); -@@ -65,6 +66,8 @@ - */ - bool irq_work_queue_on(struct irq_work *work, int cpu) - { -+ struct llist_head *list; -+ - /* All work should have been flushed before going offline */ - WARN_ON_ONCE(cpu_is_offline(cpu)); + /* + * idr_alloc() is likely to succeed w/o full idr_layer buffer and +@@ -413,9 +443,9 @@ + while (__this_cpu_read(idr_preload_cnt) < MAX_IDR_FREE) { + struct idr_layer *new; -@@ -75,7 +78,12 @@ - if (!irq_work_claim(work)) - return false; +- preempt_enable(); ++ idr_preload_unlock(); + new = kmem_cache_zalloc(idr_layer_cache, gfp_mask); +- preempt_disable(); ++ idr_preload_lock(); + if (!new) + break; -- if (llist_add(&work->llnode, &per_cpu(raised_list, cpu))) -+ if (IS_ENABLED(CONFIG_PREEMPT_RT_FULL) && !(work->flags & IRQ_WORK_HARD_IRQ)) -+ list = &per_cpu(lazy_list, cpu); -+ else -+ list = &per_cpu(raised_list, cpu); -+ -+ if (llist_add(&work->llnode, list)) - arch_send_call_function_single_ipi(cpu); +diff -Nur linux-4.1.10.orig/lib/Kconfig linux-4.1.10/lib/Kconfig +--- linux-4.1.10.orig/lib/Kconfig 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/lib/Kconfig 2015-10-12 22:33:32.324673582 +0200 +@@ -391,6 +391,7 @@ - return true; -@@ -86,6 +94,9 @@ - /* Enqueue the irq work @work on the current CPU */ - bool irq_work_queue(struct irq_work *work) - { -+ struct llist_head *list; -+ bool lazy_work, realtime = IS_ENABLED(CONFIG_PREEMPT_RT_FULL); -+ - /* Only queue if not already pending */ - if (!irq_work_claim(work)) - return false; -@@ -93,13 +104,15 @@ - /* Queue the entry and raise the IPI if needed. */ - preempt_disable(); + config CPUMASK_OFFSTACK + bool "Force CPU masks off stack" if DEBUG_PER_CPU_MAPS ++ depends on !PREEMPT_RT_FULL + help + Use dynamic allocation for cpumask_var_t, instead of putting + them on the stack. This is a bit more expensive, but avoids +diff -Nur linux-4.1.10.orig/lib/locking-selftest.c linux-4.1.10/lib/locking-selftest.c +--- linux-4.1.10.orig/lib/locking-selftest.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/lib/locking-selftest.c 2015-10-12 22:33:32.324673582 +0200 +@@ -590,6 +590,8 @@ + #include "locking-selftest-spin-hardirq.h" + GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_hard_spin) -- /* If the work is "lazy", handle it from next tick if any */ -- if (work->flags & IRQ_WORK_LAZY) { -- if (llist_add(&work->llnode, this_cpu_ptr(&lazy_list)) && -- tick_nohz_tick_stopped()) -- arch_irq_work_raise(); -- } else { -- if (llist_add(&work->llnode, this_cpu_ptr(&raised_list))) -+ lazy_work = work->flags & IRQ_WORK_LAZY; -+ -+ if (lazy_work || (realtime && !(work->flags & IRQ_WORK_HARD_IRQ))) -+ list = this_cpu_ptr(&lazy_list); -+ else -+ list = this_cpu_ptr(&raised_list); ++#ifndef CONFIG_PREEMPT_RT_FULL + -+ if (llist_add(&work->llnode, list)) { -+ if (!lazy_work || tick_nohz_tick_stopped()) - arch_irq_work_raise(); - } + #include "locking-selftest-rlock-hardirq.h" + GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_hard_rlock) -@@ -116,9 +129,8 @@ - raised = this_cpu_ptr(&raised_list); - lazy = this_cpu_ptr(&lazy_list); +@@ -605,9 +607,12 @@ + #include "locking-selftest-wlock-softirq.h" + GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_soft_wlock) -- if (llist_empty(raised) || arch_irq_work_has_interrupt()) -- if (llist_empty(lazy)) -- return false; -+ if (llist_empty(raised) && llist_empty(lazy)) -+ return false; ++#endif ++ + #undef E1 + #undef E2 - /* All work should have been flushed before going offline */ - WARN_ON_ONCE(cpu_is_offline(smp_processor_id())); -@@ -132,7 +144,7 @@ - struct irq_work *work; - struct llist_node *llnode; ++#ifndef CONFIG_PREEMPT_RT_FULL + /* + * Enabling hardirqs with a softirq-safe lock held: + */ +@@ -640,6 +645,8 @@ + #undef E1 + #undef E2 -- BUG_ON(!irqs_disabled()); -+ BUG_ON_NONRT(!irqs_disabled()); ++#endif ++ + /* + * Enabling irqs with an irq-safe lock held: + */ +@@ -663,6 +670,8 @@ + #include "locking-selftest-spin-hardirq.h" + GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_hard_spin) - if (llist_empty(list)) - return; -@@ -169,7 +181,16 @@ - void irq_work_run(void) - { - irq_work_run_list(this_cpu_ptr(&raised_list)); -- irq_work_run_list(this_cpu_ptr(&lazy_list)); -+ if (IS_ENABLED(CONFIG_PREEMPT_RT_FULL)) { -+ /* -+ * NOTE: we raise softirq via IPI for safety, -+ * and execute in irq_work_tick() to move the -+ * overhead from hard to soft irq context. -+ */ -+ if (!llist_empty(this_cpu_ptr(&lazy_list))) -+ raise_softirq(TIMER_SOFTIRQ); -+ } else -+ irq_work_run_list(this_cpu_ptr(&lazy_list)); - } - EXPORT_SYMBOL_GPL(irq_work_run); ++#ifndef CONFIG_PREEMPT_RT_FULL ++ + #include "locking-selftest-rlock-hardirq.h" + GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_hard_rlock) -diff -Nur linux-4.1.10.orig/kernel/Kconfig.locks linux-4.1.10/kernel/Kconfig.locks ---- linux-4.1.10.orig/kernel/Kconfig.locks 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/Kconfig.locks 2015-10-07 18:00:08.000000000 +0200 -@@ -225,11 +225,11 @@ +@@ -678,6 +687,8 @@ + #include "locking-selftest-wlock-softirq.h" + GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_soft_wlock) - config MUTEX_SPIN_ON_OWNER - def_bool y -- depends on SMP && !DEBUG_MUTEXES && ARCH_SUPPORTS_ATOMIC_RMW -+ depends on SMP && !DEBUG_MUTEXES && ARCH_SUPPORTS_ATOMIC_RMW && !PREEMPT_RT_FULL ++#endif ++ + #undef E1 + #undef E2 - config RWSEM_SPIN_ON_OWNER - def_bool y -- depends on SMP && RWSEM_XCHGADD_ALGORITHM && ARCH_SUPPORTS_ATOMIC_RMW -+ depends on SMP && RWSEM_XCHGADD_ALGORITHM && ARCH_SUPPORTS_ATOMIC_RMW && !PREEMPT_RT_FULL +@@ -709,6 +720,8 @@ + #include "locking-selftest-spin-hardirq.h" + GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_hard_spin) - config LOCK_SPIN_ON_OWNER - def_bool y -diff -Nur linux-4.1.10.orig/kernel/Kconfig.preempt linux-4.1.10/kernel/Kconfig.preempt ---- linux-4.1.10.orig/kernel/Kconfig.preempt 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/Kconfig.preempt 2015-10-07 18:00:08.000000000 +0200 -@@ -1,3 +1,16 @@ -+config PREEMPT -+ bool -+ select PREEMPT_COUNT ++#ifndef CONFIG_PREEMPT_RT_FULL + -+config PREEMPT_RT_BASE -+ bool -+ select PREEMPT + #include "locking-selftest-rlock-hardirq.h" + GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_hard_rlock) + +@@ -724,6 +737,8 @@ + #include "locking-selftest-wlock-softirq.h" + GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_soft_wlock) + ++#endif + -+config HAVE_PREEMPT_LAZY -+ bool + #undef E1 + #undef E2 + #undef E3 +@@ -757,6 +772,8 @@ + #include "locking-selftest-spin-hardirq.h" + GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_hard_spin) + ++#ifndef CONFIG_PREEMPT_RT_FULL + -+config PREEMPT_LAZY -+ def_bool y if HAVE_PREEMPT_LAZY && PREEMPT_RT_FULL + #include "locking-selftest-rlock-hardirq.h" + GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_hard_rlock) - choice - prompt "Preemption Model" -@@ -33,9 +46,9 @@ +@@ -772,10 +789,14 @@ + #include "locking-selftest-wlock-softirq.h" + GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_soft_wlock) - Select this if you are building a kernel for a desktop system. ++#endif ++ + #undef E1 + #undef E2 + #undef E3 --config PREEMPT -+config PREEMPT__LL - bool "Preemptible Kernel (Low-Latency Desktop)" -- select PREEMPT_COUNT -+ select PREEMPT - select UNINLINE_SPIN_UNLOCK if !ARCH_INLINE_SPIN_UNLOCK - help - This option reduces the latency of the kernel by making -@@ -52,6 +65,22 @@ - embedded system with latency requirements in the milliseconds - range. ++#ifndef CONFIG_PREEMPT_RT_FULL ++ + /* + * read-lock / write-lock irq inversion. + * +@@ -838,6 +859,10 @@ + #undef E2 + #undef E3 -+config PREEMPT_RTB -+ bool "Preemptible Kernel (Basic RT)" -+ select PREEMPT_RT_BASE -+ help -+ This option is basically the same as (Low-Latency Desktop) but -+ enables changes which are preliminary for the full preemptible -+ RT kernel. ++#endif + -+config PREEMPT_RT_FULL -+ bool "Fully Preemptible Kernel (RT)" -+ depends on IRQ_FORCED_THREADING -+ select PREEMPT_RT_BASE -+ select PREEMPT_RCU -+ help -+ All and everything ++#ifndef CONFIG_PREEMPT_RT_FULL + - endchoice - - config PREEMPT_COUNT -diff -Nur linux-4.1.10.orig/kernel/ksysfs.c linux-4.1.10/kernel/ksysfs.c ---- linux-4.1.10.orig/kernel/ksysfs.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/ksysfs.c 2015-10-07 18:00:08.000000000 +0200 -@@ -136,6 +136,15 @@ - - #endif /* CONFIG_KEXEC */ + /* + * read-lock / write-lock recursion that is actually safe. + */ +@@ -876,6 +901,8 @@ + #undef E2 + #undef E3 -+#if defined(CONFIG_PREEMPT_RT_FULL) -+static ssize_t realtime_show(struct kobject *kobj, -+ struct kobj_attribute *attr, char *buf) -+{ -+ return sprintf(buf, "%d\n", 1); -+} -+KERNEL_ATTR_RO(realtime); +#endif + - /* whether file capabilities are enabled */ - static ssize_t fscaps_show(struct kobject *kobj, - struct kobj_attribute *attr, char *buf) -@@ -203,6 +212,9 @@ - &vmcoreinfo_attr.attr, - #endif - &rcu_expedited_attr.attr, -+#ifdef CONFIG_PREEMPT_RT_FULL -+ &realtime_attr.attr, -+#endif - NULL - }; + /* + * read-lock / write-lock recursion that is unsafe. + */ +@@ -1858,6 +1885,7 @@ -diff -Nur linux-4.1.10.orig/kernel/locking/lglock.c linux-4.1.10/kernel/locking/lglock.c ---- linux-4.1.10.orig/kernel/locking/lglock.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/locking/lglock.c 2015-10-07 18:00:08.000000000 +0200 -@@ -4,6 +4,15 @@ - #include - #include + printk(" --------------------------------------------------------------------------\n"); +#ifndef CONFIG_PREEMPT_RT_FULL -+# define lg_lock_ptr arch_spinlock_t -+# define lg_do_lock(l) arch_spin_lock(l) -+# define lg_do_unlock(l) arch_spin_unlock(l) -+#else -+# define lg_lock_ptr struct rt_mutex -+# define lg_do_lock(l) __rt_spin_lock(l) -+# define lg_do_unlock(l) __rt_spin_unlock(l) -+#endif - /* - * Note there is no uninit, so lglocks cannot be defined in - * modules (but it's fine to use them from there) -@@ -12,51 +21,60 @@ + /* + * irq-context testcases: + */ +@@ -1870,6 +1898,28 @@ - void lg_lock_init(struct lglock *lg, char *name) - { -+#ifdef CONFIG_PREEMPT_RT_FULL -+ int i; + DO_TESTCASE_6x2("irq read-recursion", irq_read_recursion); + // DO_TESTCASE_6x2B("irq read-recursion #2", irq_read_recursion2); ++#else ++ /* On -rt, we only do hardirq context test for raw spinlock */ ++ DO_TESTCASE_1B("hard-irqs-on + irq-safe-A", irqsafe1_hard_spin, 12); ++ DO_TESTCASE_1B("hard-irqs-on + irq-safe-A", irqsafe1_hard_spin, 21); + -+ for_each_possible_cpu(i) { -+ struct rt_mutex *lock = per_cpu_ptr(lg->lock, i); ++ DO_TESTCASE_1B("hard-safe-A + irqs-on", irqsafe2B_hard_spin, 12); ++ DO_TESTCASE_1B("hard-safe-A + irqs-on", irqsafe2B_hard_spin, 21); + -+ rt_mutex_init(lock); -+ } ++ DO_TESTCASE_1B("hard-safe-A + unsafe-B #1", irqsafe3_hard_spin, 123); ++ DO_TESTCASE_1B("hard-safe-A + unsafe-B #1", irqsafe3_hard_spin, 132); ++ DO_TESTCASE_1B("hard-safe-A + unsafe-B #1", irqsafe3_hard_spin, 213); ++ DO_TESTCASE_1B("hard-safe-A + unsafe-B #1", irqsafe3_hard_spin, 231); ++ DO_TESTCASE_1B("hard-safe-A + unsafe-B #1", irqsafe3_hard_spin, 312); ++ DO_TESTCASE_1B("hard-safe-A + unsafe-B #1", irqsafe3_hard_spin, 321); ++ ++ DO_TESTCASE_1B("hard-safe-A + unsafe-B #2", irqsafe4_hard_spin, 123); ++ DO_TESTCASE_1B("hard-safe-A + unsafe-B #2", irqsafe4_hard_spin, 132); ++ DO_TESTCASE_1B("hard-safe-A + unsafe-B #2", irqsafe4_hard_spin, 213); ++ DO_TESTCASE_1B("hard-safe-A + unsafe-B #2", irqsafe4_hard_spin, 231); ++ DO_TESTCASE_1B("hard-safe-A + unsafe-B #2", irqsafe4_hard_spin, 312); ++ DO_TESTCASE_1B("hard-safe-A + unsafe-B #2", irqsafe4_hard_spin, 321); +#endif - LOCKDEP_INIT_MAP(&lg->lock_dep_map, name, &lg->lock_key, 0); - } - EXPORT_SYMBOL(lg_lock_init); - void lg_local_lock(struct lglock *lg) - { -- arch_spinlock_t *lock; -+ lg_lock_ptr *lock; + ww_tests(); -- preempt_disable(); -+ migrate_disable(); - lock_acquire_shared(&lg->lock_dep_map, 0, 0, NULL, _RET_IP_); - lock = this_cpu_ptr(lg->lock); -- arch_spin_lock(lock); -+ lg_do_lock(lock); - } - EXPORT_SYMBOL(lg_local_lock); +diff -Nur linux-4.1.10.orig/lib/percpu_ida.c linux-4.1.10/lib/percpu_ida.c +--- linux-4.1.10.orig/lib/percpu_ida.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/lib/percpu_ida.c 2015-10-12 22:33:32.324673582 +0200 +@@ -26,6 +26,9 @@ + #include + #include + #include ++#include ++ ++static DEFINE_LOCAL_IRQ_LOCK(irq_off_lock); - void lg_local_unlock(struct lglock *lg) - { -- arch_spinlock_t *lock; -+ lg_lock_ptr *lock; + struct percpu_ida_cpu { + /* +@@ -148,13 +151,13 @@ + unsigned long flags; + int tag; - lock_release(&lg->lock_dep_map, 1, _RET_IP_); - lock = this_cpu_ptr(lg->lock); -- arch_spin_unlock(lock); -- preempt_enable(); -+ lg_do_unlock(lock); -+ migrate_enable(); - } - EXPORT_SYMBOL(lg_local_unlock); +- local_irq_save(flags); ++ local_lock_irqsave(irq_off_lock, flags); + tags = this_cpu_ptr(pool->tag_cpu); - void lg_local_lock_cpu(struct lglock *lg, int cpu) - { -- arch_spinlock_t *lock; -+ lg_lock_ptr *lock; + /* Fastpath */ + tag = alloc_local_tag(tags); + if (likely(tag >= 0)) { +- local_irq_restore(flags); ++ local_unlock_irqrestore(irq_off_lock, flags); + return tag; + } -- preempt_disable(); -+ preempt_disable_nort(); - lock_acquire_shared(&lg->lock_dep_map, 0, 0, NULL, _RET_IP_); - lock = per_cpu_ptr(lg->lock, cpu); -- arch_spin_lock(lock); -+ lg_do_lock(lock); - } - EXPORT_SYMBOL(lg_local_lock_cpu); +@@ -173,6 +176,7 @@ - void lg_local_unlock_cpu(struct lglock *lg, int cpu) - { -- arch_spinlock_t *lock; -+ lg_lock_ptr *lock; + if (!tags->nr_free) + alloc_global_tags(pool, tags); ++ + if (!tags->nr_free) + steal_tags(pool, tags); - lock_release(&lg->lock_dep_map, 1, _RET_IP_); - lock = per_cpu_ptr(lg->lock, cpu); -- arch_spin_unlock(lock); -- preempt_enable(); -+ lg_do_unlock(lock); -+ preempt_enable_nort(); - } - EXPORT_SYMBOL(lg_local_unlock_cpu); +@@ -184,7 +188,7 @@ + } -@@ -64,12 +82,12 @@ - { - int i; + spin_unlock(&pool->lock); +- local_irq_restore(flags); ++ local_unlock_irqrestore(irq_off_lock, flags); -- preempt_disable(); -+ preempt_disable_nort(); - lock_acquire_exclusive(&lg->lock_dep_map, 0, 0, NULL, _RET_IP_); - for_each_possible_cpu(i) { -- arch_spinlock_t *lock; -+ lg_lock_ptr *lock; - lock = per_cpu_ptr(lg->lock, i); -- arch_spin_lock(lock); -+ lg_do_lock(lock); + if (tag >= 0 || state == TASK_RUNNING) + break; +@@ -196,7 +200,7 @@ + + schedule(); + +- local_irq_save(flags); ++ local_lock_irqsave(irq_off_lock, flags); + tags = this_cpu_ptr(pool->tag_cpu); + } + if (state != TASK_RUNNING) +@@ -221,7 +225,7 @@ + + BUG_ON(tag >= pool->nr_tags); + +- local_irq_save(flags); ++ local_lock_irqsave(irq_off_lock, flags); + tags = this_cpu_ptr(pool->tag_cpu); + + spin_lock(&tags->lock); +@@ -253,7 +257,7 @@ + spin_unlock(&pool->lock); } + +- local_irq_restore(flags); ++ local_unlock_irqrestore(irq_off_lock, flags); } - EXPORT_SYMBOL(lg_global_lock); -@@ -80,10 +98,35 @@ + EXPORT_SYMBOL_GPL(percpu_ida_free); - lock_release(&lg->lock_dep_map, 1, _RET_IP_); - for_each_possible_cpu(i) { -- arch_spinlock_t *lock; -+ lg_lock_ptr *lock; - lock = per_cpu_ptr(lg->lock, i); -- arch_spin_unlock(lock); -+ lg_do_unlock(lock); +@@ -345,7 +349,7 @@ + struct percpu_ida_cpu *remote; + unsigned cpu, i, err = 0; + +- local_irq_save(flags); ++ local_lock_irqsave(irq_off_lock, flags); + for_each_possible_cpu(cpu) { + remote = per_cpu_ptr(pool->tag_cpu, cpu); + spin_lock(&remote->lock); +@@ -367,7 +371,7 @@ } -- preempt_enable(); -+ preempt_enable_nort(); + spin_unlock(&pool->lock); + out: +- local_irq_restore(flags); ++ local_unlock_irqrestore(irq_off_lock, flags); + return err; } - EXPORT_SYMBOL(lg_global_unlock); -+ -+#ifdef CONFIG_PREEMPT_RT_FULL -+/* -+ * HACK: If you use this, you get to keep the pieces. -+ * Used in queue_stop_cpus_work() when stop machinery -+ * is called from inactive CPU, so we can't schedule. -+ */ -+# define lg_do_trylock_relax(l) \ -+ do { \ -+ while (!__rt_spin_trylock(l)) \ -+ cpu_relax(); \ -+ } while (0) -+ -+void lg_global_trylock_relax(struct lglock *lg) -+{ -+ int i; -+ -+ lock_acquire_exclusive(&lg->lock_dep_map, 0, 0, NULL, _RET_IP_); -+ for_each_possible_cpu(i) { -+ lg_lock_ptr *lock; -+ lock = per_cpu_ptr(lg->lock, i); -+ lg_do_trylock_relax(lock); -+ } -+} -+#endif -diff -Nur linux-4.1.10.orig/kernel/locking/lockdep.c linux-4.1.10/kernel/locking/lockdep.c ---- linux-4.1.10.orig/kernel/locking/lockdep.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/locking/lockdep.c 2015-10-07 18:00:08.000000000 +0200 -@@ -3563,6 +3563,7 @@ + EXPORT_SYMBOL_GPL(percpu_ida_for_each_free); +diff -Nur linux-4.1.10.orig/lib/radix-tree.c linux-4.1.10/lib/radix-tree.c +--- linux-4.1.10.orig/lib/radix-tree.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/lib/radix-tree.c 2015-10-12 22:33:32.324673582 +0200 +@@ -195,12 +195,13 @@ + * succeed in getting a node here (and never reach + * kmem_cache_alloc) + */ +- rtp = this_cpu_ptr(&radix_tree_preloads); ++ rtp = &get_cpu_var(radix_tree_preloads); + if (rtp->nr) { + ret = rtp->nodes[rtp->nr - 1]; + rtp->nodes[rtp->nr - 1] = NULL; + rtp->nr--; } - } ++ put_cpu_var(radix_tree_preloads); + /* + * Update the allocation stack trace as this is more useful + * for debugging. +@@ -240,6 +241,7 @@ + call_rcu(&node->rcu_head, radix_tree_node_rcu_free); + } +#ifndef CONFIG_PREEMPT_RT_FULL - /* - * We dont accurately track softirq state in e.g. - * hardirq contexts (such as on 4KSTACKS), so only -@@ -3577,6 +3578,7 @@ - DEBUG_LOCKS_WARN_ON(!current->softirqs_enabled); - } + /* + * Load up this CPU's radix_tree_node buffer with sufficient objects to + * ensure that the addition of a single element in the tree cannot fail. On +@@ -305,6 +307,7 @@ + return 0; + } + EXPORT_SYMBOL(radix_tree_maybe_preload); ++#endif + + /* + * Return the maximum key which can be store into a +diff -Nur linux-4.1.10.orig/lib/scatterlist.c linux-4.1.10/lib/scatterlist.c +--- linux-4.1.10.orig/lib/scatterlist.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/lib/scatterlist.c 2015-10-12 22:33:32.324673582 +0200 +@@ -592,7 +592,7 @@ + flush_kernel_dcache_page(miter->page); + + if (miter->__flags & SG_MITER_ATOMIC) { +- WARN_ON_ONCE(preemptible()); ++ WARN_ON_ONCE(!pagefault_disabled()); + kunmap_atomic(miter->addr); + } else + kunmap(miter->page); +@@ -637,7 +637,7 @@ + if (!sg_miter_skip(&miter, skip)) + return false; + +- local_irq_save(flags); ++ local_irq_save_nort(flags); + + while (sg_miter_next(&miter) && offset < buflen) { + unsigned int len; +@@ -654,7 +654,7 @@ + + sg_miter_stop(&miter); + +- local_irq_restore(flags); ++ local_irq_restore_nort(flags); + return offset; + } + +diff -Nur linux-4.1.10.orig/lib/smp_processor_id.c linux-4.1.10/lib/smp_processor_id.c +--- linux-4.1.10.orig/lib/smp_processor_id.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/lib/smp_processor_id.c 2015-10-12 22:33:32.324673582 +0200 +@@ -39,8 +39,9 @@ + if (!printk_ratelimit()) + goto out_enable; + +- printk(KERN_ERR "BUG: using %s%s() in preemptible [%08x] code: %s/%d\n", +- what1, what2, preempt_count() - 1, current->comm, current->pid); ++ printk(KERN_ERR "BUG: using %s%s() in preemptible [%08x %08x] code: %s/%d\n", ++ what1, what2, preempt_count() - 1, __migrate_disabled(current), ++ current->comm, current->pid); + + print_symbol("caller is %s\n", (long)__builtin_return_address(0)); + dump_stack(); +diff -Nur linux-4.1.10.orig/lib/strnlen_user.c linux-4.1.10/lib/strnlen_user.c +--- linux-4.1.10.orig/lib/strnlen_user.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/lib/strnlen_user.c 2015-10-12 22:33:32.324673582 +0200 +@@ -85,7 +85,8 @@ + * @str: The string to measure. + * @count: Maximum count (including NUL character) + * +- * Context: User context only. This function may sleep. ++ * Context: User context only. This function may sleep if pagefaults are ++ * enabled. + * + * Get the size of a NUL-terminated string in user space. + * +@@ -121,7 +122,8 @@ + * strlen_user: - Get the size of a user string INCLUDING final NUL. + * @str: The string to measure. + * +- * Context: User context only. This function may sleep. ++ * Context: User context only. This function may sleep if pagefaults are ++ * enabled. + * + * Get the size of a NUL-terminated string in user space. + * +diff -Nur linux-4.1.10.orig/mm/compaction.c linux-4.1.10/mm/compaction.c +--- linux-4.1.10.orig/mm/compaction.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/mm/compaction.c 2015-10-12 22:33:32.324673582 +0200 +@@ -1406,10 +1406,12 @@ + cc->migrate_pfn & ~((1UL << cc->order) - 1); + + if (last_migrated_pfn < current_block_start) { +- cpu = get_cpu(); ++ cpu = get_cpu_light(); ++ local_lock_irq(swapvec_lock); + lru_add_drain_cpu(cpu); ++ local_unlock_irq(swapvec_lock); + drain_local_pages(zone); +- put_cpu(); ++ put_cpu_light(); + /* No more flushing until we migrate again */ + last_migrated_pfn = 0; + } +diff -Nur linux-4.1.10.orig/mm/filemap.c linux-4.1.10/mm/filemap.c +--- linux-4.1.10.orig/mm/filemap.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/mm/filemap.c 2015-10-12 22:33:32.324673582 +0200 +@@ -167,7 +167,9 @@ + if (!workingset_node_pages(node) && + list_empty(&node->private_list)) { + node->private_data = mapping; +- list_lru_add(&workingset_shadow_nodes, &node->private_list); ++ local_lock(workingset_shadow_lock); ++ list_lru_add(&__workingset_shadow_nodes, &node->private_list); ++ local_unlock(workingset_shadow_lock); + } + } + +@@ -533,9 +535,12 @@ + * node->private_list is protected by + * mapping->tree_lock. + */ +- if (!list_empty(&node->private_list)) +- list_lru_del(&workingset_shadow_nodes, ++ if (!list_empty(&node->private_list)) { ++ local_lock(workingset_shadow_lock); ++ list_lru_del(&__workingset_shadow_nodes, + &node->private_list); ++ local_unlock(workingset_shadow_lock); ++ } } + return 0; + } +diff -Nur linux-4.1.10.orig/mm/highmem.c linux-4.1.10/mm/highmem.c +--- linux-4.1.10.orig/mm/highmem.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/mm/highmem.c 2015-10-12 22:33:32.324673582 +0200 +@@ -29,10 +29,11 @@ + #include + #include + +- ++#ifndef CONFIG_PREEMPT_RT_FULL + #if defined(CONFIG_HIGHMEM) || defined(CONFIG_X86_32) + DEFINE_PER_CPU(int, __kmap_atomic_idx); + #endif +#endif - if (!debug_locks) - print_irqtrace_events(current); -diff -Nur linux-4.1.10.orig/kernel/locking/locktorture.c linux-4.1.10/kernel/locking/locktorture.c ---- linux-4.1.10.orig/kernel/locking/locktorture.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/locking/locktorture.c 2015-10-07 18:00:08.000000000 +0200 -@@ -24,7 +24,6 @@ - #include - #include - #include --#include - #include - #include - #include -diff -Nur linux-4.1.10.orig/kernel/locking/Makefile linux-4.1.10/kernel/locking/Makefile ---- linux-4.1.10.orig/kernel/locking/Makefile 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/locking/Makefile 2015-10-07 18:00:08.000000000 +0200 -@@ -1,5 +1,5 @@ + /* + * Virtual_count is not a pure "count". +@@ -107,8 +108,9 @@ + unsigned long totalhigh_pages __read_mostly; + EXPORT_SYMBOL(totalhigh_pages); --obj-y += mutex.o semaphore.o rwsem.o -+obj-y += semaphore.o +- ++#ifndef CONFIG_PREEMPT_RT_FULL + EXPORT_PER_CPU_SYMBOL(__kmap_atomic_idx); ++#endif - ifdef CONFIG_FUNCTION_TRACER - CFLAGS_REMOVE_lockdep.o = $(CC_FLAGS_FTRACE) -@@ -8,7 +8,11 @@ - CFLAGS_REMOVE_rtmutex-debug.o = $(CC_FLAGS_FTRACE) - endif + unsigned int nr_free_highpages (void) + { +diff -Nur linux-4.1.10.orig/mm/Kconfig linux-4.1.10/mm/Kconfig +--- linux-4.1.10.orig/mm/Kconfig 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/mm/Kconfig 2015-10-12 22:33:32.324673582 +0200 +@@ -409,7 +409,7 @@ -+ifneq ($(CONFIG_PREEMPT_RT_FULL),y) -+obj-y += mutex.o - obj-$(CONFIG_DEBUG_MUTEXES) += mutex-debug.o -+obj-y += rwsem.o -+endif - obj-$(CONFIG_LOCKDEP) += lockdep.o - ifeq ($(CONFIG_PROC_FS),y) - obj-$(CONFIG_LOCKDEP) += lockdep_proc.o -@@ -22,8 +26,11 @@ - obj-$(CONFIG_RT_MUTEX_TESTER) += rtmutex-tester.o - obj-$(CONFIG_DEBUG_SPINLOCK) += spinlock.o - obj-$(CONFIG_DEBUG_SPINLOCK) += spinlock_debug.o -+ifneq ($(CONFIG_PREEMPT_RT_FULL),y) - obj-$(CONFIG_RWSEM_GENERIC_SPINLOCK) += rwsem-spinlock.o - obj-$(CONFIG_RWSEM_XCHGADD_ALGORITHM) += rwsem-xadd.o -+endif - obj-$(CONFIG_PERCPU_RWSEM) += percpu-rwsem.o -+obj-$(CONFIG_PREEMPT_RT_FULL) += rt.o - obj-$(CONFIG_QUEUE_RWLOCK) += qrwlock.o - obj-$(CONFIG_LOCK_TORTURE_TEST) += locktorture.o -diff -Nur linux-4.1.10.orig/kernel/locking/rt.c linux-4.1.10/kernel/locking/rt.c ---- linux-4.1.10.orig/kernel/locking/rt.c 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.10/kernel/locking/rt.c 2015-10-07 18:00:08.000000000 +0200 -@@ -0,0 +1,461 @@ -+/* -+ * kernel/rt.c -+ * -+ * Real-Time Preemption Support -+ * -+ * started by Ingo Molnar: -+ * -+ * Copyright (C) 2004-2006 Red Hat, Inc., Ingo Molnar -+ * Copyright (C) 2006, Timesys Corp., Thomas Gleixner -+ * -+ * historic credit for proving that Linux spinlocks can be implemented via -+ * RT-aware mutexes goes to many people: The Pmutex project (Dirk Grambow -+ * and others) who prototyped it on 2.4 and did lots of comparative -+ * research and analysis; TimeSys, for proving that you can implement a -+ * fully preemptible kernel via the use of IRQ threading and mutexes; -+ * Bill Huey for persuasively arguing on lkml that the mutex model is the -+ * right one; and to MontaVista, who ported pmutexes to 2.6. -+ * -+ * This code is a from-scratch implementation and is not based on pmutexes, -+ * but the idea of converting spinlocks to mutexes is used here too. -+ * -+ * lock debugging, locking tree, deadlock detection: -+ * -+ * Copyright (C) 2004, LynuxWorks, Inc., Igor Manyilov, Bill Huey -+ * Released under the General Public License (GPL). -+ * -+ * Includes portions of the generic R/W semaphore implementation from: -+ * -+ * Copyright (c) 2001 David Howells (dhowells@redhat.com). -+ * - Derived partially from idea by Andrea Arcangeli -+ * - Derived also from comments by Linus -+ * -+ * Pending ownership of locks and ownership stealing: -+ * -+ * Copyright (C) 2005, Kihon Technologies Inc., Steven Rostedt -+ * -+ * (also by Steven Rostedt) -+ * - Converted single pi_lock to individual task locks. -+ * -+ * By Esben Nielsen: -+ * Doing priority inheritance with help of the scheduler. -+ * -+ * Copyright (C) 2006, Timesys Corp., Thomas Gleixner -+ * - major rework based on Esben Nielsens initial patch -+ * - replaced thread_info references by task_struct refs -+ * - removed task->pending_owner dependency -+ * - BKL drop/reacquire for semaphore style locks to avoid deadlocks -+ * in the scheduler return path as discussed with Steven Rostedt -+ * -+ * Copyright (C) 2006, Kihon Technologies Inc. -+ * Steven Rostedt -+ * - debugged and patched Thomas Gleixner's rework. -+ * - added back the cmpxchg to the rework. -+ * - turned atomic require back on for SMP. -+ */ + config TRANSPARENT_HUGEPAGE + bool "Transparent Hugepage Support" +- depends on HAVE_ARCH_TRANSPARENT_HUGEPAGE ++ depends on HAVE_ARCH_TRANSPARENT_HUGEPAGE && !PREEMPT_RT_FULL + select COMPACTION + help + Transparent Hugepages allows the kernel to use huge pages and +diff -Nur linux-4.1.10.orig/mm/memcontrol.c linux-4.1.10/mm/memcontrol.c +--- linux-4.1.10.orig/mm/memcontrol.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/mm/memcontrol.c 2015-10-12 22:33:32.328673318 +0200 +@@ -66,6 +66,8 @@ + #include + #include + #include ++#include + -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include + #include "slab.h" + + #include +@@ -85,6 +87,7 @@ + #define do_swap_account 0 + #endif + ++static DEFINE_LOCAL_IRQ_LOCK(event_lock); + static const char * const mem_cgroup_stat_names[] = { + "cache", + "rss", +@@ -2124,14 +2127,17 @@ + */ + static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages) + { +- struct memcg_stock_pcp *stock = &get_cpu_var(memcg_stock); ++ struct memcg_stock_pcp *stock; ++ int cpu = get_cpu_light(); + -+#include "rtmutex_common.h" ++ stock = &per_cpu(memcg_stock, cpu); + + if (stock->cached != memcg) { /* reset if necessary */ + drain_stock(stock); + stock->cached = memcg; + } + stock->nr_pages += nr_pages; +- put_cpu_var(memcg_stock); ++ put_cpu_light(); + } + + /* +@@ -2147,7 +2153,7 @@ + return; + /* Notify other cpus that system-wide "drain" is running */ + get_online_cpus(); +- curcpu = get_cpu(); ++ curcpu = get_cpu_light(); + for_each_online_cpu(cpu) { + struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu); + struct mem_cgroup *memcg; +@@ -2164,7 +2170,7 @@ + schedule_work_on(cpu, &stock->work); + } + } +- put_cpu(); ++ put_cpu_light(); + put_online_cpus(); + mutex_unlock(&percpu_charge_mutex); + } +@@ -4801,12 +4807,12 @@ + + ret = 0; + +- local_irq_disable(); ++ local_lock_irq(event_lock); + mem_cgroup_charge_statistics(to, page, nr_pages); + memcg_check_events(to, page); + mem_cgroup_charge_statistics(from, page, -nr_pages); + memcg_check_events(from, page); +- local_irq_enable(); ++ local_unlock_irq(event_lock); + out_unlock: + unlock_page(page); + out: +@@ -5543,10 +5549,10 @@ + VM_BUG_ON_PAGE(!PageTransHuge(page), page); + } + +- local_irq_disable(); ++ local_lock_irq(event_lock); + mem_cgroup_charge_statistics(memcg, page, nr_pages); + memcg_check_events(memcg, page); +- local_irq_enable(); ++ local_unlock_irq(event_lock); + + if (do_swap_account && PageSwapCache(page)) { + swp_entry_t entry = { .val = page_private(page) }; +@@ -5602,14 +5608,14 @@ + memcg_oom_recover(memcg); + } + +- local_irq_save(flags); ++ local_lock_irqsave(event_lock, flags); + __this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_RSS], nr_anon); + __this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_CACHE], nr_file); + __this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_RSS_HUGE], nr_huge); + __this_cpu_add(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGOUT], pgpgout); + __this_cpu_add(memcg->stat->nr_page_events, nr_pages); + memcg_check_events(memcg, dummy_page); +- local_irq_restore(flags); ++ local_unlock_irqrestore(event_lock, flags); + + if (!mem_cgroup_is_root(memcg)) + css_put_many(&memcg->css, nr_pages); +@@ -5813,6 +5819,7 @@ + { + struct mem_cgroup *memcg; + unsigned short oldid; ++ unsigned long flags; + + VM_BUG_ON_PAGE(PageLRU(page), page); + VM_BUG_ON_PAGE(page_count(page), page); +@@ -5835,9 +5842,11 @@ + if (!mem_cgroup_is_root(memcg)) + page_counter_uncharge(&memcg->memory, 1); + ++ local_lock_irqsave(event_lock, flags); + /* Caller disabled preemption with mapping->tree_lock */ + mem_cgroup_charge_statistics(memcg, page, -1); + memcg_check_events(memcg, page); ++ local_unlock_irqrestore(event_lock, flags); + } + + /** +diff -Nur linux-4.1.10.orig/mm/memory.c linux-4.1.10/mm/memory.c +--- linux-4.1.10.orig/mm/memory.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/mm/memory.c 2015-10-12 22:33:32.328673318 +0200 +@@ -3743,7 +3743,7 @@ + } + + #if defined(CONFIG_PROVE_LOCKING) || defined(CONFIG_DEBUG_ATOMIC_SLEEP) +-void might_fault(void) ++void __might_fault(const char *file, int line) + { + /* + * Some code (nfs/sunrpc) uses socket ops on kernel memory while +@@ -3753,21 +3753,15 @@ + */ + if (segment_eq(get_fs(), KERNEL_DS)) + return; +- +- /* +- * it would be nicer only to annotate paths which are not under +- * pagefault_disable, however that requires a larger audit and +- * providing helpers like get_user_atomic. +- */ +- if (in_atomic()) ++ if (pagefault_disabled()) + return; +- +- __might_sleep(__FILE__, __LINE__, 0); +- ++ __might_sleep(file, line, 0); ++#if defined(CONFIG_DEBUG_ATOMIC_SLEEP) + if (current->mm) + might_lock_read(¤t->mm->mmap_sem); ++#endif + } +-EXPORT_SYMBOL(might_fault); ++EXPORT_SYMBOL(__might_fault); + #endif + + #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS) +diff -Nur linux-4.1.10.orig/mm/mmu_context.c linux-4.1.10/mm/mmu_context.c +--- linux-4.1.10.orig/mm/mmu_context.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/mm/mmu_context.c 2015-10-12 22:33:32.328673318 +0200 +@@ -23,6 +23,7 @@ + struct task_struct *tsk = current; + + task_lock(tsk); ++ preempt_disable_rt(); + active_mm = tsk->active_mm; + if (active_mm != mm) { + atomic_inc(&mm->mm_count); +@@ -30,6 +31,7 @@ + } + tsk->mm = mm; + switch_mm(active_mm, mm, tsk); ++ preempt_enable_rt(); + task_unlock(tsk); + #ifdef finish_arch_post_lock_switch + finish_arch_post_lock_switch(); +diff -Nur linux-4.1.10.orig/mm/page_alloc.c linux-4.1.10/mm/page_alloc.c +--- linux-4.1.10.orig/mm/page_alloc.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/mm/page_alloc.c 2015-10-12 22:33:32.328673318 +0200 +@@ -60,6 +60,7 @@ + #include + #include + #include ++#include + #include + + #include +@@ -233,6 +234,18 @@ + EXPORT_SYMBOL(nr_online_nodes); + #endif + ++static DEFINE_LOCAL_IRQ_LOCK(pa_lock); + -+/* -+ * struct mutex functions -+ */ -+void __mutex_do_init(struct mutex *mutex, const char *name, -+ struct lock_class_key *key) -+{ -+#ifdef CONFIG_DEBUG_LOCK_ALLOC -+ /* -+ * Make sure we are not reinitializing a held lock: -+ */ -+ debug_check_no_locks_freed((void *)mutex, sizeof(*mutex)); -+ lockdep_init_map(&mutex->dep_map, name, key, 0); ++#ifdef CONFIG_PREEMPT_RT_BASE ++# define cpu_lock_irqsave(cpu, flags) \ ++ local_lock_irqsave_on(pa_lock, flags, cpu) ++# define cpu_unlock_irqrestore(cpu, flags) \ ++ local_unlock_irqrestore_on(pa_lock, flags, cpu) ++#else ++# define cpu_lock_irqsave(cpu, flags) local_irq_save(flags) ++# define cpu_unlock_irqrestore(cpu, flags) local_irq_restore(flags) +#endif -+ mutex->lock.save_state = 0; -+} -+EXPORT_SYMBOL(__mutex_do_init); + -+void __lockfunc _mutex_lock(struct mutex *lock) -+{ -+ mutex_acquire(&lock->dep_map, 0, 0, _RET_IP_); -+ rt_mutex_lock(&lock->lock); -+} -+EXPORT_SYMBOL(_mutex_lock); + int page_group_by_mobility_disabled __read_mostly; + + void set_pageblock_migratetype(struct page *page, int migratetype) +@@ -681,7 +694,7 @@ + } + + /* +- * Frees a number of pages from the PCP lists ++ * Frees a number of pages which have been collected from the pcp lists. + * Assumes all pages on list are in same zone, and of same order. + * count is the number of pages to free. + * +@@ -692,18 +705,51 @@ + * pinned" detection logic. + */ + static void free_pcppages_bulk(struct zone *zone, int count, +- struct per_cpu_pages *pcp) ++ struct list_head *list) + { +- int migratetype = 0; +- int batch_free = 0; + int to_free = count; + unsigned long nr_scanned; ++ unsigned long flags; + -+int __lockfunc _mutex_lock_interruptible(struct mutex *lock) -+{ -+ int ret; -+ -+ mutex_acquire(&lock->dep_map, 0, 0, _RET_IP_); -+ ret = rt_mutex_lock_interruptible(&lock->lock); -+ if (ret) -+ mutex_release(&lock->dep_map, 1, _RET_IP_); -+ return ret; -+} -+EXPORT_SYMBOL(_mutex_lock_interruptible); -+ -+int __lockfunc _mutex_lock_killable(struct mutex *lock) -+{ -+ int ret; -+ -+ mutex_acquire(&lock->dep_map, 0, 0, _RET_IP_); -+ ret = rt_mutex_lock_killable(&lock->lock); -+ if (ret) -+ mutex_release(&lock->dep_map, 1, _RET_IP_); -+ return ret; -+} -+EXPORT_SYMBOL(_mutex_lock_killable); -+ -+#ifdef CONFIG_DEBUG_LOCK_ALLOC -+void __lockfunc _mutex_lock_nested(struct mutex *lock, int subclass) -+{ -+ mutex_acquire_nest(&lock->dep_map, subclass, 0, NULL, _RET_IP_); -+ rt_mutex_lock(&lock->lock); -+} -+EXPORT_SYMBOL(_mutex_lock_nested); -+ -+void __lockfunc _mutex_lock_nest_lock(struct mutex *lock, struct lockdep_map *nest) -+{ -+ mutex_acquire_nest(&lock->dep_map, 0, 0, nest, _RET_IP_); -+ rt_mutex_lock(&lock->lock); -+} -+EXPORT_SYMBOL(_mutex_lock_nest_lock); -+ -+int __lockfunc _mutex_lock_interruptible_nested(struct mutex *lock, int subclass) -+{ -+ int ret; -+ -+ mutex_acquire_nest(&lock->dep_map, subclass, 0, NULL, _RET_IP_); -+ ret = rt_mutex_lock_interruptible(&lock->lock); -+ if (ret) -+ mutex_release(&lock->dep_map, 1, _RET_IP_); -+ return ret; -+} -+EXPORT_SYMBOL(_mutex_lock_interruptible_nested); -+ -+int __lockfunc _mutex_lock_killable_nested(struct mutex *lock, int subclass) -+{ -+ int ret; -+ -+ mutex_acquire(&lock->dep_map, subclass, 0, _RET_IP_); -+ ret = rt_mutex_lock_killable(&lock->lock); -+ if (ret) -+ mutex_release(&lock->dep_map, 1, _RET_IP_); -+ return ret; -+} -+EXPORT_SYMBOL(_mutex_lock_killable_nested); -+#endif -+ -+int __lockfunc _mutex_trylock(struct mutex *lock) -+{ -+ int ret = rt_mutex_trylock(&lock->lock); -+ -+ if (ret) -+ mutex_acquire(&lock->dep_map, 0, 1, _RET_IP_); -+ -+ return ret; -+} -+EXPORT_SYMBOL(_mutex_trylock); -+ -+void __lockfunc _mutex_unlock(struct mutex *lock) -+{ -+ mutex_release(&lock->dep_map, 1, _RET_IP_); -+ rt_mutex_unlock(&lock->lock); -+} -+EXPORT_SYMBOL(_mutex_unlock); -+ -+/* -+ * rwlock_t functions -+ */ -+int __lockfunc rt_write_trylock(rwlock_t *rwlock) -+{ -+ int ret; -+ -+ migrate_disable(); -+ ret = rt_mutex_trylock(&rwlock->lock); -+ if (ret) -+ rwlock_acquire(&rwlock->dep_map, 0, 1, _RET_IP_); -+ else -+ migrate_enable(); -+ -+ return ret; -+} -+EXPORT_SYMBOL(rt_write_trylock); -+ -+int __lockfunc rt_write_trylock_irqsave(rwlock_t *rwlock, unsigned long *flags) -+{ -+ int ret; -+ -+ *flags = 0; -+ ret = rt_write_trylock(rwlock); -+ return ret; -+} -+EXPORT_SYMBOL(rt_write_trylock_irqsave); -+ -+int __lockfunc rt_read_trylock(rwlock_t *rwlock) -+{ -+ struct rt_mutex *lock = &rwlock->lock; -+ int ret = 1; -+ -+ /* -+ * recursive read locks succeed when current owns the lock, -+ * but not when read_depth == 0 which means that the lock is -+ * write locked. -+ */ -+ if (rt_mutex_owner(lock) != current) { -+ migrate_disable(); -+ ret = rt_mutex_trylock(lock); -+ if (ret) -+ rwlock_acquire(&rwlock->dep_map, 0, 1, _RET_IP_); -+ else -+ migrate_enable(); -+ -+ } else if (!rwlock->read_depth) { -+ ret = 0; -+ } -+ -+ if (ret) -+ rwlock->read_depth++; -+ -+ return ret; -+} -+EXPORT_SYMBOL(rt_read_trylock); -+ -+void __lockfunc rt_write_lock(rwlock_t *rwlock) -+{ -+ rwlock_acquire(&rwlock->dep_map, 0, 0, _RET_IP_); -+ migrate_disable(); -+ __rt_spin_lock(&rwlock->lock); -+} -+EXPORT_SYMBOL(rt_write_lock); -+ -+void __lockfunc rt_read_lock(rwlock_t *rwlock) -+{ -+ struct rt_mutex *lock = &rwlock->lock; -+ -+ -+ /* -+ * recursive read locks succeed when current owns the lock -+ */ -+ if (rt_mutex_owner(lock) != current) { -+ migrate_disable(); -+ rwlock_acquire(&rwlock->dep_map, 0, 0, _RET_IP_); -+ __rt_spin_lock(lock); -+ } -+ rwlock->read_depth++; -+} ++ spin_lock_irqsave(&zone->lock, flags); + +- spin_lock(&zone->lock); + nr_scanned = zone_page_state(zone, NR_PAGES_SCANNED); + if (nr_scanned) + __mod_zone_page_state(zone, NR_PAGES_SCANNED, -nr_scanned); + ++ while (!list_empty(list)) { ++ struct page *page = list_first_entry(list, struct page, lru); ++ int mt; /* migratetype of the to-be-freed page */ + -+EXPORT_SYMBOL(rt_read_lock); ++ /* must delete as __free_one_page list manipulates */ ++ list_del(&page->lru); + -+void __lockfunc rt_write_unlock(rwlock_t *rwlock) -+{ -+ /* NOTE: we always pass in '1' for nested, for simplicity */ -+ rwlock_release(&rwlock->dep_map, 1, _RET_IP_); -+ __rt_spin_unlock(&rwlock->lock); -+ migrate_enable(); -+} -+EXPORT_SYMBOL(rt_write_unlock); ++ mt = get_freepage_migratetype(page); ++ if (unlikely(has_isolate_pageblock(zone))) ++ mt = get_pageblock_migratetype(page); + -+void __lockfunc rt_read_unlock(rwlock_t *rwlock) -+{ -+ /* Release the lock only when read_depth is down to 0 */ -+ if (--rwlock->read_depth == 0) { -+ rwlock_release(&rwlock->dep_map, 1, _RET_IP_); -+ __rt_spin_unlock(&rwlock->lock); -+ migrate_enable(); ++ /* MIGRATE_MOVABLE list may include MIGRATE_RESERVEs */ ++ __free_one_page(page, page_to_pfn(page), zone, 0, mt); ++ trace_mm_page_pcpu_drain(page, 0, mt); ++ to_free--; + } ++ WARN_ON(to_free != 0); ++ spin_unlock_irqrestore(&zone->lock, flags); +} -+EXPORT_SYMBOL(rt_read_unlock); -+ -+unsigned long __lockfunc rt_write_lock_irqsave(rwlock_t *rwlock) -+{ -+ rt_write_lock(rwlock); -+ -+ return 0; -+} -+EXPORT_SYMBOL(rt_write_lock_irqsave); -+ -+unsigned long __lockfunc rt_read_lock_irqsave(rwlock_t *rwlock) -+{ -+ rt_read_lock(rwlock); -+ -+ return 0; -+} -+EXPORT_SYMBOL(rt_read_lock_irqsave); -+ -+void __rt_rwlock_init(rwlock_t *rwlock, char *name, struct lock_class_key *key) -+{ -+#ifdef CONFIG_DEBUG_LOCK_ALLOC -+ /* -+ * Make sure we are not reinitializing a held lock: -+ */ -+ debug_check_no_locks_freed((void *)rwlock, sizeof(*rwlock)); -+ lockdep_init_map(&rwlock->dep_map, name, key, 0); -+#endif -+ rwlock->lock.save_state = 1; -+ rwlock->read_depth = 0; -+} -+EXPORT_SYMBOL(__rt_rwlock_init); -+ -+/* -+ * rw_semaphores -+ */ -+ -+void rt_up_write(struct rw_semaphore *rwsem) -+{ -+ rwsem_release(&rwsem->dep_map, 1, _RET_IP_); -+ rt_mutex_unlock(&rwsem->lock); -+} -+EXPORT_SYMBOL(rt_up_write); -+ -+void __rt_up_read(struct rw_semaphore *rwsem) -+{ -+ if (--rwsem->read_depth == 0) -+ rt_mutex_unlock(&rwsem->lock); -+} -+ -+void rt_up_read(struct rw_semaphore *rwsem) -+{ -+ rwsem_release(&rwsem->dep_map, 1, _RET_IP_); -+ __rt_up_read(rwsem); -+} -+EXPORT_SYMBOL(rt_up_read); + +/* -+ * downgrade a write lock into a read lock -+ * - just wake up any readers at the front of the queue -+ */ -+void rt_downgrade_write(struct rw_semaphore *rwsem) -+{ -+ BUG_ON(rt_mutex_owner(&rwsem->lock) != current); -+ rwsem->read_depth = 1; -+} -+EXPORT_SYMBOL(rt_downgrade_write); -+ -+int rt_down_write_trylock(struct rw_semaphore *rwsem) -+{ -+ int ret = rt_mutex_trylock(&rwsem->lock); -+ -+ if (ret) -+ rwsem_acquire(&rwsem->dep_map, 0, 1, _RET_IP_); -+ return ret; -+} -+EXPORT_SYMBOL(rt_down_write_trylock); -+ -+void rt_down_write(struct rw_semaphore *rwsem) -+{ -+ rwsem_acquire(&rwsem->dep_map, 0, 0, _RET_IP_); -+ rt_mutex_lock(&rwsem->lock); -+} -+EXPORT_SYMBOL(rt_down_write); -+ -+void rt_down_write_nested(struct rw_semaphore *rwsem, int subclass) -+{ -+ rwsem_acquire(&rwsem->dep_map, subclass, 0, _RET_IP_); -+ rt_mutex_lock(&rwsem->lock); -+} -+EXPORT_SYMBOL(rt_down_write_nested); -+ -+void rt_down_write_nested_lock(struct rw_semaphore *rwsem, -+ struct lockdep_map *nest) -+{ -+ rwsem_acquire_nest(&rwsem->dep_map, 0, 0, nest, _RET_IP_); -+ rt_mutex_lock(&rwsem->lock); -+} -+EXPORT_SYMBOL(rt_down_write_nested_lock); -+ -+int rt_down_read_trylock(struct rw_semaphore *rwsem) -+{ -+ struct rt_mutex *lock = &rwsem->lock; -+ int ret = 1; -+ -+ /* -+ * recursive read locks succeed when current owns the rwsem, -+ * but not when read_depth == 0 which means that the rwsem is -+ * write locked. -+ */ -+ if (rt_mutex_owner(lock) != current) -+ ret = rt_mutex_trylock(&rwsem->lock); -+ else if (!rwsem->read_depth) -+ ret = 0; -+ -+ if (ret) { -+ rwsem->read_depth++; -+ rwsem_acquire(&rwsem->dep_map, 0, 1, _RET_IP_); -+ } -+ return ret; -+} -+EXPORT_SYMBOL(rt_down_read_trylock); -+ -+static void __rt_down_read(struct rw_semaphore *rwsem, int subclass) -+{ -+ struct rt_mutex *lock = &rwsem->lock; -+ -+ rwsem_acquire_read(&rwsem->dep_map, subclass, 0, _RET_IP_); -+ -+ if (rt_mutex_owner(lock) != current) -+ rt_mutex_lock(&rwsem->lock); -+ rwsem->read_depth++; -+} -+ -+void rt_down_read(struct rw_semaphore *rwsem) -+{ -+ __rt_down_read(rwsem, 0); -+} -+EXPORT_SYMBOL(rt_down_read); -+ -+void rt_down_read_nested(struct rw_semaphore *rwsem, int subclass) -+{ -+ __rt_down_read(rwsem, subclass); -+} -+EXPORT_SYMBOL(rt_down_read_nested); -+ -+void __rt_rwsem_init(struct rw_semaphore *rwsem, const char *name, -+ struct lock_class_key *key) -+{ -+#ifdef CONFIG_DEBUG_LOCK_ALLOC -+ /* -+ * Make sure we are not reinitializing a held lock: -+ */ -+ debug_check_no_locks_freed((void *)rwsem, sizeof(*rwsem)); -+ lockdep_init_map(&rwsem->dep_map, name, key, 0); -+#endif -+ rwsem->read_depth = 0; -+ rwsem->lock.save_state = 0; -+} -+EXPORT_SYMBOL(__rt_rwsem_init); -+ -+/** -+ * atomic_dec_and_mutex_lock - return holding mutex if we dec to 0 -+ * @cnt: the atomic which we are to dec -+ * @lock: the mutex to return holding if we dec to 0 ++ * Moves a number of pages from the PCP lists to free list which ++ * is freed outside of the locked region. + * -+ * return true and hold lock if we dec to 0, return false otherwise ++ * Assumes all pages on list are in same zone, and of same order. ++ * count is the number of pages to free. + */ -+int atomic_dec_and_mutex_lock(atomic_t *cnt, struct mutex *lock) ++static void isolate_pcp_pages(int to_free, struct per_cpu_pages *src, ++ struct list_head *dst) +{ -+ /* dec if we can't possibly hit 0 */ -+ if (atomic_add_unless(cnt, -1, 1)) -+ return 0; -+ /* we might hit 0, so take the lock */ -+ mutex_lock(lock); -+ if (!atomic_dec_and_test(cnt)) { -+ /* when we actually did the dec, we didn't hit 0 */ -+ mutex_unlock(lock); -+ return 0; -+ } -+ /* we hit 0, and we hold the lock */ -+ return 1; -+} -+EXPORT_SYMBOL(atomic_dec_and_mutex_lock); -diff -Nur linux-4.1.10.orig/kernel/locking/rtmutex.c linux-4.1.10/kernel/locking/rtmutex.c ---- linux-4.1.10.orig/kernel/locking/rtmutex.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/locking/rtmutex.c 2015-10-07 18:00:08.000000000 +0200 -@@ -7,6 +7,11 @@ - * Copyright (C) 2005-2006 Timesys Corp., Thomas Gleixner - * Copyright (C) 2005 Kihon Technologies Inc., Steven Rostedt - * Copyright (C) 2006 Esben Nielsen -+ * Adaptive Spinlocks: -+ * Copyright (C) 2008 Novell, Inc., Gregory Haskins, Sven Dietrich, -+ * and Peter Morreale, -+ * Adaptive Spinlocks simplification: -+ * Copyright (C) 2008 Red Hat, Inc., Steven Rostedt - * - * See Documentation/locking/rt-mutex-design.txt for details. - */ -@@ -16,6 +21,7 @@ - #include - #include - #include -+#include ++ int migratetype = 0; ++ int batch_free = 0; ++ + while (to_free) { + struct page *page; + struct list_head *list; +@@ -719,7 +765,7 @@ + batch_free++; + if (++migratetype == MIGRATE_PCPTYPES) + migratetype = 0; +- list = &pcp->lists[migratetype]; ++ list = &src->lists[migratetype]; + } while (list_empty(list)); - #include "rtmutex_common.h" + /* This is the only non-empty list. Free them all. */ +@@ -727,21 +773,11 @@ + batch_free = to_free; -@@ -69,6 +75,12 @@ - clear_rt_mutex_waiters(lock); + do { +- int mt; /* migratetype of the to-be-freed page */ +- +- page = list_entry(list->prev, struct page, lru); +- /* must delete as __free_one_page list manipulates */ ++ page = list_last_entry(list, struct page, lru); + list_del(&page->lru); +- mt = get_freepage_migratetype(page); +- if (unlikely(has_isolate_pageblock(zone))) +- mt = get_pageblock_migratetype(page); +- +- /* MIGRATE_MOVABLE list may include MIGRATE_RESERVEs */ +- __free_one_page(page, page_to_pfn(page), zone, 0, mt); +- trace_mm_page_pcpu_drain(page, 0, mt); ++ list_add(&page->lru, dst); + } while (--to_free && --batch_free && !list_empty(list)); + } +- spin_unlock(&zone->lock); } -+static int rt_mutex_real_waiter(struct rt_mutex_waiter *waiter) -+{ -+ return waiter && waiter != PI_WAKEUP_INPROGRESS && -+ waiter != PI_REQUEUE_INPROGRESS; -+} -+ - /* - * We can speed up the acquire/release, if the architecture - * supports cmpxchg and if there's no debugging state to be set up -@@ -300,7 +312,7 @@ - * of task. We do not use the spin_xx_mutex() variants here as we are - * outside of the debug path.) - */ --static void rt_mutex_adjust_prio(struct task_struct *task) -+void rt_mutex_adjust_prio(struct task_struct *task) + static void free_one_page(struct zone *zone, +@@ -750,7 +786,9 @@ + int migratetype) { - unsigned long flags; - -@@ -335,6 +347,14 @@ - return debug_rt_mutex_detect_deadlock(waiter, chwalk); + unsigned long nr_scanned; +- spin_lock(&zone->lock); ++ unsigned long flags; ++ ++ spin_lock_irqsave(&zone->lock, flags); + nr_scanned = zone_page_state(zone, NR_PAGES_SCANNED); + if (nr_scanned) + __mod_zone_page_state(zone, NR_PAGES_SCANNED, -nr_scanned); +@@ -760,7 +798,7 @@ + migratetype = get_pfnblock_migratetype(page, pfn); + } + __free_one_page(page, pfn, zone, order, migratetype); +- spin_unlock(&zone->lock); ++ spin_unlock_irqrestore(&zone->lock, flags); } -+static void rt_mutex_wake_waiter(struct rt_mutex_waiter *waiter) -+{ -+ if (waiter->savestate) -+ wake_up_lock_sleeper(waiter->task); -+ else -+ wake_up_process(waiter->task); -+} -+ - /* - * Max number of times we'll walk the boosting chain: - */ -@@ -342,7 +362,8 @@ + static int free_tail_pages_check(struct page *head_page, struct page *page) +@@ -825,11 +863,11 @@ + return; - static inline struct rt_mutex *task_blocked_on_lock(struct task_struct *p) - { -- return p->pi_blocked_on ? p->pi_blocked_on->lock : NULL; -+ return rt_mutex_real_waiter(p->pi_blocked_on) ? -+ p->pi_blocked_on->lock : NULL; + migratetype = get_pfnblock_migratetype(page, pfn); +- local_irq_save(flags); ++ local_lock_irqsave(pa_lock, flags); + __count_vm_events(PGFREE, 1 << order); + set_freepage_migratetype(page, migratetype); + free_one_page(page_zone(page), page, pfn, order, migratetype); +- local_irq_restore(flags); ++ local_unlock_irqrestore(pa_lock, flags); } - /* -@@ -479,7 +500,7 @@ - * reached or the state of the chain has changed while we - * dropped the locks. - */ -- if (!waiter) -+ if (!rt_mutex_real_waiter(waiter)) - goto out_unlock_pi; - - /* -@@ -641,13 +662,16 @@ - * follow here. This is the end of the chain we are walking. - */ - if (!rt_mutex_owner(lock)) { -+ struct rt_mutex_waiter *lock_top_waiter; -+ - /* - * If the requeue [7] above changed the top waiter, - * then we need to wake the new top waiter up to try - * to get the lock. - */ -- if (prerequeue_top_waiter != rt_mutex_top_waiter(lock)) -- wake_up_process(rt_mutex_top_waiter(lock)->task); -+ lock_top_waiter = rt_mutex_top_waiter(lock); -+ if (prerequeue_top_waiter != lock_top_waiter) -+ rt_mutex_wake_waiter(lock_top_waiter); - raw_spin_unlock(&lock->wait_lock); - return 0; + void __init __free_pages_bootmem(struct page *page, unsigned int order) +@@ -1371,16 +1409,18 @@ + void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp) + { + unsigned long flags; ++ LIST_HEAD(dst); + int to_drain, batch; + +- local_irq_save(flags); ++ local_lock_irqsave(pa_lock, flags); + batch = READ_ONCE(pcp->batch); + to_drain = min(pcp->count, batch); + if (to_drain > 0) { +- free_pcppages_bulk(zone, to_drain, pcp); ++ isolate_pcp_pages(to_drain, pcp, &dst); + pcp->count -= to_drain; } -@@ -740,6 +764,25 @@ - return ret; +- local_irq_restore(flags); ++ local_unlock_irqrestore(pa_lock, flags); ++ free_pcppages_bulk(zone, to_drain, &dst); } + #endif +@@ -1396,16 +1436,21 @@ + unsigned long flags; + struct per_cpu_pageset *pset; + struct per_cpu_pages *pcp; ++ LIST_HEAD(dst); ++ int count; + +- local_irq_save(flags); ++ cpu_lock_irqsave(cpu, flags); + pset = per_cpu_ptr(zone->pageset, cpu); + + pcp = &pset->pcp; +- if (pcp->count) { +- free_pcppages_bulk(zone, pcp->count, pcp); ++ count = pcp->count; ++ if (count) { ++ isolate_pcp_pages(count, pcp, &dst); + pcp->count = 0; + } +- local_irq_restore(flags); ++ cpu_unlock_irqrestore(cpu, flags); ++ if (count) ++ free_pcppages_bulk(zone, count, &dst); + } + + /* +@@ -1491,8 +1536,17 @@ + else + cpumask_clear_cpu(cpu, &cpus_with_pcps); + } ++#ifndef CONFIG_PREEMPT_RT_BASE + on_each_cpu_mask(&cpus_with_pcps, (smp_call_func_t) drain_local_pages, + zone, 1); ++#else ++ for_each_cpu(cpu, &cpus_with_pcps) { ++ if (zone) ++ drain_pages_zone(cpu, zone); ++ else ++ drain_pages(cpu); ++ } ++#endif + } + + #ifdef CONFIG_HIBERNATION +@@ -1548,7 +1602,7 @@ + + migratetype = get_pfnblock_migratetype(page, pfn); + set_freepage_migratetype(page, migratetype); +- local_irq_save(flags); ++ local_lock_irqsave(pa_lock, flags); + __count_vm_event(PGFREE); + + /* +@@ -1574,12 +1628,17 @@ + pcp->count++; + if (pcp->count >= pcp->high) { + unsigned long batch = READ_ONCE(pcp->batch); +- free_pcppages_bulk(zone, batch, pcp); ++ LIST_HEAD(dst); + -+#define STEAL_NORMAL 0 -+#define STEAL_LATERAL 1 -+ -+/* -+ * Note that RT tasks are excluded from lateral-steals to prevent the -+ * introduction of an unbounded latency -+ */ -+static inline int lock_is_stealable(struct task_struct *task, -+ struct task_struct *pendowner, int mode) -+{ -+ if (mode == STEAL_NORMAL || rt_task(task)) { -+ if (task->prio >= pendowner->prio) -+ return 0; -+ } else if (task->prio > pendowner->prio) -+ return 0; -+ return 1; -+} -+ ++ isolate_pcp_pages(batch, pcp, &dst); + pcp->count -= batch; ++ local_unlock_irqrestore(pa_lock, flags); ++ free_pcppages_bulk(zone, batch, &dst); ++ return; + } + + out: +- local_irq_restore(flags); ++ local_unlock_irqrestore(pa_lock, flags); + } + /* - * Try to take an rt-mutex - * -@@ -750,8 +793,9 @@ - * @waiter: The waiter that is queued to the lock's wait list if the - * callsite called task_blocked_on_lock(), otherwise NULL +@@ -1710,7 +1769,7 @@ + struct per_cpu_pages *pcp; + struct list_head *list; + +- local_irq_save(flags); ++ local_lock_irqsave(pa_lock, flags); + pcp = &this_cpu_ptr(zone->pageset)->pcp; + list = &pcp->lists[migratetype]; + if (list_empty(list)) { +@@ -1742,13 +1801,15 @@ + */ + WARN_ON_ONCE(order > 1); + } +- spin_lock_irqsave(&zone->lock, flags); ++ local_spin_lock_irqsave(pa_lock, &zone->lock, flags); + page = __rmqueue(zone, order, migratetype); +- spin_unlock(&zone->lock); +- if (!page) ++ if (!page) { ++ spin_unlock(&zone->lock); + goto failed; ++ } + __mod_zone_freepage_state(zone, -(1 << order), + get_freepage_migratetype(page)); ++ spin_unlock(&zone->lock); + } + + __mod_zone_page_state(zone, NR_ALLOC_BATCH, -(1 << order)); +@@ -1758,13 +1819,13 @@ + + __count_zone_vm_events(PGALLOC, zone, 1 << order); + zone_statistics(preferred_zone, zone, gfp_flags); +- local_irq_restore(flags); ++ local_unlock_irqrestore(pa_lock, flags); + + VM_BUG_ON_PAGE(bad_range(zone, page), page); + return page; + + failed: +- local_irq_restore(flags); ++ local_unlock_irqrestore(pa_lock, flags); + return NULL; + } + +@@ -5653,6 +5714,7 @@ + void __init page_alloc_init(void) + { + hotcpu_notifier(page_alloc_cpu_notify, 0); ++ local_irq_lock_init(pa_lock); + } + + /* +@@ -6547,7 +6609,7 @@ + struct per_cpu_pageset *pset; + + /* avoid races with drain_pages() */ +- local_irq_save(flags); ++ local_lock_irqsave(pa_lock, flags); + if (zone->pageset != &boot_pageset) { + for_each_online_cpu(cpu) { + pset = per_cpu_ptr(zone->pageset, cpu); +@@ -6556,7 +6618,7 @@ + free_percpu(zone->pageset); + zone->pageset = &boot_pageset; + } +- local_irq_restore(flags); ++ local_unlock_irqrestore(pa_lock, flags); + } + + #ifdef CONFIG_MEMORY_HOTREMOVE +diff -Nur linux-4.1.10.orig/mm/slab.h linux-4.1.10/mm/slab.h +--- linux-4.1.10.orig/mm/slab.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/mm/slab.h 2015-10-12 22:33:32.332673054 +0200 +@@ -330,7 +330,11 @@ + * The slab lists for all objects. */ --static int try_to_take_rt_mutex(struct rt_mutex *lock, struct task_struct *task, -- struct rt_mutex_waiter *waiter) -+static int __try_to_take_rt_mutex(struct rt_mutex *lock, -+ struct task_struct *task, -+ struct rt_mutex_waiter *waiter, int mode) + struct kmem_cache_node { ++#ifdef CONFIG_SLUB ++ raw_spinlock_t list_lock; ++#else + spinlock_t list_lock; ++#endif + + #ifdef CONFIG_SLAB + struct list_head slabs_partial; /* partial list first, better asm code */ +diff -Nur linux-4.1.10.orig/mm/slub.c linux-4.1.10/mm/slub.c +--- linux-4.1.10.orig/mm/slub.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/mm/slub.c 2015-10-12 22:33:32.332673054 +0200 +@@ -1069,7 +1069,7 @@ { - unsigned long flags; + struct kmem_cache_node *n = get_node(s, page_to_nid(page)); -@@ -790,8 +834,10 @@ - * If waiter is not the highest priority waiter of - * @lock, give up. - */ -- if (waiter != rt_mutex_top_waiter(lock)) -+ if (waiter != rt_mutex_top_waiter(lock)) { -+ /* XXX lock_is_stealable() ? */ - return 0; -+ } +- spin_lock_irqsave(&n->list_lock, *flags); ++ raw_spin_lock_irqsave(&n->list_lock, *flags); + slab_lock(page); - /* - * We can acquire the lock. Remove the waiter from the -@@ -809,14 +855,10 @@ - * not need to be dequeued. - */ - if (rt_mutex_has_waiters(lock)) { -- /* -- * If @task->prio is greater than or equal to -- * the top waiter priority (kernel view), -- * @task lost. -- */ -- if (task->prio >= rt_mutex_top_waiter(lock)->prio) -- return 0; -+ struct task_struct *pown = rt_mutex_top_waiter(lock)->task; + if (!check_slab(s, page)) +@@ -1116,7 +1116,7 @@ -+ if (task != pown && !lock_is_stealable(task, pown, mode)) -+ return 0; - /* - * The current top waiter stays enqueued. We - * don't have to change anything in the lock -@@ -865,6 +907,347 @@ - return 1; + fail: + slab_unlock(page); +- spin_unlock_irqrestore(&n->list_lock, *flags); ++ raw_spin_unlock_irqrestore(&n->list_lock, *flags); + slab_fix(s, "Object at 0x%p not freed", object); + return NULL; } +@@ -1242,6 +1242,12 @@ -+#ifdef CONFIG_PREEMPT_RT_FULL -+/* -+ * preemptible spin_lock functions: -+ */ -+static inline void rt_spin_lock_fastlock(struct rt_mutex *lock, -+ void (*slowfn)(struct rt_mutex *lock)) -+{ -+ might_sleep_no_state_check(); -+ -+ if (likely(rt_mutex_cmpxchg(lock, NULL, current))) -+ rt_mutex_deadlock_account_lock(lock, current); -+ else -+ slowfn(lock); -+} + #endif /* CONFIG_SLUB_DEBUG */ + ++struct slub_free_list { ++ raw_spinlock_t lock; ++ struct list_head list; ++}; ++static DEFINE_PER_CPU(struct slub_free_list, slub_free_list); + -+static inline void rt_spin_lock_fastunlock(struct rt_mutex *lock, -+ void (*slowfn)(struct rt_mutex *lock)) -+{ -+ if (likely(rt_mutex_cmpxchg(lock, current, NULL))) -+ rt_mutex_deadlock_account_unlock(current); -+ else -+ slowfn(lock); -+} -+#ifdef CONFIG_SMP -+/* -+ * Note that owner is a speculative pointer and dereferencing relies -+ * on rcu_read_lock() and the check against the lock owner. -+ */ -+static int adaptive_wait(struct rt_mutex *lock, -+ struct task_struct *owner) + /* + * Hooks for other subsystems that check memory allocations. In a typical + * production configuration these hooks all should produce no code at all. +@@ -1306,6 +1312,17 @@ + kasan_slab_free(s, x); + } + ++static void setup_object(struct kmem_cache *s, struct page *page, ++ void *object) +{ -+ int res = 0; -+ -+ rcu_read_lock(); -+ for (;;) { -+ if (owner != rt_mutex_owner(lock)) -+ break; -+ /* -+ * Ensure that owner->on_cpu is dereferenced _after_ -+ * checking the above to be valid. -+ */ -+ barrier(); -+ if (!owner->on_cpu) { -+ res = 1; -+ break; -+ } -+ cpu_relax(); ++ setup_object_debug(s, page, object); ++ if (unlikely(s->ctor)) { ++ kasan_unpoison_object_data(s, object); ++ s->ctor(object); ++ kasan_poison_object_data(s, object); + } -+ rcu_read_unlock(); -+ return res; -+} -+#else -+static int adaptive_wait(struct rt_mutex *lock, -+ struct task_struct *orig_owner) -+{ -+ return 1; +} -+#endif + -+# define pi_lock(lock) raw_spin_lock_irq(lock) -+# define pi_unlock(lock) raw_spin_unlock_irq(lock) + /* + * Slab allocation and freeing + */ +@@ -1336,10 +1353,17 @@ + struct page *page; + struct kmem_cache_order_objects oo = s->oo; + gfp_t alloc_gfp; ++ void *start, *p; ++ int idx, order; ++ bool enableirqs; + + flags &= gfp_allowed_mask; + +- if (flags & __GFP_WAIT) ++ enableirqs = (flags & __GFP_WAIT) != 0; ++#ifdef CONFIG_PREEMPT_RT_FULL ++ enableirqs |= system_state == SYSTEM_RUNNING; ++#endif ++ if (enableirqs) + local_irq_enable(); + + flags |= s->allocflags; +@@ -1359,13 +1383,13 @@ + * Try a lower order alloc if possible + */ + page = alloc_slab_page(s, alloc_gfp, node, oo); +- +- if (page) +- stat(s, ORDER_FALLBACK); ++ if (unlikely(!page)) ++ goto out; ++ stat(s, ORDER_FALLBACK); + } + +- if (kmemcheck_enabled && page +- && !(s->flags & (SLAB_NOTRACK | DEBUG_DEFAULT_FLAGS))) { ++ if (kmemcheck_enabled && ++ !(s->flags & (SLAB_NOTRACK | DEBUG_DEFAULT_FLAGS))) { + int pages = 1 << oo_order(oo); + + kmemcheck_alloc_shadow(page, oo_order(oo), alloc_gfp, node); +@@ -1380,51 +1404,9 @@ + kmemcheck_mark_unallocated_pages(page, pages); + } + +- if (flags & __GFP_WAIT) +- local_irq_disable(); +- if (!page) +- return NULL; +- + page->objects = oo_objects(oo); +- mod_zone_page_state(page_zone(page), +- (s->flags & SLAB_RECLAIM_ACCOUNT) ? +- NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE, +- 1 << oo_order(oo)); +- +- return page; +-} +- +-static void setup_object(struct kmem_cache *s, struct page *page, +- void *object) +-{ +- setup_object_debug(s, page, object); +- if (unlikely(s->ctor)) { +- kasan_unpoison_object_data(s, object); +- s->ctor(object); +- kasan_poison_object_data(s, object); +- } +-} +- +-static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node) +-{ +- struct page *page; +- void *start; +- void *p; +- int order; +- int idx; +- +- if (unlikely(flags & GFP_SLAB_BUG_MASK)) { +- pr_emerg("gfp: %u\n", flags & GFP_SLAB_BUG_MASK); +- BUG(); +- } +- +- page = allocate_slab(s, +- flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node); +- if (!page) +- goto out; + + order = compound_order(page); +- inc_slabs_node(s, page_to_nid(page), page->objects); + page->slab_cache = s; + __SetPageSlab(page); + if (page_is_pfmemalloc(page)) +@@ -1448,10 +1430,34 @@ + page->freelist = start; + page->inuse = page->objects; + page->frozen = 1; + -+static int task_blocks_on_rt_mutex(struct rt_mutex *lock, -+ struct rt_mutex_waiter *waiter, -+ struct task_struct *task, -+ enum rtmutex_chainwalk chwalk); -+/* -+ * Slow path lock function spin_lock style: this variant is very -+ * careful not to miss any non-lock wakeups. -+ * -+ * We store the current state under p->pi_lock in p->saved_state and -+ * the try_to_wake_up() code handles this accordingly. -+ */ -+static void noinline __sched rt_spin_lock_slowlock(struct rt_mutex *lock) -+{ -+ struct task_struct *lock_owner, *self = current; -+ struct rt_mutex_waiter waiter, *top_waiter; -+ int ret; + out: ++ if (enableirqs) ++ local_irq_disable(); ++ if (!page) ++ return NULL; + -+ rt_mutex_init_waiter(&waiter, true); ++ mod_zone_page_state(page_zone(page), ++ (s->flags & SLAB_RECLAIM_ACCOUNT) ? ++ NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE, ++ 1 << oo_order(oo)); + -+ raw_spin_lock(&lock->wait_lock); ++ inc_slabs_node(s, page_to_nid(page), page->objects); + -+ if (__try_to_take_rt_mutex(lock, self, NULL, STEAL_LATERAL)) { -+ raw_spin_unlock(&lock->wait_lock); -+ return; + return page; + } + ++static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node) ++{ ++ if (unlikely(flags & GFP_SLAB_BUG_MASK)) { ++ pr_emerg("gfp: %u\n", flags & GFP_SLAB_BUG_MASK); ++ BUG(); + } + -+ BUG_ON(rt_mutex_owner(lock) == self); ++ return allocate_slab(s, ++ flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node); ++} + -+ /* -+ * We save whatever state the task is in and we'll restore it -+ * after acquiring the lock taking real wakeups into account -+ * as well. We are serialized via pi_lock against wakeups. See -+ * try_to_wake_up(). -+ */ -+ pi_lock(&self->pi_lock); -+ self->saved_state = self->state; -+ __set_current_state_no_track(TASK_UNINTERRUPTIBLE); -+ pi_unlock(&self->pi_lock); + static void __free_slab(struct kmem_cache *s, struct page *page) + { + int order = compound_order(page); +@@ -1483,6 +1489,16 @@ + memcg_uncharge_slab(s, order); + } + ++static void free_delayed(struct list_head *h) ++{ ++ while(!list_empty(h)) { ++ struct page *page = list_first_entry(h, struct page, lru); + -+ ret = task_blocks_on_rt_mutex(lock, &waiter, self, 0); -+ BUG_ON(ret); ++ list_del(&page->lru); ++ __free_slab(page->slab_cache, page); ++ } ++} + -+ for (;;) { -+ /* Try to acquire the lock again. */ -+ if (__try_to_take_rt_mutex(lock, self, &waiter, STEAL_LATERAL)) -+ break; -+ -+ top_waiter = rt_mutex_top_waiter(lock); -+ lock_owner = rt_mutex_owner(lock); -+ -+ raw_spin_unlock(&lock->wait_lock); -+ -+ debug_rt_mutex_print_deadlock(&waiter); -+ -+ if (top_waiter != &waiter || adaptive_wait(lock, lock_owner)) -+ schedule_rt_mutex(lock); -+ -+ raw_spin_lock(&lock->wait_lock); -+ -+ pi_lock(&self->pi_lock); -+ __set_current_state_no_track(TASK_UNINTERRUPTIBLE); -+ pi_unlock(&self->pi_lock); -+ } -+ -+ /* -+ * Restore the task state to current->saved_state. We set it -+ * to the original state above and the try_to_wake_up() code -+ * has possibly updated it when a real (non-rtmutex) wakeup -+ * happened while we were blocked. Clear saved_state so -+ * try_to_wakeup() does not get confused. -+ */ -+ pi_lock(&self->pi_lock); -+ __set_current_state_no_track(self->saved_state); -+ self->saved_state = TASK_RUNNING; -+ pi_unlock(&self->pi_lock); -+ -+ /* -+ * try_to_take_rt_mutex() sets the waiter bit -+ * unconditionally. We might have to fix that up: -+ */ -+ fixup_rt_mutex_waiters(lock); -+ -+ BUG_ON(rt_mutex_has_waiters(lock) && &waiter == rt_mutex_top_waiter(lock)); -+ BUG_ON(!RB_EMPTY_NODE(&waiter.tree_entry)); -+ -+ raw_spin_unlock(&lock->wait_lock); -+ -+ debug_rt_mutex_free_waiter(&waiter); -+} -+ -+static void wakeup_next_waiter(struct rt_mutex *lock); -+/* -+ * Slow path to release a rt_mutex spin_lock style -+ */ -+static void noinline __sched rt_spin_lock_slowunlock(struct rt_mutex *lock) -+{ -+ raw_spin_lock(&lock->wait_lock); -+ -+ debug_rt_mutex_unlock(lock); -+ -+ rt_mutex_deadlock_account_unlock(current); -+ -+ if (!rt_mutex_has_waiters(lock)) { -+ lock->owner = NULL; -+ raw_spin_unlock(&lock->wait_lock); -+ return; -+ } -+ -+ wakeup_next_waiter(lock); -+ -+ raw_spin_unlock(&lock->wait_lock); -+ -+ /* Undo pi boosting.when necessary */ -+ rt_mutex_adjust_prio(current); -+} -+ -+void __lockfunc rt_spin_lock(spinlock_t *lock) -+{ -+ rt_spin_lock_fastlock(&lock->lock, rt_spin_lock_slowlock); -+ spin_acquire(&lock->dep_map, 0, 0, _RET_IP_); -+} -+EXPORT_SYMBOL(rt_spin_lock); -+ -+void __lockfunc __rt_spin_lock(struct rt_mutex *lock) -+{ -+ rt_spin_lock_fastlock(lock, rt_spin_lock_slowlock); -+} -+EXPORT_SYMBOL(__rt_spin_lock); -+ -+#ifdef CONFIG_DEBUG_LOCK_ALLOC -+void __lockfunc rt_spin_lock_nested(spinlock_t *lock, int subclass) -+{ -+ rt_spin_lock_fastlock(&lock->lock, rt_spin_lock_slowlock); -+ spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_); -+} -+EXPORT_SYMBOL(rt_spin_lock_nested); -+#endif -+ -+void __lockfunc rt_spin_unlock(spinlock_t *lock) -+{ -+ /* NOTE: we always pass in '1' for nested, for simplicity */ -+ spin_release(&lock->dep_map, 1, _RET_IP_); -+ rt_spin_lock_fastunlock(&lock->lock, rt_spin_lock_slowunlock); -+} -+EXPORT_SYMBOL(rt_spin_unlock); -+ -+void __lockfunc __rt_spin_unlock(struct rt_mutex *lock) -+{ -+ rt_spin_lock_fastunlock(lock, rt_spin_lock_slowunlock); -+} -+EXPORT_SYMBOL(__rt_spin_unlock); -+ -+/* -+ * Wait for the lock to get unlocked: instead of polling for an unlock -+ * (like raw spinlocks do), we lock and unlock, to force the kernel to -+ * schedule if there's contention: -+ */ -+void __lockfunc rt_spin_unlock_wait(spinlock_t *lock) -+{ -+ spin_lock(lock); -+ spin_unlock(lock); -+} -+EXPORT_SYMBOL(rt_spin_unlock_wait); -+ -+int __lockfunc __rt_spin_trylock(struct rt_mutex *lock) -+{ -+ return rt_mutex_trylock(lock); -+} -+ -+int __lockfunc rt_spin_trylock(spinlock_t *lock) -+{ -+ int ret = rt_mutex_trylock(&lock->lock); -+ -+ if (ret) -+ spin_acquire(&lock->dep_map, 0, 1, _RET_IP_); -+ return ret; -+} -+EXPORT_SYMBOL(rt_spin_trylock); -+ -+int __lockfunc rt_spin_trylock_bh(spinlock_t *lock) -+{ -+ int ret; -+ -+ local_bh_disable(); -+ ret = rt_mutex_trylock(&lock->lock); -+ if (ret) { -+ migrate_disable(); -+ spin_acquire(&lock->dep_map, 0, 1, _RET_IP_); -+ } else -+ local_bh_enable(); -+ return ret; -+} -+EXPORT_SYMBOL(rt_spin_trylock_bh); -+ -+int __lockfunc rt_spin_trylock_irqsave(spinlock_t *lock, unsigned long *flags) -+{ -+ int ret; -+ -+ *flags = 0; -+ ret = rt_mutex_trylock(&lock->lock); -+ if (ret) { -+ migrate_disable(); -+ spin_acquire(&lock->dep_map, 0, 1, _RET_IP_); -+ } -+ return ret; -+} -+EXPORT_SYMBOL(rt_spin_trylock_irqsave); -+ -+int atomic_dec_and_spin_lock(atomic_t *atomic, spinlock_t *lock) -+{ -+ /* Subtract 1 from counter unless that drops it to 0 (ie. it was 1) */ -+ if (atomic_add_unless(atomic, -1, 1)) -+ return 0; -+ migrate_disable(); -+ rt_spin_lock(lock); -+ if (atomic_dec_and_test(atomic)) -+ return 1; -+ rt_spin_unlock(lock); -+ migrate_enable(); -+ return 0; -+} -+EXPORT_SYMBOL(atomic_dec_and_spin_lock); -+ -+ void -+__rt_spin_lock_init(spinlock_t *lock, char *name, struct lock_class_key *key) -+{ -+#ifdef CONFIG_DEBUG_LOCK_ALLOC -+ /* -+ * Make sure we are not reinitializing a held lock: -+ */ -+ debug_check_no_locks_freed((void *)lock, sizeof(*lock)); -+ lockdep_init_map(&lock->dep_map, name, key, 0); -+#endif -+} -+EXPORT_SYMBOL(__rt_spin_lock_init); -+ -+#endif /* PREEMPT_RT_FULL */ -+ -+#ifdef CONFIG_PREEMPT_RT_FULL -+ static inline int __sched -+__mutex_lock_check_stamp(struct rt_mutex *lock, struct ww_acquire_ctx *ctx) -+{ -+ struct ww_mutex *ww = container_of(lock, struct ww_mutex, base.lock); -+ struct ww_acquire_ctx *hold_ctx = ACCESS_ONCE(ww->ctx); -+ -+ if (!hold_ctx) -+ return 0; -+ -+ if (unlikely(ctx == hold_ctx)) -+ return -EALREADY; -+ -+ if (ctx->stamp - hold_ctx->stamp <= LONG_MAX && -+ (ctx->stamp != hold_ctx->stamp || ctx > hold_ctx)) { -+#ifdef CONFIG_DEBUG_MUTEXES -+ DEBUG_LOCKS_WARN_ON(ctx->contending_lock); -+ ctx->contending_lock = ww; -+#endif -+ return -EDEADLK; -+ } -+ -+ return 0; -+} -+#else -+ static inline int __sched -+__mutex_lock_check_stamp(struct rt_mutex *lock, struct ww_acquire_ctx *ctx) -+{ -+ BUG(); -+ return 0; -+} -+ -+#endif -+ -+static inline int -+try_to_take_rt_mutex(struct rt_mutex *lock, struct task_struct *task, -+ struct rt_mutex_waiter *waiter) -+{ -+ return __try_to_take_rt_mutex(lock, task, waiter, STEAL_NORMAL); -+} -+ - /* - * Task blocks on lock. - * -@@ -896,6 +1279,23 @@ - return -EDEADLK; + #define need_reserve_slab_rcu \ + (sizeof(((struct page *)NULL)->lru) < sizeof(struct rcu_head)) - raw_spin_lock_irqsave(&task->pi_lock, flags); -+ -+ /* -+ * In the case of futex requeue PI, this will be a proxy -+ * lock. The task will wake unaware that it is enqueueed on -+ * this lock. Avoid blocking on two locks and corrupting -+ * pi_blocked_on via the PI_WAKEUP_INPROGRESS -+ * flag. futex_wait_requeue_pi() sets this when it wakes up -+ * before requeue (due to a signal or timeout). Do not enqueue -+ * the task if PI_WAKEUP_INPROGRESS is set. -+ */ -+ if (task != current && task->pi_blocked_on == PI_WAKEUP_INPROGRESS) { -+ raw_spin_unlock_irqrestore(&task->pi_lock, flags); -+ return -EAGAIN; -+ } -+ -+ BUG_ON(rt_mutex_real_waiter(task->pi_blocked_on)); +@@ -1517,6 +1533,12 @@ + } + + call_rcu(head, rcu_free_slab); ++ } else if (irqs_disabled()) { ++ struct slub_free_list *f = this_cpu_ptr(&slub_free_list); + - __rt_mutex_adjust_prio(task); - waiter->task = task; - waiter->lock = lock; -@@ -919,7 +1319,7 @@ - rt_mutex_enqueue_pi(owner, waiter); ++ raw_spin_lock(&f->lock); ++ list_add(&page->lru, &f->list); ++ raw_spin_unlock(&f->lock); + } else + __free_slab(s, page); + } +@@ -1630,7 +1652,7 @@ + if (!n || !n->nr_partial) + return NULL; - __rt_mutex_adjust_prio(owner); -- if (owner->pi_blocked_on) -+ if (rt_mutex_real_waiter(owner->pi_blocked_on)) - chain_walk = 1; - } else if (rt_mutex_cond_detect_deadlock(waiter, chwalk)) { - chain_walk = 1; -@@ -957,8 +1357,9 @@ - /* - * Wake up the next waiter on the lock. - * -- * Remove the top waiter from the current tasks pi waiter list and -- * wake it up. -+ * Remove the top waiter from the current tasks pi waiter list, -+ * wake it up and return whether the current task needs to undo -+ * a potential priority boosting. - * - * Called with lock->wait_lock held. - */ -@@ -996,7 +1397,7 @@ - * long as we hold lock->wait_lock. The waiter task needs to - * acquire it in order to dequeue the waiter. - */ -- wake_up_process(waiter->task); -+ rt_mutex_wake_waiter(waiter); +- spin_lock(&n->list_lock); ++ raw_spin_lock(&n->list_lock); + list_for_each_entry_safe(page, page2, &n->partial, lru) { + void *t; + +@@ -1655,7 +1677,7 @@ + break; + + } +- spin_unlock(&n->list_lock); ++ raw_spin_unlock(&n->list_lock); + return object; } - /* -@@ -1010,7 +1411,7 @@ - { - bool is_top_waiter = (waiter == rt_mutex_top_waiter(lock)); - struct task_struct *owner = rt_mutex_owner(lock); -- struct rt_mutex *next_lock; -+ struct rt_mutex *next_lock = NULL; - unsigned long flags; +@@ -1901,7 +1923,7 @@ + * that acquire_slab() will see a slab page that + * is frozen + */ +- spin_lock(&n->list_lock); ++ raw_spin_lock(&n->list_lock); + } + } else { + m = M_FULL; +@@ -1912,7 +1934,7 @@ + * slabs from diagnostic functions will not see + * any frozen slabs. + */ +- spin_lock(&n->list_lock); ++ raw_spin_lock(&n->list_lock); + } + } - raw_spin_lock_irqsave(¤t->pi_lock, flags); -@@ -1035,7 +1436,8 @@ - __rt_mutex_adjust_prio(owner); +@@ -1947,7 +1969,7 @@ + goto redo; - /* Store the lock on which owner is blocked or NULL */ -- next_lock = task_blocked_on_lock(owner); -+ if (rt_mutex_real_waiter(owner->pi_blocked_on)) -+ next_lock = task_blocked_on_lock(owner); + if (lock) +- spin_unlock(&n->list_lock); ++ raw_spin_unlock(&n->list_lock); - raw_spin_unlock_irqrestore(&owner->pi_lock, flags); + if (m == M_FREE) { + stat(s, DEACTIVATE_EMPTY); +@@ -1979,10 +2001,10 @@ + n2 = get_node(s, page_to_nid(page)); + if (n != n2) { + if (n) +- spin_unlock(&n->list_lock); ++ raw_spin_unlock(&n->list_lock); -@@ -1071,17 +1473,17 @@ - raw_spin_lock_irqsave(&task->pi_lock, flags); + n = n2; +- spin_lock(&n->list_lock); ++ raw_spin_lock(&n->list_lock); + } - waiter = task->pi_blocked_on; -- if (!waiter || (waiter->prio == task->prio && -+ if (!rt_mutex_real_waiter(waiter) || (waiter->prio == task->prio && - !dl_prio(task->prio))) { - raw_spin_unlock_irqrestore(&task->pi_lock, flags); - return; + do { +@@ -2011,7 +2033,7 @@ } - next_lock = waiter->lock; -- raw_spin_unlock_irqrestore(&task->pi_lock, flags); - /* gets dropped in rt_mutex_adjust_prio_chain()! */ - get_task_struct(task); + if (n) +- spin_unlock(&n->list_lock); ++ raw_spin_unlock(&n->list_lock); -+ raw_spin_unlock_irqrestore(&task->pi_lock, flags); - rt_mutex_adjust_prio_chain(task, RT_MUTEX_MIN_CHAINWALK, NULL, - next_lock, NULL, task); + while (discard_page) { + page = discard_page; +@@ -2050,14 +2072,21 @@ + pobjects = oldpage->pobjects; + pages = oldpage->pages; + if (drain && pobjects > s->cpu_partial) { ++ struct slub_free_list *f; + unsigned long flags; ++ LIST_HEAD(tofree); + /* + * partial array is full. Move the existing + * set to the per node partial list. + */ + local_irq_save(flags); + unfreeze_partials(s, this_cpu_ptr(s->cpu_slab)); ++ f = this_cpu_ptr(&slub_free_list); ++ raw_spin_lock(&f->lock); ++ list_splice_init(&f->list, &tofree); ++ raw_spin_unlock(&f->lock); + local_irq_restore(flags); ++ free_delayed(&tofree); + oldpage = NULL; + pobjects = 0; + pages = 0; +@@ -2129,7 +2158,22 @@ + + static void flush_all(struct kmem_cache *s) + { ++ LIST_HEAD(tofree); ++ int cpu; ++ + on_each_cpu_cond(has_cpu_slab, flush_cpu_slab, s, 1, GFP_ATOMIC); ++ for_each_online_cpu(cpu) { ++ struct slub_free_list *f; ++ ++ if (!has_cpu_slab(cpu, s)) ++ continue; ++ ++ f = &per_cpu(slub_free_list, cpu); ++ raw_spin_lock_irq(&f->lock); ++ list_splice_init(&f->list, &tofree); ++ raw_spin_unlock_irq(&f->lock); ++ free_delayed(&tofree); ++ } } -@@ -1099,7 +1501,8 @@ - static int __sched - __rt_mutex_slowlock(struct rt_mutex *lock, int state, - struct hrtimer_sleeper *timeout, -- struct rt_mutex_waiter *waiter) -+ struct rt_mutex_waiter *waiter, -+ struct ww_acquire_ctx *ww_ctx) + + /* +@@ -2165,10 +2209,10 @@ + unsigned long x = 0; + struct page *page; + +- spin_lock_irqsave(&n->list_lock, flags); ++ raw_spin_lock_irqsave(&n->list_lock, flags); + list_for_each_entry(page, &n->partial, lru) + x += get_count(page); +- spin_unlock_irqrestore(&n->list_lock, flags); ++ raw_spin_unlock_irqrestore(&n->list_lock, flags); + return x; + } + #endif /* CONFIG_SLUB_DEBUG || CONFIG_SYSFS */ +@@ -2305,9 +2349,11 @@ + static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, + unsigned long addr, struct kmem_cache_cpu *c) { - int ret = 0; ++ struct slub_free_list *f; + void *freelist; + struct page *page; + unsigned long flags; ++ LIST_HEAD(tofree); -@@ -1122,6 +1525,12 @@ - break; - } + local_irq_save(flags); + #ifdef CONFIG_PREEMPT +@@ -2375,7 +2421,13 @@ + VM_BUG_ON(!c->page->frozen); + c->freelist = get_freepointer(s, freelist); + c->tid = next_tid(c->tid); ++out: ++ f = this_cpu_ptr(&slub_free_list); ++ raw_spin_lock(&f->lock); ++ list_splice_init(&f->list, &tofree); ++ raw_spin_unlock(&f->lock); + local_irq_restore(flags); ++ free_delayed(&tofree); + return freelist; -+ if (ww_ctx && ww_ctx->acquired > 0) { -+ ret = __mutex_lock_check_stamp(lock, ww_ctx); -+ if (ret) -+ break; -+ } -+ - raw_spin_unlock(&lock->wait_lock); + new_slab: +@@ -2392,8 +2444,7 @@ - debug_rt_mutex_print_deadlock(waiter); -@@ -1156,25 +1565,102 @@ + if (unlikely(!freelist)) { + slab_out_of_memory(s, gfpflags, node); +- local_irq_restore(flags); +- return NULL; ++ goto out; } + + page = c->page; +@@ -2408,8 +2459,7 @@ + deactivate_slab(s, page, get_freepointer(s, freelist)); + c->page = NULL; + c->freelist = NULL; +- local_irq_restore(flags); +- return freelist; ++ goto out; } -+static __always_inline void ww_mutex_lock_acquired(struct ww_mutex *ww, -+ struct ww_acquire_ctx *ww_ctx) -+{ -+#ifdef CONFIG_DEBUG_MUTEXES -+ /* -+ * If this WARN_ON triggers, you used ww_mutex_lock to acquire, -+ * but released with a normal mutex_unlock in this call. -+ * -+ * This should never happen, always use ww_mutex_unlock. -+ */ -+ DEBUG_LOCKS_WARN_ON(ww->ctx); -+ -+ /* -+ * Not quite done after calling ww_acquire_done() ? -+ */ -+ DEBUG_LOCKS_WARN_ON(ww_ctx->done_acquire); -+ -+ if (ww_ctx->contending_lock) { -+ /* -+ * After -EDEADLK you tried to -+ * acquire a different ww_mutex? Bad! -+ */ -+ DEBUG_LOCKS_WARN_ON(ww_ctx->contending_lock != ww); -+ -+ /* -+ * You called ww_mutex_lock after receiving -EDEADLK, -+ * but 'forgot' to unlock everything else first? -+ */ -+ DEBUG_LOCKS_WARN_ON(ww_ctx->acquired > 0); -+ ww_ctx->contending_lock = NULL; -+ } -+ -+ /* -+ * Naughty, using a different class will lead to undefined behavior! -+ */ -+ DEBUG_LOCKS_WARN_ON(ww_ctx->ww_class != ww->ww_class); -+#endif -+ ww_ctx->acquired++; -+} -+ -+#ifdef CONFIG_PREEMPT_RT_FULL -+static void ww_mutex_account_lock(struct rt_mutex *lock, -+ struct ww_acquire_ctx *ww_ctx) -+{ -+ struct ww_mutex *ww = container_of(lock, struct ww_mutex, base.lock); -+ struct rt_mutex_waiter *waiter, *n; -+ -+ /* -+ * This branch gets optimized out for the common case, -+ * and is only important for ww_mutex_lock. -+ */ -+ ww_mutex_lock_acquired(ww, ww_ctx); -+ ww->ctx = ww_ctx; -+ -+ /* -+ * Give any possible sleeping processes the chance to wake up, -+ * so they can recheck if they have to back off. -+ */ -+ rbtree_postorder_for_each_entry_safe(waiter, n, &lock->waiters, -+ tree_entry) { -+ /* XXX debug rt mutex waiter wakeup */ -+ -+ BUG_ON(waiter->lock != lock); -+ rt_mutex_wake_waiter(waiter); -+ } -+} -+ -+#else -+ -+static void ww_mutex_account_lock(struct rt_mutex *lock, -+ struct ww_acquire_ctx *ww_ctx) -+{ -+ BUG(); -+} -+#endif -+ /* - * Slow path lock function: - */ - static int __sched - rt_mutex_slowlock(struct rt_mutex *lock, int state, - struct hrtimer_sleeper *timeout, -- enum rtmutex_chainwalk chwalk) -+ enum rtmutex_chainwalk chwalk, -+ struct ww_acquire_ctx *ww_ctx) - { - struct rt_mutex_waiter waiter; - int ret = 0; - -- debug_rt_mutex_init_waiter(&waiter); -- RB_CLEAR_NODE(&waiter.pi_tree_entry); -- RB_CLEAR_NODE(&waiter.tree_entry); -+ rt_mutex_init_waiter(&waiter, false); +@@ -2593,7 +2643,7 @@ - raw_spin_lock(&lock->wait_lock); + do { + if (unlikely(n)) { +- spin_unlock_irqrestore(&n->list_lock, flags); ++ raw_spin_unlock_irqrestore(&n->list_lock, flags); + n = NULL; + } + prior = page->freelist; +@@ -2625,7 +2675,7 @@ + * Otherwise the list_lock will synchronize with + * other processors updating the list of slabs. + */ +- spin_lock_irqsave(&n->list_lock, flags); ++ raw_spin_lock_irqsave(&n->list_lock, flags); - /* Try to acquire the lock again: */ - if (try_to_take_rt_mutex(lock, current, NULL)) { -+ if (ww_ctx) -+ ww_mutex_account_lock(lock, ww_ctx); - raw_spin_unlock(&lock->wait_lock); - return 0; + } + } +@@ -2667,7 +2717,7 @@ + add_partial(n, page, DEACTIVATE_TO_TAIL); + stat(s, FREE_ADD_PARTIAL); } -@@ -1192,13 +1678,23 @@ - - if (likely(!ret)) - /* sleep on the mutex */ -- ret = __rt_mutex_slowlock(lock, state, timeout, &waiter); -+ ret = __rt_mutex_slowlock(lock, state, timeout, &waiter, -+ ww_ctx); -+ else if (ww_ctx) { -+ /* ww_mutex received EDEADLK, let it become EALREADY */ -+ ret = __mutex_lock_check_stamp(lock, ww_ctx); -+ BUG_ON(!ret); -+ } +- spin_unlock_irqrestore(&n->list_lock, flags); ++ raw_spin_unlock_irqrestore(&n->list_lock, flags); + return; - if (unlikely(ret)) { - __set_current_state(TASK_RUNNING); - if (rt_mutex_has_waiters(lock)) - remove_waiter(lock, &waiter); -- rt_mutex_handle_deadlock(ret, chwalk, &waiter); -+ /* ww_mutex want to report EDEADLK/EALREADY, let them */ -+ if (!ww_ctx) -+ rt_mutex_handle_deadlock(ret, chwalk, &waiter); -+ } else if (ww_ctx) { -+ ww_mutex_account_lock(lock, ww_ctx); + slab_empty: +@@ -2682,7 +2732,7 @@ + remove_full(s, n, page); } - /* -@@ -1255,7 +1751,7 @@ - /* - * Slow path to release a rt-mutex: - */ --static void __sched -+static bool __sched - rt_mutex_slowunlock(struct rt_mutex *lock) +- spin_unlock_irqrestore(&n->list_lock, flags); ++ raw_spin_unlock_irqrestore(&n->list_lock, flags); + stat(s, FREE_SLAB); + discard_slab(s, page); + } +@@ -2881,7 +2931,7 @@ + init_kmem_cache_node(struct kmem_cache_node *n) { - raw_spin_lock(&lock->wait_lock); -@@ -1298,7 +1794,7 @@ - while (!rt_mutex_has_waiters(lock)) { - /* Drops lock->wait_lock ! */ - if (unlock_rt_mutex_safe(lock) == true) -- return; -+ return false; - /* Relock the rtmutex and try again */ - raw_spin_lock(&lock->wait_lock); - } -@@ -1311,8 +1807,7 @@ + n->nr_partial = 0; +- spin_lock_init(&n->list_lock); ++ raw_spin_lock_init(&n->list_lock); + INIT_LIST_HEAD(&n->partial); + #ifdef CONFIG_SLUB_DEBUG + atomic_long_set(&n->nr_slabs, 0); +@@ -3463,7 +3513,7 @@ + for (i = 0; i < SHRINK_PROMOTE_MAX; i++) + INIT_LIST_HEAD(promote + i); - raw_spin_unlock(&lock->wait_lock); +- spin_lock_irqsave(&n->list_lock, flags); ++ raw_spin_lock_irqsave(&n->list_lock, flags); -- /* Undo pi boosting if necessary: */ -- rt_mutex_adjust_prio(current); -+ return true; - } + /* + * Build lists of slabs to discard or promote. +@@ -3494,7 +3544,7 @@ + for (i = SHRINK_PROMOTE_MAX - 1; i >= 0; i--) + list_splice(promote + i, &n->partial); - /* -@@ -1323,31 +1818,36 @@ - */ - static inline int - rt_mutex_fastlock(struct rt_mutex *lock, int state, -+ struct ww_acquire_ctx *ww_ctx, - int (*slowfn)(struct rt_mutex *lock, int state, - struct hrtimer_sleeper *timeout, -- enum rtmutex_chainwalk chwalk)) -+ enum rtmutex_chainwalk chwalk, -+ struct ww_acquire_ctx *ww_ctx)) - { - if (likely(rt_mutex_cmpxchg(lock, NULL, current))) { - rt_mutex_deadlock_account_lock(lock, current); - return 0; - } else -- return slowfn(lock, state, NULL, RT_MUTEX_MIN_CHAINWALK); -+ return slowfn(lock, state, NULL, RT_MUTEX_MIN_CHAINWALK, -+ ww_ctx); - } +- spin_unlock_irqrestore(&n->list_lock, flags); ++ raw_spin_unlock_irqrestore(&n->list_lock, flags); - static inline int - rt_mutex_timed_fastlock(struct rt_mutex *lock, int state, - struct hrtimer_sleeper *timeout, - enum rtmutex_chainwalk chwalk, -+ struct ww_acquire_ctx *ww_ctx, - int (*slowfn)(struct rt_mutex *lock, int state, - struct hrtimer_sleeper *timeout, -- enum rtmutex_chainwalk chwalk)) -+ enum rtmutex_chainwalk chwalk, -+ struct ww_acquire_ctx *ww_ctx)) + /* Release empty slabs */ + list_for_each_entry_safe(page, t, &discard, lru) +@@ -3670,6 +3720,12 @@ { - if (chwalk == RT_MUTEX_MIN_CHAINWALK && - likely(rt_mutex_cmpxchg(lock, NULL, current))) { - rt_mutex_deadlock_account_lock(lock, current); - return 0; - } else -- return slowfn(lock, state, timeout, chwalk); -+ return slowfn(lock, state, timeout, chwalk, ww_ctx); - } + static __initdata struct kmem_cache boot_kmem_cache, + boot_kmem_cache_node; ++ int cpu; ++ ++ for_each_possible_cpu(cpu) { ++ raw_spin_lock_init(&per_cpu(slub_free_list, cpu).lock); ++ INIT_LIST_HEAD(&per_cpu(slub_free_list, cpu).list); ++ } - static inline int -@@ -1363,12 +1863,14 @@ + if (debug_guardpage_minorder()) + slub_max_order = 0; +@@ -3912,7 +3968,7 @@ + struct page *page; + unsigned long flags; - static inline void - rt_mutex_fastunlock(struct rt_mutex *lock, -- void (*slowfn)(struct rt_mutex *lock)) -+ bool (*slowfn)(struct rt_mutex *lock)) - { -- if (likely(rt_mutex_cmpxchg(lock, current, NULL))) -+ if (likely(rt_mutex_cmpxchg(lock, current, NULL))) { - rt_mutex_deadlock_account_unlock(current); -- else -- slowfn(lock); -+ } else if (slowfn(lock)) { -+ /* Undo pi boosting if necessary: */ -+ rt_mutex_adjust_prio(current); -+ } +- spin_lock_irqsave(&n->list_lock, flags); ++ raw_spin_lock_irqsave(&n->list_lock, flags); + + list_for_each_entry(page, &n->partial, lru) { + validate_slab_slab(s, page, map); +@@ -3934,7 +3990,7 @@ + s->name, count, atomic_long_read(&n->nr_slabs)); + + out: +- spin_unlock_irqrestore(&n->list_lock, flags); ++ raw_spin_unlock_irqrestore(&n->list_lock, flags); + return count; } - /** -@@ -1380,7 +1882,7 @@ - { - might_sleep(); +@@ -4122,12 +4178,12 @@ + if (!atomic_long_read(&n->nr_slabs)) + continue; -- rt_mutex_fastlock(lock, TASK_UNINTERRUPTIBLE, rt_mutex_slowlock); -+ rt_mutex_fastlock(lock, TASK_UNINTERRUPTIBLE, NULL, rt_mutex_slowlock); +- spin_lock_irqsave(&n->list_lock, flags); ++ raw_spin_lock_irqsave(&n->list_lock, flags); + list_for_each_entry(page, &n->partial, lru) + process_slab(&t, s, page, alloc, map); + list_for_each_entry(page, &n->full, lru) + process_slab(&t, s, page, alloc, map); +- spin_unlock_irqrestore(&n->list_lock, flags); ++ raw_spin_unlock_irqrestore(&n->list_lock, flags); + } + + for (i = 0; i < t.count; i++) { +diff -Nur linux-4.1.10.orig/mm/swap.c linux-4.1.10/mm/swap.c +--- linux-4.1.10.orig/mm/swap.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/mm/swap.c 2015-10-12 22:33:32.336672790 +0200 +@@ -32,6 +32,7 @@ + #include + #include + #include ++#include + + #include "internal.h" + +@@ -45,6 +46,9 @@ + static DEFINE_PER_CPU(struct pagevec, lru_rotate_pvecs); + static DEFINE_PER_CPU(struct pagevec, lru_deactivate_file_pvecs); + ++static DEFINE_LOCAL_IRQ_LOCK(rotate_lock); ++DEFINE_LOCAL_IRQ_LOCK(swapvec_lock); ++ + /* + * This path almost never happens for VM activity - pages are normally + * freed via pagevecs. But it gets used by networking. +@@ -481,11 +485,11 @@ + unsigned long flags; + + page_cache_get(page); +- local_irq_save(flags); ++ local_lock_irqsave(rotate_lock, flags); + pvec = this_cpu_ptr(&lru_rotate_pvecs); + if (!pagevec_add(pvec, page)) + pagevec_move_tail(pvec); +- local_irq_restore(flags); ++ local_unlock_irqrestore(rotate_lock, flags); + } } - EXPORT_SYMBOL_GPL(rt_mutex_lock); -@@ -1397,7 +1899,7 @@ +@@ -536,12 +540,13 @@ + void activate_page(struct page *page) { - might_sleep(); + if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) { +- struct pagevec *pvec = &get_cpu_var(activate_page_pvecs); ++ struct pagevec *pvec = &get_locked_var(swapvec_lock, ++ activate_page_pvecs); -- return rt_mutex_fastlock(lock, TASK_INTERRUPTIBLE, rt_mutex_slowlock); -+ return rt_mutex_fastlock(lock, TASK_INTERRUPTIBLE, NULL, rt_mutex_slowlock); + page_cache_get(page); + if (!pagevec_add(pvec, page)) + pagevec_lru_move_fn(pvec, __activate_page, NULL); +- put_cpu_var(activate_page_pvecs); ++ put_locked_var(swapvec_lock, activate_page_pvecs); + } } - EXPORT_SYMBOL_GPL(rt_mutex_lock_interruptible); -@@ -1410,11 +1912,30 @@ - might_sleep(); +@@ -567,7 +572,7 @@ - return rt_mutex_timed_fastlock(lock, TASK_INTERRUPTIBLE, timeout, -- RT_MUTEX_FULL_CHAINWALK, -+ RT_MUTEX_FULL_CHAINWALK, NULL, - rt_mutex_slowlock); - } + static void __lru_cache_activate_page(struct page *page) + { +- struct pagevec *pvec = &get_cpu_var(lru_add_pvec); ++ struct pagevec *pvec = &get_locked_var(swapvec_lock, lru_add_pvec); + int i; - /** -+ * rt_mutex_lock_killable - lock a rt_mutex killable -+ * -+ * @lock: the rt_mutex to be locked -+ * @detect_deadlock: deadlock detection on/off -+ * -+ * Returns: -+ * 0 on success -+ * -EINTR when interrupted by a signal -+ * -EDEADLK when the lock would deadlock (when deadlock detection is on) -+ */ -+int __sched rt_mutex_lock_killable(struct rt_mutex *lock) -+{ -+ might_sleep(); -+ -+ return rt_mutex_fastlock(lock, TASK_KILLABLE, NULL, rt_mutex_slowlock); -+} -+EXPORT_SYMBOL_GPL(rt_mutex_lock_killable); -+ -+/** - * rt_mutex_timed_lock - lock a rt_mutex interruptible - * the timeout structure is provided - * by the caller -@@ -1434,6 +1955,7 @@ + /* +@@ -589,7 +594,7 @@ + } + } - return rt_mutex_timed_fastlock(lock, TASK_INTERRUPTIBLE, timeout, - RT_MUTEX_MIN_CHAINWALK, -+ NULL, - rt_mutex_slowlock); +- put_cpu_var(lru_add_pvec); ++ put_locked_var(swapvec_lock, lru_add_pvec); } - EXPORT_SYMBOL_GPL(rt_mutex_timed_lock); -@@ -1463,6 +1985,22 @@ - EXPORT_SYMBOL_GPL(rt_mutex_unlock); - /** -+ * rt_mutex_futex_unlock - Futex variant of rt_mutex_unlock -+ * @lock: the rt_mutex to be unlocked -+ * -+ * Returns: true/false indicating whether priority adjustment is -+ * required or not. -+ */ -+bool __sched rt_mutex_futex_unlock(struct rt_mutex *lock) -+{ -+ if (likely(rt_mutex_cmpxchg(lock, current, NULL))) { -+ rt_mutex_deadlock_account_unlock(current); -+ return false; -+ } -+ return rt_mutex_slowunlock(lock); -+} -+ -+/** - * rt_mutex_destroy - mark a mutex unusable - * @lock: the mutex to be destroyed - * -@@ -1492,13 +2030,12 @@ - void __rt_mutex_init(struct rt_mutex *lock, const char *name) + /* +@@ -628,13 +633,13 @@ + + static void __lru_cache_add(struct page *page) { - lock->owner = NULL; -- raw_spin_lock_init(&lock->wait_lock); - lock->waiters = RB_ROOT; - lock->waiters_leftmost = NULL; +- struct pagevec *pvec = &get_cpu_var(lru_add_pvec); ++ struct pagevec *pvec = &get_locked_var(swapvec_lock, lru_add_pvec); - debug_rt_mutex_init(lock, name); + page_cache_get(page); + if (!pagevec_space(pvec)) + __pagevec_lru_add(pvec); + pagevec_add(pvec, page); +- put_cpu_var(lru_add_pvec); ++ put_locked_var(swapvec_lock, lru_add_pvec); } --EXPORT_SYMBOL_GPL(__rt_mutex_init); -+EXPORT_SYMBOL(__rt_mutex_init); /** - * rt_mutex_init_proxy_locked - initialize and lock a rt_mutex on behalf of a -@@ -1513,7 +2050,7 @@ - void rt_mutex_init_proxy_locked(struct rt_mutex *lock, - struct task_struct *proxy_owner) - { -- __rt_mutex_init(lock, NULL); -+ rt_mutex_init(lock); - debug_rt_mutex_proxy_lock(lock, proxy_owner); - rt_mutex_set_owner(lock, proxy_owner); - rt_mutex_deadlock_account_lock(lock, proxy_owner); -@@ -1561,6 +2098,35 @@ - return 1; +@@ -814,9 +819,9 @@ + unsigned long flags; + + /* No harm done if a racing interrupt already did this */ +- local_irq_save(flags); ++ local_lock_irqsave(rotate_lock, flags); + pagevec_move_tail(pvec); +- local_irq_restore(flags); ++ local_unlock_irqrestore(rotate_lock, flags); } -+#ifdef CONFIG_PREEMPT_RT_FULL -+ /* -+ * In PREEMPT_RT there's an added race. -+ * If the task, that we are about to requeue, times out, -+ * it can set the PI_WAKEUP_INPROGRESS. This tells the requeue -+ * to skip this task. But right after the task sets -+ * its pi_blocked_on to PI_WAKEUP_INPROGRESS it can then -+ * block on the spin_lock(&hb->lock), which in RT is an rtmutex. -+ * This will replace the PI_WAKEUP_INPROGRESS with the actual -+ * lock that it blocks on. We *must not* place this task -+ * on this proxy lock in that case. -+ * -+ * To prevent this race, we first take the task's pi_lock -+ * and check if it has updated its pi_blocked_on. If it has, -+ * we assume that it woke up and we return -EAGAIN. -+ * Otherwise, we set the task's pi_blocked_on to -+ * PI_REQUEUE_INPROGRESS, so that if the task is waking up -+ * it will know that we are in the process of requeuing it. -+ */ -+ raw_spin_lock_irq(&task->pi_lock); -+ if (task->pi_blocked_on) { -+ raw_spin_unlock_irq(&task->pi_lock); -+ raw_spin_unlock(&lock->wait_lock); -+ return -EAGAIN; -+ } -+ task->pi_blocked_on = PI_REQUEUE_INPROGRESS; -+ raw_spin_unlock_irq(&task->pi_lock); -+#endif -+ - /* We enforce deadlock detection for futexes */ - ret = task_blocks_on_rt_mutex(lock, waiter, task, - RT_MUTEX_FULL_CHAINWALK); -@@ -1631,7 +2197,7 @@ - set_current_state(TASK_INTERRUPTIBLE); + pvec = &per_cpu(lru_deactivate_file_pvecs, cpu); +@@ -844,18 +849,19 @@ + return; - /* sleep on the mutex */ -- ret = __rt_mutex_slowlock(lock, TASK_INTERRUPTIBLE, to, waiter); -+ ret = __rt_mutex_slowlock(lock, TASK_INTERRUPTIBLE, to, waiter, NULL); + if (likely(get_page_unless_zero(page))) { +- struct pagevec *pvec = &get_cpu_var(lru_deactivate_file_pvecs); ++ struct pagevec *pvec = &get_locked_var(swapvec_lock, ++ lru_deactivate_file_pvecs); - if (unlikely(ret)) - remove_waiter(lock, waiter); -@@ -1646,3 +2212,89 @@ + if (!pagevec_add(pvec, page)) + pagevec_lru_move_fn(pvec, lru_deactivate_file_fn, NULL); +- put_cpu_var(lru_deactivate_file_pvecs); ++ put_locked_var(swapvec_lock, lru_deactivate_file_pvecs); + } + } - return ret; + void lru_add_drain(void) + { +- lru_add_drain_cpu(get_cpu()); +- put_cpu(); ++ lru_add_drain_cpu(local_lock_cpu(swapvec_lock)); ++ local_unlock_cpu(swapvec_lock); } -+ -+static inline int -+ww_mutex_deadlock_injection(struct ww_mutex *lock, struct ww_acquire_ctx *ctx) -+{ -+#ifdef CONFIG_DEBUG_WW_MUTEX_SLOWPATH -+ unsigned tmp; -+ -+ if (ctx->deadlock_inject_countdown-- == 0) { -+ tmp = ctx->deadlock_inject_interval; -+ if (tmp > UINT_MAX/4) -+ tmp = UINT_MAX; -+ else -+ tmp = tmp*2 + tmp + tmp/2; -+ -+ ctx->deadlock_inject_interval = tmp; -+ ctx->deadlock_inject_countdown = tmp; -+ ctx->contending_lock = lock; -+ -+ ww_mutex_unlock(lock); -+ -+ return -EDEADLK; -+ } -+#endif -+ -+ return 0; -+} -+ -+#ifdef CONFIG_PREEMPT_RT_FULL -+int __sched -+__ww_mutex_lock_interruptible(struct ww_mutex *lock, struct ww_acquire_ctx *ww_ctx) -+{ -+ int ret; -+ -+ might_sleep(); -+ -+ mutex_acquire_nest(&lock->base.dep_map, 0, 0, &ww_ctx->dep_map, _RET_IP_); -+ ret = rt_mutex_slowlock(&lock->base.lock, TASK_INTERRUPTIBLE, NULL, 0, ww_ctx); -+ if (ret) -+ mutex_release(&lock->base.dep_map, 1, _RET_IP_); -+ else if (!ret && ww_ctx->acquired > 1) -+ return ww_mutex_deadlock_injection(lock, ww_ctx); -+ -+ return ret; -+} -+EXPORT_SYMBOL_GPL(__ww_mutex_lock_interruptible); -+ -+int __sched -+__ww_mutex_lock(struct ww_mutex *lock, struct ww_acquire_ctx *ww_ctx) -+{ -+ int ret; -+ -+ might_sleep(); -+ -+ mutex_acquire_nest(&lock->base.dep_map, 0, 0, &ww_ctx->dep_map, _RET_IP_); -+ ret = rt_mutex_slowlock(&lock->base.lock, TASK_UNINTERRUPTIBLE, NULL, 0, ww_ctx); -+ if (ret) -+ mutex_release(&lock->base.dep_map, 1, _RET_IP_); -+ else if (!ret && ww_ctx->acquired > 1) -+ return ww_mutex_deadlock_injection(lock, ww_ctx); -+ -+ return ret; -+} -+EXPORT_SYMBOL_GPL(__ww_mutex_lock); -+ -+void __sched ww_mutex_unlock(struct ww_mutex *lock) -+{ -+ int nest = !!lock->ctx; -+ -+ /* -+ * The unlocking fastpath is the 0->1 transition from 'locked' -+ * into 'unlocked' state: -+ */ -+ if (nest) { -+#ifdef CONFIG_DEBUG_MUTEXES -+ DEBUG_LOCKS_WARN_ON(!lock->ctx->acquired); -+#endif -+ if (lock->ctx->acquired > 0) -+ lock->ctx->acquired--; -+ lock->ctx = NULL; -+ } -+ -+ mutex_release(&lock->base.dep_map, nest, _RET_IP_); -+ rt_mutex_unlock(&lock->base.lock); -+} -+EXPORT_SYMBOL(ww_mutex_unlock); -+#endif -diff -Nur linux-4.1.10.orig/kernel/locking/rtmutex_common.h linux-4.1.10/kernel/locking/rtmutex_common.h ---- linux-4.1.10.orig/kernel/locking/rtmutex_common.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/locking/rtmutex_common.h 2015-10-07 18:00:08.000000000 +0200 -@@ -49,6 +49,7 @@ - struct rb_node pi_tree_entry; - struct task_struct *task; - struct rt_mutex *lock; -+ bool savestate; - #ifdef CONFIG_DEBUG_RT_MUTEXES - unsigned long ip; - struct pid *deadlock_task_pid; -@@ -119,6 +120,9 @@ - /* - * PI-futex support (proxy locking functions, etc.): - */ -+#define PI_WAKEUP_INPROGRESS ((struct rt_mutex_waiter *) 1) -+#define PI_REQUEUE_INPROGRESS ((struct rt_mutex_waiter *) 2) -+ - extern struct task_struct *rt_mutex_next_owner(struct rt_mutex *lock); - extern void rt_mutex_init_proxy_locked(struct rt_mutex *lock, - struct task_struct *proxy_owner); -@@ -132,10 +136,24 @@ - struct rt_mutex_waiter *waiter); - extern int rt_mutex_timed_futex_lock(struct rt_mutex *l, struct hrtimer_sleeper *to); -+extern bool rt_mutex_futex_unlock(struct rt_mutex *lock); -+ -+extern void rt_mutex_adjust_prio(struct task_struct *task); -+ - #ifdef CONFIG_DEBUG_RT_MUTEXES - # include "rtmutex-debug.h" - #else - # include "rtmutex.h" - #endif + static void lru_add_drain_per_cpu(struct work_struct *dummy) +diff -Nur linux-4.1.10.orig/mm/truncate.c linux-4.1.10/mm/truncate.c +--- linux-4.1.10.orig/mm/truncate.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/mm/truncate.c 2015-10-12 22:33:32.336672790 +0200 +@@ -56,8 +56,11 @@ + * protected by mapping->tree_lock. + */ + if (!workingset_node_shadows(node) && +- !list_empty(&node->private_list)) +- list_lru_del(&workingset_shadow_nodes, &node->private_list); ++ !list_empty(&node->private_list)) { ++ local_lock(workingset_shadow_lock); ++ list_lru_del(&__workingset_shadow_nodes, &node->private_list); ++ local_unlock(workingset_shadow_lock); ++ } + __radix_tree_delete_node(&mapping->page_tree, node); + unlock: + spin_unlock_irq(&mapping->tree_lock); +diff -Nur linux-4.1.10.orig/mm/vmalloc.c linux-4.1.10/mm/vmalloc.c +--- linux-4.1.10.orig/mm/vmalloc.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/mm/vmalloc.c 2015-10-12 22:33:32.336672790 +0200 +@@ -819,7 +819,7 @@ + struct vmap_block *vb; + struct vmap_area *va; + unsigned long vb_idx; +- int node, err; ++ int node, err, cpu; + void *vaddr; -+static inline void -+rt_mutex_init_waiter(struct rt_mutex_waiter *waiter, bool savestate) -+{ -+ debug_rt_mutex_init_waiter(waiter); -+ waiter->task = NULL; -+ waiter->savestate = savestate; -+ RB_CLEAR_NODE(&waiter->pi_tree_entry); -+ RB_CLEAR_NODE(&waiter->tree_entry); -+} -+ - #endif -diff -Nur linux-4.1.10.orig/kernel/locking/spinlock.c linux-4.1.10/kernel/locking/spinlock.c ---- linux-4.1.10.orig/kernel/locking/spinlock.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/locking/spinlock.c 2015-10-07 18:00:08.000000000 +0200 -@@ -124,8 +124,11 @@ - * __[spin|read|write]_lock_bh() - */ - BUILD_LOCK_OPS(spin, raw_spinlock); -+ -+#ifndef CONFIG_PREEMPT_RT_FULL - BUILD_LOCK_OPS(read, rwlock); - BUILD_LOCK_OPS(write, rwlock); -+#endif + node = numa_node_id(); +@@ -862,11 +862,12 @@ + BUG_ON(err); + radix_tree_preload_end(); - #endif +- vbq = &get_cpu_var(vmap_block_queue); ++ cpu = get_cpu_light(); ++ vbq = this_cpu_ptr(&vmap_block_queue); + spin_lock(&vbq->lock); + list_add_tail_rcu(&vb->free_list, &vbq->free); + spin_unlock(&vbq->lock); +- put_cpu_var(vmap_block_queue); ++ put_cpu_light(); -@@ -209,6 +212,8 @@ - EXPORT_SYMBOL(_raw_spin_unlock_bh); - #endif + return vaddr; + } +@@ -935,6 +936,7 @@ + struct vmap_block *vb; + void *vaddr = NULL; + unsigned int order; ++ int cpu; -+#ifndef CONFIG_PREEMPT_RT_FULL -+ - #ifndef CONFIG_INLINE_READ_TRYLOCK - int __lockfunc _raw_read_trylock(rwlock_t *lock) - { -@@ -353,6 +358,8 @@ - EXPORT_SYMBOL(_raw_write_unlock_bh); - #endif + BUG_ON(size & ~PAGE_MASK); + BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC); +@@ -949,7 +951,8 @@ + order = get_order(size); -+#endif /* !PREEMPT_RT_FULL */ -+ - #ifdef CONFIG_DEBUG_LOCK_ALLOC + rcu_read_lock(); +- vbq = &get_cpu_var(vmap_block_queue); ++ cpu = get_cpu_light(); ++ vbq = this_cpu_ptr(&vmap_block_queue); + list_for_each_entry_rcu(vb, &vbq->free, free_list) { + unsigned long pages_off; - void __lockfunc _raw_spin_lock_nested(raw_spinlock_t *lock, int subclass) -diff -Nur linux-4.1.10.orig/kernel/locking/spinlock_debug.c linux-4.1.10/kernel/locking/spinlock_debug.c ---- linux-4.1.10.orig/kernel/locking/spinlock_debug.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/locking/spinlock_debug.c 2015-10-07 18:00:08.000000000 +0200 -@@ -31,6 +31,7 @@ +@@ -972,7 +975,7 @@ + break; + } - EXPORT_SYMBOL(__raw_spin_lock_init); +- put_cpu_var(vmap_block_queue); ++ put_cpu_light(); + rcu_read_unlock(); -+#ifndef CONFIG_PREEMPT_RT_FULL - void __rwlock_init(rwlock_t *lock, const char *name, - struct lock_class_key *key) - { -@@ -48,6 +49,7 @@ - } + /* Allocate new block if nothing was found */ +diff -Nur linux-4.1.10.orig/mm/vmstat.c linux-4.1.10/mm/vmstat.c +--- linux-4.1.10.orig/mm/vmstat.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/mm/vmstat.c 2015-10-12 22:33:32.336672790 +0200 +@@ -226,6 +226,7 @@ + long x; + long t; - EXPORT_SYMBOL(__rwlock_init); -+#endif ++ preempt_disable_rt(); + x = delta + __this_cpu_read(*p); - static void spin_dump(raw_spinlock_t *lock, const char *msg) - { -@@ -159,6 +161,7 @@ - arch_spin_unlock(&lock->raw_lock); + t = __this_cpu_read(pcp->stat_threshold); +@@ -235,6 +236,7 @@ + x = 0; + } + __this_cpu_write(*p, x); ++ preempt_enable_rt(); } + EXPORT_SYMBOL(__mod_zone_page_state); -+#ifndef CONFIG_PREEMPT_RT_FULL - static void rwlock_bug(rwlock_t *lock, const char *msg) - { - if (!debug_locks_off()) -@@ -300,3 +303,5 @@ - debug_write_unlock(lock); - arch_write_unlock(&lock->raw_lock); +@@ -267,6 +269,7 @@ + s8 __percpu *p = pcp->vm_stat_diff + item; + s8 v, t; + ++ preempt_disable_rt(); + v = __this_cpu_inc_return(*p); + t = __this_cpu_read(pcp->stat_threshold); + if (unlikely(v > t)) { +@@ -275,6 +278,7 @@ + zone_page_state_add(v + overstep, zone, item); + __this_cpu_write(*p, -overstep); + } ++ preempt_enable_rt(); } -+ -+#endif -diff -Nur linux-4.1.10.orig/kernel/panic.c linux-4.1.10/kernel/panic.c ---- linux-4.1.10.orig/kernel/panic.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/panic.c 2015-10-07 18:00:08.000000000 +0200 -@@ -387,9 +387,11 @@ - static int init_oops_id(void) - { -+#ifndef CONFIG_PREEMPT_RT_FULL - if (!oops_id) - get_random_bytes(&oops_id, sizeof(oops_id)); - else -+#endif - oops_id++; + void __inc_zone_page_state(struct page *page, enum zone_stat_item item) +@@ -289,6 +293,7 @@ + s8 __percpu *p = pcp->vm_stat_diff + item; + s8 v, t; - return 0; -diff -Nur linux-4.1.10.orig/kernel/power/hibernate.c linux-4.1.10/kernel/power/hibernate.c ---- linux-4.1.10.orig/kernel/power/hibernate.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/power/hibernate.c 2015-10-07 18:00:08.000000000 +0200 -@@ -285,6 +285,8 @@ ++ preempt_disable_rt(); + v = __this_cpu_dec_return(*p); + t = __this_cpu_read(pcp->stat_threshold); + if (unlikely(v < - t)) { +@@ -297,6 +302,7 @@ + zone_page_state_add(v - overstep, zone, item); + __this_cpu_write(*p, overstep); + } ++ preempt_enable_rt(); + } - local_irq_disable(); + void __dec_zone_page_state(struct page *page, enum zone_stat_item item) +diff -Nur linux-4.1.10.orig/mm/workingset.c linux-4.1.10/mm/workingset.c +--- linux-4.1.10.orig/mm/workingset.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/mm/workingset.c 2015-10-12 22:33:32.336672790 +0200 +@@ -264,7 +264,8 @@ + * point where they would still be useful. + */ -+ system_state = SYSTEM_SUSPEND; -+ - error = syscore_suspend(); - if (error) { - printk(KERN_ERR "PM: Some system devices failed to power down, " -@@ -314,6 +316,7 @@ - syscore_resume(); +-struct list_lru workingset_shadow_nodes; ++struct list_lru __workingset_shadow_nodes; ++DEFINE_LOCAL_IRQ_LOCK(workingset_shadow_lock); - Enable_irqs: -+ system_state = SYSTEM_RUNNING; - local_irq_enable(); + static unsigned long count_shadow_nodes(struct shrinker *shrinker, + struct shrink_control *sc) +@@ -274,9 +275,9 @@ + unsigned long pages; - Enable_cpus: -@@ -437,6 +440,7 @@ - goto Enable_cpus; + /* list_lru lock nests inside IRQ-safe mapping->tree_lock */ +- local_irq_disable(); +- shadow_nodes = list_lru_shrink_count(&workingset_shadow_nodes, sc); +- local_irq_enable(); ++ local_lock_irq(workingset_shadow_lock); ++ shadow_nodes = list_lru_shrink_count(&__workingset_shadow_nodes, sc); ++ local_unlock_irq(workingset_shadow_lock); - local_irq_disable(); -+ system_state = SYSTEM_SUSPEND; + pages = node_present_pages(sc->nid); + /* +@@ -363,9 +364,9 @@ + spin_unlock(&mapping->tree_lock); + ret = LRU_REMOVED_RETRY; + out: +- local_irq_enable(); ++ local_unlock_irq(workingset_shadow_lock); + cond_resched(); +- local_irq_disable(); ++ local_lock_irq(workingset_shadow_lock); + spin_lock(lru_lock); + return ret; + } +@@ -376,10 +377,10 @@ + unsigned long ret; - error = syscore_suspend(); - if (error) -@@ -470,6 +474,7 @@ - syscore_resume(); + /* list_lru lock nests inside IRQ-safe mapping->tree_lock */ +- local_irq_disable(); +- ret = list_lru_shrink_walk(&workingset_shadow_nodes, sc, ++ local_lock_irq(workingset_shadow_lock); ++ ret = list_lru_shrink_walk(&__workingset_shadow_nodes, sc, + shadow_lru_isolate, NULL); +- local_irq_enable(); ++ local_unlock_irq(workingset_shadow_lock); + return ret; + } - Enable_irqs: -+ system_state = SYSTEM_RUNNING; - local_irq_enable(); +@@ -400,7 +401,7 @@ + { + int ret; - Enable_cpus: -@@ -555,6 +560,7 @@ - goto Platform_finish; +- ret = list_lru_init_key(&workingset_shadow_nodes, &shadow_nodes_key); ++ ret = list_lru_init_key(&__workingset_shadow_nodes, &shadow_nodes_key); + if (ret) + goto err; + ret = register_shrinker(&workingset_shadow_shrinker); +@@ -408,7 +409,7 @@ + goto err_list_lru; + return 0; + err_list_lru: +- list_lru_destroy(&workingset_shadow_nodes); ++ list_lru_destroy(&__workingset_shadow_nodes); + err: + return ret; + } +diff -Nur linux-4.1.10.orig/net/core/dev.c linux-4.1.10/net/core/dev.c +--- linux-4.1.10.orig/net/core/dev.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/net/core/dev.c 2015-10-12 22:33:32.340672525 +0200 +@@ -184,6 +184,7 @@ + static DEFINE_HASHTABLE(napi_hash, 8); - local_irq_disable(); -+ system_state = SYSTEM_SUSPEND; - syscore_suspend(); - if (pm_wakeup_pending()) { - error = -EAGAIN; -@@ -567,6 +573,7 @@ + static seqcount_t devnet_rename_seq; ++static DEFINE_MUTEX(devnet_rename_mutex); - Power_up: - syscore_resume(); -+ system_state = SYSTEM_RUNNING; - local_irq_enable(); - enable_nonboot_cpus(); + static inline void dev_base_seq_inc(struct net *net) + { +@@ -205,14 +206,14 @@ + static inline void rps_lock(struct softnet_data *sd) + { + #ifdef CONFIG_RPS +- spin_lock(&sd->input_pkt_queue.lock); ++ raw_spin_lock(&sd->input_pkt_queue.raw_lock); + #endif + } -diff -Nur linux-4.1.10.orig/kernel/power/suspend.c linux-4.1.10/kernel/power/suspend.c ---- linux-4.1.10.orig/kernel/power/suspend.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/power/suspend.c 2015-10-07 18:00:08.000000000 +0200 -@@ -356,6 +356,8 @@ - arch_suspend_disable_irqs(); - BUG_ON(!irqs_disabled()); + static inline void rps_unlock(struct softnet_data *sd) + { + #ifdef CONFIG_RPS +- spin_unlock(&sd->input_pkt_queue.lock); ++ raw_spin_unlock(&sd->input_pkt_queue.raw_lock); + #endif + } -+ system_state = SYSTEM_SUSPEND; -+ - error = syscore_suspend(); - if (!error) { - *wakeup = pm_wakeup_pending(); -@@ -370,6 +372,8 @@ - syscore_resume(); +@@ -852,7 +853,8 @@ + strcpy(name, dev->name); + rcu_read_unlock(); + if (read_seqcount_retry(&devnet_rename_seq, seq)) { +- cond_resched(); ++ mutex_lock(&devnet_rename_mutex); ++ mutex_unlock(&devnet_rename_mutex); + goto retry; } -+ system_state = SYSTEM_RUNNING; -+ - arch_suspend_enable_irqs(); - BUG_ON(irqs_disabled()); +@@ -1121,20 +1123,17 @@ + if (dev->flags & IFF_UP) + return -EBUSY; -diff -Nur linux-4.1.10.orig/kernel/printk/printk.c linux-4.1.10/kernel/printk/printk.c ---- linux-4.1.10.orig/kernel/printk/printk.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/printk/printk.c 2015-10-07 18:00:08.000000000 +0200 -@@ -1163,6 +1163,7 @@ - { - char *text; - int len = 0; -+ int attempts = 0; +- write_seqcount_begin(&devnet_rename_seq); ++ mutex_lock(&devnet_rename_mutex); ++ __raw_write_seqcount_begin(&devnet_rename_seq); - text = kmalloc(LOG_LINE_MAX + PREFIX_MAX, GFP_KERNEL); - if (!text) -@@ -1174,7 +1175,14 @@ - u64 seq; - u32 idx; - enum log_flags prev; -- -+ int num_msg; -+try_again: -+ attempts++; -+ if (attempts > 10) { -+ len = -EBUSY; -+ goto out; -+ } -+ num_msg = 0; - if (clear_seq < log_first_seq) { - /* messages are gone, move to first available one */ - clear_seq = log_first_seq; -@@ -1195,6 +1203,14 @@ - prev = msg->flags; - idx = log_next(idx); - seq++; -+ num_msg++; -+ if (num_msg > 5) { -+ num_msg = 0; -+ raw_spin_unlock_irq(&logbuf_lock); -+ raw_spin_lock_irq(&logbuf_lock); -+ if (clear_seq < log_first_seq) -+ goto try_again; -+ } - } +- if (strncmp(newname, dev->name, IFNAMSIZ) == 0) { +- write_seqcount_end(&devnet_rename_seq); +- return 0; +- } ++ if (strncmp(newname, dev->name, IFNAMSIZ) == 0) ++ goto outunlock; - /* move first record forward until length fits into the buffer */ -@@ -1208,6 +1224,14 @@ - prev = msg->flags; - idx = log_next(idx); - seq++; -+ num_msg++; -+ if (num_msg > 5) { -+ num_msg = 0; -+ raw_spin_unlock_irq(&logbuf_lock); -+ raw_spin_lock_irq(&logbuf_lock); -+ if (clear_seq < log_first_seq) -+ goto try_again; -+ } - } + memcpy(oldname, dev->name, IFNAMSIZ); - /* last message fitting into this dump */ -@@ -1248,6 +1272,7 @@ - clear_seq = log_next_seq; - clear_idx = log_next_idx; - } -+out: - raw_spin_unlock_irq(&logbuf_lock); - - kfree(text); -@@ -1401,6 +1426,7 @@ - if (!console_drivers) - return; - -+ migrate_disable(); - for_each_console(con) { - if (exclusive_console && con != exclusive_console) - continue; -@@ -1413,6 +1439,7 @@ - continue; - con->write(con, text, len); - } -+ migrate_enable(); - } - - /* -@@ -1473,6 +1500,15 @@ - static int console_trylock_for_printk(void) - { - unsigned int cpu = smp_processor_id(); -+#ifdef CONFIG_PREEMPT_RT_FULL -+ int lock = !early_boot_irqs_disabled && (preempt_count() == 0) && -+ !irqs_disabled(); -+#else -+ int lock = 1; -+#endif -+ -+ if (!lock) -+ return 0; - - if (!console_trylock()) - return 0; -@@ -1607,6 +1643,62 @@ - return textlen; - } - -+#ifdef CONFIG_EARLY_PRINTK -+struct console *early_console; -+ -+static void early_vprintk(const char *fmt, va_list ap) -+{ -+ if (early_console) { -+ char buf[512]; -+ int n = vscnprintf(buf, sizeof(buf), fmt, ap); -+ -+ early_console->write(early_console, buf, n); -+ } -+} -+ -+asmlinkage void early_printk(const char *fmt, ...) -+{ -+ va_list ap; -+ -+ va_start(ap, fmt); -+ early_vprintk(fmt, ap); -+ va_end(ap); -+} -+ -+/* -+ * This is independent of any log levels - a global -+ * kill switch that turns off all of printk. -+ * -+ * Used by the NMI watchdog if early-printk is enabled. -+ */ -+static bool __read_mostly printk_killswitch; -+ -+static int __init force_early_printk_setup(char *str) -+{ -+ printk_killswitch = true; -+ return 0; -+} -+early_param("force_early_printk", force_early_printk_setup); -+ -+void printk_kill(void) -+{ -+ printk_killswitch = true; -+} -+ -+static int forced_early_printk(const char *fmt, va_list ap) -+{ -+ if (!printk_killswitch) -+ return 0; -+ early_vprintk(fmt, ap); -+ return 1; -+} -+#else -+static inline int forced_early_printk(const char *fmt, va_list ap) -+{ -+ return 0; -+} -+#endif -+ - asmlinkage int vprintk_emit(int facility, int level, - const char *dict, size_t dictlen, - const char *fmt, va_list args) -@@ -1623,6 +1715,13 @@ - /* cpu currently holding logbuf_lock in this function */ - static unsigned int logbuf_cpu = UINT_MAX; - -+ /* -+ * Fall back to early_printk if a debugging subsystem has -+ * killed printk output -+ */ -+ if (unlikely(forced_early_printk(fmt, args))) -+ return 1; -+ - if (level == LOGLEVEL_SCHED) { - level = LOGLEVEL_DEFAULT; - in_sched = true; -@@ -1764,8 +1863,7 @@ - * console_sem which would prevent anyone from printing to - * console - */ -- preempt_disable(); -- -+ migrate_disable(); - /* - * Try to acquire and then immediately release the console - * semaphore. The release will print out buffers and wake up -@@ -1773,7 +1871,7 @@ - */ - if (console_trylock_for_printk()) - console_unlock(); -- preempt_enable(); -+ migrate_enable(); - lockdep_on(); - } - -@@ -1902,26 +2000,6 @@ - - #endif /* CONFIG_PRINTK */ - --#ifdef CONFIG_EARLY_PRINTK --struct console *early_console; -- --asmlinkage __visible void early_printk(const char *fmt, ...) --{ -- va_list ap; -- char buf[512]; -- int n; -- -- if (!early_console) -- return; -- -- va_start(ap, fmt); -- n = vscnprintf(buf, sizeof(buf), fmt, ap); -- va_end(ap); -- -- early_console->write(early_console, buf, n); --} --#endif -- - static int __add_preferred_console(char *name, int idx, char *options, - char *brl_options) - { -@@ -2143,11 +2221,16 @@ - goto out; - - len = cont_print_text(text, size); -+#ifndef CONFIG_PREEMPT_RT_FULL - raw_spin_unlock(&logbuf_lock); - stop_critical_timings(); - call_console_drivers(cont.level, text, len); - start_critical_timings(); - local_irq_restore(flags); -+#else -+ raw_spin_unlock_irqrestore(&logbuf_lock, flags); -+ call_console_drivers(cont.level, text, len); -+#endif - return; - out: - raw_spin_unlock_irqrestore(&logbuf_lock, flags); -@@ -2235,12 +2318,17 @@ - console_idx = log_next(console_idx); - console_seq++; - console_prev = msg->flags; -+#ifdef CONFIG_PREEMPT_RT_FULL -+ raw_spin_unlock_irqrestore(&logbuf_lock, flags); -+ call_console_drivers(level, text, len); -+#else - raw_spin_unlock(&logbuf_lock); - - stop_critical_timings(); /* don't trace print latency */ - call_console_drivers(level, text, len); - start_critical_timings(); - local_irq_restore(flags); -+#endif - } - console_locked = 0; - -diff -Nur linux-4.1.10.orig/kernel/ptrace.c linux-4.1.10/kernel/ptrace.c ---- linux-4.1.10.orig/kernel/ptrace.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/ptrace.c 2015-10-07 18:00:08.000000000 +0200 -@@ -129,7 +129,12 @@ - - spin_lock_irq(&task->sighand->siglock); - if (task_is_traced(task) && !__fatal_signal_pending(task)) { -- task->state = __TASK_TRACED; -+ raw_spin_lock_irq(&task->pi_lock); -+ if (task->state & __TASK_TRACED) -+ task->state = __TASK_TRACED; -+ else -+ task->saved_state = __TASK_TRACED; -+ raw_spin_unlock_irq(&task->pi_lock); - ret = true; - } - spin_unlock_irq(&task->sighand->siglock); -diff -Nur linux-4.1.10.orig/kernel/rcu/tree.c linux-4.1.10/kernel/rcu/tree.c ---- linux-4.1.10.orig/kernel/rcu/tree.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/rcu/tree.c 2015-10-07 18:00:08.000000000 +0200 -@@ -56,6 +56,11 @@ - #include - #include - #include -+#include -+#include -+#include -+#include -+#include "../time/tick-internal.h" - - #include "tree.h" - #include "rcu.h" -@@ -220,6 +225,19 @@ - } - } - -+#ifdef CONFIG_PREEMPT_RT_FULL -+static void rcu_preempt_qs(void); -+ -+void rcu_bh_qs(void) -+{ -+ unsigned long flags; -+ -+ /* Callers to this function, rcu_preempt_qs(), must disable irqs. */ -+ local_irq_save(flags); -+ rcu_preempt_qs(); -+ local_irq_restore(flags); -+} -+#else - void rcu_bh_qs(void) - { - if (!__this_cpu_read(rcu_bh_data.passed_quiesce)) { -@@ -229,6 +247,7 @@ - __this_cpu_write(rcu_bh_data.passed_quiesce, 1); - } - } -+#endif - - static DEFINE_PER_CPU(int, rcu_sched_qs_mask); - -@@ -404,6 +423,7 @@ - } - EXPORT_SYMBOL_GPL(rcu_batches_completed_sched); - -+#ifndef CONFIG_PREEMPT_RT_FULL - /* - * Return the number of RCU BH batches completed thus far for debug & stats. - */ -@@ -431,6 +451,13 @@ - } - EXPORT_SYMBOL_GPL(rcu_bh_force_quiescent_state); - -+#else -+void rcu_force_quiescent_state(void) -+{ -+} -+EXPORT_SYMBOL_GPL(rcu_force_quiescent_state); -+#endif -+ - /* - * Force a quiescent state for RCU-sched. - */ -@@ -1545,7 +1572,7 @@ - !ACCESS_ONCE(rsp->gp_flags) || - !rsp->gp_kthread) - return; -- wake_up(&rsp->gp_wq); -+ swait_wake(&rsp->gp_wq); - } - - /* -@@ -1986,7 +2013,7 @@ - ACCESS_ONCE(rsp->gpnum), - TPS("reqwait")); - rsp->gp_state = RCU_GP_WAIT_GPS; -- wait_event_interruptible(rsp->gp_wq, -+ swait_event_interruptible(rsp->gp_wq, - ACCESS_ONCE(rsp->gp_flags) & - RCU_GP_FLAG_INIT); - /* Locking provides needed memory barrier. */ -@@ -2015,7 +2042,7 @@ - ACCESS_ONCE(rsp->gpnum), - TPS("fqswait")); - rsp->gp_state = RCU_GP_WAIT_FQS; -- ret = wait_event_interruptible_timeout(rsp->gp_wq, -+ ret = swait_event_interruptible_timeout(rsp->gp_wq, - ((gf = ACCESS_ONCE(rsp->gp_flags)) & - RCU_GP_FLAG_FQS) || - (!ACCESS_ONCE(rnp->qsmask) && -@@ -2860,18 +2887,17 @@ - /* - * Do RCU core processing for the current CPU. - */ --static void rcu_process_callbacks(struct softirq_action *unused) -+static void rcu_process_callbacks(void) - { - struct rcu_state *rsp; - - if (cpu_is_offline(smp_processor_id())) - return; -- trace_rcu_utilization(TPS("Start RCU core")); - for_each_rcu_flavor(rsp) - __rcu_process_callbacks(rsp); -- trace_rcu_utilization(TPS("End RCU core")); - } - -+static DEFINE_PER_CPU(struct task_struct *, rcu_cpu_kthread_task); - /* - * Schedule RCU callback invocation. If the specified type of RCU - * does not support RCU priority boosting, just do a direct call, -@@ -2883,18 +2909,105 @@ - { - if (unlikely(!ACCESS_ONCE(rcu_scheduler_fully_active))) - return; -- if (likely(!rsp->boost)) { -- rcu_do_batch(rsp, rdp); -+ rcu_do_batch(rsp, rdp); -+} -+ -+static void rcu_wake_cond(struct task_struct *t, int status) -+{ -+ /* -+ * If the thread is yielding, only wake it when this -+ * is invoked from idle -+ */ -+ if (t && (status != RCU_KTHREAD_YIELDING || is_idle_task(current))) -+ wake_up_process(t); -+} -+ -+/* -+ * Wake up this CPU's rcuc kthread to do RCU core processing. -+ */ -+static void invoke_rcu_core(void) -+{ -+ unsigned long flags; -+ struct task_struct *t; -+ -+ if (!cpu_online(smp_processor_id())) - return; -+ local_irq_save(flags); -+ __this_cpu_write(rcu_cpu_has_work, 1); -+ t = __this_cpu_read(rcu_cpu_kthread_task); -+ if (t != NULL && current != t) -+ rcu_wake_cond(t, __this_cpu_read(rcu_cpu_kthread_status)); -+ local_irq_restore(flags); -+} -+ -+static void rcu_cpu_kthread_park(unsigned int cpu) -+{ -+ per_cpu(rcu_cpu_kthread_status, cpu) = RCU_KTHREAD_OFFCPU; -+} -+ -+static int rcu_cpu_kthread_should_run(unsigned int cpu) -+{ -+ return __this_cpu_read(rcu_cpu_has_work); -+} -+ -+/* -+ * Per-CPU kernel thread that invokes RCU callbacks. This replaces the -+ * RCU softirq used in flavors and configurations of RCU that do not -+ * support RCU priority boosting. -+ */ -+static void rcu_cpu_kthread(unsigned int cpu) -+{ -+ unsigned int *statusp = this_cpu_ptr(&rcu_cpu_kthread_status); -+ char work, *workp = this_cpu_ptr(&rcu_cpu_has_work); -+ int spincnt; -+ -+ for (spincnt = 0; spincnt < 10; spincnt++) { -+ trace_rcu_utilization(TPS("Start CPU kthread@rcu_wait")); -+ local_bh_disable(); -+ *statusp = RCU_KTHREAD_RUNNING; -+ this_cpu_inc(rcu_cpu_kthread_loops); -+ local_irq_disable(); -+ work = *workp; -+ *workp = 0; -+ local_irq_enable(); -+ if (work) -+ rcu_process_callbacks(); -+ local_bh_enable(); -+ if (*workp == 0) { -+ trace_rcu_utilization(TPS("End CPU kthread@rcu_wait")); -+ *statusp = RCU_KTHREAD_WAITING; -+ return; -+ } - } -- invoke_rcu_callbacks_kthread(); -+ *statusp = RCU_KTHREAD_YIELDING; -+ trace_rcu_utilization(TPS("Start CPU kthread@rcu_yield")); -+ schedule_timeout_interruptible(2); -+ trace_rcu_utilization(TPS("End CPU kthread@rcu_yield")); -+ *statusp = RCU_KTHREAD_WAITING; - } - --static void invoke_rcu_core(void) -+static struct smp_hotplug_thread rcu_cpu_thread_spec = { -+ .store = &rcu_cpu_kthread_task, -+ .thread_should_run = rcu_cpu_kthread_should_run, -+ .thread_fn = rcu_cpu_kthread, -+ .thread_comm = "rcuc/%u", -+ .setup = rcu_cpu_kthread_setup, -+ .park = rcu_cpu_kthread_park, -+}; -+ -+/* -+ * Spawn per-CPU RCU core processing kthreads. -+ */ -+static int __init rcu_spawn_core_kthreads(void) - { -- if (cpu_online(smp_processor_id())) -- raise_softirq(RCU_SOFTIRQ); -+ int cpu; -+ -+ for_each_possible_cpu(cpu) -+ per_cpu(rcu_cpu_has_work, cpu) = 0; -+ BUG_ON(smpboot_register_percpu_thread(&rcu_cpu_thread_spec)); -+ return 0; - } -+early_initcall(rcu_spawn_core_kthreads); - - /* - * Handle any core-RCU processing required by a call_rcu() invocation. -@@ -3040,6 +3153,7 @@ - } - EXPORT_SYMBOL_GPL(call_rcu_sched); - -+#ifndef CONFIG_PREEMPT_RT_FULL - /* - * Queue an RCU callback for invocation after a quicker grace period. - */ -@@ -3048,6 +3162,7 @@ - __call_rcu(head, func, &rcu_bh_state, -1, 0); - } - EXPORT_SYMBOL_GPL(call_rcu_bh); -+#endif - - /* - * Queue an RCU callback for lazy invocation after a grace period. -@@ -3139,6 +3254,7 @@ - } - EXPORT_SYMBOL_GPL(synchronize_sched); - -+#ifndef CONFIG_PREEMPT_RT_FULL - /** - * synchronize_rcu_bh - wait until an rcu_bh grace period has elapsed. - * -@@ -3165,6 +3281,7 @@ - wait_rcu_gp(call_rcu_bh); - } - EXPORT_SYMBOL_GPL(synchronize_rcu_bh); -+#endif - - /** - * get_state_synchronize_rcu - Snapshot current RCU state -@@ -3677,6 +3794,7 @@ - mutex_unlock(&rsp->barrier_mutex); - } - -+#ifndef CONFIG_PREEMPT_RT_FULL - /** - * rcu_barrier_bh - Wait until all in-flight call_rcu_bh() callbacks complete. - */ -@@ -3685,6 +3803,7 @@ - _rcu_barrier(&rcu_bh_state); - } - EXPORT_SYMBOL_GPL(rcu_barrier_bh); -+#endif - - /** - * rcu_barrier_sched - Wait for in-flight call_rcu_sched() callbacks. -@@ -4021,7 +4140,7 @@ - } - } - -- init_waitqueue_head(&rsp->gp_wq); -+ init_swait_head(&rsp->gp_wq); - rnp = rsp->level[rcu_num_lvls - 1]; - for_each_possible_cpu(i) { - while (i > rnp->grphi) -@@ -4120,7 +4239,6 @@ - rcu_init_one(&rcu_bh_state, &rcu_bh_data); - rcu_init_one(&rcu_sched_state, &rcu_sched_data); - __rcu_init_preempt(); -- open_softirq(RCU_SOFTIRQ, rcu_process_callbacks); - - /* - * We don't need protection against CPU-hotplug here because -diff -Nur linux-4.1.10.orig/kernel/rcu/tree.h linux-4.1.10/kernel/rcu/tree.h ---- linux-4.1.10.orig/kernel/rcu/tree.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/rcu/tree.h 2015-10-07 18:00:08.000000000 +0200 -@@ -27,6 +27,7 @@ - #include - #include - #include -+#include - - /* - * Define shape of hierarchy based on NR_CPUS, CONFIG_RCU_FANOUT, and -@@ -210,7 +211,7 @@ - /* This can happen due to race conditions. */ - #endif /* #ifdef CONFIG_RCU_BOOST */ - #ifdef CONFIG_RCU_NOCB_CPU -- wait_queue_head_t nocb_gp_wq[2]; -+ struct swait_head nocb_gp_wq[2]; - /* Place for rcu_nocb_kthread() to wait GP. */ - #endif /* #ifdef CONFIG_RCU_NOCB_CPU */ - int need_future_gp[2]; -@@ -349,7 +350,7 @@ - atomic_long_t nocb_q_count_lazy; /* invocation (all stages). */ - struct rcu_head *nocb_follower_head; /* CBs ready to invoke. */ - struct rcu_head **nocb_follower_tail; -- wait_queue_head_t nocb_wq; /* For nocb kthreads to sleep on. */ -+ struct swait_head nocb_wq; /* For nocb kthreads to sleep on. */ - struct task_struct *nocb_kthread; - int nocb_defer_wakeup; /* Defer wakeup of nocb_kthread. */ - -@@ -438,7 +439,7 @@ - unsigned long gpnum; /* Current gp number. */ - unsigned long completed; /* # of last completed gp. */ - struct task_struct *gp_kthread; /* Task for grace periods. */ -- wait_queue_head_t gp_wq; /* Where GP task waits. */ -+ struct swait_head gp_wq; /* Where GP task waits. */ - short gp_flags; /* Commands for GP task. */ - short gp_state; /* GP kthread sleep state. */ - -@@ -529,12 +530,10 @@ - DECLARE_PER_CPU(struct rcu_data, rcu_preempt_data); - #endif /* #ifdef CONFIG_PREEMPT_RCU */ - --#ifdef CONFIG_RCU_BOOST - DECLARE_PER_CPU(unsigned int, rcu_cpu_kthread_status); - DECLARE_PER_CPU(int, rcu_cpu_kthread_cpu); - DECLARE_PER_CPU(unsigned int, rcu_cpu_kthread_loops); - DECLARE_PER_CPU(char, rcu_cpu_has_work); --#endif /* #ifdef CONFIG_RCU_BOOST */ - - #ifndef RCU_TREE_NONCORE - -@@ -553,10 +552,9 @@ - static void __init __rcu_init_preempt(void); - static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags); - static void rcu_preempt_boost_start_gp(struct rcu_node *rnp); --static void invoke_rcu_callbacks_kthread(void); - static bool rcu_is_callbacks_kthread(void); -+static void rcu_cpu_kthread_setup(unsigned int cpu); - #ifdef CONFIG_RCU_BOOST --static void rcu_preempt_do_callbacks(void); - static int rcu_spawn_one_boost_kthread(struct rcu_state *rsp, - struct rcu_node *rnp); - #endif /* #ifdef CONFIG_RCU_BOOST */ -diff -Nur linux-4.1.10.orig/kernel/rcu/tree_plugin.h linux-4.1.10/kernel/rcu/tree_plugin.h ---- linux-4.1.10.orig/kernel/rcu/tree_plugin.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/rcu/tree_plugin.h 2015-10-07 18:00:08.000000000 +0200 -@@ -24,27 +24,20 @@ - * Paul E. McKenney - */ - --#include --#include --#include --#include --#include "../time/tick-internal.h" -- - #ifdef CONFIG_RCU_BOOST - - #include "../locking/rtmutex_common.h" - -+#endif /* #ifdef CONFIG_RCU_BOOST */ -+ - /* - * Control variables for per-CPU and per-rcu_node kthreads. These - * handle all flavors of RCU. - */ --static DEFINE_PER_CPU(struct task_struct *, rcu_cpu_kthread_task); - DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_status); - DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_loops); - DEFINE_PER_CPU(char, rcu_cpu_has_work); - --#endif /* #ifdef CONFIG_RCU_BOOST */ -- - #ifdef CONFIG_RCU_NOCB_CPU - static cpumask_var_t rcu_nocb_mask; /* CPUs to have callbacks offloaded. */ - static bool have_rcu_nocb_mask; /* Was rcu_nocb_mask allocated? */ -@@ -291,7 +284,7 @@ - } - - /* Hardware IRQ handlers cannot block, complain if they get here. */ -- if (in_irq() || in_serving_softirq()) { -+ if (preempt_count() & (HARDIRQ_MASK | SOFTIRQ_OFFSET)) { - lockdep_rcu_suspicious(__FILE__, __LINE__, - "rcu_read_unlock() from irq or softirq with blocking in critical section!!!\n"); - pr_alert("->rcu_read_unlock_special: %#x (b: %d, nq: %d)\n", -@@ -496,15 +489,6 @@ - t->rcu_read_unlock_special.b.need_qs = true; - } - --#ifdef CONFIG_RCU_BOOST -- --static void rcu_preempt_do_callbacks(void) --{ -- rcu_do_batch(&rcu_preempt_state, this_cpu_ptr(&rcu_preempt_data)); --} -- --#endif /* #ifdef CONFIG_RCU_BOOST */ -- - /* - * Queue a preemptible-RCU callback for invocation after a grace period. - */ -@@ -939,6 +923,19 @@ - - #endif /* #else #ifdef CONFIG_PREEMPT_RCU */ - -+/* -+ * If boosting, set rcuc kthreads to realtime priority. -+ */ -+static void rcu_cpu_kthread_setup(unsigned int cpu) -+{ -+#ifdef CONFIG_RCU_BOOST -+ struct sched_param sp; -+ -+ sp.sched_priority = kthread_prio; -+ sched_setscheduler_nocheck(current, SCHED_FIFO, &sp); -+#endif /* #ifdef CONFIG_RCU_BOOST */ -+} -+ - #ifdef CONFIG_RCU_BOOST - - #include "../locking/rtmutex_common.h" -@@ -970,16 +967,6 @@ - - #endif /* #else #ifdef CONFIG_RCU_TRACE */ - --static void rcu_wake_cond(struct task_struct *t, int status) --{ -- /* -- * If the thread is yielding, only wake it when this -- * is invoked from idle -- */ -- if (status != RCU_KTHREAD_YIELDING || is_idle_task(current)) -- wake_up_process(t); --} -- - /* - * Carry out RCU priority boosting on the task indicated by ->exp_tasks - * or ->boost_tasks, advancing the pointer to the next task in the -@@ -1125,23 +1112,6 @@ - } - - /* -- * Wake up the per-CPU kthread to invoke RCU callbacks. -- */ --static void invoke_rcu_callbacks_kthread(void) --{ -- unsigned long flags; -- -- local_irq_save(flags); -- __this_cpu_write(rcu_cpu_has_work, 1); -- if (__this_cpu_read(rcu_cpu_kthread_task) != NULL && -- current != __this_cpu_read(rcu_cpu_kthread_task)) { -- rcu_wake_cond(__this_cpu_read(rcu_cpu_kthread_task), -- __this_cpu_read(rcu_cpu_kthread_status)); -- } -- local_irq_restore(flags); --} -- --/* - * Is the current CPU running the RCU-callbacks kthread? - * Caller must have preemption disabled. - */ -@@ -1196,67 +1166,6 @@ - return 0; - } - --static void rcu_kthread_do_work(void) --{ -- rcu_do_batch(&rcu_sched_state, this_cpu_ptr(&rcu_sched_data)); -- rcu_do_batch(&rcu_bh_state, this_cpu_ptr(&rcu_bh_data)); -- rcu_preempt_do_callbacks(); --} -- --static void rcu_cpu_kthread_setup(unsigned int cpu) --{ -- struct sched_param sp; -- -- sp.sched_priority = kthread_prio; -- sched_setscheduler_nocheck(current, SCHED_FIFO, &sp); --} -- --static void rcu_cpu_kthread_park(unsigned int cpu) --{ -- per_cpu(rcu_cpu_kthread_status, cpu) = RCU_KTHREAD_OFFCPU; --} -- --static int rcu_cpu_kthread_should_run(unsigned int cpu) --{ -- return __this_cpu_read(rcu_cpu_has_work); --} -- --/* -- * Per-CPU kernel thread that invokes RCU callbacks. This replaces the -- * RCU softirq used in flavors and configurations of RCU that do not -- * support RCU priority boosting. -- */ --static void rcu_cpu_kthread(unsigned int cpu) --{ -- unsigned int *statusp = this_cpu_ptr(&rcu_cpu_kthread_status); -- char work, *workp = this_cpu_ptr(&rcu_cpu_has_work); -- int spincnt; -- -- for (spincnt = 0; spincnt < 10; spincnt++) { -- trace_rcu_utilization(TPS("Start CPU kthread@rcu_wait")); -- local_bh_disable(); -- *statusp = RCU_KTHREAD_RUNNING; -- this_cpu_inc(rcu_cpu_kthread_loops); -- local_irq_disable(); -- work = *workp; -- *workp = 0; -- local_irq_enable(); -- if (work) -- rcu_kthread_do_work(); -- local_bh_enable(); -- if (*workp == 0) { -- trace_rcu_utilization(TPS("End CPU kthread@rcu_wait")); -- *statusp = RCU_KTHREAD_WAITING; -- return; -- } -- } -- *statusp = RCU_KTHREAD_YIELDING; -- trace_rcu_utilization(TPS("Start CPU kthread@rcu_yield")); -- schedule_timeout_interruptible(2); -- trace_rcu_utilization(TPS("End CPU kthread@rcu_yield")); -- *statusp = RCU_KTHREAD_WAITING; --} -- - /* - * Set the per-rcu_node kthread's affinity to cover all CPUs that are - * served by the rcu_node in question. The CPU hotplug lock is still -@@ -1286,26 +1195,12 @@ - free_cpumask_var(cm); - } - --static struct smp_hotplug_thread rcu_cpu_thread_spec = { -- .store = &rcu_cpu_kthread_task, -- .thread_should_run = rcu_cpu_kthread_should_run, -- .thread_fn = rcu_cpu_kthread, -- .thread_comm = "rcuc/%u", -- .setup = rcu_cpu_kthread_setup, -- .park = rcu_cpu_kthread_park, --}; -- - /* - * Spawn boost kthreads -- called as soon as the scheduler is running. - */ - static void __init rcu_spawn_boost_kthreads(void) - { - struct rcu_node *rnp; -- int cpu; -- -- for_each_possible_cpu(cpu) -- per_cpu(rcu_cpu_has_work, cpu) = 0; -- BUG_ON(smpboot_register_percpu_thread(&rcu_cpu_thread_spec)); - rcu_for_each_leaf_node(rcu_state_p, rnp) - (void)rcu_spawn_one_boost_kthread(rcu_state_p, rnp); - } -@@ -1328,11 +1223,6 @@ - raw_spin_unlock_irqrestore(&rnp->lock, flags); - } - --static void invoke_rcu_callbacks_kthread(void) --{ -- WARN_ON_ONCE(1); --} -- - static bool rcu_is_callbacks_kthread(void) - { - return false; -@@ -1356,7 +1246,7 @@ - - #endif /* #else #ifdef CONFIG_RCU_BOOST */ - --#if !defined(CONFIG_RCU_FAST_NO_HZ) -+#if !defined(CONFIG_RCU_FAST_NO_HZ) || defined(CONFIG_PREEMPT_RT_FULL) - - /* - * Check to see if any future RCU-related work will need to be done -@@ -1374,7 +1264,9 @@ - return rcu_cpu_has_callbacks(NULL); - } - #endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */ -+#endif /* !defined(CONFIG_RCU_FAST_NO_HZ) || defined(CONFIG_PREEMPT_RT_FULL) */ - -+#if !defined(CONFIG_RCU_FAST_NO_HZ) - /* - * Because we do not have RCU_FAST_NO_HZ, don't bother cleaning up - * after it. -@@ -1472,6 +1364,8 @@ - return cbs_ready; - } - -+#ifndef CONFIG_PREEMPT_RT_FULL -+ - /* - * Allow the CPU to enter dyntick-idle mode unless it has callbacks ready - * to invoke. If the CPU has callbacks, try to advance them. Tell the -@@ -1512,7 +1406,7 @@ - return 0; - } - #endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */ -- -+#endif /* #ifndef CONFIG_PREEMPT_RT_FULL */ - /* - * Prepare a CPU for idle from an RCU perspective. The first major task - * is to sense whether nohz mode has been enabled or disabled via sysfs. -@@ -1859,7 +1753,7 @@ - */ - static void rcu_nocb_gp_cleanup(struct rcu_state *rsp, struct rcu_node *rnp) - { -- wake_up_all(&rnp->nocb_gp_wq[rnp->completed & 0x1]); -+ swait_wake_all(&rnp->nocb_gp_wq[rnp->completed & 0x1]); - } - - /* -@@ -1877,8 +1771,8 @@ - - static void rcu_init_one_nocb(struct rcu_node *rnp) - { -- init_waitqueue_head(&rnp->nocb_gp_wq[0]); -- init_waitqueue_head(&rnp->nocb_gp_wq[1]); -+ init_swait_head(&rnp->nocb_gp_wq[0]); -+ init_swait_head(&rnp->nocb_gp_wq[1]); - } - - #ifndef CONFIG_RCU_NOCB_CPU_ALL -@@ -1903,7 +1797,7 @@ - if (ACCESS_ONCE(rdp_leader->nocb_leader_sleep) || force) { - /* Prior smp_mb__after_atomic() orders against prior enqueue. */ - ACCESS_ONCE(rdp_leader->nocb_leader_sleep) = false; -- wake_up(&rdp_leader->nocb_wq); -+ swait_wake(&rdp_leader->nocb_wq); - } - } - -@@ -2116,7 +2010,7 @@ - */ - trace_rcu_future_gp(rnp, rdp, c, TPS("StartWait")); - for (;;) { -- wait_event_interruptible( -+ swait_event_interruptible( - rnp->nocb_gp_wq[c & 0x1], - (d = ULONG_CMP_GE(ACCESS_ONCE(rnp->completed), c))); - if (likely(d)) -@@ -2144,7 +2038,7 @@ - /* Wait for callbacks to appear. */ - if (!rcu_nocb_poll) { - trace_rcu_nocb_wake(my_rdp->rsp->name, my_rdp->cpu, "Sleep"); -- wait_event_interruptible(my_rdp->nocb_wq, -+ swait_event_interruptible(my_rdp->nocb_wq, - !ACCESS_ONCE(my_rdp->nocb_leader_sleep)); - /* Memory barrier handled by smp_mb() calls below and repoll. */ - } else if (firsttime) { -@@ -2219,7 +2113,7 @@ - * List was empty, wake up the follower. - * Memory barriers supplied by atomic_long_add(). - */ -- wake_up(&rdp->nocb_wq); -+ swait_wake(&rdp->nocb_wq); - } - } - -@@ -2240,7 +2134,7 @@ - if (!rcu_nocb_poll) { - trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, - "FollowerSleep"); -- wait_event_interruptible(rdp->nocb_wq, -+ swait_event_interruptible(rdp->nocb_wq, - ACCESS_ONCE(rdp->nocb_follower_head)); - } else if (firsttime) { - /* Don't drown trace log with "Poll"! */ -@@ -2399,7 +2293,7 @@ - static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp) - { - rdp->nocb_tail = &rdp->nocb_head; -- init_waitqueue_head(&rdp->nocb_wq); -+ init_swait_head(&rdp->nocb_wq); - rdp->nocb_follower_tail = &rdp->nocb_follower_head; - } - -diff -Nur linux-4.1.10.orig/kernel/rcu/update.c linux-4.1.10/kernel/rcu/update.c ---- linux-4.1.10.orig/kernel/rcu/update.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/rcu/update.c 2015-10-07 18:00:08.000000000 +0200 -@@ -227,6 +227,7 @@ - } - EXPORT_SYMBOL_GPL(rcu_read_lock_held); - -+#ifndef CONFIG_PREEMPT_RT_FULL - /** - * rcu_read_lock_bh_held() - might we be in RCU-bh read-side critical section? - * -@@ -253,6 +254,7 @@ - return in_softirq() || irqs_disabled(); - } - EXPORT_SYMBOL_GPL(rcu_read_lock_bh_held); -+#endif - - #endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ - -diff -Nur linux-4.1.10.orig/kernel/relay.c linux-4.1.10/kernel/relay.c ---- linux-4.1.10.orig/kernel/relay.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/relay.c 2015-10-07 18:00:08.000000000 +0200 -@@ -339,6 +339,10 @@ - { - struct rchan_buf *buf = (struct rchan_buf *)data; - wake_up_interruptible(&buf->read_wait); -+ /* -+ * Stupid polling for now: -+ */ -+ mod_timer(&buf->timer, jiffies + 1); - } - - /** -@@ -356,6 +360,7 @@ - init_waitqueue_head(&buf->read_wait); - kref_init(&buf->kref); - setup_timer(&buf->timer, wakeup_readers, (unsigned long)buf); -+ mod_timer(&buf->timer, jiffies + 1); - } else - del_timer_sync(&buf->timer); - -@@ -739,15 +744,6 @@ - else - buf->early_bytes += buf->chan->subbuf_size - - buf->padding[old_subbuf]; -- smp_mb(); -- if (waitqueue_active(&buf->read_wait)) -- /* -- * Calling wake_up_interruptible() from here -- * will deadlock if we happen to be logging -- * from the scheduler (trying to re-grab -- * rq->lock), so defer it. -- */ -- mod_timer(&buf->timer, jiffies + 1); - } - - old = buf->data; -diff -Nur linux-4.1.10.orig/kernel/sched/completion.c linux-4.1.10/kernel/sched/completion.c ---- linux-4.1.10.orig/kernel/sched/completion.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/sched/completion.c 2015-10-07 18:00:08.000000000 +0200 -@@ -30,10 +30,10 @@ - { - unsigned long flags; - -- spin_lock_irqsave(&x->wait.lock, flags); -+ raw_spin_lock_irqsave(&x->wait.lock, flags); - x->done++; -- __wake_up_locked(&x->wait, TASK_NORMAL, 1); -- spin_unlock_irqrestore(&x->wait.lock, flags); -+ __swait_wake_locked(&x->wait, TASK_NORMAL, 1); -+ raw_spin_unlock_irqrestore(&x->wait.lock, flags); - } - EXPORT_SYMBOL(complete); - -@@ -50,10 +50,10 @@ - { - unsigned long flags; - -- spin_lock_irqsave(&x->wait.lock, flags); -+ raw_spin_lock_irqsave(&x->wait.lock, flags); - x->done += UINT_MAX/2; -- __wake_up_locked(&x->wait, TASK_NORMAL, 0); -- spin_unlock_irqrestore(&x->wait.lock, flags); -+ __swait_wake_locked(&x->wait, TASK_NORMAL, 0); -+ raw_spin_unlock_irqrestore(&x->wait.lock, flags); - } - EXPORT_SYMBOL(complete_all); - -@@ -62,20 +62,20 @@ - long (*action)(long), long timeout, int state) - { - if (!x->done) { -- DECLARE_WAITQUEUE(wait, current); -+ DEFINE_SWAITER(wait); - -- __add_wait_queue_tail_exclusive(&x->wait, &wait); -+ swait_prepare_locked(&x->wait, &wait); - do { - if (signal_pending_state(state, current)) { - timeout = -ERESTARTSYS; - break; - } - __set_current_state(state); -- spin_unlock_irq(&x->wait.lock); -+ raw_spin_unlock_irq(&x->wait.lock); - timeout = action(timeout); -- spin_lock_irq(&x->wait.lock); -+ raw_spin_lock_irq(&x->wait.lock); - } while (!x->done && timeout); -- __remove_wait_queue(&x->wait, &wait); -+ swait_finish_locked(&x->wait, &wait); - if (!x->done) - return timeout; - } -@@ -89,9 +89,9 @@ - { - might_sleep(); - -- spin_lock_irq(&x->wait.lock); -+ raw_spin_lock_irq(&x->wait.lock); - timeout = do_wait_for_common(x, action, timeout, state); -- spin_unlock_irq(&x->wait.lock); -+ raw_spin_unlock_irq(&x->wait.lock); - return timeout; - } - -@@ -277,12 +277,12 @@ - if (!READ_ONCE(x->done)) - return 0; - -- spin_lock_irqsave(&x->wait.lock, flags); -+ raw_spin_lock_irqsave(&x->wait.lock, flags); - if (!x->done) - ret = 0; - else - x->done--; -- spin_unlock_irqrestore(&x->wait.lock, flags); -+ raw_spin_unlock_irqrestore(&x->wait.lock, flags); - return ret; - } - EXPORT_SYMBOL(try_wait_for_completion); -@@ -311,7 +311,7 @@ - * after it's acquired the lock. - */ - smp_rmb(); -- spin_unlock_wait(&x->wait.lock); -+ raw_spin_unlock_wait(&x->wait.lock); - return true; - } - EXPORT_SYMBOL(completion_done); -diff -Nur linux-4.1.10.orig/kernel/sched/core.c linux-4.1.10/kernel/sched/core.c ---- linux-4.1.10.orig/kernel/sched/core.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/sched/core.c 2015-10-07 18:00:08.000000000 +0200 -@@ -282,7 +282,11 @@ - * Number of tasks to iterate in a single balance run. - * Limited because this is done with IRQs disabled. - */ -+#ifndef CONFIG_PREEMPT_RT_FULL - const_debug unsigned int sysctl_sched_nr_migrate = 32; -+#else -+const_debug unsigned int sysctl_sched_nr_migrate = 8; -+#endif - - /* - * period over which we average the RT time consumption, measured -@@ -461,6 +465,7 @@ - - hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); - rq->hrtick_timer.function = hrtick; -+ rq->hrtick_timer.irqsafe = 1; - } - #else /* CONFIG_SCHED_HRTICK */ - static inline void hrtick_clear(struct rq *rq) -@@ -541,6 +546,52 @@ - #endif - #endif - -+void wake_q_add(struct wake_q_head *head, struct task_struct *task) -+{ -+ struct wake_q_node *node = &task->wake_q; -+ -+ /* -+ * Atomically grab the task, if ->wake_q is !nil already it means -+ * its already queued (either by us or someone else) and will get the -+ * wakeup due to that. -+ * -+ * This cmpxchg() implies a full barrier, which pairs with the write -+ * barrier implied by the wakeup in wake_up_list(). -+ */ -+ if (cmpxchg(&node->next, NULL, WAKE_Q_TAIL)) -+ return; -+ -+ get_task_struct(task); -+ -+ /* -+ * The head is context local, there can be no concurrency. -+ */ -+ *head->lastp = node; -+ head->lastp = &node->next; -+} -+ -+void wake_up_q(struct wake_q_head *head) -+{ -+ struct wake_q_node *node = head->first; -+ -+ while (node != WAKE_Q_TAIL) { -+ struct task_struct *task; -+ -+ task = container_of(node, struct task_struct, wake_q); -+ BUG_ON(!task); -+ /* task can safely be re-inserted now */ -+ node = node->next; -+ task->wake_q.next = NULL; -+ -+ /* -+ * wake_up_process() implies a wmb() to pair with the queueing -+ * in wake_q_add() so as not to miss wakeups. -+ */ -+ wake_up_process(task); -+ put_task_struct(task); -+ } -+} -+ - /* - * resched_curr - mark rq's current task 'to be rescheduled now'. - * -@@ -572,6 +623,38 @@ - trace_sched_wake_idle_without_ipi(cpu); - } - -+#ifdef CONFIG_PREEMPT_LAZY -+void resched_curr_lazy(struct rq *rq) -+{ -+ struct task_struct *curr = rq->curr; -+ int cpu; -+ -+ if (!sched_feat(PREEMPT_LAZY)) { -+ resched_curr(rq); -+ return; -+ } -+ -+ lockdep_assert_held(&rq->lock); -+ -+ if (test_tsk_need_resched(curr)) -+ return; -+ -+ if (test_tsk_need_resched_lazy(curr)) -+ return; -+ -+ set_tsk_need_resched_lazy(curr); -+ -+ cpu = cpu_of(rq); -+ if (cpu == smp_processor_id()) -+ return; -+ -+ /* NEED_RESCHED_LAZY must be visible before we test polling */ -+ smp_mb(); -+ if (!tsk_is_polling(curr)) -+ smp_send_reschedule(cpu); -+} -+#endif -+ - void resched_cpu(int cpu) - { - struct rq *rq = cpu_rq(cpu); -@@ -595,12 +678,14 @@ - */ - int get_nohz_timer_target(int pinned) - { -- int cpu = smp_processor_id(); -+ int cpu; - int i; - struct sched_domain *sd; - -+ preempt_disable_rt(); -+ cpu = smp_processor_id(); - if (pinned || !get_sysctl_timer_migration() || !idle_cpu(cpu)) -- return cpu; -+ goto preempt_en_rt; - - rcu_read_lock(); - for_each_domain(cpu, sd) { -@@ -613,6 +698,8 @@ - } - unlock: - rcu_read_unlock(); -+preempt_en_rt: -+ preempt_enable_rt(); - return cpu; - } - /* -@@ -1164,6 +1251,18 @@ - - static int migration_cpu_stop(void *data); - -+static bool check_task_state(struct task_struct *p, long match_state) -+{ -+ bool match = false; -+ -+ raw_spin_lock_irq(&p->pi_lock); -+ if (p->state == match_state || p->saved_state == match_state) -+ match = true; -+ raw_spin_unlock_irq(&p->pi_lock); -+ -+ return match; -+} -+ - /* - * wait_task_inactive - wait for a thread to unschedule. - * -@@ -1208,7 +1307,7 @@ - * is actually now running somewhere else! - */ - while (task_running(rq, p)) { -- if (match_state && unlikely(p->state != match_state)) -+ if (match_state && !check_task_state(p, match_state)) - return 0; - cpu_relax(); - } -@@ -1223,7 +1322,8 @@ - running = task_running(rq, p); - queued = task_on_rq_queued(p); - ncsw = 0; -- if (!match_state || p->state == match_state) -+ if (!match_state || p->state == match_state || -+ p->saved_state == match_state) - ncsw = p->nvcsw | LONG_MIN; /* sets MSB */ - task_rq_unlock(rq, p, &flags); - -@@ -1449,10 +1549,6 @@ - { - activate_task(rq, p, en_flags); - p->on_rq = TASK_ON_RQ_QUEUED; -- -- /* if a worker is waking up, notify workqueue */ -- if (p->flags & PF_WQ_WORKER) -- wq_worker_waking_up(p, cpu_of(rq)); - } - - /* -@@ -1666,8 +1762,27 @@ - */ - smp_mb__before_spinlock(); - raw_spin_lock_irqsave(&p->pi_lock, flags); -- if (!(p->state & state)) -+ if (!(p->state & state)) { -+ /* -+ * The task might be running due to a spinlock sleeper -+ * wakeup. Check the saved state and set it to running -+ * if the wakeup condition is true. -+ */ -+ if (!(wake_flags & WF_LOCK_SLEEPER)) { -+ if (p->saved_state & state) { -+ p->saved_state = TASK_RUNNING; -+ success = 1; -+ } -+ } - goto out; -+ } -+ -+ /* -+ * If this is a regular wakeup, then we can unconditionally -+ * clear the saved state of a "lock sleeper". -+ */ -+ if (!(wake_flags & WF_LOCK_SLEEPER)) -+ p->saved_state = TASK_RUNNING; - - success = 1; /* we're going to change ->state */ - cpu = task_cpu(p); -@@ -1710,42 +1825,6 @@ - } - - /** -- * try_to_wake_up_local - try to wake up a local task with rq lock held -- * @p: the thread to be awakened -- * -- * Put @p on the run-queue if it's not already there. The caller must -- * ensure that this_rq() is locked, @p is bound to this_rq() and not -- * the current task. -- */ --static void try_to_wake_up_local(struct task_struct *p) --{ -- struct rq *rq = task_rq(p); -- -- if (WARN_ON_ONCE(rq != this_rq()) || -- WARN_ON_ONCE(p == current)) -- return; -- -- lockdep_assert_held(&rq->lock); -- -- if (!raw_spin_trylock(&p->pi_lock)) { -- raw_spin_unlock(&rq->lock); -- raw_spin_lock(&p->pi_lock); -- raw_spin_lock(&rq->lock); -- } -- -- if (!(p->state & TASK_NORMAL)) -- goto out; -- -- if (!task_on_rq_queued(p)) -- ttwu_activate(rq, p, ENQUEUE_WAKEUP); -- -- ttwu_do_wakeup(rq, p, 0); -- ttwu_stat(p, smp_processor_id(), 0); --out: -- raw_spin_unlock(&p->pi_lock); --} -- --/** - * wake_up_process - Wake up a specific process - * @p: The process to be woken up. - * -@@ -1759,11 +1838,23 @@ - */ - int wake_up_process(struct task_struct *p) - { -- WARN_ON(task_is_stopped_or_traced(p)); -+ WARN_ON(__task_is_stopped_or_traced(p)); - return try_to_wake_up(p, TASK_NORMAL, 0); - } - EXPORT_SYMBOL(wake_up_process); - -+/** -+ * wake_up_lock_sleeper - Wake up a specific process blocked on a "sleeping lock" -+ * @p: The process to be woken up. -+ * -+ * Same as wake_up_process() above, but wake_flags=WF_LOCK_SLEEPER to indicate -+ * the nature of the wakeup. -+ */ -+int wake_up_lock_sleeper(struct task_struct *p) -+{ -+ return try_to_wake_up(p, TASK_ALL, WF_LOCK_SLEEPER); -+} -+ - int wake_up_state(struct task_struct *p, unsigned int state) - { - return try_to_wake_up(p, state, 0); -@@ -1959,6 +2050,9 @@ - p->on_cpu = 0; - #endif - init_task_preempt_count(p); -+#ifdef CONFIG_HAVE_PREEMPT_LAZY -+ task_thread_info(p)->preempt_lazy_count = 0; -+#endif - #ifdef CONFIG_SMP - plist_node_init(&p->pushable_tasks, MAX_PRIO); - RB_CLEAR_NODE(&p->pushable_dl_tasks); -@@ -2231,8 +2325,12 @@ - finish_arch_post_lock_switch(); - - fire_sched_in_preempt_notifiers(current); -+ /* -+ * We use mmdrop_delayed() here so we don't have to do the -+ * full __mmdrop() when we are the last user. -+ */ - if (mm) -- mmdrop(mm); -+ mmdrop_delayed(mm); - if (unlikely(prev_state == TASK_DEAD)) { - if (prev->sched_class->task_dead) - prev->sched_class->task_dead(prev); -@@ -2650,6 +2748,133 @@ - schedstat_inc(this_rq(), sched_count); - } - -+#if defined(CONFIG_PREEMPT_RT_FULL) && defined(CONFIG_SMP) -+#define MIGRATE_DISABLE_SET_AFFIN (1<<30) /* Can't make a negative */ -+#define migrate_disabled_updated(p) ((p)->migrate_disable & MIGRATE_DISABLE_SET_AFFIN) -+#define migrate_disable_count(p) ((p)->migrate_disable & ~MIGRATE_DISABLE_SET_AFFIN) -+ -+static inline void update_migrate_disable(struct task_struct *p) -+{ -+ const struct cpumask *mask; -+ -+ if (likely(!p->migrate_disable)) -+ return; -+ -+ /* Did we already update affinity? */ -+ if (unlikely(migrate_disabled_updated(p))) -+ return; -+ -+ /* -+ * Since this is always current we can get away with only locking -+ * rq->lock, the ->cpus_allowed value can normally only be changed -+ * while holding both p->pi_lock and rq->lock, but seeing that this -+ * is current, we cannot actually be waking up, so all code that -+ * relies on serialization against p->pi_lock is out of scope. -+ * -+ * Having rq->lock serializes us against things like -+ * set_cpus_allowed_ptr() that can still happen concurrently. -+ */ -+ mask = tsk_cpus_allowed(p); -+ -+ if (p->sched_class->set_cpus_allowed) -+ p->sched_class->set_cpus_allowed(p, mask); -+ /* mask==cpumask_of(task_cpu(p)) which has a cpumask_weight==1 */ -+ p->nr_cpus_allowed = 1; -+ -+ /* Let migrate_enable know to fix things back up */ -+ p->migrate_disable |= MIGRATE_DISABLE_SET_AFFIN; -+} -+ -+void migrate_disable(void) -+{ -+ struct task_struct *p = current; -+ -+ if (in_atomic()) { -+#ifdef CONFIG_SCHED_DEBUG -+ p->migrate_disable_atomic++; -+#endif -+ return; -+ } -+ -+#ifdef CONFIG_SCHED_DEBUG -+ if (unlikely(p->migrate_disable_atomic)) { -+ tracing_off(); -+ WARN_ON_ONCE(1); -+ } -+#endif -+ -+ if (p->migrate_disable) { -+ p->migrate_disable++; -+ return; -+ } -+ -+ preempt_disable(); -+ preempt_lazy_disable(); -+ pin_current_cpu(); -+ p->migrate_disable = 1; -+ preempt_enable(); -+} -+EXPORT_SYMBOL(migrate_disable); -+ -+void migrate_enable(void) -+{ -+ struct task_struct *p = current; -+ const struct cpumask *mask; -+ unsigned long flags; -+ struct rq *rq; -+ -+ if (in_atomic()) { -+#ifdef CONFIG_SCHED_DEBUG -+ p->migrate_disable_atomic--; -+#endif -+ return; -+ } -+ -+#ifdef CONFIG_SCHED_DEBUG -+ if (unlikely(p->migrate_disable_atomic)) { -+ tracing_off(); -+ WARN_ON_ONCE(1); -+ } -+#endif -+ WARN_ON_ONCE(p->migrate_disable <= 0); -+ -+ if (migrate_disable_count(p) > 1) { -+ p->migrate_disable--; -+ return; -+ } -+ -+ preempt_disable(); -+ if (unlikely(migrate_disabled_updated(p))) { -+ /* -+ * Undo whatever update_migrate_disable() did, also see there -+ * about locking. -+ */ -+ rq = this_rq(); -+ raw_spin_lock_irqsave(&rq->lock, flags); -+ -+ /* -+ * Clearing migrate_disable causes tsk_cpus_allowed to -+ * show the tasks original cpu affinity. -+ */ -+ p->migrate_disable = 0; -+ mask = tsk_cpus_allowed(p); -+ if (p->sched_class->set_cpus_allowed) -+ p->sched_class->set_cpus_allowed(p, mask); -+ p->nr_cpus_allowed = cpumask_weight(mask); -+ raw_spin_unlock_irqrestore(&rq->lock, flags); -+ } else -+ p->migrate_disable = 0; -+ -+ unpin_current_cpu(); -+ preempt_enable(); -+ preempt_lazy_enable(); -+} -+EXPORT_SYMBOL(migrate_enable); -+#else -+static inline void update_migrate_disable(struct task_struct *p) { } -+#define migrate_disabled_updated(p) 0 -+#endif -+ - /* - * Pick up the highest-prio task: - */ -@@ -2756,6 +2981,8 @@ - smp_mb__before_spinlock(); - raw_spin_lock_irq(&rq->lock); - -+ update_migrate_disable(prev); -+ - rq->clock_skip_update <<= 1; /* promote REQ to ACT */ - - switch_count = &prev->nivcsw; -@@ -2765,19 +2992,6 @@ - } else { - deactivate_task(rq, prev, DEQUEUE_SLEEP); - prev->on_rq = 0; -- -- /* -- * If a worker went to sleep, notify and ask workqueue -- * whether it wants to wake up a task to maintain -- * concurrency. -- */ -- if (prev->flags & PF_WQ_WORKER) { -- struct task_struct *to_wakeup; -- -- to_wakeup = wq_worker_sleeping(prev, cpu); -- if (to_wakeup) -- try_to_wake_up_local(to_wakeup); -- } - } - switch_count = &prev->nvcsw; - } -@@ -2787,6 +3001,7 @@ - - next = pick_next_task(rq, prev); - clear_tsk_need_resched(prev); -+ clear_tsk_need_resched_lazy(prev); - clear_preempt_need_resched(); - rq->clock_skip_update = 0; - -@@ -2807,8 +3022,19 @@ - - static inline void sched_submit_work(struct task_struct *tsk) - { -- if (!tsk->state || tsk_is_pi_blocked(tsk)) -+ if (!tsk->state) -+ return; -+ /* -+ * If a worker went to sleep, notify and ask workqueue whether -+ * it wants to wake up a task to maintain concurrency. -+ */ -+ if (tsk->flags & PF_WQ_WORKER) -+ wq_worker_sleeping(tsk); -+ -+ -+ if (tsk_is_pi_blocked(tsk)) - return; -+ - /* - * If we are going to sleep and we have plugged IO queued, - * make sure to submit it to avoid deadlocks. -@@ -2817,6 +3043,12 @@ - blk_schedule_flush_plug(tsk); - } - -+static void sched_update_worker(struct task_struct *tsk) -+{ -+ if (tsk->flags & PF_WQ_WORKER) -+ wq_worker_running(tsk); -+} -+ - asmlinkage __visible void __sched schedule(void) - { - struct task_struct *tsk = current; -@@ -2825,6 +3057,7 @@ - do { - __schedule(); - } while (need_resched()); -+ sched_update_worker(tsk); - } - EXPORT_SYMBOL(schedule); - -@@ -2916,6 +3149,14 @@ - if (likely(!preemptible())) - return; - -+#ifdef CONFIG_PREEMPT_LAZY -+ /* -+ * Check for lazy preemption -+ */ -+ if (current_thread_info()->preempt_lazy_count && -+ !test_thread_flag(TIF_NEED_RESCHED)) -+ return; -+#endif - do { - __preempt_count_add(PREEMPT_ACTIVE); - /* -@@ -2924,7 +3165,16 @@ - * an infinite recursion. - */ - prev_ctx = exception_enter(); -+ /* -+ * The add/subtract must not be traced by the function -+ * tracer. But we still want to account for the -+ * preempt off latency tracer. Since the _notrace versions -+ * of add/subtract skip the accounting for latency tracer -+ * we must force it manually. -+ */ -+ start_critical_timings(); - __schedule(); -+ stop_critical_timings(); - exception_exit(prev_ctx); - - __preempt_count_sub(PREEMPT_ACTIVE); -@@ -4261,6 +4511,7 @@ - } - EXPORT_SYMBOL(__cond_resched_lock); - -+#ifndef CONFIG_PREEMPT_RT_FULL - int __sched __cond_resched_softirq(void) - { - BUG_ON(!in_softirq()); -@@ -4274,6 +4525,7 @@ - return 0; - } - EXPORT_SYMBOL(__cond_resched_softirq); -+#endif - - /** - * yield - yield the current processor to other threads. -@@ -4628,7 +4880,9 @@ - - /* Set the preempt count _outside_ the spinlocks! */ - init_idle_preempt_count(idle, cpu); -- -+#ifdef CONFIG_HAVE_PREEMPT_LAZY -+ task_thread_info(idle)->preempt_lazy_count = 0; -+#endif - /* - * The idle tasks have their own, simple scheduling class: - */ -@@ -4748,11 +5002,91 @@ - - void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask) - { -- if (p->sched_class->set_cpus_allowed) -- p->sched_class->set_cpus_allowed(p, new_mask); -+ if (!migrate_disabled_updated(p)) { -+ if (p->sched_class->set_cpus_allowed) -+ p->sched_class->set_cpus_allowed(p, new_mask); -+ p->nr_cpus_allowed = cpumask_weight(new_mask); -+ } - - cpumask_copy(&p->cpus_allowed, new_mask); -- p->nr_cpus_allowed = cpumask_weight(new_mask); -+} -+ -+static DEFINE_PER_CPU(struct cpumask, sched_cpumasks); -+static DEFINE_MUTEX(sched_down_mutex); -+static cpumask_t sched_down_cpumask; -+ -+void tell_sched_cpu_down_begin(int cpu) -+{ -+ mutex_lock(&sched_down_mutex); -+ cpumask_set_cpu(cpu, &sched_down_cpumask); -+ mutex_unlock(&sched_down_mutex); -+} -+ -+void tell_sched_cpu_down_done(int cpu) -+{ -+ mutex_lock(&sched_down_mutex); -+ cpumask_clear_cpu(cpu, &sched_down_cpumask); -+ mutex_unlock(&sched_down_mutex); -+} -+ -+/** -+ * migrate_me - try to move the current task off this cpu -+ * -+ * Used by the pin_current_cpu() code to try to get tasks -+ * to move off the current CPU as it is going down. -+ * It will only move the task if the task isn't pinned to -+ * the CPU (with migrate_disable, affinity or NO_SETAFFINITY) -+ * and the task has to be in a RUNNING state. Otherwise the -+ * movement of the task will wake it up (change its state -+ * to running) when the task did not expect it. -+ * -+ * Returns 1 if it succeeded in moving the current task -+ * 0 otherwise. -+ */ -+int migrate_me(void) -+{ -+ struct task_struct *p = current; -+ struct migration_arg arg; -+ struct cpumask *cpumask; -+ struct cpumask *mask; -+ unsigned long flags; -+ unsigned int dest_cpu; -+ struct rq *rq; -+ -+ /* -+ * We can not migrate tasks bounded to a CPU or tasks not -+ * running. The movement of the task will wake it up. -+ */ -+ if (p->flags & PF_NO_SETAFFINITY || p->state) -+ return 0; -+ -+ mutex_lock(&sched_down_mutex); -+ rq = task_rq_lock(p, &flags); -+ -+ cpumask = this_cpu_ptr(&sched_cpumasks); -+ mask = &p->cpus_allowed; -+ -+ cpumask_andnot(cpumask, mask, &sched_down_cpumask); -+ -+ if (!cpumask_weight(cpumask)) { -+ /* It's only on this CPU? */ -+ task_rq_unlock(rq, p, &flags); -+ mutex_unlock(&sched_down_mutex); -+ return 0; -+ } -+ -+ dest_cpu = cpumask_any_and(cpu_active_mask, cpumask); -+ -+ arg.task = p; -+ arg.dest_cpu = dest_cpu; -+ -+ task_rq_unlock(rq, p, &flags); -+ -+ stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg); -+ tlb_migrate_finish(p->mm); -+ mutex_unlock(&sched_down_mutex); -+ -+ return 1; - } - - /* -@@ -4798,7 +5132,7 @@ - do_set_cpus_allowed(p, new_mask); - - /* Can the task run on the task's current CPU? If so, we're done */ -- if (cpumask_test_cpu(task_cpu(p), new_mask)) -+ if (cpumask_test_cpu(task_cpu(p), new_mask) || __migrate_disabled(p)) - goto out; - - dest_cpu = cpumask_any_and(cpu_active_mask, new_mask); -@@ -4938,6 +5272,8 @@ - - #ifdef CONFIG_HOTPLUG_CPU - -+static DEFINE_PER_CPU(struct mm_struct *, idle_last_mm); -+ - /* - * Ensures that the idle task is using init_mm right before its cpu goes - * offline. -@@ -4952,7 +5288,11 @@ - switch_mm(mm, &init_mm, current); - finish_arch_post_lock_switch(); - } -- mmdrop(mm); -+ /* -+ * Defer the cleanup to an alive cpu. On RT we can neither -+ * call mmdrop() nor mmdrop_delayed() from here. -+ */ -+ per_cpu(idle_last_mm, smp_processor_id()) = mm; - } - - /* -@@ -5295,6 +5635,10 @@ - - case CPU_DEAD: - calc_load_migrate(rq); -+ if (per_cpu(idle_last_mm, cpu)) { -+ mmdrop(per_cpu(idle_last_mm, cpu)); -+ per_cpu(idle_last_mm, cpu) = NULL; -+ } - break; - #endif - } -@@ -7274,7 +7618,8 @@ - #ifdef CONFIG_DEBUG_ATOMIC_SLEEP - static inline int preempt_count_equals(int preempt_offset) - { -- int nested = (preempt_count() & ~PREEMPT_ACTIVE) + rcu_preempt_depth(); -+ int nested = (preempt_count() & ~PREEMPT_ACTIVE) + -+ sched_rcu_preempt_depth(); - - return (nested == preempt_offset); - } -diff -Nur linux-4.1.10.orig/kernel/sched/core.c.orig linux-4.1.10/kernel/sched/core.c.orig ---- linux-4.1.10.orig/kernel/sched/core.c.orig 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.10/kernel/sched/core.c.orig 2015-10-03 13:49:38.000000000 +0200 -@@ -0,0 +1,8389 @@ -+/* -+ * kernel/sched/core.c -+ * -+ * Kernel scheduler and related syscalls -+ * -+ * Copyright (C) 1991-2002 Linus Torvalds -+ * -+ * 1996-12-23 Modified by Dave Grothe to fix bugs in semaphores and -+ * make semaphores SMP safe -+ * 1998-11-19 Implemented schedule_timeout() and related stuff -+ * by Andrea Arcangeli -+ * 2002-01-04 New ultra-scalable O(1) scheduler by Ingo Molnar: -+ * hybrid priority-list and round-robin design with -+ * an array-switch method of distributing timeslices -+ * and per-CPU runqueues. Cleanups and useful suggestions -+ * by Davide Libenzi, preemptible kernel bits by Robert Love. -+ * 2003-09-03 Interactivity tuning by Con Kolivas. -+ * 2004-04-02 Scheduler domains code by Nick Piggin -+ * 2007-04-15 Work begun on replacing all interactivity tuning with a -+ * fair scheduling design by Con Kolivas. -+ * 2007-05-05 Load balancing (smp-nice) and other improvements -+ * by Peter Williams -+ * 2007-05-06 Interactivity improvements to CFS by Mike Galbraith -+ * 2007-07-01 Group scheduling enhancements by Srivatsa Vaddagiri -+ * 2007-11-29 RT balancing improvements by Steven Rostedt, Gregory Haskins, -+ * Thomas Gleixner, Mike Kravetz -+ */ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include -+#include -+#include -+#include -+#ifdef CONFIG_PARAVIRT -+#include -+#endif -+ -+#include "sched.h" -+#include "../workqueue_internal.h" -+#include "../smpboot.h" -+ -+#define CREATE_TRACE_POINTS -+#include -+ -+void start_bandwidth_timer(struct hrtimer *period_timer, ktime_t period) -+{ -+ unsigned long delta; -+ ktime_t soft, hard, now; -+ -+ for (;;) { -+ if (hrtimer_active(period_timer)) -+ break; -+ -+ now = hrtimer_cb_get_time(period_timer); -+ hrtimer_forward(period_timer, now, period); -+ -+ soft = hrtimer_get_softexpires(period_timer); -+ hard = hrtimer_get_expires(period_timer); -+ delta = ktime_to_ns(ktime_sub(hard, soft)); -+ __hrtimer_start_range_ns(period_timer, soft, delta, -+ HRTIMER_MODE_ABS_PINNED, 0); -+ } -+} -+ -+DEFINE_MUTEX(sched_domains_mutex); -+DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues); -+ -+static void update_rq_clock_task(struct rq *rq, s64 delta); -+ -+void update_rq_clock(struct rq *rq) -+{ -+ s64 delta; -+ -+ lockdep_assert_held(&rq->lock); -+ -+ if (rq->clock_skip_update & RQCF_ACT_SKIP) -+ return; -+ -+ delta = sched_clock_cpu(cpu_of(rq)) - rq->clock; -+ if (delta < 0) -+ return; -+ rq->clock += delta; -+ update_rq_clock_task(rq, delta); -+} -+ -+/* -+ * Debugging: various feature bits -+ */ -+ -+#define SCHED_FEAT(name, enabled) \ -+ (1UL << __SCHED_FEAT_##name) * enabled | -+ -+const_debug unsigned int sysctl_sched_features = -+#include "features.h" -+ 0; -+ -+#undef SCHED_FEAT -+ -+#ifdef CONFIG_SCHED_DEBUG -+#define SCHED_FEAT(name, enabled) \ -+ #name , -+ -+static const char * const sched_feat_names[] = { -+#include "features.h" -+}; -+ -+#undef SCHED_FEAT -+ -+static int sched_feat_show(struct seq_file *m, void *v) -+{ -+ int i; -+ -+ for (i = 0; i < __SCHED_FEAT_NR; i++) { -+ if (!(sysctl_sched_features & (1UL << i))) -+ seq_puts(m, "NO_"); -+ seq_printf(m, "%s ", sched_feat_names[i]); -+ } -+ seq_puts(m, "\n"); -+ -+ return 0; -+} -+ -+#ifdef HAVE_JUMP_LABEL -+ -+#define jump_label_key__true STATIC_KEY_INIT_TRUE -+#define jump_label_key__false STATIC_KEY_INIT_FALSE -+ -+#define SCHED_FEAT(name, enabled) \ -+ jump_label_key__##enabled , -+ -+struct static_key sched_feat_keys[__SCHED_FEAT_NR] = { -+#include "features.h" -+}; -+ -+#undef SCHED_FEAT -+ -+static void sched_feat_disable(int i) -+{ -+ if (static_key_enabled(&sched_feat_keys[i])) -+ static_key_slow_dec(&sched_feat_keys[i]); -+} -+ -+static void sched_feat_enable(int i) -+{ -+ if (!static_key_enabled(&sched_feat_keys[i])) -+ static_key_slow_inc(&sched_feat_keys[i]); -+} -+#else -+static void sched_feat_disable(int i) { }; -+static void sched_feat_enable(int i) { }; -+#endif /* HAVE_JUMP_LABEL */ -+ -+static int sched_feat_set(char *cmp) -+{ -+ int i; -+ int neg = 0; -+ -+ if (strncmp(cmp, "NO_", 3) == 0) { -+ neg = 1; -+ cmp += 3; -+ } -+ -+ for (i = 0; i < __SCHED_FEAT_NR; i++) { -+ if (strcmp(cmp, sched_feat_names[i]) == 0) { -+ if (neg) { -+ sysctl_sched_features &= ~(1UL << i); -+ sched_feat_disable(i); -+ } else { -+ sysctl_sched_features |= (1UL << i); -+ sched_feat_enable(i); -+ } -+ break; -+ } -+ } -+ -+ return i; -+} -+ -+static ssize_t -+sched_feat_write(struct file *filp, const char __user *ubuf, -+ size_t cnt, loff_t *ppos) -+{ -+ char buf[64]; -+ char *cmp; -+ int i; -+ struct inode *inode; -+ -+ if (cnt > 63) -+ cnt = 63; -+ -+ if (copy_from_user(&buf, ubuf, cnt)) -+ return -EFAULT; -+ -+ buf[cnt] = 0; -+ cmp = strstrip(buf); -+ -+ /* Ensure the static_key remains in a consistent state */ -+ inode = file_inode(filp); -+ mutex_lock(&inode->i_mutex); -+ i = sched_feat_set(cmp); -+ mutex_unlock(&inode->i_mutex); -+ if (i == __SCHED_FEAT_NR) -+ return -EINVAL; -+ -+ *ppos += cnt; -+ -+ return cnt; -+} -+ -+static int sched_feat_open(struct inode *inode, struct file *filp) -+{ -+ return single_open(filp, sched_feat_show, NULL); -+} -+ -+static const struct file_operations sched_feat_fops = { -+ .open = sched_feat_open, -+ .write = sched_feat_write, -+ .read = seq_read, -+ .llseek = seq_lseek, -+ .release = single_release, -+}; -+ -+static __init int sched_init_debug(void) -+{ -+ debugfs_create_file("sched_features", 0644, NULL, NULL, -+ &sched_feat_fops); -+ -+ return 0; -+} -+late_initcall(sched_init_debug); -+#endif /* CONFIG_SCHED_DEBUG */ -+ -+/* -+ * Number of tasks to iterate in a single balance run. -+ * Limited because this is done with IRQs disabled. -+ */ -+const_debug unsigned int sysctl_sched_nr_migrate = 32; -+ -+/* -+ * period over which we average the RT time consumption, measured -+ * in ms. -+ * -+ * default: 1s -+ */ -+const_debug unsigned int sysctl_sched_time_avg = MSEC_PER_SEC; -+ -+/* -+ * period over which we measure -rt task cpu usage in us. -+ * default: 1s -+ */ -+unsigned int sysctl_sched_rt_period = 1000000; -+ -+__read_mostly int scheduler_running; -+ -+/* -+ * part of the period that we allow rt tasks to run in us. -+ * default: 0.95s -+ */ -+int sysctl_sched_rt_runtime = 950000; -+ -+/* cpus with isolated domains */ -+cpumask_var_t cpu_isolated_map; -+ -+/* -+ * this_rq_lock - lock this runqueue and disable interrupts. -+ */ -+static struct rq *this_rq_lock(void) -+ __acquires(rq->lock) -+{ -+ struct rq *rq; -+ -+ local_irq_disable(); -+ rq = this_rq(); -+ raw_spin_lock(&rq->lock); -+ -+ return rq; -+} -+ -+#ifdef CONFIG_SCHED_HRTICK -+/* -+ * Use HR-timers to deliver accurate preemption points. -+ */ -+ -+static void hrtick_clear(struct rq *rq) -+{ -+ if (hrtimer_active(&rq->hrtick_timer)) -+ hrtimer_cancel(&rq->hrtick_timer); -+} -+ -+/* -+ * High-resolution timer tick. -+ * Runs from hardirq context with interrupts disabled. -+ */ -+static enum hrtimer_restart hrtick(struct hrtimer *timer) -+{ -+ struct rq *rq = container_of(timer, struct rq, hrtick_timer); -+ -+ WARN_ON_ONCE(cpu_of(rq) != smp_processor_id()); -+ -+ raw_spin_lock(&rq->lock); -+ update_rq_clock(rq); -+ rq->curr->sched_class->task_tick(rq, rq->curr, 1); -+ raw_spin_unlock(&rq->lock); -+ -+ return HRTIMER_NORESTART; -+} -+ -+#ifdef CONFIG_SMP -+ -+static int __hrtick_restart(struct rq *rq) -+{ -+ struct hrtimer *timer = &rq->hrtick_timer; -+ ktime_t time = hrtimer_get_softexpires(timer); -+ -+ return __hrtimer_start_range_ns(timer, time, 0, HRTIMER_MODE_ABS_PINNED, 0); -+} -+ -+/* -+ * called from hardirq (IPI) context -+ */ -+static void __hrtick_start(void *arg) -+{ -+ struct rq *rq = arg; -+ -+ raw_spin_lock(&rq->lock); -+ __hrtick_restart(rq); -+ rq->hrtick_csd_pending = 0; -+ raw_spin_unlock(&rq->lock); -+} -+ -+/* -+ * Called to set the hrtick timer state. -+ * -+ * called with rq->lock held and irqs disabled -+ */ -+void hrtick_start(struct rq *rq, u64 delay) -+{ -+ struct hrtimer *timer = &rq->hrtick_timer; -+ ktime_t time; -+ s64 delta; -+ -+ /* -+ * Don't schedule slices shorter than 10000ns, that just -+ * doesn't make sense and can cause timer DoS. -+ */ -+ delta = max_t(s64, delay, 10000LL); -+ time = ktime_add_ns(timer->base->get_time(), delta); -+ -+ hrtimer_set_expires(timer, time); -+ -+ if (rq == this_rq()) { -+ __hrtick_restart(rq); -+ } else if (!rq->hrtick_csd_pending) { -+ smp_call_function_single_async(cpu_of(rq), &rq->hrtick_csd); -+ rq->hrtick_csd_pending = 1; -+ } -+} -+ -+static int -+hotplug_hrtick(struct notifier_block *nfb, unsigned long action, void *hcpu) -+{ -+ int cpu = (int)(long)hcpu; -+ -+ switch (action) { -+ case CPU_UP_CANCELED: -+ case CPU_UP_CANCELED_FROZEN: -+ case CPU_DOWN_PREPARE: -+ case CPU_DOWN_PREPARE_FROZEN: -+ case CPU_DEAD: -+ case CPU_DEAD_FROZEN: -+ hrtick_clear(cpu_rq(cpu)); -+ return NOTIFY_OK; -+ } -+ -+ return NOTIFY_DONE; -+} -+ -+static __init void init_hrtick(void) -+{ -+ hotcpu_notifier(hotplug_hrtick, 0); -+} -+#else -+/* -+ * Called to set the hrtick timer state. -+ * -+ * called with rq->lock held and irqs disabled -+ */ -+void hrtick_start(struct rq *rq, u64 delay) -+{ -+ /* -+ * Don't schedule slices shorter than 10000ns, that just -+ * doesn't make sense. Rely on vruntime for fairness. -+ */ -+ delay = max_t(u64, delay, 10000LL); -+ __hrtimer_start_range_ns(&rq->hrtick_timer, ns_to_ktime(delay), 0, -+ HRTIMER_MODE_REL_PINNED, 0); -+} -+ -+static inline void init_hrtick(void) -+{ -+} -+#endif /* CONFIG_SMP */ -+ -+static void init_rq_hrtick(struct rq *rq) -+{ -+#ifdef CONFIG_SMP -+ rq->hrtick_csd_pending = 0; -+ -+ rq->hrtick_csd.flags = 0; -+ rq->hrtick_csd.func = __hrtick_start; -+ rq->hrtick_csd.info = rq; -+#endif -+ -+ hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); -+ rq->hrtick_timer.function = hrtick; -+} -+#else /* CONFIG_SCHED_HRTICK */ -+static inline void hrtick_clear(struct rq *rq) -+{ -+} -+ -+static inline void init_rq_hrtick(struct rq *rq) -+{ -+} -+ -+static inline void init_hrtick(void) -+{ -+} -+#endif /* CONFIG_SCHED_HRTICK */ -+ -+/* -+ * cmpxchg based fetch_or, macro so it works for different integer types -+ */ -+#define fetch_or(ptr, val) \ -+({ typeof(*(ptr)) __old, __val = *(ptr); \ -+ for (;;) { \ -+ __old = cmpxchg((ptr), __val, __val | (val)); \ -+ if (__old == __val) \ -+ break; \ -+ __val = __old; \ -+ } \ -+ __old; \ -+}) -+ -+#if defined(CONFIG_SMP) && defined(TIF_POLLING_NRFLAG) -+/* -+ * Atomically set TIF_NEED_RESCHED and test for TIF_POLLING_NRFLAG, -+ * this avoids any races wrt polling state changes and thereby avoids -+ * spurious IPIs. -+ */ -+static bool set_nr_and_not_polling(struct task_struct *p) -+{ -+ struct thread_info *ti = task_thread_info(p); -+ return !(fetch_or(&ti->flags, _TIF_NEED_RESCHED) & _TIF_POLLING_NRFLAG); -+} -+ -+/* -+ * Atomically set TIF_NEED_RESCHED if TIF_POLLING_NRFLAG is set. -+ * -+ * If this returns true, then the idle task promises to call -+ * sched_ttwu_pending() and reschedule soon. -+ */ -+static bool set_nr_if_polling(struct task_struct *p) -+{ -+ struct thread_info *ti = task_thread_info(p); -+ typeof(ti->flags) old, val = ACCESS_ONCE(ti->flags); -+ -+ for (;;) { -+ if (!(val & _TIF_POLLING_NRFLAG)) -+ return false; -+ if (val & _TIF_NEED_RESCHED) -+ return true; -+ old = cmpxchg(&ti->flags, val, val | _TIF_NEED_RESCHED); -+ if (old == val) -+ break; -+ val = old; -+ } -+ return true; -+} -+ -+#else -+static bool set_nr_and_not_polling(struct task_struct *p) -+{ -+ set_tsk_need_resched(p); -+ return true; -+} -+ -+#ifdef CONFIG_SMP -+static bool set_nr_if_polling(struct task_struct *p) -+{ -+ return false; -+} -+#endif -+#endif -+ -+/* -+ * resched_curr - mark rq's current task 'to be rescheduled now'. -+ * -+ * On UP this means the setting of the need_resched flag, on SMP it -+ * might also involve a cross-CPU call to trigger the scheduler on -+ * the target CPU. -+ */ -+void resched_curr(struct rq *rq) -+{ -+ struct task_struct *curr = rq->curr; -+ int cpu; -+ -+ lockdep_assert_held(&rq->lock); -+ -+ if (test_tsk_need_resched(curr)) -+ return; -+ -+ cpu = cpu_of(rq); -+ -+ if (cpu == smp_processor_id()) { -+ set_tsk_need_resched(curr); -+ set_preempt_need_resched(); -+ return; -+ } -+ -+ if (set_nr_and_not_polling(curr)) -+ smp_send_reschedule(cpu); -+ else -+ trace_sched_wake_idle_without_ipi(cpu); -+} -+ -+void resched_cpu(int cpu) -+{ -+ struct rq *rq = cpu_rq(cpu); -+ unsigned long flags; -+ -+ if (!raw_spin_trylock_irqsave(&rq->lock, flags)) -+ return; -+ resched_curr(rq); -+ raw_spin_unlock_irqrestore(&rq->lock, flags); -+} -+ -+#ifdef CONFIG_SMP -+#ifdef CONFIG_NO_HZ_COMMON -+/* -+ * In the semi idle case, use the nearest busy cpu for migrating timers -+ * from an idle cpu. This is good for power-savings. -+ * -+ * We don't do similar optimization for completely idle system, as -+ * selecting an idle cpu will add more delays to the timers than intended -+ * (as that cpu's timer base may not be uptodate wrt jiffies etc). -+ */ -+int get_nohz_timer_target(int pinned) -+{ -+ int cpu = smp_processor_id(); -+ int i; -+ struct sched_domain *sd; -+ -+ if (pinned || !get_sysctl_timer_migration() || !idle_cpu(cpu)) -+ return cpu; -+ -+ rcu_read_lock(); -+ for_each_domain(cpu, sd) { -+ for_each_cpu(i, sched_domain_span(sd)) { -+ if (!idle_cpu(i)) { -+ cpu = i; -+ goto unlock; -+ } -+ } -+ } -+unlock: -+ rcu_read_unlock(); -+ return cpu; -+} -+/* -+ * When add_timer_on() enqueues a timer into the timer wheel of an -+ * idle CPU then this timer might expire before the next timer event -+ * which is scheduled to wake up that CPU. In case of a completely -+ * idle system the next event might even be infinite time into the -+ * future. wake_up_idle_cpu() ensures that the CPU is woken up and -+ * leaves the inner idle loop so the newly added timer is taken into -+ * account when the CPU goes back to idle and evaluates the timer -+ * wheel for the next timer event. -+ */ -+static void wake_up_idle_cpu(int cpu) -+{ -+ struct rq *rq = cpu_rq(cpu); -+ -+ if (cpu == smp_processor_id()) -+ return; -+ -+ if (set_nr_and_not_polling(rq->idle)) -+ smp_send_reschedule(cpu); -+ else -+ trace_sched_wake_idle_without_ipi(cpu); -+} -+ -+static bool wake_up_full_nohz_cpu(int cpu) -+{ -+ /* -+ * We just need the target to call irq_exit() and re-evaluate -+ * the next tick. The nohz full kick at least implies that. -+ * If needed we can still optimize that later with an -+ * empty IRQ. -+ */ -+ if (tick_nohz_full_cpu(cpu)) { -+ if (cpu != smp_processor_id() || -+ tick_nohz_tick_stopped()) -+ tick_nohz_full_kick_cpu(cpu); -+ return true; -+ } -+ -+ return false; -+} -+ -+void wake_up_nohz_cpu(int cpu) -+{ -+ if (!wake_up_full_nohz_cpu(cpu)) -+ wake_up_idle_cpu(cpu); -+} -+ -+static inline bool got_nohz_idle_kick(void) -+{ -+ int cpu = smp_processor_id(); -+ -+ if (!test_bit(NOHZ_BALANCE_KICK, nohz_flags(cpu))) -+ return false; -+ -+ if (idle_cpu(cpu) && !need_resched()) -+ return true; -+ -+ /* -+ * We can't run Idle Load Balance on this CPU for this time so we -+ * cancel it and clear NOHZ_BALANCE_KICK -+ */ -+ clear_bit(NOHZ_BALANCE_KICK, nohz_flags(cpu)); -+ return false; -+} -+ -+#else /* CONFIG_NO_HZ_COMMON */ -+ -+static inline bool got_nohz_idle_kick(void) -+{ -+ return false; -+} -+ -+#endif /* CONFIG_NO_HZ_COMMON */ -+ -+#ifdef CONFIG_NO_HZ_FULL -+bool sched_can_stop_tick(void) -+{ -+ /* -+ * FIFO realtime policy runs the highest priority task. Other runnable -+ * tasks are of a lower priority. The scheduler tick does nothing. -+ */ -+ if (current->policy == SCHED_FIFO) -+ return true; -+ -+ /* -+ * Round-robin realtime tasks time slice with other tasks at the same -+ * realtime priority. Is this task the only one at this priority? -+ */ -+ if (current->policy == SCHED_RR) { -+ struct sched_rt_entity *rt_se = ¤t->rt; -+ -+ return rt_se->run_list.prev == rt_se->run_list.next; -+ } -+ -+ /* -+ * More than one running task need preemption. -+ * nr_running update is assumed to be visible -+ * after IPI is sent from wakers. -+ */ -+ if (this_rq()->nr_running > 1) -+ return false; -+ -+ return true; -+} -+#endif /* CONFIG_NO_HZ_FULL */ -+ -+void sched_avg_update(struct rq *rq) -+{ -+ s64 period = sched_avg_period(); -+ -+ while ((s64)(rq_clock(rq) - rq->age_stamp) > period) { -+ /* -+ * Inline assembly required to prevent the compiler -+ * optimising this loop into a divmod call. -+ * See __iter_div_u64_rem() for another example of this. -+ */ -+ asm("" : "+rm" (rq->age_stamp)); -+ rq->age_stamp += period; -+ rq->rt_avg /= 2; -+ } -+} -+ -+#endif /* CONFIG_SMP */ -+ -+#if defined(CONFIG_RT_GROUP_SCHED) || (defined(CONFIG_FAIR_GROUP_SCHED) && \ -+ (defined(CONFIG_SMP) || defined(CONFIG_CFS_BANDWIDTH))) -+/* -+ * Iterate task_group tree rooted at *from, calling @down when first entering a -+ * node and @up when leaving it for the final time. -+ * -+ * Caller must hold rcu_lock or sufficient equivalent. -+ */ -+int walk_tg_tree_from(struct task_group *from, -+ tg_visitor down, tg_visitor up, void *data) -+{ -+ struct task_group *parent, *child; -+ int ret; -+ -+ parent = from; -+ -+down: -+ ret = (*down)(parent, data); -+ if (ret) -+ goto out; -+ list_for_each_entry_rcu(child, &parent->children, siblings) { -+ parent = child; -+ goto down; -+ -+up: -+ continue; -+ } -+ ret = (*up)(parent, data); -+ if (ret || parent == from) -+ goto out; -+ -+ child = parent; -+ parent = parent->parent; -+ if (parent) -+ goto up; -+out: -+ return ret; -+} -+ -+int tg_nop(struct task_group *tg, void *data) -+{ -+ return 0; -+} -+#endif -+ -+static void set_load_weight(struct task_struct *p) -+{ -+ int prio = p->static_prio - MAX_RT_PRIO; -+ struct load_weight *load = &p->se.load; -+ -+ /* -+ * SCHED_IDLE tasks get minimal weight: -+ */ -+ if (p->policy == SCHED_IDLE) { -+ load->weight = scale_load(WEIGHT_IDLEPRIO); -+ load->inv_weight = WMULT_IDLEPRIO; -+ return; -+ } -+ -+ load->weight = scale_load(prio_to_weight[prio]); -+ load->inv_weight = prio_to_wmult[prio]; -+} -+ -+static void enqueue_task(struct rq *rq, struct task_struct *p, int flags) -+{ -+ update_rq_clock(rq); -+ sched_info_queued(rq, p); -+ p->sched_class->enqueue_task(rq, p, flags); -+} -+ -+static void dequeue_task(struct rq *rq, struct task_struct *p, int flags) -+{ -+ update_rq_clock(rq); -+ sched_info_dequeued(rq, p); -+ p->sched_class->dequeue_task(rq, p, flags); -+} -+ -+void activate_task(struct rq *rq, struct task_struct *p, int flags) -+{ -+ if (task_contributes_to_load(p)) -+ rq->nr_uninterruptible--; -+ -+ enqueue_task(rq, p, flags); -+} -+ -+void deactivate_task(struct rq *rq, struct task_struct *p, int flags) -+{ -+ if (task_contributes_to_load(p)) -+ rq->nr_uninterruptible++; -+ -+ dequeue_task(rq, p, flags); -+} -+ -+static void update_rq_clock_task(struct rq *rq, s64 delta) -+{ -+/* -+ * In theory, the compile should just see 0 here, and optimize out the call -+ * to sched_rt_avg_update. But I don't trust it... -+ */ -+#if defined(CONFIG_IRQ_TIME_ACCOUNTING) || defined(CONFIG_PARAVIRT_TIME_ACCOUNTING) -+ s64 steal = 0, irq_delta = 0; -+#endif -+#ifdef CONFIG_IRQ_TIME_ACCOUNTING -+ irq_delta = irq_time_read(cpu_of(rq)) - rq->prev_irq_time; -+ -+ /* -+ * Since irq_time is only updated on {soft,}irq_exit, we might run into -+ * this case when a previous update_rq_clock() happened inside a -+ * {soft,}irq region. -+ * -+ * When this happens, we stop ->clock_task and only update the -+ * prev_irq_time stamp to account for the part that fit, so that a next -+ * update will consume the rest. This ensures ->clock_task is -+ * monotonic. -+ * -+ * It does however cause some slight miss-attribution of {soft,}irq -+ * time, a more accurate solution would be to update the irq_time using -+ * the current rq->clock timestamp, except that would require using -+ * atomic ops. -+ */ -+ if (irq_delta > delta) -+ irq_delta = delta; -+ -+ rq->prev_irq_time += irq_delta; -+ delta -= irq_delta; -+#endif -+#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING -+ if (static_key_false((¶virt_steal_rq_enabled))) { -+ steal = paravirt_steal_clock(cpu_of(rq)); -+ steal -= rq->prev_steal_time_rq; -+ -+ if (unlikely(steal > delta)) -+ steal = delta; -+ -+ rq->prev_steal_time_rq += steal; -+ delta -= steal; -+ } -+#endif -+ -+ rq->clock_task += delta; -+ -+#if defined(CONFIG_IRQ_TIME_ACCOUNTING) || defined(CONFIG_PARAVIRT_TIME_ACCOUNTING) -+ if ((irq_delta + steal) && sched_feat(NONTASK_CAPACITY)) -+ sched_rt_avg_update(rq, irq_delta + steal); -+#endif -+} -+ -+void sched_set_stop_task(int cpu, struct task_struct *stop) -+{ -+ struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 }; -+ struct task_struct *old_stop = cpu_rq(cpu)->stop; -+ -+ if (stop) { -+ /* -+ * Make it appear like a SCHED_FIFO task, its something -+ * userspace knows about and won't get confused about. -+ * -+ * Also, it will make PI more or less work without too -+ * much confusion -- but then, stop work should not -+ * rely on PI working anyway. -+ */ -+ sched_setscheduler_nocheck(stop, SCHED_FIFO, ¶m); -+ -+ stop->sched_class = &stop_sched_class; -+ } -+ -+ cpu_rq(cpu)->stop = stop; -+ -+ if (old_stop) { -+ /* -+ * Reset it back to a normal scheduling class so that -+ * it can die in pieces. -+ */ -+ old_stop->sched_class = &rt_sched_class; -+ } -+} -+ -+/* -+ * __normal_prio - return the priority that is based on the static prio -+ */ -+static inline int __normal_prio(struct task_struct *p) -+{ -+ return p->static_prio; -+} -+ -+/* -+ * Calculate the expected normal priority: i.e. priority -+ * without taking RT-inheritance into account. Might be -+ * boosted by interactivity modifiers. Changes upon fork, -+ * setprio syscalls, and whenever the interactivity -+ * estimator recalculates. -+ */ -+static inline int normal_prio(struct task_struct *p) -+{ -+ int prio; -+ -+ if (task_has_dl_policy(p)) -+ prio = MAX_DL_PRIO-1; -+ else if (task_has_rt_policy(p)) -+ prio = MAX_RT_PRIO-1 - p->rt_priority; -+ else -+ prio = __normal_prio(p); -+ return prio; -+} -+ -+/* -+ * Calculate the current priority, i.e. the priority -+ * taken into account by the scheduler. This value might -+ * be boosted by RT tasks, or might be boosted by -+ * interactivity modifiers. Will be RT if the task got -+ * RT-boosted. If not then it returns p->normal_prio. -+ */ -+static int effective_prio(struct task_struct *p) -+{ -+ p->normal_prio = normal_prio(p); -+ /* -+ * If we are RT tasks or we were boosted to RT priority, -+ * keep the priority unchanged. Otherwise, update priority -+ * to the normal priority: -+ */ -+ if (!rt_prio(p->prio)) -+ return p->normal_prio; -+ return p->prio; -+} -+ -+/** -+ * task_curr - is this task currently executing on a CPU? -+ * @p: the task in question. -+ * -+ * Return: 1 if the task is currently executing. 0 otherwise. -+ */ -+inline int task_curr(const struct task_struct *p) -+{ -+ return cpu_curr(task_cpu(p)) == p; -+} -+ -+/* -+ * Can drop rq->lock because from sched_class::switched_from() methods drop it. -+ */ -+static inline void check_class_changed(struct rq *rq, struct task_struct *p, -+ const struct sched_class *prev_class, -+ int oldprio) -+{ -+ if (prev_class != p->sched_class) { -+ if (prev_class->switched_from) -+ prev_class->switched_from(rq, p); -+ /* Possble rq->lock 'hole'. */ -+ p->sched_class->switched_to(rq, p); -+ } else if (oldprio != p->prio || dl_task(p)) -+ p->sched_class->prio_changed(rq, p, oldprio); -+} -+ -+void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags) -+{ -+ const struct sched_class *class; -+ -+ if (p->sched_class == rq->curr->sched_class) { -+ rq->curr->sched_class->check_preempt_curr(rq, p, flags); -+ } else { -+ for_each_class(class) { -+ if (class == rq->curr->sched_class) -+ break; -+ if (class == p->sched_class) { -+ resched_curr(rq); -+ break; -+ } -+ } -+ } -+ -+ /* -+ * A queue event has occurred, and we're going to schedule. In -+ * this case, we can save a useless back to back clock update. -+ */ -+ if (task_on_rq_queued(rq->curr) && test_tsk_need_resched(rq->curr)) -+ rq_clock_skip_update(rq, true); -+} -+ -+#ifdef CONFIG_SMP -+void set_task_cpu(struct task_struct *p, unsigned int new_cpu) -+{ -+#ifdef CONFIG_SCHED_DEBUG -+ /* -+ * We should never call set_task_cpu() on a blocked task, -+ * ttwu() will sort out the placement. -+ */ -+ WARN_ON_ONCE(p->state != TASK_RUNNING && p->state != TASK_WAKING && -+ !p->on_rq); -+ -+#ifdef CONFIG_LOCKDEP -+ /* -+ * The caller should hold either p->pi_lock or rq->lock, when changing -+ * a task's CPU. ->pi_lock for waking tasks, rq->lock for runnable tasks. -+ * -+ * sched_move_task() holds both and thus holding either pins the cgroup, -+ * see task_group(). -+ * -+ * Furthermore, all task_rq users should acquire both locks, see -+ * task_rq_lock(). -+ */ -+ WARN_ON_ONCE(debug_locks && !(lockdep_is_held(&p->pi_lock) || -+ lockdep_is_held(&task_rq(p)->lock))); -+#endif -+#endif -+ -+ trace_sched_migrate_task(p, new_cpu); -+ -+ if (task_cpu(p) != new_cpu) { -+ if (p->sched_class->migrate_task_rq) -+ p->sched_class->migrate_task_rq(p, new_cpu); -+ p->se.nr_migrations++; -+ perf_sw_event_sched(PERF_COUNT_SW_CPU_MIGRATIONS, 1, 0); -+ } -+ -+ __set_task_cpu(p, new_cpu); -+} -+ -+static void __migrate_swap_task(struct task_struct *p, int cpu) -+{ -+ if (task_on_rq_queued(p)) { -+ struct rq *src_rq, *dst_rq; -+ -+ src_rq = task_rq(p); -+ dst_rq = cpu_rq(cpu); -+ -+ deactivate_task(src_rq, p, 0); -+ set_task_cpu(p, cpu); -+ activate_task(dst_rq, p, 0); -+ check_preempt_curr(dst_rq, p, 0); -+ } else { -+ /* -+ * Task isn't running anymore; make it appear like we migrated -+ * it before it went to sleep. This means on wakeup we make the -+ * previous cpu our targer instead of where it really is. -+ */ -+ p->wake_cpu = cpu; -+ } -+} -+ -+struct migration_swap_arg { -+ struct task_struct *src_task, *dst_task; -+ int src_cpu, dst_cpu; -+}; -+ -+static int migrate_swap_stop(void *data) -+{ -+ struct migration_swap_arg *arg = data; -+ struct rq *src_rq, *dst_rq; -+ int ret = -EAGAIN; -+ -+ src_rq = cpu_rq(arg->src_cpu); -+ dst_rq = cpu_rq(arg->dst_cpu); -+ -+ double_raw_lock(&arg->src_task->pi_lock, -+ &arg->dst_task->pi_lock); -+ double_rq_lock(src_rq, dst_rq); -+ if (task_cpu(arg->dst_task) != arg->dst_cpu) -+ goto unlock; -+ -+ if (task_cpu(arg->src_task) != arg->src_cpu) -+ goto unlock; -+ -+ if (!cpumask_test_cpu(arg->dst_cpu, tsk_cpus_allowed(arg->src_task))) -+ goto unlock; -+ -+ if (!cpumask_test_cpu(arg->src_cpu, tsk_cpus_allowed(arg->dst_task))) -+ goto unlock; -+ -+ __migrate_swap_task(arg->src_task, arg->dst_cpu); -+ __migrate_swap_task(arg->dst_task, arg->src_cpu); -+ -+ ret = 0; -+ -+unlock: -+ double_rq_unlock(src_rq, dst_rq); -+ raw_spin_unlock(&arg->dst_task->pi_lock); -+ raw_spin_unlock(&arg->src_task->pi_lock); -+ -+ return ret; -+} -+ -+/* -+ * Cross migrate two tasks -+ */ -+int migrate_swap(struct task_struct *cur, struct task_struct *p) -+{ -+ struct migration_swap_arg arg; -+ int ret = -EINVAL; -+ -+ arg = (struct migration_swap_arg){ -+ .src_task = cur, -+ .src_cpu = task_cpu(cur), -+ .dst_task = p, -+ .dst_cpu = task_cpu(p), -+ }; -+ -+ if (arg.src_cpu == arg.dst_cpu) -+ goto out; -+ -+ /* -+ * These three tests are all lockless; this is OK since all of them -+ * will be re-checked with proper locks held further down the line. -+ */ -+ if (!cpu_active(arg.src_cpu) || !cpu_active(arg.dst_cpu)) -+ goto out; -+ -+ if (!cpumask_test_cpu(arg.dst_cpu, tsk_cpus_allowed(arg.src_task))) -+ goto out; -+ -+ if (!cpumask_test_cpu(arg.src_cpu, tsk_cpus_allowed(arg.dst_task))) -+ goto out; -+ -+ trace_sched_swap_numa(cur, arg.src_cpu, p, arg.dst_cpu); -+ ret = stop_two_cpus(arg.dst_cpu, arg.src_cpu, migrate_swap_stop, &arg); -+ -+out: -+ return ret; -+} -+ -+struct migration_arg { -+ struct task_struct *task; -+ int dest_cpu; -+}; -+ -+static int migration_cpu_stop(void *data); -+ -+/* -+ * wait_task_inactive - wait for a thread to unschedule. -+ * -+ * If @match_state is nonzero, it's the @p->state value just checked and -+ * not expected to change. If it changes, i.e. @p might have woken up, -+ * then return zero. When we succeed in waiting for @p to be off its CPU, -+ * we return a positive number (its total switch count). If a second call -+ * a short while later returns the same number, the caller can be sure that -+ * @p has remained unscheduled the whole time. -+ * -+ * The caller must ensure that the task *will* unschedule sometime soon, -+ * else this function might spin for a *long* time. This function can't -+ * be called with interrupts off, or it may introduce deadlock with -+ * smp_call_function() if an IPI is sent by the same process we are -+ * waiting to become inactive. -+ */ -+unsigned long wait_task_inactive(struct task_struct *p, long match_state) -+{ -+ unsigned long flags; -+ int running, queued; -+ unsigned long ncsw; -+ struct rq *rq; -+ -+ for (;;) { -+ /* -+ * We do the initial early heuristics without holding -+ * any task-queue locks at all. We'll only try to get -+ * the runqueue lock when things look like they will -+ * work out! -+ */ -+ rq = task_rq(p); -+ -+ /* -+ * If the task is actively running on another CPU -+ * still, just relax and busy-wait without holding -+ * any locks. -+ * -+ * NOTE! Since we don't hold any locks, it's not -+ * even sure that "rq" stays as the right runqueue! -+ * But we don't care, since "task_running()" will -+ * return false if the runqueue has changed and p -+ * is actually now running somewhere else! -+ */ -+ while (task_running(rq, p)) { -+ if (match_state && unlikely(p->state != match_state)) -+ return 0; -+ cpu_relax(); -+ } -+ -+ /* -+ * Ok, time to look more closely! We need the rq -+ * lock now, to be *sure*. If we're wrong, we'll -+ * just go back and repeat. -+ */ -+ rq = task_rq_lock(p, &flags); -+ trace_sched_wait_task(p); -+ running = task_running(rq, p); -+ queued = task_on_rq_queued(p); -+ ncsw = 0; -+ if (!match_state || p->state == match_state) -+ ncsw = p->nvcsw | LONG_MIN; /* sets MSB */ -+ task_rq_unlock(rq, p, &flags); -+ -+ /* -+ * If it changed from the expected state, bail out now. -+ */ -+ if (unlikely(!ncsw)) -+ break; -+ -+ /* -+ * Was it really running after all now that we -+ * checked with the proper locks actually held? -+ * -+ * Oops. Go back and try again.. -+ */ -+ if (unlikely(running)) { -+ cpu_relax(); -+ continue; -+ } -+ -+ /* -+ * It's not enough that it's not actively running, -+ * it must be off the runqueue _entirely_, and not -+ * preempted! -+ * -+ * So if it was still runnable (but just not actively -+ * running right now), it's preempted, and we should -+ * yield - it could be a while. -+ */ -+ if (unlikely(queued)) { -+ ktime_t to = ktime_set(0, NSEC_PER_SEC/HZ); -+ -+ set_current_state(TASK_UNINTERRUPTIBLE); -+ schedule_hrtimeout(&to, HRTIMER_MODE_REL); -+ continue; -+ } -+ -+ /* -+ * Ahh, all good. It wasn't running, and it wasn't -+ * runnable, which means that it will never become -+ * running in the future either. We're all done! -+ */ -+ break; -+ } -+ -+ return ncsw; -+} -+ -+/*** -+ * kick_process - kick a running thread to enter/exit the kernel -+ * @p: the to-be-kicked thread -+ * -+ * Cause a process which is running on another CPU to enter -+ * kernel-mode, without any delay. (to get signals handled.) -+ * -+ * NOTE: this function doesn't have to take the runqueue lock, -+ * because all it wants to ensure is that the remote task enters -+ * the kernel. If the IPI races and the task has been migrated -+ * to another CPU then no harm is done and the purpose has been -+ * achieved as well. -+ */ -+void kick_process(struct task_struct *p) -+{ -+ int cpu; -+ -+ preempt_disable(); -+ cpu = task_cpu(p); -+ if ((cpu != smp_processor_id()) && task_curr(p)) -+ smp_send_reschedule(cpu); -+ preempt_enable(); -+} -+EXPORT_SYMBOL_GPL(kick_process); -+#endif /* CONFIG_SMP */ -+ -+#ifdef CONFIG_SMP -+/* -+ * ->cpus_allowed is protected by both rq->lock and p->pi_lock -+ */ -+static int select_fallback_rq(int cpu, struct task_struct *p) -+{ -+ int nid = cpu_to_node(cpu); -+ const struct cpumask *nodemask = NULL; -+ enum { cpuset, possible, fail } state = cpuset; -+ int dest_cpu; -+ -+ /* -+ * If the node that the cpu is on has been offlined, cpu_to_node() -+ * will return -1. There is no cpu on the node, and we should -+ * select the cpu on the other node. -+ */ -+ if (nid != -1) { -+ nodemask = cpumask_of_node(nid); -+ -+ /* Look for allowed, online CPU in same node. */ -+ for_each_cpu(dest_cpu, nodemask) { -+ if (!cpu_online(dest_cpu)) -+ continue; -+ if (!cpu_active(dest_cpu)) -+ continue; -+ if (cpumask_test_cpu(dest_cpu, tsk_cpus_allowed(p))) -+ return dest_cpu; -+ } -+ } -+ -+ for (;;) { -+ /* Any allowed, online CPU? */ -+ for_each_cpu(dest_cpu, tsk_cpus_allowed(p)) { -+ if (!cpu_online(dest_cpu)) -+ continue; -+ if (!cpu_active(dest_cpu)) -+ continue; -+ goto out; -+ } -+ -+ switch (state) { -+ case cpuset: -+ /* No more Mr. Nice Guy. */ -+ cpuset_cpus_allowed_fallback(p); -+ state = possible; -+ break; -+ -+ case possible: -+ do_set_cpus_allowed(p, cpu_possible_mask); -+ state = fail; -+ break; -+ -+ case fail: -+ BUG(); -+ break; -+ } -+ } -+ -+out: -+ if (state != cpuset) { -+ /* -+ * Don't tell them about moving exiting tasks or -+ * kernel threads (both mm NULL), since they never -+ * leave kernel. -+ */ -+ if (p->mm && printk_ratelimit()) { -+ printk_deferred("process %d (%s) no longer affine to cpu%d\n", -+ task_pid_nr(p), p->comm, cpu); -+ } -+ } -+ -+ return dest_cpu; -+} -+ -+/* -+ * The caller (fork, wakeup) owns p->pi_lock, ->cpus_allowed is stable. -+ */ -+static inline -+int select_task_rq(struct task_struct *p, int cpu, int sd_flags, int wake_flags) -+{ -+ if (p->nr_cpus_allowed > 1) -+ cpu = p->sched_class->select_task_rq(p, cpu, sd_flags, wake_flags); -+ -+ /* -+ * In order not to call set_task_cpu() on a blocking task we need -+ * to rely on ttwu() to place the task on a valid ->cpus_allowed -+ * cpu. -+ * -+ * Since this is common to all placement strategies, this lives here. -+ * -+ * [ this allows ->select_task() to simply return task_cpu(p) and -+ * not worry about this generic constraint ] -+ */ -+ if (unlikely(!cpumask_test_cpu(cpu, tsk_cpus_allowed(p)) || -+ !cpu_online(cpu))) -+ cpu = select_fallback_rq(task_cpu(p), p); -+ -+ return cpu; -+} -+ -+static void update_avg(u64 *avg, u64 sample) -+{ -+ s64 diff = sample - *avg; -+ *avg += diff >> 3; -+} -+#endif -+ -+static void -+ttwu_stat(struct task_struct *p, int cpu, int wake_flags) -+{ -+#ifdef CONFIG_SCHEDSTATS -+ struct rq *rq = this_rq(); -+ -+#ifdef CONFIG_SMP -+ int this_cpu = smp_processor_id(); -+ -+ if (cpu == this_cpu) { -+ schedstat_inc(rq, ttwu_local); -+ schedstat_inc(p, se.statistics.nr_wakeups_local); -+ } else { -+ struct sched_domain *sd; -+ -+ schedstat_inc(p, se.statistics.nr_wakeups_remote); -+ rcu_read_lock(); -+ for_each_domain(this_cpu, sd) { -+ if (cpumask_test_cpu(cpu, sched_domain_span(sd))) { -+ schedstat_inc(sd, ttwu_wake_remote); -+ break; -+ } -+ } -+ rcu_read_unlock(); -+ } -+ -+ if (wake_flags & WF_MIGRATED) -+ schedstat_inc(p, se.statistics.nr_wakeups_migrate); -+ -+#endif /* CONFIG_SMP */ -+ -+ schedstat_inc(rq, ttwu_count); -+ schedstat_inc(p, se.statistics.nr_wakeups); -+ -+ if (wake_flags & WF_SYNC) -+ schedstat_inc(p, se.statistics.nr_wakeups_sync); -+ -+#endif /* CONFIG_SCHEDSTATS */ -+} -+ -+static void ttwu_activate(struct rq *rq, struct task_struct *p, int en_flags) -+{ -+ activate_task(rq, p, en_flags); -+ p->on_rq = TASK_ON_RQ_QUEUED; -+ -+ /* if a worker is waking up, notify workqueue */ -+ if (p->flags & PF_WQ_WORKER) -+ wq_worker_waking_up(p, cpu_of(rq)); -+} -+ -+/* -+ * Mark the task runnable and perform wakeup-preemption. -+ */ -+static void -+ttwu_do_wakeup(struct rq *rq, struct task_struct *p, int wake_flags) -+{ -+ check_preempt_curr(rq, p, wake_flags); -+ trace_sched_wakeup(p, true); -+ -+ p->state = TASK_RUNNING; -+#ifdef CONFIG_SMP -+ if (p->sched_class->task_woken) -+ p->sched_class->task_woken(rq, p); -+ -+ if (rq->idle_stamp) { -+ u64 delta = rq_clock(rq) - rq->idle_stamp; -+ u64 max = 2*rq->max_idle_balance_cost; -+ -+ update_avg(&rq->avg_idle, delta); -+ -+ if (rq->avg_idle > max) -+ rq->avg_idle = max; -+ -+ rq->idle_stamp = 0; -+ } -+#endif -+} -+ -+static void -+ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags) -+{ -+#ifdef CONFIG_SMP -+ if (p->sched_contributes_to_load) -+ rq->nr_uninterruptible--; -+#endif -+ -+ ttwu_activate(rq, p, ENQUEUE_WAKEUP | ENQUEUE_WAKING); -+ ttwu_do_wakeup(rq, p, wake_flags); -+} -+ -+/* -+ * Called in case the task @p isn't fully descheduled from its runqueue, -+ * in this case we must do a remote wakeup. Its a 'light' wakeup though, -+ * since all we need to do is flip p->state to TASK_RUNNING, since -+ * the task is still ->on_rq. -+ */ -+static int ttwu_remote(struct task_struct *p, int wake_flags) -+{ -+ struct rq *rq; -+ int ret = 0; -+ -+ rq = __task_rq_lock(p); -+ if (task_on_rq_queued(p)) { -+ /* check_preempt_curr() may use rq clock */ -+ update_rq_clock(rq); -+ ttwu_do_wakeup(rq, p, wake_flags); -+ ret = 1; -+ } -+ __task_rq_unlock(rq); -+ -+ return ret; -+} -+ -+#ifdef CONFIG_SMP -+void sched_ttwu_pending(void) -+{ -+ struct rq *rq = this_rq(); -+ struct llist_node *llist = llist_del_all(&rq->wake_list); -+ struct task_struct *p; -+ unsigned long flags; -+ -+ if (!llist) -+ return; -+ -+ raw_spin_lock_irqsave(&rq->lock, flags); -+ -+ while (llist) { -+ p = llist_entry(llist, struct task_struct, wake_entry); -+ llist = llist_next(llist); -+ ttwu_do_activate(rq, p, 0); -+ } -+ -+ raw_spin_unlock_irqrestore(&rq->lock, flags); -+} -+ -+void scheduler_ipi(void) -+{ -+ /* -+ * Fold TIF_NEED_RESCHED into the preempt_count; anybody setting -+ * TIF_NEED_RESCHED remotely (for the first time) will also send -+ * this IPI. -+ */ -+ preempt_fold_need_resched(); -+ -+ if (llist_empty(&this_rq()->wake_list) && !got_nohz_idle_kick()) -+ return; -+ -+ /* -+ * Not all reschedule IPI handlers call irq_enter/irq_exit, since -+ * traditionally all their work was done from the interrupt return -+ * path. Now that we actually do some work, we need to make sure -+ * we do call them. -+ * -+ * Some archs already do call them, luckily irq_enter/exit nest -+ * properly. -+ * -+ * Arguably we should visit all archs and update all handlers, -+ * however a fair share of IPIs are still resched only so this would -+ * somewhat pessimize the simple resched case. -+ */ -+ irq_enter(); -+ sched_ttwu_pending(); -+ -+ /* -+ * Check if someone kicked us for doing the nohz idle load balance. -+ */ -+ if (unlikely(got_nohz_idle_kick())) { -+ this_rq()->idle_balance = 1; -+ raise_softirq_irqoff(SCHED_SOFTIRQ); -+ } -+ irq_exit(); -+} -+ -+static void ttwu_queue_remote(struct task_struct *p, int cpu) -+{ -+ struct rq *rq = cpu_rq(cpu); -+ -+ if (llist_add(&p->wake_entry, &cpu_rq(cpu)->wake_list)) { -+ if (!set_nr_if_polling(rq->idle)) -+ smp_send_reschedule(cpu); -+ else -+ trace_sched_wake_idle_without_ipi(cpu); -+ } -+} -+ -+void wake_up_if_idle(int cpu) -+{ -+ struct rq *rq = cpu_rq(cpu); -+ unsigned long flags; -+ -+ rcu_read_lock(); -+ -+ if (!is_idle_task(rcu_dereference(rq->curr))) -+ goto out; -+ -+ if (set_nr_if_polling(rq->idle)) { -+ trace_sched_wake_idle_without_ipi(cpu); -+ } else { -+ raw_spin_lock_irqsave(&rq->lock, flags); -+ if (is_idle_task(rq->curr)) -+ smp_send_reschedule(cpu); -+ /* Else cpu is not in idle, do nothing here */ -+ raw_spin_unlock_irqrestore(&rq->lock, flags); -+ } -+ -+out: -+ rcu_read_unlock(); -+} -+ -+bool cpus_share_cache(int this_cpu, int that_cpu) -+{ -+ return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu); -+} -+#endif /* CONFIG_SMP */ -+ -+static void ttwu_queue(struct task_struct *p, int cpu) -+{ -+ struct rq *rq = cpu_rq(cpu); -+ -+#if defined(CONFIG_SMP) -+ if (sched_feat(TTWU_QUEUE) && !cpus_share_cache(smp_processor_id(), cpu)) { -+ sched_clock_cpu(cpu); /* sync clocks x-cpu */ -+ ttwu_queue_remote(p, cpu); -+ return; -+ } -+#endif -+ -+ raw_spin_lock(&rq->lock); -+ ttwu_do_activate(rq, p, 0); -+ raw_spin_unlock(&rq->lock); -+} -+ -+/** -+ * try_to_wake_up - wake up a thread -+ * @p: the thread to be awakened -+ * @state: the mask of task states that can be woken -+ * @wake_flags: wake modifier flags (WF_*) -+ * -+ * Put it on the run-queue if it's not already there. The "current" -+ * thread is always on the run-queue (except when the actual -+ * re-schedule is in progress), and as such you're allowed to do -+ * the simpler "current->state = TASK_RUNNING" to mark yourself -+ * runnable without the overhead of this. -+ * -+ * Return: %true if @p was woken up, %false if it was already running. -+ * or @state didn't match @p's state. -+ */ -+static int -+try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) -+{ -+ unsigned long flags; -+ int cpu, success = 0; -+ -+ /* -+ * If we are going to wake up a thread waiting for CONDITION we -+ * need to ensure that CONDITION=1 done by the caller can not be -+ * reordered with p->state check below. This pairs with mb() in -+ * set_current_state() the waiting thread does. -+ */ -+ smp_mb__before_spinlock(); -+ raw_spin_lock_irqsave(&p->pi_lock, flags); -+ if (!(p->state & state)) -+ goto out; -+ -+ success = 1; /* we're going to change ->state */ -+ cpu = task_cpu(p); -+ -+ if (p->on_rq && ttwu_remote(p, wake_flags)) -+ goto stat; -+ -+#ifdef CONFIG_SMP -+ /* -+ * If the owning (remote) cpu is still in the middle of schedule() with -+ * this task as prev, wait until its done referencing the task. -+ */ -+ while (p->on_cpu) -+ cpu_relax(); -+ /* -+ * Pairs with the smp_wmb() in finish_lock_switch(). -+ */ -+ smp_rmb(); -+ -+ p->sched_contributes_to_load = !!task_contributes_to_load(p); -+ p->state = TASK_WAKING; -+ -+ if (p->sched_class->task_waking) -+ p->sched_class->task_waking(p); -+ -+ cpu = select_task_rq(p, p->wake_cpu, SD_BALANCE_WAKE, wake_flags); -+ if (task_cpu(p) != cpu) { -+ wake_flags |= WF_MIGRATED; -+ set_task_cpu(p, cpu); -+ } -+#endif /* CONFIG_SMP */ -+ -+ ttwu_queue(p, cpu); -+stat: -+ ttwu_stat(p, cpu, wake_flags); -+out: -+ raw_spin_unlock_irqrestore(&p->pi_lock, flags); -+ -+ return success; -+} -+ -+/** -+ * try_to_wake_up_local - try to wake up a local task with rq lock held -+ * @p: the thread to be awakened -+ * -+ * Put @p on the run-queue if it's not already there. The caller must -+ * ensure that this_rq() is locked, @p is bound to this_rq() and not -+ * the current task. -+ */ -+static void try_to_wake_up_local(struct task_struct *p) -+{ -+ struct rq *rq = task_rq(p); -+ -+ if (WARN_ON_ONCE(rq != this_rq()) || -+ WARN_ON_ONCE(p == current)) -+ return; -+ -+ lockdep_assert_held(&rq->lock); -+ -+ if (!raw_spin_trylock(&p->pi_lock)) { -+ raw_spin_unlock(&rq->lock); -+ raw_spin_lock(&p->pi_lock); -+ raw_spin_lock(&rq->lock); -+ } -+ -+ if (!(p->state & TASK_NORMAL)) -+ goto out; -+ -+ if (!task_on_rq_queued(p)) -+ ttwu_activate(rq, p, ENQUEUE_WAKEUP); -+ -+ ttwu_do_wakeup(rq, p, 0); -+ ttwu_stat(p, smp_processor_id(), 0); -+out: -+ raw_spin_unlock(&p->pi_lock); -+} -+ -+/** -+ * wake_up_process - Wake up a specific process -+ * @p: The process to be woken up. -+ * -+ * Attempt to wake up the nominated process and move it to the set of runnable -+ * processes. -+ * -+ * Return: 1 if the process was woken up, 0 if it was already running. -+ * -+ * It may be assumed that this function implies a write memory barrier before -+ * changing the task state if and only if any tasks are woken up. -+ */ -+int wake_up_process(struct task_struct *p) -+{ -+ WARN_ON(task_is_stopped_or_traced(p)); -+ return try_to_wake_up(p, TASK_NORMAL, 0); -+} -+EXPORT_SYMBOL(wake_up_process); -+ -+int wake_up_state(struct task_struct *p, unsigned int state) -+{ -+ return try_to_wake_up(p, state, 0); -+} -+ -+/* -+ * This function clears the sched_dl_entity static params. -+ */ -+void __dl_clear_params(struct task_struct *p) -+{ -+ struct sched_dl_entity *dl_se = &p->dl; -+ -+ dl_se->dl_runtime = 0; -+ dl_se->dl_deadline = 0; -+ dl_se->dl_period = 0; -+ dl_se->flags = 0; -+ dl_se->dl_bw = 0; -+ -+ dl_se->dl_throttled = 0; -+ dl_se->dl_new = 1; -+ dl_se->dl_yielded = 0; -+} -+ -+/* -+ * Perform scheduler related setup for a newly forked process p. -+ * p is forked by current. -+ * -+ * __sched_fork() is basic setup used by init_idle() too: -+ */ -+static void __sched_fork(unsigned long clone_flags, struct task_struct *p) -+{ -+ p->on_rq = 0; -+ -+ p->se.on_rq = 0; -+ p->se.exec_start = 0; -+ p->se.sum_exec_runtime = 0; -+ p->se.prev_sum_exec_runtime = 0; -+ p->se.nr_migrations = 0; -+ p->se.vruntime = 0; -+#ifdef CONFIG_SMP -+ p->se.avg.decay_count = 0; -+#endif -+ INIT_LIST_HEAD(&p->se.group_node); -+ -+#ifdef CONFIG_SCHEDSTATS -+ memset(&p->se.statistics, 0, sizeof(p->se.statistics)); -+#endif -+ -+ RB_CLEAR_NODE(&p->dl.rb_node); -+ init_dl_task_timer(&p->dl); -+ __dl_clear_params(p); -+ -+ INIT_LIST_HEAD(&p->rt.run_list); -+ -+#ifdef CONFIG_PREEMPT_NOTIFIERS -+ INIT_HLIST_HEAD(&p->preempt_notifiers); -+#endif -+ -+#ifdef CONFIG_NUMA_BALANCING -+ if (p->mm && atomic_read(&p->mm->mm_users) == 1) { -+ p->mm->numa_next_scan = jiffies + msecs_to_jiffies(sysctl_numa_balancing_scan_delay); -+ p->mm->numa_scan_seq = 0; -+ } -+ -+ if (clone_flags & CLONE_VM) -+ p->numa_preferred_nid = current->numa_preferred_nid; -+ else -+ p->numa_preferred_nid = -1; -+ -+ p->node_stamp = 0ULL; -+ p->numa_scan_seq = p->mm ? p->mm->numa_scan_seq : 0; -+ p->numa_scan_period = sysctl_numa_balancing_scan_delay; -+ p->numa_work.next = &p->numa_work; -+ p->numa_faults = NULL; -+ p->last_task_numa_placement = 0; -+ p->last_sum_exec_runtime = 0; -+ -+ p->numa_group = NULL; -+#endif /* CONFIG_NUMA_BALANCING */ -+} -+ -+#ifdef CONFIG_NUMA_BALANCING -+#ifdef CONFIG_SCHED_DEBUG -+void set_numabalancing_state(bool enabled) -+{ -+ if (enabled) -+ sched_feat_set("NUMA"); -+ else -+ sched_feat_set("NO_NUMA"); -+} -+#else -+__read_mostly bool numabalancing_enabled; -+ -+void set_numabalancing_state(bool enabled) -+{ -+ numabalancing_enabled = enabled; -+} -+#endif /* CONFIG_SCHED_DEBUG */ -+ -+#ifdef CONFIG_PROC_SYSCTL -+int sysctl_numa_balancing(struct ctl_table *table, int write, -+ void __user *buffer, size_t *lenp, loff_t *ppos) -+{ -+ struct ctl_table t; -+ int err; -+ int state = numabalancing_enabled; -+ -+ if (write && !capable(CAP_SYS_ADMIN)) -+ return -EPERM; -+ -+ t = *table; -+ t.data = &state; -+ err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos); -+ if (err < 0) -+ return err; -+ if (write) -+ set_numabalancing_state(state); -+ return err; -+} -+#endif -+#endif -+ -+/* -+ * fork()/clone()-time setup: -+ */ -+int sched_fork(unsigned long clone_flags, struct task_struct *p) -+{ -+ unsigned long flags; -+ int cpu = get_cpu(); -+ -+ __sched_fork(clone_flags, p); -+ /* -+ * We mark the process as running here. This guarantees that -+ * nobody will actually run it, and a signal or other external -+ * event cannot wake it up and insert it on the runqueue either. -+ */ -+ p->state = TASK_RUNNING; -+ -+ /* -+ * Make sure we do not leak PI boosting priority to the child. -+ */ -+ p->prio = current->normal_prio; -+ -+ /* -+ * Revert to default priority/policy on fork if requested. -+ */ -+ if (unlikely(p->sched_reset_on_fork)) { -+ if (task_has_dl_policy(p) || task_has_rt_policy(p)) { -+ p->policy = SCHED_NORMAL; -+ p->static_prio = NICE_TO_PRIO(0); -+ p->rt_priority = 0; -+ } else if (PRIO_TO_NICE(p->static_prio) < 0) -+ p->static_prio = NICE_TO_PRIO(0); -+ -+ p->prio = p->normal_prio = __normal_prio(p); -+ set_load_weight(p); -+ -+ /* -+ * We don't need the reset flag anymore after the fork. It has -+ * fulfilled its duty: -+ */ -+ p->sched_reset_on_fork = 0; -+ } -+ -+ if (dl_prio(p->prio)) { -+ put_cpu(); -+ return -EAGAIN; -+ } else if (rt_prio(p->prio)) { -+ p->sched_class = &rt_sched_class; -+ } else { -+ p->sched_class = &fair_sched_class; -+ } -+ -+ if (p->sched_class->task_fork) -+ p->sched_class->task_fork(p); -+ -+ /* -+ * The child is not yet in the pid-hash so no cgroup attach races, -+ * and the cgroup is pinned to this child due to cgroup_fork() -+ * is ran before sched_fork(). -+ * -+ * Silence PROVE_RCU. -+ */ -+ raw_spin_lock_irqsave(&p->pi_lock, flags); -+ set_task_cpu(p, cpu); -+ raw_spin_unlock_irqrestore(&p->pi_lock, flags); -+ -+#if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT) -+ if (likely(sched_info_on())) -+ memset(&p->sched_info, 0, sizeof(p->sched_info)); -+#endif -+#if defined(CONFIG_SMP) -+ p->on_cpu = 0; -+#endif -+ init_task_preempt_count(p); -+#ifdef CONFIG_SMP -+ plist_node_init(&p->pushable_tasks, MAX_PRIO); -+ RB_CLEAR_NODE(&p->pushable_dl_tasks); -+#endif -+ -+ put_cpu(); -+ return 0; -+} -+ -+unsigned long to_ratio(u64 period, u64 runtime) -+{ -+ if (runtime == RUNTIME_INF) -+ return 1ULL << 20; -+ -+ /* -+ * Doing this here saves a lot of checks in all -+ * the calling paths, and returning zero seems -+ * safe for them anyway. -+ */ -+ if (period == 0) -+ return 0; -+ -+ return div64_u64(runtime << 20, period); -+} -+ -+#ifdef CONFIG_SMP -+inline struct dl_bw *dl_bw_of(int i) -+{ -+ rcu_lockdep_assert(rcu_read_lock_sched_held(), -+ "sched RCU must be held"); -+ return &cpu_rq(i)->rd->dl_bw; -+} -+ -+static inline int dl_bw_cpus(int i) -+{ -+ struct root_domain *rd = cpu_rq(i)->rd; -+ int cpus = 0; -+ -+ rcu_lockdep_assert(rcu_read_lock_sched_held(), -+ "sched RCU must be held"); -+ for_each_cpu_and(i, rd->span, cpu_active_mask) -+ cpus++; -+ -+ return cpus; -+} -+#else -+inline struct dl_bw *dl_bw_of(int i) -+{ -+ return &cpu_rq(i)->dl.dl_bw; -+} -+ -+static inline int dl_bw_cpus(int i) -+{ -+ return 1; -+} -+#endif -+ -+/* -+ * We must be sure that accepting a new task (or allowing changing the -+ * parameters of an existing one) is consistent with the bandwidth -+ * constraints. If yes, this function also accordingly updates the currently -+ * allocated bandwidth to reflect the new situation. -+ * -+ * This function is called while holding p's rq->lock. -+ * -+ * XXX we should delay bw change until the task's 0-lag point, see -+ * __setparam_dl(). -+ */ -+static int dl_overflow(struct task_struct *p, int policy, -+ const struct sched_attr *attr) -+{ -+ -+ struct dl_bw *dl_b = dl_bw_of(task_cpu(p)); -+ u64 period = attr->sched_period ?: attr->sched_deadline; -+ u64 runtime = attr->sched_runtime; -+ u64 new_bw = dl_policy(policy) ? to_ratio(period, runtime) : 0; -+ int cpus, err = -1; -+ -+ if (new_bw == p->dl.dl_bw) -+ return 0; -+ -+ /* -+ * Either if a task, enters, leave, or stays -deadline but changes -+ * its parameters, we may need to update accordingly the total -+ * allocated bandwidth of the container. -+ */ -+ raw_spin_lock(&dl_b->lock); -+ cpus = dl_bw_cpus(task_cpu(p)); -+ if (dl_policy(policy) && !task_has_dl_policy(p) && -+ !__dl_overflow(dl_b, cpus, 0, new_bw)) { -+ __dl_add(dl_b, new_bw); -+ err = 0; -+ } else if (dl_policy(policy) && task_has_dl_policy(p) && -+ !__dl_overflow(dl_b, cpus, p->dl.dl_bw, new_bw)) { -+ __dl_clear(dl_b, p->dl.dl_bw); -+ __dl_add(dl_b, new_bw); -+ err = 0; -+ } else if (!dl_policy(policy) && task_has_dl_policy(p)) { -+ __dl_clear(dl_b, p->dl.dl_bw); -+ err = 0; -+ } -+ raw_spin_unlock(&dl_b->lock); -+ -+ return err; -+} -+ -+extern void init_dl_bw(struct dl_bw *dl_b); -+ -+/* -+ * wake_up_new_task - wake up a newly created task for the first time. -+ * -+ * This function will do some initial scheduler statistics housekeeping -+ * that must be done for every newly created context, then puts the task -+ * on the runqueue and wakes it. -+ */ -+void wake_up_new_task(struct task_struct *p) -+{ -+ unsigned long flags; -+ struct rq *rq; -+ -+ raw_spin_lock_irqsave(&p->pi_lock, flags); -+#ifdef CONFIG_SMP -+ /* -+ * Fork balancing, do it here and not earlier because: -+ * - cpus_allowed can change in the fork path -+ * - any previously selected cpu might disappear through hotplug -+ */ -+ set_task_cpu(p, select_task_rq(p, task_cpu(p), SD_BALANCE_FORK, 0)); -+#endif -+ -+ /* Initialize new task's runnable average */ -+ init_task_runnable_average(p); -+ rq = __task_rq_lock(p); -+ activate_task(rq, p, 0); -+ p->on_rq = TASK_ON_RQ_QUEUED; -+ trace_sched_wakeup_new(p, true); -+ check_preempt_curr(rq, p, WF_FORK); -+#ifdef CONFIG_SMP -+ if (p->sched_class->task_woken) -+ p->sched_class->task_woken(rq, p); -+#endif -+ task_rq_unlock(rq, p, &flags); -+} -+ -+#ifdef CONFIG_PREEMPT_NOTIFIERS -+ -+/** -+ * preempt_notifier_register - tell me when current is being preempted & rescheduled -+ * @notifier: notifier struct to register -+ */ -+void preempt_notifier_register(struct preempt_notifier *notifier) -+{ -+ hlist_add_head(¬ifier->link, ¤t->preempt_notifiers); -+} -+EXPORT_SYMBOL_GPL(preempt_notifier_register); -+ -+/** -+ * preempt_notifier_unregister - no longer interested in preemption notifications -+ * @notifier: notifier struct to unregister -+ * -+ * This is safe to call from within a preemption notifier. -+ */ -+void preempt_notifier_unregister(struct preempt_notifier *notifier) -+{ -+ hlist_del(¬ifier->link); -+} -+EXPORT_SYMBOL_GPL(preempt_notifier_unregister); -+ -+static void fire_sched_in_preempt_notifiers(struct task_struct *curr) -+{ -+ struct preempt_notifier *notifier; -+ -+ hlist_for_each_entry(notifier, &curr->preempt_notifiers, link) -+ notifier->ops->sched_in(notifier, raw_smp_processor_id()); -+} -+ -+static void -+fire_sched_out_preempt_notifiers(struct task_struct *curr, -+ struct task_struct *next) -+{ -+ struct preempt_notifier *notifier; -+ -+ hlist_for_each_entry(notifier, &curr->preempt_notifiers, link) -+ notifier->ops->sched_out(notifier, next); -+} -+ -+#else /* !CONFIG_PREEMPT_NOTIFIERS */ -+ -+static void fire_sched_in_preempt_notifiers(struct task_struct *curr) -+{ -+} -+ -+static void -+fire_sched_out_preempt_notifiers(struct task_struct *curr, -+ struct task_struct *next) -+{ -+} -+ -+#endif /* CONFIG_PREEMPT_NOTIFIERS */ -+ -+/** -+ * prepare_task_switch - prepare to switch tasks -+ * @rq: the runqueue preparing to switch -+ * @prev: the current task that is being switched out -+ * @next: the task we are going to switch to. -+ * -+ * This is called with the rq lock held and interrupts off. It must -+ * be paired with a subsequent finish_task_switch after the context -+ * switch. -+ * -+ * prepare_task_switch sets up locking and calls architecture specific -+ * hooks. -+ */ -+static inline void -+prepare_task_switch(struct rq *rq, struct task_struct *prev, -+ struct task_struct *next) -+{ -+ trace_sched_switch(prev, next); -+ sched_info_switch(rq, prev, next); -+ perf_event_task_sched_out(prev, next); -+ fire_sched_out_preempt_notifiers(prev, next); -+ prepare_lock_switch(rq, next); -+ prepare_arch_switch(next); -+} -+ -+/** -+ * finish_task_switch - clean up after a task-switch -+ * @prev: the thread we just switched away from. -+ * -+ * finish_task_switch must be called after the context switch, paired -+ * with a prepare_task_switch call before the context switch. -+ * finish_task_switch will reconcile locking set up by prepare_task_switch, -+ * and do any other architecture-specific cleanup actions. -+ * -+ * Note that we may have delayed dropping an mm in context_switch(). If -+ * so, we finish that here outside of the runqueue lock. (Doing it -+ * with the lock held can cause deadlocks; see schedule() for -+ * details.) -+ * -+ * The context switch have flipped the stack from under us and restored the -+ * local variables which were saved when this task called schedule() in the -+ * past. prev == current is still correct but we need to recalculate this_rq -+ * because prev may have moved to another CPU. -+ */ -+static struct rq *finish_task_switch(struct task_struct *prev) -+ __releases(rq->lock) -+{ -+ struct rq *rq = this_rq(); -+ struct mm_struct *mm = rq->prev_mm; -+ long prev_state; -+ -+ rq->prev_mm = NULL; -+ -+ /* -+ * A task struct has one reference for the use as "current". -+ * If a task dies, then it sets TASK_DEAD in tsk->state and calls -+ * schedule one last time. The schedule call will never return, and -+ * the scheduled task must drop that reference. -+ * The test for TASK_DEAD must occur while the runqueue locks are -+ * still held, otherwise prev could be scheduled on another cpu, die -+ * there before we look at prev->state, and then the reference would -+ * be dropped twice. -+ * Manfred Spraul -+ */ -+ prev_state = prev->state; -+ vtime_task_switch(prev); -+ finish_arch_switch(prev); -+ perf_event_task_sched_in(prev, current); -+ finish_lock_switch(rq, prev); -+ finish_arch_post_lock_switch(); -+ -+ fire_sched_in_preempt_notifiers(current); -+ if (mm) -+ mmdrop(mm); -+ if (unlikely(prev_state == TASK_DEAD)) { -+ if (prev->sched_class->task_dead) -+ prev->sched_class->task_dead(prev); -+ -+ /* -+ * Remove function-return probe instances associated with this -+ * task and put them back on the free list. -+ */ -+ kprobe_flush_task(prev); -+ put_task_struct(prev); -+ } -+ -+ tick_nohz_task_switch(current); -+ return rq; -+} -+ -+#ifdef CONFIG_SMP -+ -+/* rq->lock is NOT held, but preemption is disabled */ -+static inline void post_schedule(struct rq *rq) -+{ -+ if (rq->post_schedule) { -+ unsigned long flags; -+ -+ raw_spin_lock_irqsave(&rq->lock, flags); -+ if (rq->curr->sched_class->post_schedule) -+ rq->curr->sched_class->post_schedule(rq); -+ raw_spin_unlock_irqrestore(&rq->lock, flags); -+ -+ rq->post_schedule = 0; -+ } -+} -+ -+#else -+ -+static inline void post_schedule(struct rq *rq) -+{ -+} -+ -+#endif -+ -+/** -+ * schedule_tail - first thing a freshly forked thread must call. -+ * @prev: the thread we just switched away from. -+ */ -+asmlinkage __visible void schedule_tail(struct task_struct *prev) -+ __releases(rq->lock) -+{ -+ struct rq *rq; -+ -+ /* finish_task_switch() drops rq->lock and enables preemtion */ -+ preempt_disable(); -+ rq = finish_task_switch(prev); -+ post_schedule(rq); -+ preempt_enable(); -+ -+ if (current->set_child_tid) -+ put_user(task_pid_vnr(current), current->set_child_tid); -+} -+ -+/* -+ * context_switch - switch to the new MM and the new thread's register state. -+ */ -+static inline struct rq * -+context_switch(struct rq *rq, struct task_struct *prev, -+ struct task_struct *next) -+{ -+ struct mm_struct *mm, *oldmm; -+ -+ prepare_task_switch(rq, prev, next); -+ -+ mm = next->mm; -+ oldmm = prev->active_mm; -+ /* -+ * For paravirt, this is coupled with an exit in switch_to to -+ * combine the page table reload and the switch backend into -+ * one hypercall. -+ */ -+ arch_start_context_switch(prev); -+ -+ if (!mm) { -+ next->active_mm = oldmm; -+ atomic_inc(&oldmm->mm_count); -+ enter_lazy_tlb(oldmm, next); -+ } else -+ switch_mm(oldmm, mm, next); -+ -+ if (!prev->mm) { -+ prev->active_mm = NULL; -+ rq->prev_mm = oldmm; -+ } -+ /* -+ * Since the runqueue lock will be released by the next -+ * task (which is an invalid locking op but in the case -+ * of the scheduler it's an obvious special-case), so we -+ * do an early lockdep release here: -+ */ -+ spin_release(&rq->lock.dep_map, 1, _THIS_IP_); -+ -+ context_tracking_task_switch(prev, next); -+ /* Here we just switch the register state and the stack. */ -+ switch_to(prev, next, prev); -+ barrier(); -+ -+ return finish_task_switch(prev); -+} -+ -+/* -+ * nr_running and nr_context_switches: -+ * -+ * externally visible scheduler statistics: current number of runnable -+ * threads, total number of context switches performed since bootup. -+ */ -+unsigned long nr_running(void) -+{ -+ unsigned long i, sum = 0; -+ -+ for_each_online_cpu(i) -+ sum += cpu_rq(i)->nr_running; -+ -+ return sum; -+} -+ -+/* -+ * Check if only the current task is running on the cpu. -+ */ -+bool single_task_running(void) -+{ -+ if (cpu_rq(smp_processor_id())->nr_running == 1) -+ return true; -+ else -+ return false; -+} -+EXPORT_SYMBOL(single_task_running); -+ -+unsigned long long nr_context_switches(void) -+{ -+ int i; -+ unsigned long long sum = 0; -+ -+ for_each_possible_cpu(i) -+ sum += cpu_rq(i)->nr_switches; -+ -+ return sum; -+} -+ -+unsigned long nr_iowait(void) -+{ -+ unsigned long i, sum = 0; -+ -+ for_each_possible_cpu(i) -+ sum += atomic_read(&cpu_rq(i)->nr_iowait); -+ -+ return sum; -+} -+ -+unsigned long nr_iowait_cpu(int cpu) -+{ -+ struct rq *this = cpu_rq(cpu); -+ return atomic_read(&this->nr_iowait); -+} -+ -+void get_iowait_load(unsigned long *nr_waiters, unsigned long *load) -+{ -+ struct rq *this = this_rq(); -+ *nr_waiters = atomic_read(&this->nr_iowait); -+ *load = this->cpu_load[0]; -+} -+ -+#ifdef CONFIG_SMP -+ -+/* -+ * sched_exec - execve() is a valuable balancing opportunity, because at -+ * this point the task has the smallest effective memory and cache footprint. -+ */ -+void sched_exec(void) -+{ -+ struct task_struct *p = current; -+ unsigned long flags; -+ int dest_cpu; -+ -+ raw_spin_lock_irqsave(&p->pi_lock, flags); -+ dest_cpu = p->sched_class->select_task_rq(p, task_cpu(p), SD_BALANCE_EXEC, 0); -+ if (dest_cpu == smp_processor_id()) -+ goto unlock; -+ -+ if (likely(cpu_active(dest_cpu))) { -+ struct migration_arg arg = { p, dest_cpu }; -+ -+ raw_spin_unlock_irqrestore(&p->pi_lock, flags); -+ stop_one_cpu(task_cpu(p), migration_cpu_stop, &arg); -+ return; -+ } -+unlock: -+ raw_spin_unlock_irqrestore(&p->pi_lock, flags); -+} -+ -+#endif -+ -+DEFINE_PER_CPU(struct kernel_stat, kstat); -+DEFINE_PER_CPU(struct kernel_cpustat, kernel_cpustat); -+ -+EXPORT_PER_CPU_SYMBOL(kstat); -+EXPORT_PER_CPU_SYMBOL(kernel_cpustat); -+ -+/* -+ * Return accounted runtime for the task. -+ * In case the task is currently running, return the runtime plus current's -+ * pending runtime that have not been accounted yet. -+ */ -+unsigned long long task_sched_runtime(struct task_struct *p) -+{ -+ unsigned long flags; -+ struct rq *rq; -+ u64 ns; -+ -+#if defined(CONFIG_64BIT) && defined(CONFIG_SMP) -+ /* -+ * 64-bit doesn't need locks to atomically read a 64bit value. -+ * So we have a optimization chance when the task's delta_exec is 0. -+ * Reading ->on_cpu is racy, but this is ok. -+ * -+ * If we race with it leaving cpu, we'll take a lock. So we're correct. -+ * If we race with it entering cpu, unaccounted time is 0. This is -+ * indistinguishable from the read occurring a few cycles earlier. -+ * If we see ->on_cpu without ->on_rq, the task is leaving, and has -+ * been accounted, so we're correct here as well. -+ */ -+ if (!p->on_cpu || !task_on_rq_queued(p)) -+ return p->se.sum_exec_runtime; -+#endif -+ -+ rq = task_rq_lock(p, &flags); -+ /* -+ * Must be ->curr _and_ ->on_rq. If dequeued, we would -+ * project cycles that may never be accounted to this -+ * thread, breaking clock_gettime(). -+ */ -+ if (task_current(rq, p) && task_on_rq_queued(p)) { -+ update_rq_clock(rq); -+ p->sched_class->update_curr(rq); -+ } -+ ns = p->se.sum_exec_runtime; -+ task_rq_unlock(rq, p, &flags); -+ -+ return ns; -+} -+ -+/* -+ * This function gets called by the timer code, with HZ frequency. -+ * We call it with interrupts disabled. -+ */ -+void scheduler_tick(void) -+{ -+ int cpu = smp_processor_id(); -+ struct rq *rq = cpu_rq(cpu); -+ struct task_struct *curr = rq->curr; -+ -+ sched_clock_tick(); -+ -+ raw_spin_lock(&rq->lock); -+ update_rq_clock(rq); -+ curr->sched_class->task_tick(rq, curr, 0); -+ update_cpu_load_active(rq); -+ raw_spin_unlock(&rq->lock); -+ -+ perf_event_task_tick(); -+ -+#ifdef CONFIG_SMP -+ rq->idle_balance = idle_cpu(cpu); -+ trigger_load_balance(rq); -+#endif -+ rq_last_tick_reset(rq); -+} -+ -+#ifdef CONFIG_NO_HZ_FULL -+/** -+ * scheduler_tick_max_deferment -+ * -+ * Keep at least one tick per second when a single -+ * active task is running because the scheduler doesn't -+ * yet completely support full dynticks environment. -+ * -+ * This makes sure that uptime, CFS vruntime, load -+ * balancing, etc... continue to move forward, even -+ * with a very low granularity. -+ * -+ * Return: Maximum deferment in nanoseconds. -+ */ -+u64 scheduler_tick_max_deferment(void) -+{ -+ struct rq *rq = this_rq(); -+ unsigned long next, now = ACCESS_ONCE(jiffies); -+ -+ next = rq->last_sched_tick + HZ; -+ -+ if (time_before_eq(next, now)) -+ return 0; -+ -+ return jiffies_to_nsecs(next - now); -+} -+#endif -+ -+notrace unsigned long get_parent_ip(unsigned long addr) -+{ -+ if (in_lock_functions(addr)) { -+ addr = CALLER_ADDR2; -+ if (in_lock_functions(addr)) -+ addr = CALLER_ADDR3; -+ } -+ return addr; -+} -+ -+#if defined(CONFIG_PREEMPT) && (defined(CONFIG_DEBUG_PREEMPT) || \ -+ defined(CONFIG_PREEMPT_TRACER)) -+ -+void preempt_count_add(int val) -+{ -+#ifdef CONFIG_DEBUG_PREEMPT -+ /* -+ * Underflow? -+ */ -+ if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0))) -+ return; -+#endif -+ __preempt_count_add(val); -+#ifdef CONFIG_DEBUG_PREEMPT -+ /* -+ * Spinlock count overflowing soon? -+ */ -+ DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >= -+ PREEMPT_MASK - 10); -+#endif -+ if (preempt_count() == val) { -+ unsigned long ip = get_parent_ip(CALLER_ADDR1); -+#ifdef CONFIG_DEBUG_PREEMPT -+ current->preempt_disable_ip = ip; -+#endif -+ trace_preempt_off(CALLER_ADDR0, ip); -+ } -+} -+EXPORT_SYMBOL(preempt_count_add); -+NOKPROBE_SYMBOL(preempt_count_add); -+ -+void preempt_count_sub(int val) -+{ -+#ifdef CONFIG_DEBUG_PREEMPT -+ /* -+ * Underflow? -+ */ -+ if (DEBUG_LOCKS_WARN_ON(val > preempt_count())) -+ return; -+ /* -+ * Is the spinlock portion underflowing? -+ */ -+ if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) && -+ !(preempt_count() & PREEMPT_MASK))) -+ return; -+#endif -+ -+ if (preempt_count() == val) -+ trace_preempt_on(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1)); -+ __preempt_count_sub(val); -+} -+EXPORT_SYMBOL(preempt_count_sub); -+NOKPROBE_SYMBOL(preempt_count_sub); -+ -+#endif -+ -+/* -+ * Print scheduling while atomic bug: -+ */ -+static noinline void __schedule_bug(struct task_struct *prev) -+{ -+ if (oops_in_progress) -+ return; -+ -+ printk(KERN_ERR "BUG: scheduling while atomic: %s/%d/0x%08x\n", -+ prev->comm, prev->pid, preempt_count()); -+ -+ debug_show_held_locks(prev); -+ print_modules(); -+ if (irqs_disabled()) -+ print_irqtrace_events(prev); -+#ifdef CONFIG_DEBUG_PREEMPT -+ if (in_atomic_preempt_off()) { -+ pr_err("Preemption disabled at:"); -+ print_ip_sym(current->preempt_disable_ip); -+ pr_cont("\n"); -+ } -+#endif -+ dump_stack(); -+ add_taint(TAINT_WARN, LOCKDEP_STILL_OK); -+} -+ -+/* -+ * Various schedule()-time debugging checks and statistics: -+ */ -+static inline void schedule_debug(struct task_struct *prev) -+{ -+#ifdef CONFIG_SCHED_STACK_END_CHECK -+ BUG_ON(unlikely(task_stack_end_corrupted(prev))); -+#endif -+ /* -+ * Test if we are atomic. Since do_exit() needs to call into -+ * schedule() atomically, we ignore that path. Otherwise whine -+ * if we are scheduling when we should not. -+ */ -+ if (unlikely(in_atomic_preempt_off() && prev->state != TASK_DEAD)) -+ __schedule_bug(prev); -+ rcu_sleep_check(); -+ -+ profile_hit(SCHED_PROFILING, __builtin_return_address(0)); -+ -+ schedstat_inc(this_rq(), sched_count); -+} -+ -+/* -+ * Pick up the highest-prio task: -+ */ -+static inline struct task_struct * -+pick_next_task(struct rq *rq, struct task_struct *prev) -+{ -+ const struct sched_class *class = &fair_sched_class; -+ struct task_struct *p; -+ -+ /* -+ * Optimization: we know that if all tasks are in -+ * the fair class we can call that function directly: -+ */ -+ if (likely(prev->sched_class == class && -+ rq->nr_running == rq->cfs.h_nr_running)) { -+ p = fair_sched_class.pick_next_task(rq, prev); -+ if (unlikely(p == RETRY_TASK)) -+ goto again; -+ -+ /* assumes fair_sched_class->next == idle_sched_class */ -+ if (unlikely(!p)) -+ p = idle_sched_class.pick_next_task(rq, prev); -+ -+ return p; -+ } -+ -+again: -+ for_each_class(class) { -+ p = class->pick_next_task(rq, prev); -+ if (p) { -+ if (unlikely(p == RETRY_TASK)) -+ goto again; -+ return p; -+ } -+ } -+ -+ BUG(); /* the idle class will always have a runnable task */ -+} -+ -+/* -+ * __schedule() is the main scheduler function. -+ * -+ * The main means of driving the scheduler and thus entering this function are: -+ * -+ * 1. Explicit blocking: mutex, semaphore, waitqueue, etc. -+ * -+ * 2. TIF_NEED_RESCHED flag is checked on interrupt and userspace return -+ * paths. For example, see arch/x86/entry_64.S. -+ * -+ * To drive preemption between tasks, the scheduler sets the flag in timer -+ * interrupt handler scheduler_tick(). -+ * -+ * 3. Wakeups don't really cause entry into schedule(). They add a -+ * task to the run-queue and that's it. -+ * -+ * Now, if the new task added to the run-queue preempts the current -+ * task, then the wakeup sets TIF_NEED_RESCHED and schedule() gets -+ * called on the nearest possible occasion: -+ * -+ * - If the kernel is preemptible (CONFIG_PREEMPT=y): -+ * -+ * - in syscall or exception context, at the next outmost -+ * preempt_enable(). (this might be as soon as the wake_up()'s -+ * spin_unlock()!) -+ * -+ * - in IRQ context, return from interrupt-handler to -+ * preemptible context -+ * -+ * - If the kernel is not preemptible (CONFIG_PREEMPT is not set) -+ * then at the next: -+ * -+ * - cond_resched() call -+ * - explicit schedule() call -+ * - return from syscall or exception to user-space -+ * - return from interrupt-handler to user-space -+ * -+ * WARNING: all callers must re-check need_resched() afterward and reschedule -+ * accordingly in case an event triggered the need for rescheduling (such as -+ * an interrupt waking up a task) while preemption was disabled in __schedule(). -+ */ -+static void __sched __schedule(void) -+{ -+ struct task_struct *prev, *next; -+ unsigned long *switch_count; -+ struct rq *rq; -+ int cpu; -+ -+ preempt_disable(); -+ cpu = smp_processor_id(); -+ rq = cpu_rq(cpu); -+ rcu_note_context_switch(); -+ prev = rq->curr; -+ -+ schedule_debug(prev); -+ -+ if (sched_feat(HRTICK)) -+ hrtick_clear(rq); -+ -+ /* -+ * Make sure that signal_pending_state()->signal_pending() below -+ * can't be reordered with __set_current_state(TASK_INTERRUPTIBLE) -+ * done by the caller to avoid the race with signal_wake_up(). -+ */ -+ smp_mb__before_spinlock(); -+ raw_spin_lock_irq(&rq->lock); -+ -+ rq->clock_skip_update <<= 1; /* promote REQ to ACT */ -+ -+ switch_count = &prev->nivcsw; -+ if (prev->state && !(preempt_count() & PREEMPT_ACTIVE)) { -+ if (unlikely(signal_pending_state(prev->state, prev))) { -+ prev->state = TASK_RUNNING; -+ } else { -+ deactivate_task(rq, prev, DEQUEUE_SLEEP); -+ prev->on_rq = 0; -+ -+ /* -+ * If a worker went to sleep, notify and ask workqueue -+ * whether it wants to wake up a task to maintain -+ * concurrency. -+ */ -+ if (prev->flags & PF_WQ_WORKER) { -+ struct task_struct *to_wakeup; -+ -+ to_wakeup = wq_worker_sleeping(prev, cpu); -+ if (to_wakeup) -+ try_to_wake_up_local(to_wakeup); -+ } -+ } -+ switch_count = &prev->nvcsw; -+ } -+ -+ if (task_on_rq_queued(prev)) -+ update_rq_clock(rq); -+ -+ next = pick_next_task(rq, prev); -+ clear_tsk_need_resched(prev); -+ clear_preempt_need_resched(); -+ rq->clock_skip_update = 0; -+ -+ if (likely(prev != next)) { -+ rq->nr_switches++; -+ rq->curr = next; -+ ++*switch_count; -+ -+ rq = context_switch(rq, prev, next); /* unlocks the rq */ -+ cpu = cpu_of(rq); -+ } else -+ raw_spin_unlock_irq(&rq->lock); -+ -+ post_schedule(rq); -+ -+ sched_preempt_enable_no_resched(); -+} -+ -+static inline void sched_submit_work(struct task_struct *tsk) -+{ -+ if (!tsk->state || tsk_is_pi_blocked(tsk)) -+ return; -+ /* -+ * If we are going to sleep and we have plugged IO queued, -+ * make sure to submit it to avoid deadlocks. -+ */ -+ if (blk_needs_flush_plug(tsk)) -+ blk_schedule_flush_plug(tsk); -+} -+ -+asmlinkage __visible void __sched schedule(void) -+{ -+ struct task_struct *tsk = current; -+ -+ sched_submit_work(tsk); -+ do { -+ __schedule(); -+ } while (need_resched()); -+} -+EXPORT_SYMBOL(schedule); -+ -+#ifdef CONFIG_CONTEXT_TRACKING -+asmlinkage __visible void __sched schedule_user(void) -+{ -+ /* -+ * If we come here after a random call to set_need_resched(), -+ * or we have been woken up remotely but the IPI has not yet arrived, -+ * we haven't yet exited the RCU idle mode. Do it here manually until -+ * we find a better solution. -+ * -+ * NB: There are buggy callers of this function. Ideally we -+ * should warn if prev_state != CONTEXT_USER, but that will trigger -+ * too frequently to make sense yet. -+ */ -+ enum ctx_state prev_state = exception_enter(); -+ schedule(); -+ exception_exit(prev_state); -+} -+#endif -+ -+/** -+ * schedule_preempt_disabled - called with preemption disabled -+ * -+ * Returns with preemption disabled. Note: preempt_count must be 1 -+ */ -+void __sched schedule_preempt_disabled(void) -+{ -+ sched_preempt_enable_no_resched(); -+ schedule(); -+ preempt_disable(); -+} -+ -+static void __sched notrace preempt_schedule_common(void) -+{ -+ do { -+ __preempt_count_add(PREEMPT_ACTIVE); -+ __schedule(); -+ __preempt_count_sub(PREEMPT_ACTIVE); -+ -+ /* -+ * Check again in case we missed a preemption opportunity -+ * between schedule and now. -+ */ -+ barrier(); -+ } while (need_resched()); -+} -+ -+#ifdef CONFIG_PREEMPT -+/* -+ * this is the entry point to schedule() from in-kernel preemption -+ * off of preempt_enable. Kernel preemptions off return from interrupt -+ * occur there and call schedule directly. -+ */ -+asmlinkage __visible void __sched notrace preempt_schedule(void) -+{ -+ /* -+ * If there is a non-zero preempt_count or interrupts are disabled, -+ * we do not want to preempt the current task. Just return.. -+ */ -+ if (likely(!preemptible())) -+ return; -+ -+ preempt_schedule_common(); -+} -+NOKPROBE_SYMBOL(preempt_schedule); -+EXPORT_SYMBOL(preempt_schedule); -+ -+#ifdef CONFIG_CONTEXT_TRACKING -+/** -+ * preempt_schedule_context - preempt_schedule called by tracing -+ * -+ * The tracing infrastructure uses preempt_enable_notrace to prevent -+ * recursion and tracing preempt enabling caused by the tracing -+ * infrastructure itself. But as tracing can happen in areas coming -+ * from userspace or just about to enter userspace, a preempt enable -+ * can occur before user_exit() is called. This will cause the scheduler -+ * to be called when the system is still in usermode. -+ * -+ * To prevent this, the preempt_enable_notrace will use this function -+ * instead of preempt_schedule() to exit user context if needed before -+ * calling the scheduler. -+ */ -+asmlinkage __visible void __sched notrace preempt_schedule_context(void) -+{ -+ enum ctx_state prev_ctx; -+ -+ if (likely(!preemptible())) -+ return; -+ -+ do { -+ __preempt_count_add(PREEMPT_ACTIVE); -+ /* -+ * Needs preempt disabled in case user_exit() is traced -+ * and the tracer calls preempt_enable_notrace() causing -+ * an infinite recursion. -+ */ -+ prev_ctx = exception_enter(); -+ __schedule(); -+ exception_exit(prev_ctx); -+ -+ __preempt_count_sub(PREEMPT_ACTIVE); -+ barrier(); -+ } while (need_resched()); -+} -+EXPORT_SYMBOL_GPL(preempt_schedule_context); -+#endif /* CONFIG_CONTEXT_TRACKING */ -+ -+#endif /* CONFIG_PREEMPT */ -+ -+/* -+ * this is the entry point to schedule() from kernel preemption -+ * off of irq context. -+ * Note, that this is called and return with irqs disabled. This will -+ * protect us against recursive calling from irq. -+ */ -+asmlinkage __visible void __sched preempt_schedule_irq(void) -+{ -+ enum ctx_state prev_state; -+ -+ /* Catch callers which need to be fixed */ -+ BUG_ON(preempt_count() || !irqs_disabled()); -+ -+ prev_state = exception_enter(); -+ -+ do { -+ __preempt_count_add(PREEMPT_ACTIVE); -+ local_irq_enable(); -+ __schedule(); -+ local_irq_disable(); -+ __preempt_count_sub(PREEMPT_ACTIVE); -+ -+ /* -+ * Check again in case we missed a preemption opportunity -+ * between schedule and now. -+ */ -+ barrier(); -+ } while (need_resched()); -+ -+ exception_exit(prev_state); -+} -+ -+int default_wake_function(wait_queue_t *curr, unsigned mode, int wake_flags, -+ void *key) -+{ -+ return try_to_wake_up(curr->private, mode, wake_flags); -+} -+EXPORT_SYMBOL(default_wake_function); -+ -+#ifdef CONFIG_RT_MUTEXES -+ -+/* -+ * rt_mutex_setprio - set the current priority of a task -+ * @p: task -+ * @prio: prio value (kernel-internal form) -+ * -+ * This function changes the 'effective' priority of a task. It does -+ * not touch ->normal_prio like __setscheduler(). -+ * -+ * Used by the rt_mutex code to implement priority inheritance -+ * logic. Call site only calls if the priority of the task changed. -+ */ -+void rt_mutex_setprio(struct task_struct *p, int prio) -+{ -+ int oldprio, queued, running, enqueue_flag = 0; -+ struct rq *rq; -+ const struct sched_class *prev_class; -+ -+ BUG_ON(prio > MAX_PRIO); -+ -+ rq = __task_rq_lock(p); -+ -+ /* -+ * Idle task boosting is a nono in general. There is one -+ * exception, when PREEMPT_RT and NOHZ is active: -+ * -+ * The idle task calls get_next_timer_interrupt() and holds -+ * the timer wheel base->lock on the CPU and another CPU wants -+ * to access the timer (probably to cancel it). We can safely -+ * ignore the boosting request, as the idle CPU runs this code -+ * with interrupts disabled and will complete the lock -+ * protected section without being interrupted. So there is no -+ * real need to boost. -+ */ -+ if (unlikely(p == rq->idle)) { -+ WARN_ON(p != rq->curr); -+ WARN_ON(p->pi_blocked_on); -+ goto out_unlock; -+ } -+ -+ trace_sched_pi_setprio(p, prio); -+ oldprio = p->prio; -+ prev_class = p->sched_class; -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ if (queued) -+ dequeue_task(rq, p, 0); -+ if (running) -+ put_prev_task(rq, p); -+ -+ /* -+ * Boosting condition are: -+ * 1. -rt task is running and holds mutex A -+ * --> -dl task blocks on mutex A -+ * -+ * 2. -dl task is running and holds mutex A -+ * --> -dl task blocks on mutex A and could preempt the -+ * running task -+ */ -+ if (dl_prio(prio)) { -+ struct task_struct *pi_task = rt_mutex_get_top_task(p); -+ if (!dl_prio(p->normal_prio) || -+ (pi_task && dl_entity_preempt(&pi_task->dl, &p->dl))) { -+ p->dl.dl_boosted = 1; -+ p->dl.dl_throttled = 0; -+ enqueue_flag = ENQUEUE_REPLENISH; -+ } else -+ p->dl.dl_boosted = 0; -+ p->sched_class = &dl_sched_class; -+ } else if (rt_prio(prio)) { -+ if (dl_prio(oldprio)) -+ p->dl.dl_boosted = 0; -+ if (oldprio < prio) -+ enqueue_flag = ENQUEUE_HEAD; -+ p->sched_class = &rt_sched_class; -+ } else { -+ if (dl_prio(oldprio)) -+ p->dl.dl_boosted = 0; -+ if (rt_prio(oldprio)) -+ p->rt.timeout = 0; -+ p->sched_class = &fair_sched_class; -+ } -+ -+ p->prio = prio; -+ -+ if (running) -+ p->sched_class->set_curr_task(rq); -+ if (queued) -+ enqueue_task(rq, p, enqueue_flag); -+ -+ check_class_changed(rq, p, prev_class, oldprio); -+out_unlock: -+ __task_rq_unlock(rq); -+} -+#endif -+ -+void set_user_nice(struct task_struct *p, long nice) -+{ -+ int old_prio, delta, queued; -+ unsigned long flags; -+ struct rq *rq; -+ -+ if (task_nice(p) == nice || nice < MIN_NICE || nice > MAX_NICE) -+ return; -+ /* -+ * We have to be careful, if called from sys_setpriority(), -+ * the task might be in the middle of scheduling on another CPU. -+ */ -+ rq = task_rq_lock(p, &flags); -+ /* -+ * The RT priorities are set via sched_setscheduler(), but we still -+ * allow the 'normal' nice value to be set - but as expected -+ * it wont have any effect on scheduling until the task is -+ * SCHED_DEADLINE, SCHED_FIFO or SCHED_RR: -+ */ -+ if (task_has_dl_policy(p) || task_has_rt_policy(p)) { -+ p->static_prio = NICE_TO_PRIO(nice); -+ goto out_unlock; -+ } -+ queued = task_on_rq_queued(p); -+ if (queued) -+ dequeue_task(rq, p, 0); -+ -+ p->static_prio = NICE_TO_PRIO(nice); -+ set_load_weight(p); -+ old_prio = p->prio; -+ p->prio = effective_prio(p); -+ delta = p->prio - old_prio; -+ -+ if (queued) { -+ enqueue_task(rq, p, 0); -+ /* -+ * If the task increased its priority or is running and -+ * lowered its priority, then reschedule its CPU: -+ */ -+ if (delta < 0 || (delta > 0 && task_running(rq, p))) -+ resched_curr(rq); -+ } -+out_unlock: -+ task_rq_unlock(rq, p, &flags); -+} -+EXPORT_SYMBOL(set_user_nice); -+ -+/* -+ * can_nice - check if a task can reduce its nice value -+ * @p: task -+ * @nice: nice value -+ */ -+int can_nice(const struct task_struct *p, const int nice) -+{ -+ /* convert nice value [19,-20] to rlimit style value [1,40] */ -+ int nice_rlim = nice_to_rlimit(nice); -+ -+ return (nice_rlim <= task_rlimit(p, RLIMIT_NICE) || -+ capable(CAP_SYS_NICE)); -+} -+ -+#ifdef __ARCH_WANT_SYS_NICE -+ -+/* -+ * sys_nice - change the priority of the current process. -+ * @increment: priority increment -+ * -+ * sys_setpriority is a more generic, but much slower function that -+ * does similar things. -+ */ -+SYSCALL_DEFINE1(nice, int, increment) -+{ -+ long nice, retval; -+ -+ /* -+ * Setpriority might change our priority at the same moment. -+ * We don't have to worry. Conceptually one call occurs first -+ * and we have a single winner. -+ */ -+ increment = clamp(increment, -NICE_WIDTH, NICE_WIDTH); -+ nice = task_nice(current) + increment; -+ -+ nice = clamp_val(nice, MIN_NICE, MAX_NICE); -+ if (increment < 0 && !can_nice(current, nice)) -+ return -EPERM; -+ -+ retval = security_task_setnice(current, nice); -+ if (retval) -+ return retval; -+ -+ set_user_nice(current, nice); -+ return 0; -+} -+ -+#endif -+ -+/** -+ * task_prio - return the priority value of a given task. -+ * @p: the task in question. -+ * -+ * Return: The priority value as seen by users in /proc. -+ * RT tasks are offset by -200. Normal tasks are centered -+ * around 0, value goes from -16 to +15. -+ */ -+int task_prio(const struct task_struct *p) -+{ -+ return p->prio - MAX_RT_PRIO; -+} -+ -+/** -+ * idle_cpu - is a given cpu idle currently? -+ * @cpu: the processor in question. -+ * -+ * Return: 1 if the CPU is currently idle. 0 otherwise. -+ */ -+int idle_cpu(int cpu) -+{ -+ struct rq *rq = cpu_rq(cpu); -+ -+ if (rq->curr != rq->idle) -+ return 0; -+ -+ if (rq->nr_running) -+ return 0; -+ -+#ifdef CONFIG_SMP -+ if (!llist_empty(&rq->wake_list)) -+ return 0; -+#endif -+ -+ return 1; -+} -+ -+/** -+ * idle_task - return the idle task for a given cpu. -+ * @cpu: the processor in question. -+ * -+ * Return: The idle task for the cpu @cpu. -+ */ -+struct task_struct *idle_task(int cpu) -+{ -+ return cpu_rq(cpu)->idle; -+} -+ -+/** -+ * find_process_by_pid - find a process with a matching PID value. -+ * @pid: the pid in question. -+ * -+ * The task of @pid, if found. %NULL otherwise. -+ */ -+static struct task_struct *find_process_by_pid(pid_t pid) -+{ -+ return pid ? find_task_by_vpid(pid) : current; -+} -+ -+/* -+ * This function initializes the sched_dl_entity of a newly becoming -+ * SCHED_DEADLINE task. -+ * -+ * Only the static values are considered here, the actual runtime and the -+ * absolute deadline will be properly calculated when the task is enqueued -+ * for the first time with its new policy. -+ */ -+static void -+__setparam_dl(struct task_struct *p, const struct sched_attr *attr) -+{ -+ struct sched_dl_entity *dl_se = &p->dl; -+ -+ dl_se->dl_runtime = attr->sched_runtime; -+ dl_se->dl_deadline = attr->sched_deadline; -+ dl_se->dl_period = attr->sched_period ?: dl_se->dl_deadline; -+ dl_se->flags = attr->sched_flags; -+ dl_se->dl_bw = to_ratio(dl_se->dl_period, dl_se->dl_runtime); -+ -+ /* -+ * Changing the parameters of a task is 'tricky' and we're not doing -+ * the correct thing -- also see task_dead_dl() and switched_from_dl(). -+ * -+ * What we SHOULD do is delay the bandwidth release until the 0-lag -+ * point. This would include retaining the task_struct until that time -+ * and change dl_overflow() to not immediately decrement the current -+ * amount. -+ * -+ * Instead we retain the current runtime/deadline and let the new -+ * parameters take effect after the current reservation period lapses. -+ * This is safe (albeit pessimistic) because the 0-lag point is always -+ * before the current scheduling deadline. -+ * -+ * We can still have temporary overloads because we do not delay the -+ * change in bandwidth until that time; so admission control is -+ * not on the safe side. It does however guarantee tasks will never -+ * consume more than promised. -+ */ -+} -+ -+/* -+ * sched_setparam() passes in -1 for its policy, to let the functions -+ * it calls know not to change it. -+ */ -+#define SETPARAM_POLICY -1 -+ -+static void __setscheduler_params(struct task_struct *p, -+ const struct sched_attr *attr) -+{ -+ int policy = attr->sched_policy; -+ -+ if (policy == SETPARAM_POLICY) -+ policy = p->policy; -+ -+ p->policy = policy; -+ -+ if (dl_policy(policy)) -+ __setparam_dl(p, attr); -+ else if (fair_policy(policy)) -+ p->static_prio = NICE_TO_PRIO(attr->sched_nice); -+ -+ /* -+ * __sched_setscheduler() ensures attr->sched_priority == 0 when -+ * !rt_policy. Always setting this ensures that things like -+ * getparam()/getattr() don't report silly values for !rt tasks. -+ */ -+ p->rt_priority = attr->sched_priority; -+ p->normal_prio = normal_prio(p); -+ set_load_weight(p); -+} -+ -+/* Actually do priority change: must hold pi & rq lock. */ -+static void __setscheduler(struct rq *rq, struct task_struct *p, -+ const struct sched_attr *attr, bool keep_boost) -+{ -+ __setscheduler_params(p, attr); -+ -+ /* -+ * Keep a potential priority boosting if called from -+ * sched_setscheduler(). -+ */ -+ if (keep_boost) -+ p->prio = rt_mutex_get_effective_prio(p, normal_prio(p)); -+ else -+ p->prio = normal_prio(p); -+ -+ if (dl_prio(p->prio)) -+ p->sched_class = &dl_sched_class; -+ else if (rt_prio(p->prio)) -+ p->sched_class = &rt_sched_class; -+ else -+ p->sched_class = &fair_sched_class; -+} -+ -+static void -+__getparam_dl(struct task_struct *p, struct sched_attr *attr) -+{ -+ struct sched_dl_entity *dl_se = &p->dl; -+ -+ attr->sched_priority = p->rt_priority; -+ attr->sched_runtime = dl_se->dl_runtime; -+ attr->sched_deadline = dl_se->dl_deadline; -+ attr->sched_period = dl_se->dl_period; -+ attr->sched_flags = dl_se->flags; -+} -+ -+/* -+ * This function validates the new parameters of a -deadline task. -+ * We ask for the deadline not being zero, and greater or equal -+ * than the runtime, as well as the period of being zero or -+ * greater than deadline. Furthermore, we have to be sure that -+ * user parameters are above the internal resolution of 1us (we -+ * check sched_runtime only since it is always the smaller one) and -+ * below 2^63 ns (we have to check both sched_deadline and -+ * sched_period, as the latter can be zero). -+ */ -+static bool -+__checkparam_dl(const struct sched_attr *attr) -+{ -+ /* deadline != 0 */ -+ if (attr->sched_deadline == 0) -+ return false; -+ -+ /* -+ * Since we truncate DL_SCALE bits, make sure we're at least -+ * that big. -+ */ -+ if (attr->sched_runtime < (1ULL << DL_SCALE)) -+ return false; -+ -+ /* -+ * Since we use the MSB for wrap-around and sign issues, make -+ * sure it's not set (mind that period can be equal to zero). -+ */ -+ if (attr->sched_deadline & (1ULL << 63) || -+ attr->sched_period & (1ULL << 63)) -+ return false; -+ -+ /* runtime <= deadline <= period (if period != 0) */ -+ if ((attr->sched_period != 0 && -+ attr->sched_period < attr->sched_deadline) || -+ attr->sched_deadline < attr->sched_runtime) -+ return false; -+ -+ return true; -+} -+ -+/* -+ * check the target process has a UID that matches the current process's -+ */ -+static bool check_same_owner(struct task_struct *p) -+{ -+ const struct cred *cred = current_cred(), *pcred; -+ bool match; -+ -+ rcu_read_lock(); -+ pcred = __task_cred(p); -+ match = (uid_eq(cred->euid, pcred->euid) || -+ uid_eq(cred->euid, pcred->uid)); -+ rcu_read_unlock(); -+ return match; -+} -+ -+static bool dl_param_changed(struct task_struct *p, -+ const struct sched_attr *attr) -+{ -+ struct sched_dl_entity *dl_se = &p->dl; -+ -+ if (dl_se->dl_runtime != attr->sched_runtime || -+ dl_se->dl_deadline != attr->sched_deadline || -+ dl_se->dl_period != attr->sched_period || -+ dl_se->flags != attr->sched_flags) -+ return true; -+ -+ return false; -+} -+ -+static int __sched_setscheduler(struct task_struct *p, -+ const struct sched_attr *attr, -+ bool user) -+{ -+ int newprio = dl_policy(attr->sched_policy) ? MAX_DL_PRIO - 1 : -+ MAX_RT_PRIO - 1 - attr->sched_priority; -+ int retval, oldprio, oldpolicy = -1, queued, running; -+ int new_effective_prio, policy = attr->sched_policy; -+ unsigned long flags; -+ const struct sched_class *prev_class; -+ struct rq *rq; -+ int reset_on_fork; -+ -+ /* may grab non-irq protected spin_locks */ -+ BUG_ON(in_interrupt()); -+recheck: -+ /* double check policy once rq lock held */ -+ if (policy < 0) { -+ reset_on_fork = p->sched_reset_on_fork; -+ policy = oldpolicy = p->policy; -+ } else { -+ reset_on_fork = !!(attr->sched_flags & SCHED_FLAG_RESET_ON_FORK); -+ -+ if (policy != SCHED_DEADLINE && -+ policy != SCHED_FIFO && policy != SCHED_RR && -+ policy != SCHED_NORMAL && policy != SCHED_BATCH && -+ policy != SCHED_IDLE) -+ return -EINVAL; -+ } -+ -+ if (attr->sched_flags & ~(SCHED_FLAG_RESET_ON_FORK)) -+ return -EINVAL; -+ -+ /* -+ * Valid priorities for SCHED_FIFO and SCHED_RR are -+ * 1..MAX_USER_RT_PRIO-1, valid priority for SCHED_NORMAL, -+ * SCHED_BATCH and SCHED_IDLE is 0. -+ */ -+ if ((p->mm && attr->sched_priority > MAX_USER_RT_PRIO-1) || -+ (!p->mm && attr->sched_priority > MAX_RT_PRIO-1)) -+ return -EINVAL; -+ if ((dl_policy(policy) && !__checkparam_dl(attr)) || -+ (rt_policy(policy) != (attr->sched_priority != 0))) -+ return -EINVAL; -+ -+ /* -+ * Allow unprivileged RT tasks to decrease priority: -+ */ -+ if (user && !capable(CAP_SYS_NICE)) { -+ if (fair_policy(policy)) { -+ if (attr->sched_nice < task_nice(p) && -+ !can_nice(p, attr->sched_nice)) -+ return -EPERM; -+ } -+ -+ if (rt_policy(policy)) { -+ unsigned long rlim_rtprio = -+ task_rlimit(p, RLIMIT_RTPRIO); -+ -+ /* can't set/change the rt policy */ -+ if (policy != p->policy && !rlim_rtprio) -+ return -EPERM; -+ -+ /* can't increase priority */ -+ if (attr->sched_priority > p->rt_priority && -+ attr->sched_priority > rlim_rtprio) -+ return -EPERM; -+ } -+ -+ /* -+ * Can't set/change SCHED_DEADLINE policy at all for now -+ * (safest behavior); in the future we would like to allow -+ * unprivileged DL tasks to increase their relative deadline -+ * or reduce their runtime (both ways reducing utilization) -+ */ -+ if (dl_policy(policy)) -+ return -EPERM; -+ -+ /* -+ * Treat SCHED_IDLE as nice 20. Only allow a switch to -+ * SCHED_NORMAL if the RLIMIT_NICE would normally permit it. -+ */ -+ if (p->policy == SCHED_IDLE && policy != SCHED_IDLE) { -+ if (!can_nice(p, task_nice(p))) -+ return -EPERM; -+ } -+ -+ /* can't change other user's priorities */ -+ if (!check_same_owner(p)) -+ return -EPERM; -+ -+ /* Normal users shall not reset the sched_reset_on_fork flag */ -+ if (p->sched_reset_on_fork && !reset_on_fork) -+ return -EPERM; -+ } -+ -+ if (user) { -+ retval = security_task_setscheduler(p); -+ if (retval) -+ return retval; -+ } -+ -+ /* -+ * make sure no PI-waiters arrive (or leave) while we are -+ * changing the priority of the task: -+ * -+ * To be able to change p->policy safely, the appropriate -+ * runqueue lock must be held. -+ */ -+ rq = task_rq_lock(p, &flags); -+ -+ /* -+ * Changing the policy of the stop threads its a very bad idea -+ */ -+ if (p == rq->stop) { -+ task_rq_unlock(rq, p, &flags); -+ return -EINVAL; -+ } -+ -+ /* -+ * If not changing anything there's no need to proceed further, -+ * but store a possible modification of reset_on_fork. -+ */ -+ if (unlikely(policy == p->policy)) { -+ if (fair_policy(policy) && attr->sched_nice != task_nice(p)) -+ goto change; -+ if (rt_policy(policy) && attr->sched_priority != p->rt_priority) -+ goto change; -+ if (dl_policy(policy) && dl_param_changed(p, attr)) -+ goto change; -+ -+ p->sched_reset_on_fork = reset_on_fork; -+ task_rq_unlock(rq, p, &flags); -+ return 0; -+ } -+change: -+ -+ if (user) { -+#ifdef CONFIG_RT_GROUP_SCHED -+ /* -+ * Do not allow realtime tasks into groups that have no runtime -+ * assigned. -+ */ -+ if (rt_bandwidth_enabled() && rt_policy(policy) && -+ task_group(p)->rt_bandwidth.rt_runtime == 0 && -+ !task_group_is_autogroup(task_group(p))) { -+ task_rq_unlock(rq, p, &flags); -+ return -EPERM; -+ } -+#endif -+#ifdef CONFIG_SMP -+ if (dl_bandwidth_enabled() && dl_policy(policy)) { -+ cpumask_t *span = rq->rd->span; -+ -+ /* -+ * Don't allow tasks with an affinity mask smaller than -+ * the entire root_domain to become SCHED_DEADLINE. We -+ * will also fail if there's no bandwidth available. -+ */ -+ if (!cpumask_subset(span, &p->cpus_allowed) || -+ rq->rd->dl_bw.bw == 0) { -+ task_rq_unlock(rq, p, &flags); -+ return -EPERM; -+ } -+ } -+#endif -+ } -+ -+ /* recheck policy now with rq lock held */ -+ if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) { -+ policy = oldpolicy = -1; -+ task_rq_unlock(rq, p, &flags); -+ goto recheck; -+ } -+ -+ /* -+ * If setscheduling to SCHED_DEADLINE (or changing the parameters -+ * of a SCHED_DEADLINE task) we need to check if enough bandwidth -+ * is available. -+ */ -+ if ((dl_policy(policy) || dl_task(p)) && dl_overflow(p, policy, attr)) { -+ task_rq_unlock(rq, p, &flags); -+ return -EBUSY; -+ } -+ -+ p->sched_reset_on_fork = reset_on_fork; -+ oldprio = p->prio; -+ -+ /* -+ * Take priority boosted tasks into account. If the new -+ * effective priority is unchanged, we just store the new -+ * normal parameters and do not touch the scheduler class and -+ * the runqueue. This will be done when the task deboost -+ * itself. -+ */ -+ new_effective_prio = rt_mutex_get_effective_prio(p, newprio); -+ if (new_effective_prio == oldprio) { -+ __setscheduler_params(p, attr); -+ task_rq_unlock(rq, p, &flags); -+ return 0; -+ } -+ -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ if (queued) -+ dequeue_task(rq, p, 0); -+ if (running) -+ put_prev_task(rq, p); -+ -+ prev_class = p->sched_class; -+ __setscheduler(rq, p, attr, true); -+ -+ if (running) -+ p->sched_class->set_curr_task(rq); -+ if (queued) { -+ /* -+ * We enqueue to tail when the priority of a task is -+ * increased (user space view). -+ */ -+ enqueue_task(rq, p, oldprio <= p->prio ? ENQUEUE_HEAD : 0); -+ } -+ -+ check_class_changed(rq, p, prev_class, oldprio); -+ task_rq_unlock(rq, p, &flags); -+ -+ rt_mutex_adjust_pi(p); -+ -+ return 0; -+} -+ -+static int _sched_setscheduler(struct task_struct *p, int policy, -+ const struct sched_param *param, bool check) -+{ -+ struct sched_attr attr = { -+ .sched_policy = policy, -+ .sched_priority = param->sched_priority, -+ .sched_nice = PRIO_TO_NICE(p->static_prio), -+ }; -+ -+ /* Fixup the legacy SCHED_RESET_ON_FORK hack. */ -+ if ((policy != SETPARAM_POLICY) && (policy & SCHED_RESET_ON_FORK)) { -+ attr.sched_flags |= SCHED_FLAG_RESET_ON_FORK; -+ policy &= ~SCHED_RESET_ON_FORK; -+ attr.sched_policy = policy; -+ } -+ -+ return __sched_setscheduler(p, &attr, check); -+} -+/** -+ * sched_setscheduler - change the scheduling policy and/or RT priority of a thread. -+ * @p: the task in question. -+ * @policy: new policy. -+ * @param: structure containing the new RT priority. -+ * -+ * Return: 0 on success. An error code otherwise. -+ * -+ * NOTE that the task may be already dead. -+ */ -+int sched_setscheduler(struct task_struct *p, int policy, -+ const struct sched_param *param) -+{ -+ return _sched_setscheduler(p, policy, param, true); -+} -+EXPORT_SYMBOL_GPL(sched_setscheduler); -+ -+int sched_setattr(struct task_struct *p, const struct sched_attr *attr) -+{ -+ return __sched_setscheduler(p, attr, true); -+} -+EXPORT_SYMBOL_GPL(sched_setattr); -+ -+/** -+ * sched_setscheduler_nocheck - change the scheduling policy and/or RT priority of a thread from kernelspace. -+ * @p: the task in question. -+ * @policy: new policy. -+ * @param: structure containing the new RT priority. -+ * -+ * Just like sched_setscheduler, only don't bother checking if the -+ * current context has permission. For example, this is needed in -+ * stop_machine(): we create temporary high priority worker threads, -+ * but our caller might not have that capability. -+ * -+ * Return: 0 on success. An error code otherwise. -+ */ -+int sched_setscheduler_nocheck(struct task_struct *p, int policy, -+ const struct sched_param *param) -+{ -+ return _sched_setscheduler(p, policy, param, false); -+} -+ -+static int -+do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param) -+{ -+ struct sched_param lparam; -+ struct task_struct *p; -+ int retval; -+ -+ if (!param || pid < 0) -+ return -EINVAL; -+ if (copy_from_user(&lparam, param, sizeof(struct sched_param))) -+ return -EFAULT; -+ -+ rcu_read_lock(); -+ retval = -ESRCH; -+ p = find_process_by_pid(pid); -+ if (p != NULL) -+ retval = sched_setscheduler(p, policy, &lparam); -+ rcu_read_unlock(); -+ -+ return retval; -+} -+ -+/* -+ * Mimics kernel/events/core.c perf_copy_attr(). -+ */ -+static int sched_copy_attr(struct sched_attr __user *uattr, -+ struct sched_attr *attr) -+{ -+ u32 size; -+ int ret; -+ -+ if (!access_ok(VERIFY_WRITE, uattr, SCHED_ATTR_SIZE_VER0)) -+ return -EFAULT; -+ -+ /* -+ * zero the full structure, so that a short copy will be nice. -+ */ -+ memset(attr, 0, sizeof(*attr)); -+ -+ ret = get_user(size, &uattr->size); -+ if (ret) -+ return ret; -+ -+ if (size > PAGE_SIZE) /* silly large */ -+ goto err_size; -+ -+ if (!size) /* abi compat */ -+ size = SCHED_ATTR_SIZE_VER0; -+ -+ if (size < SCHED_ATTR_SIZE_VER0) -+ goto err_size; -+ -+ /* -+ * If we're handed a bigger struct than we know of, -+ * ensure all the unknown bits are 0 - i.e. new -+ * user-space does not rely on any kernel feature -+ * extensions we dont know about yet. -+ */ -+ if (size > sizeof(*attr)) { -+ unsigned char __user *addr; -+ unsigned char __user *end; -+ unsigned char val; -+ -+ addr = (void __user *)uattr + sizeof(*attr); -+ end = (void __user *)uattr + size; -+ -+ for (; addr < end; addr++) { -+ ret = get_user(val, addr); -+ if (ret) -+ return ret; -+ if (val) -+ goto err_size; -+ } -+ size = sizeof(*attr); -+ } -+ -+ ret = copy_from_user(attr, uattr, size); -+ if (ret) -+ return -EFAULT; -+ -+ /* -+ * XXX: do we want to be lenient like existing syscalls; or do we want -+ * to be strict and return an error on out-of-bounds values? -+ */ -+ attr->sched_nice = clamp(attr->sched_nice, MIN_NICE, MAX_NICE); -+ -+ return 0; -+ -+err_size: -+ put_user(sizeof(*attr), &uattr->size); -+ return -E2BIG; -+} -+ -+/** -+ * sys_sched_setscheduler - set/change the scheduler policy and RT priority -+ * @pid: the pid in question. -+ * @policy: new policy. -+ * @param: structure containing the new RT priority. -+ * -+ * Return: 0 on success. An error code otherwise. -+ */ -+SYSCALL_DEFINE3(sched_setscheduler, pid_t, pid, int, policy, -+ struct sched_param __user *, param) -+{ -+ /* negative values for policy are not valid */ -+ if (policy < 0) -+ return -EINVAL; -+ -+ return do_sched_setscheduler(pid, policy, param); -+} -+ -+/** -+ * sys_sched_setparam - set/change the RT priority of a thread -+ * @pid: the pid in question. -+ * @param: structure containing the new RT priority. -+ * -+ * Return: 0 on success. An error code otherwise. -+ */ -+SYSCALL_DEFINE2(sched_setparam, pid_t, pid, struct sched_param __user *, param) -+{ -+ return do_sched_setscheduler(pid, SETPARAM_POLICY, param); -+} -+ -+/** -+ * sys_sched_setattr - same as above, but with extended sched_attr -+ * @pid: the pid in question. -+ * @uattr: structure containing the extended parameters. -+ * @flags: for future extension. -+ */ -+SYSCALL_DEFINE3(sched_setattr, pid_t, pid, struct sched_attr __user *, uattr, -+ unsigned int, flags) -+{ -+ struct sched_attr attr; -+ struct task_struct *p; -+ int retval; -+ -+ if (!uattr || pid < 0 || flags) -+ return -EINVAL; -+ -+ retval = sched_copy_attr(uattr, &attr); -+ if (retval) -+ return retval; -+ -+ if ((int)attr.sched_policy < 0) -+ return -EINVAL; -+ -+ rcu_read_lock(); -+ retval = -ESRCH; -+ p = find_process_by_pid(pid); -+ if (p != NULL) -+ retval = sched_setattr(p, &attr); -+ rcu_read_unlock(); -+ -+ return retval; -+} -+ -+/** -+ * sys_sched_getscheduler - get the policy (scheduling class) of a thread -+ * @pid: the pid in question. -+ * -+ * Return: On success, the policy of the thread. Otherwise, a negative error -+ * code. -+ */ -+SYSCALL_DEFINE1(sched_getscheduler, pid_t, pid) -+{ -+ struct task_struct *p; -+ int retval; -+ -+ if (pid < 0) -+ return -EINVAL; -+ -+ retval = -ESRCH; -+ rcu_read_lock(); -+ p = find_process_by_pid(pid); -+ if (p) { -+ retval = security_task_getscheduler(p); -+ if (!retval) -+ retval = p->policy -+ | (p->sched_reset_on_fork ? SCHED_RESET_ON_FORK : 0); -+ } -+ rcu_read_unlock(); -+ return retval; -+} -+ -+/** -+ * sys_sched_getparam - get the RT priority of a thread -+ * @pid: the pid in question. -+ * @param: structure containing the RT priority. -+ * -+ * Return: On success, 0 and the RT priority is in @param. Otherwise, an error -+ * code. -+ */ -+SYSCALL_DEFINE2(sched_getparam, pid_t, pid, struct sched_param __user *, param) -+{ -+ struct sched_param lp = { .sched_priority = 0 }; -+ struct task_struct *p; -+ int retval; -+ -+ if (!param || pid < 0) -+ return -EINVAL; -+ -+ rcu_read_lock(); -+ p = find_process_by_pid(pid); -+ retval = -ESRCH; -+ if (!p) -+ goto out_unlock; -+ -+ retval = security_task_getscheduler(p); -+ if (retval) -+ goto out_unlock; -+ -+ if (task_has_rt_policy(p)) -+ lp.sched_priority = p->rt_priority; -+ rcu_read_unlock(); -+ -+ /* -+ * This one might sleep, we cannot do it with a spinlock held ... -+ */ -+ retval = copy_to_user(param, &lp, sizeof(*param)) ? -EFAULT : 0; -+ -+ return retval; -+ -+out_unlock: -+ rcu_read_unlock(); -+ return retval; -+} -+ -+static int sched_read_attr(struct sched_attr __user *uattr, -+ struct sched_attr *attr, -+ unsigned int usize) -+{ -+ int ret; -+ -+ if (!access_ok(VERIFY_WRITE, uattr, usize)) -+ return -EFAULT; -+ -+ /* -+ * If we're handed a smaller struct than we know of, -+ * ensure all the unknown bits are 0 - i.e. old -+ * user-space does not get uncomplete information. -+ */ -+ if (usize < sizeof(*attr)) { -+ unsigned char *addr; -+ unsigned char *end; -+ -+ addr = (void *)attr + usize; -+ end = (void *)attr + sizeof(*attr); -+ -+ for (; addr < end; addr++) { -+ if (*addr) -+ return -EFBIG; -+ } -+ -+ attr->size = usize; -+ } -+ -+ ret = copy_to_user(uattr, attr, attr->size); -+ if (ret) -+ return -EFAULT; -+ -+ return 0; -+} -+ -+/** -+ * sys_sched_getattr - similar to sched_getparam, but with sched_attr -+ * @pid: the pid in question. -+ * @uattr: structure containing the extended parameters. -+ * @size: sizeof(attr) for fwd/bwd comp. -+ * @flags: for future extension. -+ */ -+SYSCALL_DEFINE4(sched_getattr, pid_t, pid, struct sched_attr __user *, uattr, -+ unsigned int, size, unsigned int, flags) -+{ -+ struct sched_attr attr = { -+ .size = sizeof(struct sched_attr), -+ }; -+ struct task_struct *p; -+ int retval; -+ -+ if (!uattr || pid < 0 || size > PAGE_SIZE || -+ size < SCHED_ATTR_SIZE_VER0 || flags) -+ return -EINVAL; -+ -+ rcu_read_lock(); -+ p = find_process_by_pid(pid); -+ retval = -ESRCH; -+ if (!p) -+ goto out_unlock; -+ -+ retval = security_task_getscheduler(p); -+ if (retval) -+ goto out_unlock; -+ -+ attr.sched_policy = p->policy; -+ if (p->sched_reset_on_fork) -+ attr.sched_flags |= SCHED_FLAG_RESET_ON_FORK; -+ if (task_has_dl_policy(p)) -+ __getparam_dl(p, &attr); -+ else if (task_has_rt_policy(p)) -+ attr.sched_priority = p->rt_priority; -+ else -+ attr.sched_nice = task_nice(p); -+ -+ rcu_read_unlock(); -+ -+ retval = sched_read_attr(uattr, &attr, size); -+ return retval; -+ -+out_unlock: -+ rcu_read_unlock(); -+ return retval; -+} -+ -+long sched_setaffinity(pid_t pid, const struct cpumask *in_mask) -+{ -+ cpumask_var_t cpus_allowed, new_mask; -+ struct task_struct *p; -+ int retval; -+ -+ rcu_read_lock(); -+ -+ p = find_process_by_pid(pid); -+ if (!p) { -+ rcu_read_unlock(); -+ return -ESRCH; -+ } -+ -+ /* Prevent p going away */ -+ get_task_struct(p); -+ rcu_read_unlock(); -+ -+ if (p->flags & PF_NO_SETAFFINITY) { -+ retval = -EINVAL; -+ goto out_put_task; -+ } -+ if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) { -+ retval = -ENOMEM; -+ goto out_put_task; -+ } -+ if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) { -+ retval = -ENOMEM; -+ goto out_free_cpus_allowed; -+ } -+ retval = -EPERM; -+ if (!check_same_owner(p)) { -+ rcu_read_lock(); -+ if (!ns_capable(__task_cred(p)->user_ns, CAP_SYS_NICE)) { -+ rcu_read_unlock(); -+ goto out_free_new_mask; -+ } -+ rcu_read_unlock(); -+ } -+ -+ retval = security_task_setscheduler(p); -+ if (retval) -+ goto out_free_new_mask; -+ -+ -+ cpuset_cpus_allowed(p, cpus_allowed); -+ cpumask_and(new_mask, in_mask, cpus_allowed); -+ -+ /* -+ * Since bandwidth control happens on root_domain basis, -+ * if admission test is enabled, we only admit -deadline -+ * tasks allowed to run on all the CPUs in the task's -+ * root_domain. -+ */ -+#ifdef CONFIG_SMP -+ if (task_has_dl_policy(p) && dl_bandwidth_enabled()) { -+ rcu_read_lock(); -+ if (!cpumask_subset(task_rq(p)->rd->span, new_mask)) { -+ retval = -EBUSY; -+ rcu_read_unlock(); -+ goto out_free_new_mask; -+ } -+ rcu_read_unlock(); -+ } -+#endif -+again: -+ retval = set_cpus_allowed_ptr(p, new_mask); -+ -+ if (!retval) { -+ cpuset_cpus_allowed(p, cpus_allowed); -+ if (!cpumask_subset(new_mask, cpus_allowed)) { -+ /* -+ * We must have raced with a concurrent cpuset -+ * update. Just reset the cpus_allowed to the -+ * cpuset's cpus_allowed -+ */ -+ cpumask_copy(new_mask, cpus_allowed); -+ goto again; -+ } -+ } -+out_free_new_mask: -+ free_cpumask_var(new_mask); -+out_free_cpus_allowed: -+ free_cpumask_var(cpus_allowed); -+out_put_task: -+ put_task_struct(p); -+ return retval; -+} -+ -+static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len, -+ struct cpumask *new_mask) -+{ -+ if (len < cpumask_size()) -+ cpumask_clear(new_mask); -+ else if (len > cpumask_size()) -+ len = cpumask_size(); -+ -+ return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0; -+} -+ -+/** -+ * sys_sched_setaffinity - set the cpu affinity of a process -+ * @pid: pid of the process -+ * @len: length in bytes of the bitmask pointed to by user_mask_ptr -+ * @user_mask_ptr: user-space pointer to the new cpu mask -+ * -+ * Return: 0 on success. An error code otherwise. -+ */ -+SYSCALL_DEFINE3(sched_setaffinity, pid_t, pid, unsigned int, len, -+ unsigned long __user *, user_mask_ptr) -+{ -+ cpumask_var_t new_mask; -+ int retval; -+ -+ if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) -+ return -ENOMEM; -+ -+ retval = get_user_cpu_mask(user_mask_ptr, len, new_mask); -+ if (retval == 0) -+ retval = sched_setaffinity(pid, new_mask); -+ free_cpumask_var(new_mask); -+ return retval; -+} -+ -+long sched_getaffinity(pid_t pid, struct cpumask *mask) -+{ -+ struct task_struct *p; -+ unsigned long flags; -+ int retval; -+ -+ rcu_read_lock(); -+ -+ retval = -ESRCH; -+ p = find_process_by_pid(pid); -+ if (!p) -+ goto out_unlock; -+ -+ retval = security_task_getscheduler(p); -+ if (retval) -+ goto out_unlock; -+ -+ raw_spin_lock_irqsave(&p->pi_lock, flags); -+ cpumask_and(mask, &p->cpus_allowed, cpu_active_mask); -+ raw_spin_unlock_irqrestore(&p->pi_lock, flags); -+ -+out_unlock: -+ rcu_read_unlock(); -+ -+ return retval; -+} -+ -+/** -+ * sys_sched_getaffinity - get the cpu affinity of a process -+ * @pid: pid of the process -+ * @len: length in bytes of the bitmask pointed to by user_mask_ptr -+ * @user_mask_ptr: user-space pointer to hold the current cpu mask -+ * -+ * Return: 0 on success. An error code otherwise. -+ */ -+SYSCALL_DEFINE3(sched_getaffinity, pid_t, pid, unsigned int, len, -+ unsigned long __user *, user_mask_ptr) -+{ -+ int ret; -+ cpumask_var_t mask; -+ -+ if ((len * BITS_PER_BYTE) < nr_cpu_ids) -+ return -EINVAL; -+ if (len & (sizeof(unsigned long)-1)) -+ return -EINVAL; -+ -+ if (!alloc_cpumask_var(&mask, GFP_KERNEL)) -+ return -ENOMEM; -+ -+ ret = sched_getaffinity(pid, mask); -+ if (ret == 0) { -+ size_t retlen = min_t(size_t, len, cpumask_size()); -+ -+ if (copy_to_user(user_mask_ptr, mask, retlen)) -+ ret = -EFAULT; -+ else -+ ret = retlen; -+ } -+ free_cpumask_var(mask); -+ -+ return ret; -+} -+ -+/** -+ * sys_sched_yield - yield the current processor to other threads. -+ * -+ * This function yields the current CPU to other tasks. If there are no -+ * other threads running on this CPU then this function will return. -+ * -+ * Return: 0. -+ */ -+SYSCALL_DEFINE0(sched_yield) -+{ -+ struct rq *rq = this_rq_lock(); -+ -+ schedstat_inc(rq, yld_count); -+ current->sched_class->yield_task(rq); -+ -+ /* -+ * Since we are going to call schedule() anyway, there's -+ * no need to preempt or enable interrupts: -+ */ -+ __release(rq->lock); -+ spin_release(&rq->lock.dep_map, 1, _THIS_IP_); -+ do_raw_spin_unlock(&rq->lock); -+ sched_preempt_enable_no_resched(); -+ -+ schedule(); -+ -+ return 0; -+} -+ -+int __sched _cond_resched(void) -+{ -+ if (should_resched()) { -+ preempt_schedule_common(); -+ return 1; -+ } -+ return 0; -+} -+EXPORT_SYMBOL(_cond_resched); -+ -+/* -+ * __cond_resched_lock() - if a reschedule is pending, drop the given lock, -+ * call schedule, and on return reacquire the lock. -+ * -+ * This works OK both with and without CONFIG_PREEMPT. We do strange low-level -+ * operations here to prevent schedule() from being called twice (once via -+ * spin_unlock(), once by hand). -+ */ -+int __cond_resched_lock(spinlock_t *lock) -+{ -+ int resched = should_resched(); -+ int ret = 0; -+ -+ lockdep_assert_held(lock); -+ -+ if (spin_needbreak(lock) || resched) { -+ spin_unlock(lock); -+ if (resched) -+ preempt_schedule_common(); -+ else -+ cpu_relax(); -+ ret = 1; -+ spin_lock(lock); -+ } -+ return ret; -+} -+EXPORT_SYMBOL(__cond_resched_lock); -+ -+int __sched __cond_resched_softirq(void) -+{ -+ BUG_ON(!in_softirq()); -+ -+ if (should_resched()) { -+ local_bh_enable(); -+ preempt_schedule_common(); -+ local_bh_disable(); -+ return 1; -+ } -+ return 0; -+} -+EXPORT_SYMBOL(__cond_resched_softirq); -+ -+/** -+ * yield - yield the current processor to other threads. -+ * -+ * Do not ever use this function, there's a 99% chance you're doing it wrong. -+ * -+ * The scheduler is at all times free to pick the calling task as the most -+ * eligible task to run, if removing the yield() call from your code breaks -+ * it, its already broken. -+ * -+ * Typical broken usage is: -+ * -+ * while (!event) -+ * yield(); -+ * -+ * where one assumes that yield() will let 'the other' process run that will -+ * make event true. If the current task is a SCHED_FIFO task that will never -+ * happen. Never use yield() as a progress guarantee!! -+ * -+ * If you want to use yield() to wait for something, use wait_event(). -+ * If you want to use yield() to be 'nice' for others, use cond_resched(). -+ * If you still want to use yield(), do not! -+ */ -+void __sched yield(void) -+{ -+ set_current_state(TASK_RUNNING); -+ sys_sched_yield(); -+} -+EXPORT_SYMBOL(yield); -+ -+/** -+ * yield_to - yield the current processor to another thread in -+ * your thread group, or accelerate that thread toward the -+ * processor it's on. -+ * @p: target task -+ * @preempt: whether task preemption is allowed or not -+ * -+ * It's the caller's job to ensure that the target task struct -+ * can't go away on us before we can do any checks. -+ * -+ * Return: -+ * true (>0) if we indeed boosted the target task. -+ * false (0) if we failed to boost the target. -+ * -ESRCH if there's no task to yield to. -+ */ -+int __sched yield_to(struct task_struct *p, bool preempt) -+{ -+ struct task_struct *curr = current; -+ struct rq *rq, *p_rq; -+ unsigned long flags; -+ int yielded = 0; -+ -+ local_irq_save(flags); -+ rq = this_rq(); -+ -+again: -+ p_rq = task_rq(p); -+ /* -+ * If we're the only runnable task on the rq and target rq also -+ * has only one task, there's absolutely no point in yielding. -+ */ -+ if (rq->nr_running == 1 && p_rq->nr_running == 1) { -+ yielded = -ESRCH; -+ goto out_irq; -+ } -+ -+ double_rq_lock(rq, p_rq); -+ if (task_rq(p) != p_rq) { -+ double_rq_unlock(rq, p_rq); -+ goto again; -+ } -+ -+ if (!curr->sched_class->yield_to_task) -+ goto out_unlock; -+ -+ if (curr->sched_class != p->sched_class) -+ goto out_unlock; -+ -+ if (task_running(p_rq, p) || p->state) -+ goto out_unlock; -+ -+ yielded = curr->sched_class->yield_to_task(rq, p, preempt); -+ if (yielded) { -+ schedstat_inc(rq, yld_count); -+ /* -+ * Make p's CPU reschedule; pick_next_entity takes care of -+ * fairness. -+ */ -+ if (preempt && rq != p_rq) -+ resched_curr(p_rq); -+ } -+ -+out_unlock: -+ double_rq_unlock(rq, p_rq); -+out_irq: -+ local_irq_restore(flags); -+ -+ if (yielded > 0) -+ schedule(); -+ -+ return yielded; -+} -+EXPORT_SYMBOL_GPL(yield_to); -+ -+/* -+ * This task is about to go to sleep on IO. Increment rq->nr_iowait so -+ * that process accounting knows that this is a task in IO wait state. -+ */ -+long __sched io_schedule_timeout(long timeout) -+{ -+ int old_iowait = current->in_iowait; -+ struct rq *rq; -+ long ret; -+ -+ current->in_iowait = 1; -+ blk_schedule_flush_plug(current); -+ -+ delayacct_blkio_start(); -+ rq = raw_rq(); -+ atomic_inc(&rq->nr_iowait); -+ ret = schedule_timeout(timeout); -+ current->in_iowait = old_iowait; -+ atomic_dec(&rq->nr_iowait); -+ delayacct_blkio_end(); -+ -+ return ret; -+} -+EXPORT_SYMBOL(io_schedule_timeout); -+ -+/** -+ * sys_sched_get_priority_max - return maximum RT priority. -+ * @policy: scheduling class. -+ * -+ * Return: On success, this syscall returns the maximum -+ * rt_priority that can be used by a given scheduling class. -+ * On failure, a negative error code is returned. -+ */ -+SYSCALL_DEFINE1(sched_get_priority_max, int, policy) -+{ -+ int ret = -EINVAL; -+ -+ switch (policy) { -+ case SCHED_FIFO: -+ case SCHED_RR: -+ ret = MAX_USER_RT_PRIO-1; -+ break; -+ case SCHED_DEADLINE: -+ case SCHED_NORMAL: -+ case SCHED_BATCH: -+ case SCHED_IDLE: -+ ret = 0; -+ break; -+ } -+ return ret; -+} -+ -+/** -+ * sys_sched_get_priority_min - return minimum RT priority. -+ * @policy: scheduling class. -+ * -+ * Return: On success, this syscall returns the minimum -+ * rt_priority that can be used by a given scheduling class. -+ * On failure, a negative error code is returned. -+ */ -+SYSCALL_DEFINE1(sched_get_priority_min, int, policy) -+{ -+ int ret = -EINVAL; -+ -+ switch (policy) { -+ case SCHED_FIFO: -+ case SCHED_RR: -+ ret = 1; -+ break; -+ case SCHED_DEADLINE: -+ case SCHED_NORMAL: -+ case SCHED_BATCH: -+ case SCHED_IDLE: -+ ret = 0; -+ } -+ return ret; -+} -+ -+/** -+ * sys_sched_rr_get_interval - return the default timeslice of a process. -+ * @pid: pid of the process. -+ * @interval: userspace pointer to the timeslice value. -+ * -+ * this syscall writes the default timeslice value of a given process -+ * into the user-space timespec buffer. A value of '0' means infinity. -+ * -+ * Return: On success, 0 and the timeslice is in @interval. Otherwise, -+ * an error code. -+ */ -+SYSCALL_DEFINE2(sched_rr_get_interval, pid_t, pid, -+ struct timespec __user *, interval) -+{ -+ struct task_struct *p; -+ unsigned int time_slice; -+ unsigned long flags; -+ struct rq *rq; -+ int retval; -+ struct timespec t; -+ -+ if (pid < 0) -+ return -EINVAL; -+ -+ retval = -ESRCH; -+ rcu_read_lock(); -+ p = find_process_by_pid(pid); -+ if (!p) -+ goto out_unlock; -+ -+ retval = security_task_getscheduler(p); -+ if (retval) -+ goto out_unlock; -+ -+ rq = task_rq_lock(p, &flags); -+ time_slice = 0; -+ if (p->sched_class->get_rr_interval) -+ time_slice = p->sched_class->get_rr_interval(rq, p); -+ task_rq_unlock(rq, p, &flags); -+ -+ rcu_read_unlock(); -+ jiffies_to_timespec(time_slice, &t); -+ retval = copy_to_user(interval, &t, sizeof(t)) ? -EFAULT : 0; -+ return retval; -+ -+out_unlock: -+ rcu_read_unlock(); -+ return retval; -+} -+ -+static const char stat_nam[] = TASK_STATE_TO_CHAR_STR; -+ -+void sched_show_task(struct task_struct *p) -+{ -+ unsigned long free = 0; -+ int ppid; -+ unsigned long state = p->state; -+ -+ if (state) -+ state = __ffs(state) + 1; -+ printk(KERN_INFO "%-15.15s %c", p->comm, -+ state < sizeof(stat_nam) - 1 ? stat_nam[state] : '?'); -+#if BITS_PER_LONG == 32 -+ if (state == TASK_RUNNING) -+ printk(KERN_CONT " running "); -+ else -+ printk(KERN_CONT " %08lx ", thread_saved_pc(p)); -+#else -+ if (state == TASK_RUNNING) -+ printk(KERN_CONT " running task "); -+ else -+ printk(KERN_CONT " %016lx ", thread_saved_pc(p)); -+#endif -+#ifdef CONFIG_DEBUG_STACK_USAGE -+ free = stack_not_used(p); -+#endif -+ ppid = 0; -+ rcu_read_lock(); -+ if (pid_alive(p)) -+ ppid = task_pid_nr(rcu_dereference(p->real_parent)); -+ rcu_read_unlock(); -+ printk(KERN_CONT "%5lu %5d %6d 0x%08lx\n", free, -+ task_pid_nr(p), ppid, -+ (unsigned long)task_thread_info(p)->flags); -+ -+ print_worker_info(KERN_INFO, p); -+ show_stack(p, NULL); -+} -+ -+void show_state_filter(unsigned long state_filter) -+{ -+ struct task_struct *g, *p; -+ -+#if BITS_PER_LONG == 32 -+ printk(KERN_INFO -+ " task PC stack pid father\n"); -+#else -+ printk(KERN_INFO -+ " task PC stack pid father\n"); -+#endif -+ rcu_read_lock(); -+ for_each_process_thread(g, p) { -+ /* -+ * reset the NMI-timeout, listing all files on a slow -+ * console might take a lot of time: -+ */ -+ touch_nmi_watchdog(); -+ if (!state_filter || (p->state & state_filter)) -+ sched_show_task(p); -+ } -+ -+ touch_all_softlockup_watchdogs(); -+ -+#ifdef CONFIG_SCHED_DEBUG -+ sysrq_sched_debug_show(); -+#endif -+ rcu_read_unlock(); -+ /* -+ * Only show locks if all tasks are dumped: -+ */ -+ if (!state_filter) -+ debug_show_all_locks(); -+} -+ -+void init_idle_bootup_task(struct task_struct *idle) -+{ -+ idle->sched_class = &idle_sched_class; -+} -+ -+/** -+ * init_idle - set up an idle thread for a given CPU -+ * @idle: task in question -+ * @cpu: cpu the idle task belongs to -+ * -+ * NOTE: this function does not set the idle thread's NEED_RESCHED -+ * flag, to make booting more robust. -+ */ -+void init_idle(struct task_struct *idle, int cpu) -+{ -+ struct rq *rq = cpu_rq(cpu); -+ unsigned long flags; -+ -+ raw_spin_lock_irqsave(&rq->lock, flags); -+ -+ __sched_fork(0, idle); -+ idle->state = TASK_RUNNING; -+ idle->se.exec_start = sched_clock(); -+ -+ do_set_cpus_allowed(idle, cpumask_of(cpu)); -+ /* -+ * We're having a chicken and egg problem, even though we are -+ * holding rq->lock, the cpu isn't yet set to this cpu so the -+ * lockdep check in task_group() will fail. -+ * -+ * Similar case to sched_fork(). / Alternatively we could -+ * use task_rq_lock() here and obtain the other rq->lock. -+ * -+ * Silence PROVE_RCU -+ */ -+ rcu_read_lock(); -+ __set_task_cpu(idle, cpu); -+ rcu_read_unlock(); -+ -+ rq->curr = rq->idle = idle; -+ idle->on_rq = TASK_ON_RQ_QUEUED; -+#if defined(CONFIG_SMP) -+ idle->on_cpu = 1; -+#endif -+ raw_spin_unlock_irqrestore(&rq->lock, flags); -+ -+ /* Set the preempt count _outside_ the spinlocks! */ -+ init_idle_preempt_count(idle, cpu); -+ -+ /* -+ * The idle tasks have their own, simple scheduling class: -+ */ -+ idle->sched_class = &idle_sched_class; -+ ftrace_graph_init_idle_task(idle, cpu); -+ vtime_init_idle(idle, cpu); -+#if defined(CONFIG_SMP) -+ sprintf(idle->comm, "%s/%d", INIT_TASK_COMM, cpu); -+#endif -+} -+ -+int cpuset_cpumask_can_shrink(const struct cpumask *cur, -+ const struct cpumask *trial) -+{ -+ int ret = 1, trial_cpus; -+ struct dl_bw *cur_dl_b; -+ unsigned long flags; -+ -+ if (!cpumask_weight(cur)) -+ return ret; -+ -+ rcu_read_lock_sched(); -+ cur_dl_b = dl_bw_of(cpumask_any(cur)); -+ trial_cpus = cpumask_weight(trial); -+ -+ raw_spin_lock_irqsave(&cur_dl_b->lock, flags); -+ if (cur_dl_b->bw != -1 && -+ cur_dl_b->bw * trial_cpus < cur_dl_b->total_bw) -+ ret = 0; -+ raw_spin_unlock_irqrestore(&cur_dl_b->lock, flags); -+ rcu_read_unlock_sched(); -+ -+ return ret; -+} -+ -+int task_can_attach(struct task_struct *p, -+ const struct cpumask *cs_cpus_allowed) -+{ -+ int ret = 0; -+ -+ /* -+ * Kthreads which disallow setaffinity shouldn't be moved -+ * to a new cpuset; we don't want to change their cpu -+ * affinity and isolating such threads by their set of -+ * allowed nodes is unnecessary. Thus, cpusets are not -+ * applicable for such threads. This prevents checking for -+ * success of set_cpus_allowed_ptr() on all attached tasks -+ * before cpus_allowed may be changed. -+ */ -+ if (p->flags & PF_NO_SETAFFINITY) { -+ ret = -EINVAL; -+ goto out; -+ } -+ -+#ifdef CONFIG_SMP -+ if (dl_task(p) && !cpumask_intersects(task_rq(p)->rd->span, -+ cs_cpus_allowed)) { -+ unsigned int dest_cpu = cpumask_any_and(cpu_active_mask, -+ cs_cpus_allowed); -+ struct dl_bw *dl_b; -+ bool overflow; -+ int cpus; -+ unsigned long flags; -+ -+ rcu_read_lock_sched(); -+ dl_b = dl_bw_of(dest_cpu); -+ raw_spin_lock_irqsave(&dl_b->lock, flags); -+ cpus = dl_bw_cpus(dest_cpu); -+ overflow = __dl_overflow(dl_b, cpus, 0, p->dl.dl_bw); -+ if (overflow) -+ ret = -EBUSY; -+ else { -+ /* -+ * We reserve space for this task in the destination -+ * root_domain, as we can't fail after this point. -+ * We will free resources in the source root_domain -+ * later on (see set_cpus_allowed_dl()). -+ */ -+ __dl_add(dl_b, p->dl.dl_bw); -+ } -+ raw_spin_unlock_irqrestore(&dl_b->lock, flags); -+ rcu_read_unlock_sched(); -+ -+ } -+#endif -+out: -+ return ret; -+} -+ -+#ifdef CONFIG_SMP -+/* -+ * move_queued_task - move a queued task to new rq. -+ * -+ * Returns (locked) new rq. Old rq's lock is released. -+ */ -+static struct rq *move_queued_task(struct task_struct *p, int new_cpu) -+{ -+ struct rq *rq = task_rq(p); -+ -+ lockdep_assert_held(&rq->lock); -+ -+ dequeue_task(rq, p, 0); -+ p->on_rq = TASK_ON_RQ_MIGRATING; -+ set_task_cpu(p, new_cpu); -+ raw_spin_unlock(&rq->lock); -+ -+ rq = cpu_rq(new_cpu); -+ -+ raw_spin_lock(&rq->lock); -+ BUG_ON(task_cpu(p) != new_cpu); -+ p->on_rq = TASK_ON_RQ_QUEUED; -+ enqueue_task(rq, p, 0); -+ check_preempt_curr(rq, p, 0); -+ -+ return rq; -+} -+ -+void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask) -+{ -+ if (p->sched_class->set_cpus_allowed) -+ p->sched_class->set_cpus_allowed(p, new_mask); -+ -+ cpumask_copy(&p->cpus_allowed, new_mask); -+ p->nr_cpus_allowed = cpumask_weight(new_mask); -+} -+ -+/* -+ * This is how migration works: -+ * -+ * 1) we invoke migration_cpu_stop() on the target CPU using -+ * stop_one_cpu(). -+ * 2) stopper starts to run (implicitly forcing the migrated thread -+ * off the CPU) -+ * 3) it checks whether the migrated task is still in the wrong runqueue. -+ * 4) if it's in the wrong runqueue then the migration thread removes -+ * it and puts it into the right queue. -+ * 5) stopper completes and stop_one_cpu() returns and the migration -+ * is done. -+ */ -+ -+/* -+ * Change a given task's CPU affinity. Migrate the thread to a -+ * proper CPU and schedule it away if the CPU it's executing on -+ * is removed from the allowed bitmask. -+ * -+ * NOTE: the caller must have a valid reference to the task, the -+ * task must not exit() & deallocate itself prematurely. The -+ * call is not atomic; no spinlocks may be held. -+ */ -+int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask) -+{ -+ unsigned long flags; -+ struct rq *rq; -+ unsigned int dest_cpu; -+ int ret = 0; -+ -+ rq = task_rq_lock(p, &flags); -+ -+ if (cpumask_equal(&p->cpus_allowed, new_mask)) -+ goto out; -+ -+ if (!cpumask_intersects(new_mask, cpu_active_mask)) { -+ ret = -EINVAL; -+ goto out; -+ } -+ -+ do_set_cpus_allowed(p, new_mask); -+ -+ /* Can the task run on the task's current CPU? If so, we're done */ -+ if (cpumask_test_cpu(task_cpu(p), new_mask)) -+ goto out; -+ -+ dest_cpu = cpumask_any_and(cpu_active_mask, new_mask); -+ if (task_running(rq, p) || p->state == TASK_WAKING) { -+ struct migration_arg arg = { p, dest_cpu }; -+ /* Need help from migration thread: drop lock and wait. */ -+ task_rq_unlock(rq, p, &flags); -+ stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg); -+ tlb_migrate_finish(p->mm); -+ return 0; -+ } else if (task_on_rq_queued(p)) -+ rq = move_queued_task(p, dest_cpu); -+out: -+ task_rq_unlock(rq, p, &flags); -+ -+ return ret; -+} -+EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr); -+ -+/* -+ * Move (not current) task off this cpu, onto dest cpu. We're doing -+ * this because either it can't run here any more (set_cpus_allowed() -+ * away from this CPU, or CPU going down), or because we're -+ * attempting to rebalance this task on exec (sched_exec). -+ * -+ * So we race with normal scheduler movements, but that's OK, as long -+ * as the task is no longer on this CPU. -+ * -+ * Returns non-zero if task was successfully migrated. -+ */ -+static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu) -+{ -+ struct rq *rq; -+ int ret = 0; -+ -+ if (unlikely(!cpu_active(dest_cpu))) -+ return ret; -+ -+ rq = cpu_rq(src_cpu); -+ -+ raw_spin_lock(&p->pi_lock); -+ raw_spin_lock(&rq->lock); -+ /* Already moved. */ -+ if (task_cpu(p) != src_cpu) -+ goto done; -+ -+ /* Affinity changed (again). */ -+ if (!cpumask_test_cpu(dest_cpu, tsk_cpus_allowed(p))) -+ goto fail; -+ -+ /* -+ * If we're not on a rq, the next wake-up will ensure we're -+ * placed properly. -+ */ -+ if (task_on_rq_queued(p)) -+ rq = move_queued_task(p, dest_cpu); -+done: -+ ret = 1; -+fail: -+ raw_spin_unlock(&rq->lock); -+ raw_spin_unlock(&p->pi_lock); -+ return ret; -+} -+ -+#ifdef CONFIG_NUMA_BALANCING -+/* Migrate current task p to target_cpu */ -+int migrate_task_to(struct task_struct *p, int target_cpu) -+{ -+ struct migration_arg arg = { p, target_cpu }; -+ int curr_cpu = task_cpu(p); -+ -+ if (curr_cpu == target_cpu) -+ return 0; -+ -+ if (!cpumask_test_cpu(target_cpu, tsk_cpus_allowed(p))) -+ return -EINVAL; -+ -+ /* TODO: This is not properly updating schedstats */ -+ -+ trace_sched_move_numa(p, curr_cpu, target_cpu); -+ return stop_one_cpu(curr_cpu, migration_cpu_stop, &arg); -+} -+ -+/* -+ * Requeue a task on a given node and accurately track the number of NUMA -+ * tasks on the runqueues -+ */ -+void sched_setnuma(struct task_struct *p, int nid) -+{ -+ struct rq *rq; -+ unsigned long flags; -+ bool queued, running; -+ -+ rq = task_rq_lock(p, &flags); -+ queued = task_on_rq_queued(p); -+ running = task_current(rq, p); -+ -+ if (queued) -+ dequeue_task(rq, p, 0); -+ if (running) -+ put_prev_task(rq, p); -+ -+ p->numa_preferred_nid = nid; -+ -+ if (running) -+ p->sched_class->set_curr_task(rq); -+ if (queued) -+ enqueue_task(rq, p, 0); -+ task_rq_unlock(rq, p, &flags); -+} -+#endif -+ -+/* -+ * migration_cpu_stop - this will be executed by a highprio stopper thread -+ * and performs thread migration by bumping thread off CPU then -+ * 'pushing' onto another runqueue. -+ */ -+static int migration_cpu_stop(void *data) -+{ -+ struct migration_arg *arg = data; -+ -+ /* -+ * The original target cpu might have gone down and we might -+ * be on another cpu but it doesn't matter. -+ */ -+ local_irq_disable(); -+ /* -+ * We need to explicitly wake pending tasks before running -+ * __migrate_task() such that we will not miss enforcing cpus_allowed -+ * during wakeups, see set_cpus_allowed_ptr()'s TASK_WAKING test. -+ */ -+ sched_ttwu_pending(); -+ __migrate_task(arg->task, raw_smp_processor_id(), arg->dest_cpu); -+ local_irq_enable(); -+ return 0; -+} -+ -+#ifdef CONFIG_HOTPLUG_CPU -+ -+/* -+ * Ensures that the idle task is using init_mm right before its cpu goes -+ * offline. -+ */ -+void idle_task_exit(void) -+{ -+ struct mm_struct *mm = current->active_mm; -+ -+ BUG_ON(cpu_online(smp_processor_id())); -+ -+ if (mm != &init_mm) { -+ switch_mm(mm, &init_mm, current); -+ finish_arch_post_lock_switch(); -+ } -+ mmdrop(mm); -+} -+ -+/* -+ * Since this CPU is going 'away' for a while, fold any nr_active delta -+ * we might have. Assumes we're called after migrate_tasks() so that the -+ * nr_active count is stable. -+ * -+ * Also see the comment "Global load-average calculations". -+ */ -+static void calc_load_migrate(struct rq *rq) -+{ -+ long delta = calc_load_fold_active(rq); -+ if (delta) -+ atomic_long_add(delta, &calc_load_tasks); -+} -+ -+static void put_prev_task_fake(struct rq *rq, struct task_struct *prev) -+{ -+} -+ -+static const struct sched_class fake_sched_class = { -+ .put_prev_task = put_prev_task_fake, -+}; -+ -+static struct task_struct fake_task = { -+ /* -+ * Avoid pull_{rt,dl}_task() -+ */ -+ .prio = MAX_PRIO + 1, -+ .sched_class = &fake_sched_class, -+}; -+ -+/* -+ * Migrate all tasks from the rq, sleeping tasks will be migrated by -+ * try_to_wake_up()->select_task_rq(). -+ * -+ * Called with rq->lock held even though we'er in stop_machine() and -+ * there's no concurrency possible, we hold the required locks anyway -+ * because of lock validation efforts. -+ */ -+static void migrate_tasks(unsigned int dead_cpu) -+{ -+ struct rq *rq = cpu_rq(dead_cpu); -+ struct task_struct *next, *stop = rq->stop; -+ int dest_cpu; -+ -+ /* -+ * Fudge the rq selection such that the below task selection loop -+ * doesn't get stuck on the currently eligible stop task. -+ * -+ * We're currently inside stop_machine() and the rq is either stuck -+ * in the stop_machine_cpu_stop() loop, or we're executing this code, -+ * either way we should never end up calling schedule() until we're -+ * done here. -+ */ -+ rq->stop = NULL; -+ -+ /* -+ * put_prev_task() and pick_next_task() sched -+ * class method both need to have an up-to-date -+ * value of rq->clock[_task] -+ */ -+ update_rq_clock(rq); -+ -+ for ( ; ; ) { -+ /* -+ * There's this thread running, bail when that's the only -+ * remaining thread. -+ */ -+ if (rq->nr_running == 1) -+ break; -+ -+ next = pick_next_task(rq, &fake_task); -+ BUG_ON(!next); -+ next->sched_class->put_prev_task(rq, next); -+ -+ /* Find suitable destination for @next, with force if needed. */ -+ dest_cpu = select_fallback_rq(dead_cpu, next); -+ raw_spin_unlock(&rq->lock); -+ -+ __migrate_task(next, dead_cpu, dest_cpu); -+ -+ raw_spin_lock(&rq->lock); -+ } -+ -+ rq->stop = stop; -+} -+ -+#endif /* CONFIG_HOTPLUG_CPU */ -+ -+#if defined(CONFIG_SCHED_DEBUG) && defined(CONFIG_SYSCTL) -+ -+static struct ctl_table sd_ctl_dir[] = { -+ { -+ .procname = "sched_domain", -+ .mode = 0555, -+ }, -+ {} -+}; -+ -+static struct ctl_table sd_ctl_root[] = { -+ { -+ .procname = "kernel", -+ .mode = 0555, -+ .child = sd_ctl_dir, -+ }, -+ {} -+}; -+ -+static struct ctl_table *sd_alloc_ctl_entry(int n) -+{ -+ struct ctl_table *entry = -+ kcalloc(n, sizeof(struct ctl_table), GFP_KERNEL); -+ -+ return entry; -+} -+ -+static void sd_free_ctl_entry(struct ctl_table **tablep) -+{ -+ struct ctl_table *entry; -+ -+ /* -+ * In the intermediate directories, both the child directory and -+ * procname are dynamically allocated and could fail but the mode -+ * will always be set. In the lowest directory the names are -+ * static strings and all have proc handlers. -+ */ -+ for (entry = *tablep; entry->mode; entry++) { -+ if (entry->child) -+ sd_free_ctl_entry(&entry->child); -+ if (entry->proc_handler == NULL) -+ kfree(entry->procname); -+ } -+ -+ kfree(*tablep); -+ *tablep = NULL; -+} -+ -+static int min_load_idx = 0; -+static int max_load_idx = CPU_LOAD_IDX_MAX-1; -+ -+static void -+set_table_entry(struct ctl_table *entry, -+ const char *procname, void *data, int maxlen, -+ umode_t mode, proc_handler *proc_handler, -+ bool load_idx) -+{ -+ entry->procname = procname; -+ entry->data = data; -+ entry->maxlen = maxlen; -+ entry->mode = mode; -+ entry->proc_handler = proc_handler; -+ -+ if (load_idx) { -+ entry->extra1 = &min_load_idx; -+ entry->extra2 = &max_load_idx; -+ } -+} -+ -+static struct ctl_table * -+sd_alloc_ctl_domain_table(struct sched_domain *sd) -+{ -+ struct ctl_table *table = sd_alloc_ctl_entry(14); -+ -+ if (table == NULL) -+ return NULL; -+ -+ set_table_entry(&table[0], "min_interval", &sd->min_interval, -+ sizeof(long), 0644, proc_doulongvec_minmax, false); -+ set_table_entry(&table[1], "max_interval", &sd->max_interval, -+ sizeof(long), 0644, proc_doulongvec_minmax, false); -+ set_table_entry(&table[2], "busy_idx", &sd->busy_idx, -+ sizeof(int), 0644, proc_dointvec_minmax, true); -+ set_table_entry(&table[3], "idle_idx", &sd->idle_idx, -+ sizeof(int), 0644, proc_dointvec_minmax, true); -+ set_table_entry(&table[4], "newidle_idx", &sd->newidle_idx, -+ sizeof(int), 0644, proc_dointvec_minmax, true); -+ set_table_entry(&table[5], "wake_idx", &sd->wake_idx, -+ sizeof(int), 0644, proc_dointvec_minmax, true); -+ set_table_entry(&table[6], "forkexec_idx", &sd->forkexec_idx, -+ sizeof(int), 0644, proc_dointvec_minmax, true); -+ set_table_entry(&table[7], "busy_factor", &sd->busy_factor, -+ sizeof(int), 0644, proc_dointvec_minmax, false); -+ set_table_entry(&table[8], "imbalance_pct", &sd->imbalance_pct, -+ sizeof(int), 0644, proc_dointvec_minmax, false); -+ set_table_entry(&table[9], "cache_nice_tries", -+ &sd->cache_nice_tries, -+ sizeof(int), 0644, proc_dointvec_minmax, false); -+ set_table_entry(&table[10], "flags", &sd->flags, -+ sizeof(int), 0644, proc_dointvec_minmax, false); -+ set_table_entry(&table[11], "max_newidle_lb_cost", -+ &sd->max_newidle_lb_cost, -+ sizeof(long), 0644, proc_doulongvec_minmax, false); -+ set_table_entry(&table[12], "name", sd->name, -+ CORENAME_MAX_SIZE, 0444, proc_dostring, false); -+ /* &table[13] is terminator */ -+ -+ return table; -+} -+ -+static struct ctl_table *sd_alloc_ctl_cpu_table(int cpu) -+{ -+ struct ctl_table *entry, *table; -+ struct sched_domain *sd; -+ int domain_num = 0, i; -+ char buf[32]; -+ -+ for_each_domain(cpu, sd) -+ domain_num++; -+ entry = table = sd_alloc_ctl_entry(domain_num + 1); -+ if (table == NULL) -+ return NULL; -+ -+ i = 0; -+ for_each_domain(cpu, sd) { -+ snprintf(buf, 32, "domain%d", i); -+ entry->procname = kstrdup(buf, GFP_KERNEL); -+ entry->mode = 0555; -+ entry->child = sd_alloc_ctl_domain_table(sd); -+ entry++; -+ i++; -+ } -+ return table; -+} -+ -+static struct ctl_table_header *sd_sysctl_header; -+static void register_sched_domain_sysctl(void) -+{ -+ int i, cpu_num = num_possible_cpus(); -+ struct ctl_table *entry = sd_alloc_ctl_entry(cpu_num + 1); -+ char buf[32]; -+ -+ WARN_ON(sd_ctl_dir[0].child); -+ sd_ctl_dir[0].child = entry; -+ -+ if (entry == NULL) -+ return; -+ -+ for_each_possible_cpu(i) { -+ snprintf(buf, 32, "cpu%d", i); -+ entry->procname = kstrdup(buf, GFP_KERNEL); -+ entry->mode = 0555; -+ entry->child = sd_alloc_ctl_cpu_table(i); -+ entry++; -+ } -+ -+ WARN_ON(sd_sysctl_header); -+ sd_sysctl_header = register_sysctl_table(sd_ctl_root); -+} -+ -+/* may be called multiple times per register */ -+static void unregister_sched_domain_sysctl(void) -+{ -+ if (sd_sysctl_header) -+ unregister_sysctl_table(sd_sysctl_header); -+ sd_sysctl_header = NULL; -+ if (sd_ctl_dir[0].child) -+ sd_free_ctl_entry(&sd_ctl_dir[0].child); -+} -+#else -+static void register_sched_domain_sysctl(void) -+{ -+} -+static void unregister_sched_domain_sysctl(void) -+{ -+} -+#endif -+ -+static void set_rq_online(struct rq *rq) -+{ -+ if (!rq->online) { -+ const struct sched_class *class; -+ -+ cpumask_set_cpu(rq->cpu, rq->rd->online); -+ rq->online = 1; -+ -+ for_each_class(class) { -+ if (class->rq_online) -+ class->rq_online(rq); -+ } -+ } -+} -+ -+static void set_rq_offline(struct rq *rq) -+{ -+ if (rq->online) { -+ const struct sched_class *class; -+ -+ for_each_class(class) { -+ if (class->rq_offline) -+ class->rq_offline(rq); -+ } -+ -+ cpumask_clear_cpu(rq->cpu, rq->rd->online); -+ rq->online = 0; -+ } -+} -+ -+/* -+ * migration_call - callback that gets triggered when a CPU is added. -+ * Here we can start up the necessary migration thread for the new CPU. -+ */ -+static int -+migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu) -+{ -+ int cpu = (long)hcpu; -+ unsigned long flags; -+ struct rq *rq = cpu_rq(cpu); -+ -+ switch (action & ~CPU_TASKS_FROZEN) { -+ -+ case CPU_UP_PREPARE: -+ rq->calc_load_update = calc_load_update; -+ break; -+ -+ case CPU_ONLINE: -+ /* Update our root-domain */ -+ raw_spin_lock_irqsave(&rq->lock, flags); -+ if (rq->rd) { -+ BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span)); -+ -+ set_rq_online(rq); -+ } -+ raw_spin_unlock_irqrestore(&rq->lock, flags); -+ break; -+ -+#ifdef CONFIG_HOTPLUG_CPU -+ case CPU_DYING: -+ sched_ttwu_pending(); -+ /* Update our root-domain */ -+ raw_spin_lock_irqsave(&rq->lock, flags); -+ if (rq->rd) { -+ BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span)); -+ set_rq_offline(rq); -+ } -+ migrate_tasks(cpu); -+ BUG_ON(rq->nr_running != 1); /* the migration thread */ -+ raw_spin_unlock_irqrestore(&rq->lock, flags); -+ break; -+ -+ case CPU_DEAD: -+ calc_load_migrate(rq); -+ break; -+#endif -+ } -+ -+ update_max_interval(); -+ -+ return NOTIFY_OK; -+} -+ -+/* -+ * Register at high priority so that task migration (migrate_all_tasks) -+ * happens before everything else. This has to be lower priority than -+ * the notifier in the perf_event subsystem, though. -+ */ -+static struct notifier_block migration_notifier = { -+ .notifier_call = migration_call, -+ .priority = CPU_PRI_MIGRATION, -+}; -+ -+static void __cpuinit set_cpu_rq_start_time(void) -+{ -+ int cpu = smp_processor_id(); -+ struct rq *rq = cpu_rq(cpu); -+ rq->age_stamp = sched_clock_cpu(cpu); -+} -+ -+static int sched_cpu_active(struct notifier_block *nfb, -+ unsigned long action, void *hcpu) -+{ -+ switch (action & ~CPU_TASKS_FROZEN) { -+ case CPU_STARTING: -+ set_cpu_rq_start_time(); -+ return NOTIFY_OK; -+ case CPU_ONLINE: -+ /* -+ * At this point a starting CPU has marked itself as online via -+ * set_cpu_online(). But it might not yet have marked itself -+ * as active, which is essential from here on. -+ * -+ * Thus, fall-through and help the starting CPU along. -+ */ -+ case CPU_DOWN_FAILED: -+ set_cpu_active((long)hcpu, true); -+ return NOTIFY_OK; -+ default: -+ return NOTIFY_DONE; -+ } -+} -+ -+static int sched_cpu_inactive(struct notifier_block *nfb, -+ unsigned long action, void *hcpu) -+{ -+ switch (action & ~CPU_TASKS_FROZEN) { -+ case CPU_DOWN_PREPARE: -+ set_cpu_active((long)hcpu, false); -+ return NOTIFY_OK; -+ default: -+ return NOTIFY_DONE; -+ } -+} -+ -+static int __init migration_init(void) -+{ -+ void *cpu = (void *)(long)smp_processor_id(); -+ int err; -+ -+ /* Initialize migration for the boot CPU */ -+ err = migration_call(&migration_notifier, CPU_UP_PREPARE, cpu); -+ BUG_ON(err == NOTIFY_BAD); -+ migration_call(&migration_notifier, CPU_ONLINE, cpu); -+ register_cpu_notifier(&migration_notifier); -+ -+ /* Register cpu active notifiers */ -+ cpu_notifier(sched_cpu_active, CPU_PRI_SCHED_ACTIVE); -+ cpu_notifier(sched_cpu_inactive, CPU_PRI_SCHED_INACTIVE); -+ -+ return 0; -+} -+early_initcall(migration_init); -+#endif -+ -+#ifdef CONFIG_SMP -+ -+static cpumask_var_t sched_domains_tmpmask; /* sched_domains_mutex */ -+ -+#ifdef CONFIG_SCHED_DEBUG -+ -+static __read_mostly int sched_debug_enabled; -+ -+static int __init sched_debug_setup(char *str) -+{ -+ sched_debug_enabled = 1; -+ -+ return 0; -+} -+early_param("sched_debug", sched_debug_setup); -+ -+static inline bool sched_debug(void) -+{ -+ return sched_debug_enabled; -+} -+ -+static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level, -+ struct cpumask *groupmask) -+{ -+ struct sched_group *group = sd->groups; -+ -+ cpumask_clear(groupmask); -+ -+ printk(KERN_DEBUG "%*s domain %d: ", level, "", level); -+ -+ if (!(sd->flags & SD_LOAD_BALANCE)) { -+ printk("does not load-balance\n"); -+ if (sd->parent) -+ printk(KERN_ERR "ERROR: !SD_LOAD_BALANCE domain" -+ " has parent"); -+ return -1; -+ } -+ -+ printk(KERN_CONT "span %*pbl level %s\n", -+ cpumask_pr_args(sched_domain_span(sd)), sd->name); -+ -+ if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) { -+ printk(KERN_ERR "ERROR: domain->span does not contain " -+ "CPU%d\n", cpu); -+ } -+ if (!cpumask_test_cpu(cpu, sched_group_cpus(group))) { -+ printk(KERN_ERR "ERROR: domain->groups does not contain" -+ " CPU%d\n", cpu); -+ } -+ -+ printk(KERN_DEBUG "%*s groups:", level + 1, ""); -+ do { -+ if (!group) { -+ printk("\n"); -+ printk(KERN_ERR "ERROR: group is NULL\n"); -+ break; -+ } -+ -+ if (!cpumask_weight(sched_group_cpus(group))) { -+ printk(KERN_CONT "\n"); -+ printk(KERN_ERR "ERROR: empty group\n"); -+ break; -+ } -+ -+ if (!(sd->flags & SD_OVERLAP) && -+ cpumask_intersects(groupmask, sched_group_cpus(group))) { -+ printk(KERN_CONT "\n"); -+ printk(KERN_ERR "ERROR: repeated CPUs\n"); -+ break; -+ } -+ -+ cpumask_or(groupmask, groupmask, sched_group_cpus(group)); -+ -+ printk(KERN_CONT " %*pbl", -+ cpumask_pr_args(sched_group_cpus(group))); -+ if (group->sgc->capacity != SCHED_CAPACITY_SCALE) { -+ printk(KERN_CONT " (cpu_capacity = %d)", -+ group->sgc->capacity); -+ } -+ -+ group = group->next; -+ } while (group != sd->groups); -+ printk(KERN_CONT "\n"); -+ -+ if (!cpumask_equal(sched_domain_span(sd), groupmask)) -+ printk(KERN_ERR "ERROR: groups don't span domain->span\n"); -+ -+ if (sd->parent && -+ !cpumask_subset(groupmask, sched_domain_span(sd->parent))) -+ printk(KERN_ERR "ERROR: parent span is not a superset " -+ "of domain->span\n"); -+ return 0; -+} -+ -+static void sched_domain_debug(struct sched_domain *sd, int cpu) -+{ -+ int level = 0; -+ -+ if (!sched_debug_enabled) -+ return; -+ -+ if (!sd) { -+ printk(KERN_DEBUG "CPU%d attaching NULL sched-domain.\n", cpu); -+ return; -+ } -+ -+ printk(KERN_DEBUG "CPU%d attaching sched-domain:\n", cpu); -+ -+ for (;;) { -+ if (sched_domain_debug_one(sd, cpu, level, sched_domains_tmpmask)) -+ break; -+ level++; -+ sd = sd->parent; -+ if (!sd) -+ break; -+ } -+} -+#else /* !CONFIG_SCHED_DEBUG */ -+# define sched_domain_debug(sd, cpu) do { } while (0) -+static inline bool sched_debug(void) -+{ -+ return false; -+} -+#endif /* CONFIG_SCHED_DEBUG */ -+ -+static int sd_degenerate(struct sched_domain *sd) -+{ -+ if (cpumask_weight(sched_domain_span(sd)) == 1) -+ return 1; -+ -+ /* Following flags need at least 2 groups */ -+ if (sd->flags & (SD_LOAD_BALANCE | -+ SD_BALANCE_NEWIDLE | -+ SD_BALANCE_FORK | -+ SD_BALANCE_EXEC | -+ SD_SHARE_CPUCAPACITY | -+ SD_SHARE_PKG_RESOURCES | -+ SD_SHARE_POWERDOMAIN)) { -+ if (sd->groups != sd->groups->next) -+ return 0; -+ } -+ -+ /* Following flags don't use groups */ -+ if (sd->flags & (SD_WAKE_AFFINE)) -+ return 0; -+ -+ return 1; -+} -+ -+static int -+sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent) -+{ -+ unsigned long cflags = sd->flags, pflags = parent->flags; -+ -+ if (sd_degenerate(parent)) -+ return 1; -+ -+ if (!cpumask_equal(sched_domain_span(sd), sched_domain_span(parent))) -+ return 0; -+ -+ /* Flags needing groups don't count if only 1 group in parent */ -+ if (parent->groups == parent->groups->next) { -+ pflags &= ~(SD_LOAD_BALANCE | -+ SD_BALANCE_NEWIDLE | -+ SD_BALANCE_FORK | -+ SD_BALANCE_EXEC | -+ SD_SHARE_CPUCAPACITY | -+ SD_SHARE_PKG_RESOURCES | -+ SD_PREFER_SIBLING | -+ SD_SHARE_POWERDOMAIN); -+ if (nr_node_ids == 1) -+ pflags &= ~SD_SERIALIZE; -+ } -+ if (~cflags & pflags) -+ return 0; -+ -+ return 1; -+} -+ -+static void free_rootdomain(struct rcu_head *rcu) -+{ -+ struct root_domain *rd = container_of(rcu, struct root_domain, rcu); -+ -+ cpupri_cleanup(&rd->cpupri); -+ cpudl_cleanup(&rd->cpudl); -+ free_cpumask_var(rd->dlo_mask); -+ free_cpumask_var(rd->rto_mask); -+ free_cpumask_var(rd->online); -+ free_cpumask_var(rd->span); -+ kfree(rd); -+} -+ -+static void rq_attach_root(struct rq *rq, struct root_domain *rd) -+{ -+ struct root_domain *old_rd = NULL; -+ unsigned long flags; -+ -+ raw_spin_lock_irqsave(&rq->lock, flags); -+ -+ if (rq->rd) { -+ old_rd = rq->rd; -+ -+ if (cpumask_test_cpu(rq->cpu, old_rd->online)) -+ set_rq_offline(rq); -+ -+ cpumask_clear_cpu(rq->cpu, old_rd->span); -+ -+ /* -+ * If we dont want to free the old_rd yet then -+ * set old_rd to NULL to skip the freeing later -+ * in this function: -+ */ -+ if (!atomic_dec_and_test(&old_rd->refcount)) -+ old_rd = NULL; -+ } -+ -+ atomic_inc(&rd->refcount); -+ rq->rd = rd; -+ -+ cpumask_set_cpu(rq->cpu, rd->span); -+ if (cpumask_test_cpu(rq->cpu, cpu_active_mask)) -+ set_rq_online(rq); -+ -+ raw_spin_unlock_irqrestore(&rq->lock, flags); -+ -+ if (old_rd) -+ call_rcu_sched(&old_rd->rcu, free_rootdomain); -+} -+ -+static int init_rootdomain(struct root_domain *rd) -+{ -+ memset(rd, 0, sizeof(*rd)); -+ -+ if (!alloc_cpumask_var(&rd->span, GFP_KERNEL)) -+ goto out; -+ if (!alloc_cpumask_var(&rd->online, GFP_KERNEL)) -+ goto free_span; -+ if (!alloc_cpumask_var(&rd->dlo_mask, GFP_KERNEL)) -+ goto free_online; -+ if (!alloc_cpumask_var(&rd->rto_mask, GFP_KERNEL)) -+ goto free_dlo_mask; -+ -+ init_dl_bw(&rd->dl_bw); -+ if (cpudl_init(&rd->cpudl) != 0) -+ goto free_dlo_mask; -+ -+ if (cpupri_init(&rd->cpupri) != 0) -+ goto free_rto_mask; -+ return 0; -+ -+free_rto_mask: -+ free_cpumask_var(rd->rto_mask); -+free_dlo_mask: -+ free_cpumask_var(rd->dlo_mask); -+free_online: -+ free_cpumask_var(rd->online); -+free_span: -+ free_cpumask_var(rd->span); -+out: -+ return -ENOMEM; -+} -+ -+/* -+ * By default the system creates a single root-domain with all cpus as -+ * members (mimicking the global state we have today). -+ */ -+struct root_domain def_root_domain; -+ -+static void init_defrootdomain(void) -+{ -+ init_rootdomain(&def_root_domain); -+ -+ atomic_set(&def_root_domain.refcount, 1); -+} -+ -+static struct root_domain *alloc_rootdomain(void) -+{ -+ struct root_domain *rd; -+ -+ rd = kmalloc(sizeof(*rd), GFP_KERNEL); -+ if (!rd) -+ return NULL; -+ -+ if (init_rootdomain(rd) != 0) { -+ kfree(rd); -+ return NULL; -+ } -+ -+ return rd; -+} -+ -+static void free_sched_groups(struct sched_group *sg, int free_sgc) -+{ -+ struct sched_group *tmp, *first; -+ -+ if (!sg) -+ return; -+ -+ first = sg; -+ do { -+ tmp = sg->next; -+ -+ if (free_sgc && atomic_dec_and_test(&sg->sgc->ref)) -+ kfree(sg->sgc); -+ -+ kfree(sg); -+ sg = tmp; -+ } while (sg != first); -+} -+ -+static void free_sched_domain(struct rcu_head *rcu) -+{ -+ struct sched_domain *sd = container_of(rcu, struct sched_domain, rcu); -+ -+ /* -+ * If its an overlapping domain it has private groups, iterate and -+ * nuke them all. -+ */ -+ if (sd->flags & SD_OVERLAP) { -+ free_sched_groups(sd->groups, 1); -+ } else if (atomic_dec_and_test(&sd->groups->ref)) { -+ kfree(sd->groups->sgc); -+ kfree(sd->groups); -+ } -+ kfree(sd); -+} -+ -+static void destroy_sched_domain(struct sched_domain *sd, int cpu) -+{ -+ call_rcu(&sd->rcu, free_sched_domain); -+} -+ -+static void destroy_sched_domains(struct sched_domain *sd, int cpu) -+{ -+ for (; sd; sd = sd->parent) -+ destroy_sched_domain(sd, cpu); -+} -+ -+/* -+ * Keep a special pointer to the highest sched_domain that has -+ * SD_SHARE_PKG_RESOURCE set (Last Level Cache Domain) for this -+ * allows us to avoid some pointer chasing select_idle_sibling(). -+ * -+ * Also keep a unique ID per domain (we use the first cpu number in -+ * the cpumask of the domain), this allows us to quickly tell if -+ * two cpus are in the same cache domain, see cpus_share_cache(). -+ */ -+DEFINE_PER_CPU(struct sched_domain *, sd_llc); -+DEFINE_PER_CPU(int, sd_llc_size); -+DEFINE_PER_CPU(int, sd_llc_id); -+DEFINE_PER_CPU(struct sched_domain *, sd_numa); -+DEFINE_PER_CPU(struct sched_domain *, sd_busy); -+DEFINE_PER_CPU(struct sched_domain *, sd_asym); -+ -+static void update_top_cache_domain(int cpu) -+{ -+ struct sched_domain *sd; -+ struct sched_domain *busy_sd = NULL; -+ int id = cpu; -+ int size = 1; -+ -+ sd = highest_flag_domain(cpu, SD_SHARE_PKG_RESOURCES); -+ if (sd) { -+ id = cpumask_first(sched_domain_span(sd)); -+ size = cpumask_weight(sched_domain_span(sd)); -+ busy_sd = sd->parent; /* sd_busy */ -+ } -+ rcu_assign_pointer(per_cpu(sd_busy, cpu), busy_sd); -+ -+ rcu_assign_pointer(per_cpu(sd_llc, cpu), sd); -+ per_cpu(sd_llc_size, cpu) = size; -+ per_cpu(sd_llc_id, cpu) = id; -+ -+ sd = lowest_flag_domain(cpu, SD_NUMA); -+ rcu_assign_pointer(per_cpu(sd_numa, cpu), sd); -+ -+ sd = highest_flag_domain(cpu, SD_ASYM_PACKING); -+ rcu_assign_pointer(per_cpu(sd_asym, cpu), sd); -+} -+ -+/* -+ * Attach the domain 'sd' to 'cpu' as its base domain. Callers must -+ * hold the hotplug lock. -+ */ -+static void -+cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu) -+{ -+ struct rq *rq = cpu_rq(cpu); -+ struct sched_domain *tmp; -+ -+ /* Remove the sched domains which do not contribute to scheduling. */ -+ for (tmp = sd; tmp; ) { -+ struct sched_domain *parent = tmp->parent; -+ if (!parent) -+ break; -+ -+ if (sd_parent_degenerate(tmp, parent)) { -+ tmp->parent = parent->parent; -+ if (parent->parent) -+ parent->parent->child = tmp; -+ /* -+ * Transfer SD_PREFER_SIBLING down in case of a -+ * degenerate parent; the spans match for this -+ * so the property transfers. -+ */ -+ if (parent->flags & SD_PREFER_SIBLING) -+ tmp->flags |= SD_PREFER_SIBLING; -+ destroy_sched_domain(parent, cpu); -+ } else -+ tmp = tmp->parent; -+ } -+ -+ if (sd && sd_degenerate(sd)) { -+ tmp = sd; -+ sd = sd->parent; -+ destroy_sched_domain(tmp, cpu); -+ if (sd) -+ sd->child = NULL; -+ } -+ -+ sched_domain_debug(sd, cpu); -+ -+ rq_attach_root(rq, rd); -+ tmp = rq->sd; -+ rcu_assign_pointer(rq->sd, sd); -+ destroy_sched_domains(tmp, cpu); -+ -+ update_top_cache_domain(cpu); -+} -+ -+/* Setup the mask of cpus configured for isolated domains */ -+static int __init isolated_cpu_setup(char *str) -+{ -+ alloc_bootmem_cpumask_var(&cpu_isolated_map); -+ cpulist_parse(str, cpu_isolated_map); -+ return 1; -+} -+ -+__setup("isolcpus=", isolated_cpu_setup); -+ -+struct s_data { -+ struct sched_domain ** __percpu sd; -+ struct root_domain *rd; -+}; -+ -+enum s_alloc { -+ sa_rootdomain, -+ sa_sd, -+ sa_sd_storage, -+ sa_none, -+}; -+ -+/* -+ * Build an iteration mask that can exclude certain CPUs from the upwards -+ * domain traversal. -+ * -+ * Asymmetric node setups can result in situations where the domain tree is of -+ * unequal depth, make sure to skip domains that already cover the entire -+ * range. -+ * -+ * In that case build_sched_domains() will have terminated the iteration early -+ * and our sibling sd spans will be empty. Domains should always include the -+ * cpu they're built on, so check that. -+ * -+ */ -+static void build_group_mask(struct sched_domain *sd, struct sched_group *sg) -+{ -+ const struct cpumask *span = sched_domain_span(sd); -+ struct sd_data *sdd = sd->private; -+ struct sched_domain *sibling; -+ int i; -+ -+ for_each_cpu(i, span) { -+ sibling = *per_cpu_ptr(sdd->sd, i); -+ if (!cpumask_test_cpu(i, sched_domain_span(sibling))) -+ continue; -+ -+ cpumask_set_cpu(i, sched_group_mask(sg)); -+ } -+} -+ -+/* -+ * Return the canonical balance cpu for this group, this is the first cpu -+ * of this group that's also in the iteration mask. -+ */ -+int group_balance_cpu(struct sched_group *sg) -+{ -+ return cpumask_first_and(sched_group_cpus(sg), sched_group_mask(sg)); -+} -+ -+static int -+build_overlap_sched_groups(struct sched_domain *sd, int cpu) -+{ -+ struct sched_group *first = NULL, *last = NULL, *groups = NULL, *sg; -+ const struct cpumask *span = sched_domain_span(sd); -+ struct cpumask *covered = sched_domains_tmpmask; -+ struct sd_data *sdd = sd->private; -+ struct sched_domain *sibling; -+ int i; -+ -+ cpumask_clear(covered); -+ -+ for_each_cpu(i, span) { -+ struct cpumask *sg_span; -+ -+ if (cpumask_test_cpu(i, covered)) -+ continue; -+ -+ sibling = *per_cpu_ptr(sdd->sd, i); -+ -+ /* See the comment near build_group_mask(). */ -+ if (!cpumask_test_cpu(i, sched_domain_span(sibling))) -+ continue; -+ -+ sg = kzalloc_node(sizeof(struct sched_group) + cpumask_size(), -+ GFP_KERNEL, cpu_to_node(cpu)); -+ -+ if (!sg) -+ goto fail; -+ -+ sg_span = sched_group_cpus(sg); -+ if (sibling->child) -+ cpumask_copy(sg_span, sched_domain_span(sibling->child)); -+ else -+ cpumask_set_cpu(i, sg_span); -+ -+ cpumask_or(covered, covered, sg_span); -+ -+ sg->sgc = *per_cpu_ptr(sdd->sgc, i); -+ if (atomic_inc_return(&sg->sgc->ref) == 1) -+ build_group_mask(sd, sg); -+ -+ /* -+ * Initialize sgc->capacity such that even if we mess up the -+ * domains and no possible iteration will get us here, we won't -+ * die on a /0 trap. -+ */ -+ sg->sgc->capacity = SCHED_CAPACITY_SCALE * cpumask_weight(sg_span); -+ -+ /* -+ * Make sure the first group of this domain contains the -+ * canonical balance cpu. Otherwise the sched_domain iteration -+ * breaks. See update_sg_lb_stats(). -+ */ -+ if ((!groups && cpumask_test_cpu(cpu, sg_span)) || -+ group_balance_cpu(sg) == cpu) -+ groups = sg; -+ -+ if (!first) -+ first = sg; -+ if (last) -+ last->next = sg; -+ last = sg; -+ last->next = first; -+ } -+ sd->groups = groups; -+ -+ return 0; -+ -+fail: -+ free_sched_groups(first, 0); -+ -+ return -ENOMEM; -+} -+ -+static int get_group(int cpu, struct sd_data *sdd, struct sched_group **sg) -+{ -+ struct sched_domain *sd = *per_cpu_ptr(sdd->sd, cpu); -+ struct sched_domain *child = sd->child; -+ -+ if (child) -+ cpu = cpumask_first(sched_domain_span(child)); -+ -+ if (sg) { -+ *sg = *per_cpu_ptr(sdd->sg, cpu); -+ (*sg)->sgc = *per_cpu_ptr(sdd->sgc, cpu); -+ atomic_set(&(*sg)->sgc->ref, 1); /* for claim_allocations */ -+ } -+ -+ return cpu; -+} -+ -+/* -+ * build_sched_groups will build a circular linked list of the groups -+ * covered by the given span, and will set each group's ->cpumask correctly, -+ * and ->cpu_capacity to 0. -+ * -+ * Assumes the sched_domain tree is fully constructed -+ */ -+static int -+build_sched_groups(struct sched_domain *sd, int cpu) -+{ -+ struct sched_group *first = NULL, *last = NULL; -+ struct sd_data *sdd = sd->private; -+ const struct cpumask *span = sched_domain_span(sd); -+ struct cpumask *covered; -+ int i; -+ -+ get_group(cpu, sdd, &sd->groups); -+ atomic_inc(&sd->groups->ref); -+ -+ if (cpu != cpumask_first(span)) -+ return 0; -+ -+ lockdep_assert_held(&sched_domains_mutex); -+ covered = sched_domains_tmpmask; -+ -+ cpumask_clear(covered); -+ -+ for_each_cpu(i, span) { -+ struct sched_group *sg; -+ int group, j; -+ -+ if (cpumask_test_cpu(i, covered)) -+ continue; -+ -+ group = get_group(i, sdd, &sg); -+ cpumask_setall(sched_group_mask(sg)); -+ -+ for_each_cpu(j, span) { -+ if (get_group(j, sdd, NULL) != group) -+ continue; -+ -+ cpumask_set_cpu(j, covered); -+ cpumask_set_cpu(j, sched_group_cpus(sg)); -+ } -+ -+ if (!first) -+ first = sg; -+ if (last) -+ last->next = sg; -+ last = sg; -+ } -+ last->next = first; -+ -+ return 0; -+} -+ -+/* -+ * Initialize sched groups cpu_capacity. -+ * -+ * cpu_capacity indicates the capacity of sched group, which is used while -+ * distributing the load between different sched groups in a sched domain. -+ * Typically cpu_capacity for all the groups in a sched domain will be same -+ * unless there are asymmetries in the topology. If there are asymmetries, -+ * group having more cpu_capacity will pickup more load compared to the -+ * group having less cpu_capacity. -+ */ -+static void init_sched_groups_capacity(int cpu, struct sched_domain *sd) -+{ -+ struct sched_group *sg = sd->groups; -+ -+ WARN_ON(!sg); -+ -+ do { -+ sg->group_weight = cpumask_weight(sched_group_cpus(sg)); -+ sg = sg->next; -+ } while (sg != sd->groups); -+ -+ if (cpu != group_balance_cpu(sg)) -+ return; -+ -+ update_group_capacity(sd, cpu); -+ atomic_set(&sg->sgc->nr_busy_cpus, sg->group_weight); -+} -+ -+/* -+ * Initializers for schedule domains -+ * Non-inlined to reduce accumulated stack pressure in build_sched_domains() -+ */ -+ -+static int default_relax_domain_level = -1; -+int sched_domain_level_max; -+ -+static int __init setup_relax_domain_level(char *str) -+{ -+ if (kstrtoint(str, 0, &default_relax_domain_level)) -+ pr_warn("Unable to set relax_domain_level\n"); -+ -+ return 1; -+} -+__setup("relax_domain_level=", setup_relax_domain_level); -+ -+static void set_domain_attribute(struct sched_domain *sd, -+ struct sched_domain_attr *attr) -+{ -+ int request; -+ -+ if (!attr || attr->relax_domain_level < 0) { -+ if (default_relax_domain_level < 0) -+ return; -+ else -+ request = default_relax_domain_level; -+ } else -+ request = attr->relax_domain_level; -+ if (request < sd->level) { -+ /* turn off idle balance on this domain */ -+ sd->flags &= ~(SD_BALANCE_WAKE|SD_BALANCE_NEWIDLE); -+ } else { -+ /* turn on idle balance on this domain */ -+ sd->flags |= (SD_BALANCE_WAKE|SD_BALANCE_NEWIDLE); -+ } -+} -+ -+static void __sdt_free(const struct cpumask *cpu_map); -+static int __sdt_alloc(const struct cpumask *cpu_map); -+ -+static void __free_domain_allocs(struct s_data *d, enum s_alloc what, -+ const struct cpumask *cpu_map) -+{ -+ switch (what) { -+ case sa_rootdomain: -+ if (!atomic_read(&d->rd->refcount)) -+ free_rootdomain(&d->rd->rcu); /* fall through */ -+ case sa_sd: -+ free_percpu(d->sd); /* fall through */ -+ case sa_sd_storage: -+ __sdt_free(cpu_map); /* fall through */ -+ case sa_none: -+ break; -+ } -+} -+ -+static enum s_alloc __visit_domain_allocation_hell(struct s_data *d, -+ const struct cpumask *cpu_map) -+{ -+ memset(d, 0, sizeof(*d)); -+ -+ if (__sdt_alloc(cpu_map)) -+ return sa_sd_storage; -+ d->sd = alloc_percpu(struct sched_domain *); -+ if (!d->sd) -+ return sa_sd_storage; -+ d->rd = alloc_rootdomain(); -+ if (!d->rd) -+ return sa_sd; -+ return sa_rootdomain; -+} -+ -+/* -+ * NULL the sd_data elements we've used to build the sched_domain and -+ * sched_group structure so that the subsequent __free_domain_allocs() -+ * will not free the data we're using. -+ */ -+static void claim_allocations(int cpu, struct sched_domain *sd) -+{ -+ struct sd_data *sdd = sd->private; -+ -+ WARN_ON_ONCE(*per_cpu_ptr(sdd->sd, cpu) != sd); -+ *per_cpu_ptr(sdd->sd, cpu) = NULL; -+ -+ if (atomic_read(&(*per_cpu_ptr(sdd->sg, cpu))->ref)) -+ *per_cpu_ptr(sdd->sg, cpu) = NULL; -+ -+ if (atomic_read(&(*per_cpu_ptr(sdd->sgc, cpu))->ref)) -+ *per_cpu_ptr(sdd->sgc, cpu) = NULL; -+} -+ -+#ifdef CONFIG_NUMA -+static int sched_domains_numa_levels; -+enum numa_topology_type sched_numa_topology_type; -+static int *sched_domains_numa_distance; -+int sched_max_numa_distance; -+static struct cpumask ***sched_domains_numa_masks; -+static int sched_domains_curr_level; -+#endif -+ -+/* -+ * SD_flags allowed in topology descriptions. -+ * -+ * SD_SHARE_CPUCAPACITY - describes SMT topologies -+ * SD_SHARE_PKG_RESOURCES - describes shared caches -+ * SD_NUMA - describes NUMA topologies -+ * SD_SHARE_POWERDOMAIN - describes shared power domain -+ * -+ * Odd one out: -+ * SD_ASYM_PACKING - describes SMT quirks -+ */ -+#define TOPOLOGY_SD_FLAGS \ -+ (SD_SHARE_CPUCAPACITY | \ -+ SD_SHARE_PKG_RESOURCES | \ -+ SD_NUMA | \ -+ SD_ASYM_PACKING | \ -+ SD_SHARE_POWERDOMAIN) -+ -+static struct sched_domain * -+sd_init(struct sched_domain_topology_level *tl, int cpu) -+{ -+ struct sched_domain *sd = *per_cpu_ptr(tl->data.sd, cpu); -+ int sd_weight, sd_flags = 0; -+ -+#ifdef CONFIG_NUMA -+ /* -+ * Ugly hack to pass state to sd_numa_mask()... -+ */ -+ sched_domains_curr_level = tl->numa_level; -+#endif -+ -+ sd_weight = cpumask_weight(tl->mask(cpu)); -+ -+ if (tl->sd_flags) -+ sd_flags = (*tl->sd_flags)(); -+ if (WARN_ONCE(sd_flags & ~TOPOLOGY_SD_FLAGS, -+ "wrong sd_flags in topology description\n")) -+ sd_flags &= ~TOPOLOGY_SD_FLAGS; -+ -+ *sd = (struct sched_domain){ -+ .min_interval = sd_weight, -+ .max_interval = 2*sd_weight, -+ .busy_factor = 32, -+ .imbalance_pct = 125, -+ -+ .cache_nice_tries = 0, -+ .busy_idx = 0, -+ .idle_idx = 0, -+ .newidle_idx = 0, -+ .wake_idx = 0, -+ .forkexec_idx = 0, -+ -+ .flags = 1*SD_LOAD_BALANCE -+ | 1*SD_BALANCE_NEWIDLE -+ | 1*SD_BALANCE_EXEC -+ | 1*SD_BALANCE_FORK -+ | 0*SD_BALANCE_WAKE -+ | 1*SD_WAKE_AFFINE -+ | 0*SD_SHARE_CPUCAPACITY -+ | 0*SD_SHARE_PKG_RESOURCES -+ | 0*SD_SERIALIZE -+ | 0*SD_PREFER_SIBLING -+ | 0*SD_NUMA -+ | sd_flags -+ , -+ -+ .last_balance = jiffies, -+ .balance_interval = sd_weight, -+ .smt_gain = 0, -+ .max_newidle_lb_cost = 0, -+ .next_decay_max_lb_cost = jiffies, -+#ifdef CONFIG_SCHED_DEBUG -+ .name = tl->name, -+#endif -+ }; -+ -+ /* -+ * Convert topological properties into behaviour. -+ */ -+ -+ if (sd->flags & SD_SHARE_CPUCAPACITY) { -+ sd->flags |= SD_PREFER_SIBLING; -+ sd->imbalance_pct = 110; -+ sd->smt_gain = 1178; /* ~15% */ -+ -+ } else if (sd->flags & SD_SHARE_PKG_RESOURCES) { -+ sd->imbalance_pct = 117; -+ sd->cache_nice_tries = 1; -+ sd->busy_idx = 2; -+ -+#ifdef CONFIG_NUMA -+ } else if (sd->flags & SD_NUMA) { -+ sd->cache_nice_tries = 2; -+ sd->busy_idx = 3; -+ sd->idle_idx = 2; -+ -+ sd->flags |= SD_SERIALIZE; -+ if (sched_domains_numa_distance[tl->numa_level] > RECLAIM_DISTANCE) { -+ sd->flags &= ~(SD_BALANCE_EXEC | -+ SD_BALANCE_FORK | -+ SD_WAKE_AFFINE); -+ } -+ -+#endif -+ } else { -+ sd->flags |= SD_PREFER_SIBLING; -+ sd->cache_nice_tries = 1; -+ sd->busy_idx = 2; -+ sd->idle_idx = 1; -+ } -+ -+ sd->private = &tl->data; -+ -+ return sd; -+} -+ -+/* -+ * Topology list, bottom-up. -+ */ -+static struct sched_domain_topology_level default_topology[] = { -+#ifdef CONFIG_SCHED_SMT -+ { cpu_smt_mask, cpu_smt_flags, SD_INIT_NAME(SMT) }, -+#endif -+#ifdef CONFIG_SCHED_MC -+ { cpu_coregroup_mask, cpu_core_flags, SD_INIT_NAME(MC) }, -+#endif -+ { cpu_cpu_mask, SD_INIT_NAME(DIE) }, -+ { NULL, }, -+}; -+ -+struct sched_domain_topology_level *sched_domain_topology = default_topology; -+ -+#define for_each_sd_topology(tl) \ -+ for (tl = sched_domain_topology; tl->mask; tl++) -+ -+void set_sched_topology(struct sched_domain_topology_level *tl) -+{ -+ sched_domain_topology = tl; -+} -+ -+#ifdef CONFIG_NUMA -+ -+static const struct cpumask *sd_numa_mask(int cpu) -+{ -+ return sched_domains_numa_masks[sched_domains_curr_level][cpu_to_node(cpu)]; -+} -+ -+static void sched_numa_warn(const char *str) -+{ -+ static int done = false; -+ int i,j; -+ -+ if (done) -+ return; -+ -+ done = true; -+ -+ printk(KERN_WARNING "ERROR: %s\n\n", str); -+ -+ for (i = 0; i < nr_node_ids; i++) { -+ printk(KERN_WARNING " "); -+ for (j = 0; j < nr_node_ids; j++) -+ printk(KERN_CONT "%02d ", node_distance(i,j)); -+ printk(KERN_CONT "\n"); -+ } -+ printk(KERN_WARNING "\n"); -+} -+ -+bool find_numa_distance(int distance) -+{ -+ int i; -+ -+ if (distance == node_distance(0, 0)) -+ return true; -+ -+ for (i = 0; i < sched_domains_numa_levels; i++) { -+ if (sched_domains_numa_distance[i] == distance) -+ return true; -+ } -+ -+ return false; -+} -+ -+/* -+ * A system can have three types of NUMA topology: -+ * NUMA_DIRECT: all nodes are directly connected, or not a NUMA system -+ * NUMA_GLUELESS_MESH: some nodes reachable through intermediary nodes -+ * NUMA_BACKPLANE: nodes can reach other nodes through a backplane -+ * -+ * The difference between a glueless mesh topology and a backplane -+ * topology lies in whether communication between not directly -+ * connected nodes goes through intermediary nodes (where programs -+ * could run), or through backplane controllers. This affects -+ * placement of programs. -+ * -+ * The type of topology can be discerned with the following tests: -+ * - If the maximum distance between any nodes is 1 hop, the system -+ * is directly connected. -+ * - If for two nodes A and B, located N > 1 hops away from each other, -+ * there is an intermediary node C, which is < N hops away from both -+ * nodes A and B, the system is a glueless mesh. -+ */ -+static void init_numa_topology_type(void) -+{ -+ int a, b, c, n; -+ -+ n = sched_max_numa_distance; -+ -+ if (n <= 1) -+ sched_numa_topology_type = NUMA_DIRECT; -+ -+ for_each_online_node(a) { -+ for_each_online_node(b) { -+ /* Find two nodes furthest removed from each other. */ -+ if (node_distance(a, b) < n) -+ continue; -+ -+ /* Is there an intermediary node between a and b? */ -+ for_each_online_node(c) { -+ if (node_distance(a, c) < n && -+ node_distance(b, c) < n) { -+ sched_numa_topology_type = -+ NUMA_GLUELESS_MESH; -+ return; -+ } -+ } -+ -+ sched_numa_topology_type = NUMA_BACKPLANE; -+ return; -+ } -+ } -+} -+ -+static void sched_init_numa(void) -+{ -+ int next_distance, curr_distance = node_distance(0, 0); -+ struct sched_domain_topology_level *tl; -+ int level = 0; -+ int i, j, k; -+ -+ sched_domains_numa_distance = kzalloc(sizeof(int) * nr_node_ids, GFP_KERNEL); -+ if (!sched_domains_numa_distance) -+ return; -+ -+ /* -+ * O(nr_nodes^2) deduplicating selection sort -- in order to find the -+ * unique distances in the node_distance() table. -+ * -+ * Assumes node_distance(0,j) includes all distances in -+ * node_distance(i,j) in order to avoid cubic time. -+ */ -+ next_distance = curr_distance; -+ for (i = 0; i < nr_node_ids; i++) { -+ for (j = 0; j < nr_node_ids; j++) { -+ for (k = 0; k < nr_node_ids; k++) { -+ int distance = node_distance(i, k); -+ -+ if (distance > curr_distance && -+ (distance < next_distance || -+ next_distance == curr_distance)) -+ next_distance = distance; -+ -+ /* -+ * While not a strong assumption it would be nice to know -+ * about cases where if node A is connected to B, B is not -+ * equally connected to A. -+ */ -+ if (sched_debug() && node_distance(k, i) != distance) -+ sched_numa_warn("Node-distance not symmetric"); -+ -+ if (sched_debug() && i && !find_numa_distance(distance)) -+ sched_numa_warn("Node-0 not representative"); -+ } -+ if (next_distance != curr_distance) { -+ sched_domains_numa_distance[level++] = next_distance; -+ sched_domains_numa_levels = level; -+ curr_distance = next_distance; -+ } else break; -+ } -+ -+ /* -+ * In case of sched_debug() we verify the above assumption. -+ */ -+ if (!sched_debug()) -+ break; -+ } -+ -+ if (!level) -+ return; -+ -+ /* -+ * 'level' contains the number of unique distances, excluding the -+ * identity distance node_distance(i,i). -+ * -+ * The sched_domains_numa_distance[] array includes the actual distance -+ * numbers. -+ */ -+ -+ /* -+ * Here, we should temporarily reset sched_domains_numa_levels to 0. -+ * If it fails to allocate memory for array sched_domains_numa_masks[][], -+ * the array will contain less then 'level' members. This could be -+ * dangerous when we use it to iterate array sched_domains_numa_masks[][] -+ * in other functions. -+ * -+ * We reset it to 'level' at the end of this function. -+ */ -+ sched_domains_numa_levels = 0; -+ -+ sched_domains_numa_masks = kzalloc(sizeof(void *) * level, GFP_KERNEL); -+ if (!sched_domains_numa_masks) -+ return; -+ -+ /* -+ * Now for each level, construct a mask per node which contains all -+ * cpus of nodes that are that many hops away from us. -+ */ -+ for (i = 0; i < level; i++) { -+ sched_domains_numa_masks[i] = -+ kzalloc(nr_node_ids * sizeof(void *), GFP_KERNEL); -+ if (!sched_domains_numa_masks[i]) -+ return; -+ -+ for (j = 0; j < nr_node_ids; j++) { -+ struct cpumask *mask = kzalloc(cpumask_size(), GFP_KERNEL); -+ if (!mask) -+ return; -+ -+ sched_domains_numa_masks[i][j] = mask; -+ -+ for (k = 0; k < nr_node_ids; k++) { -+ if (node_distance(j, k) > sched_domains_numa_distance[i]) -+ continue; -+ -+ cpumask_or(mask, mask, cpumask_of_node(k)); -+ } -+ } -+ } -+ -+ /* Compute default topology size */ -+ for (i = 0; sched_domain_topology[i].mask; i++); -+ -+ tl = kzalloc((i + level + 1) * -+ sizeof(struct sched_domain_topology_level), GFP_KERNEL); -+ if (!tl) -+ return; -+ -+ /* -+ * Copy the default topology bits.. -+ */ -+ for (i = 0; sched_domain_topology[i].mask; i++) -+ tl[i] = sched_domain_topology[i]; -+ -+ /* -+ * .. and append 'j' levels of NUMA goodness. -+ */ -+ for (j = 0; j < level; i++, j++) { -+ tl[i] = (struct sched_domain_topology_level){ -+ .mask = sd_numa_mask, -+ .sd_flags = cpu_numa_flags, -+ .flags = SDTL_OVERLAP, -+ .numa_level = j, -+ SD_INIT_NAME(NUMA) -+ }; -+ } -+ -+ sched_domain_topology = tl; -+ -+ sched_domains_numa_levels = level; -+ sched_max_numa_distance = sched_domains_numa_distance[level - 1]; -+ -+ init_numa_topology_type(); -+} -+ -+static void sched_domains_numa_masks_set(int cpu) -+{ -+ int i, j; -+ int node = cpu_to_node(cpu); -+ -+ for (i = 0; i < sched_domains_numa_levels; i++) { -+ for (j = 0; j < nr_node_ids; j++) { -+ if (node_distance(j, node) <= sched_domains_numa_distance[i]) -+ cpumask_set_cpu(cpu, sched_domains_numa_masks[i][j]); -+ } -+ } -+} -+ -+static void sched_domains_numa_masks_clear(int cpu) -+{ -+ int i, j; -+ for (i = 0; i < sched_domains_numa_levels; i++) { -+ for (j = 0; j < nr_node_ids; j++) -+ cpumask_clear_cpu(cpu, sched_domains_numa_masks[i][j]); -+ } -+} -+ -+/* -+ * Update sched_domains_numa_masks[level][node] array when new cpus -+ * are onlined. -+ */ -+static int sched_domains_numa_masks_update(struct notifier_block *nfb, -+ unsigned long action, -+ void *hcpu) -+{ -+ int cpu = (long)hcpu; -+ -+ switch (action & ~CPU_TASKS_FROZEN) { -+ case CPU_ONLINE: -+ sched_domains_numa_masks_set(cpu); -+ break; -+ -+ case CPU_DEAD: -+ sched_domains_numa_masks_clear(cpu); -+ break; -+ -+ default: -+ return NOTIFY_DONE; -+ } -+ -+ return NOTIFY_OK; -+} -+#else -+static inline void sched_init_numa(void) -+{ -+} -+ -+static int sched_domains_numa_masks_update(struct notifier_block *nfb, -+ unsigned long action, -+ void *hcpu) -+{ -+ return 0; -+} -+#endif /* CONFIG_NUMA */ -+ -+static int __sdt_alloc(const struct cpumask *cpu_map) -+{ -+ struct sched_domain_topology_level *tl; -+ int j; -+ -+ for_each_sd_topology(tl) { -+ struct sd_data *sdd = &tl->data; -+ -+ sdd->sd = alloc_percpu(struct sched_domain *); -+ if (!sdd->sd) -+ return -ENOMEM; -+ -+ sdd->sg = alloc_percpu(struct sched_group *); -+ if (!sdd->sg) -+ return -ENOMEM; -+ -+ sdd->sgc = alloc_percpu(struct sched_group_capacity *); -+ if (!sdd->sgc) -+ return -ENOMEM; -+ -+ for_each_cpu(j, cpu_map) { -+ struct sched_domain *sd; -+ struct sched_group *sg; -+ struct sched_group_capacity *sgc; -+ -+ sd = kzalloc_node(sizeof(struct sched_domain) + cpumask_size(), -+ GFP_KERNEL, cpu_to_node(j)); -+ if (!sd) -+ return -ENOMEM; -+ -+ *per_cpu_ptr(sdd->sd, j) = sd; -+ -+ sg = kzalloc_node(sizeof(struct sched_group) + cpumask_size(), -+ GFP_KERNEL, cpu_to_node(j)); -+ if (!sg) -+ return -ENOMEM; -+ -+ sg->next = sg; -+ -+ *per_cpu_ptr(sdd->sg, j) = sg; -+ -+ sgc = kzalloc_node(sizeof(struct sched_group_capacity) + cpumask_size(), -+ GFP_KERNEL, cpu_to_node(j)); -+ if (!sgc) -+ return -ENOMEM; -+ -+ *per_cpu_ptr(sdd->sgc, j) = sgc; -+ } -+ } -+ -+ return 0; -+} -+ -+static void __sdt_free(const struct cpumask *cpu_map) -+{ -+ struct sched_domain_topology_level *tl; -+ int j; -+ -+ for_each_sd_topology(tl) { -+ struct sd_data *sdd = &tl->data; -+ -+ for_each_cpu(j, cpu_map) { -+ struct sched_domain *sd; -+ -+ if (sdd->sd) { -+ sd = *per_cpu_ptr(sdd->sd, j); -+ if (sd && (sd->flags & SD_OVERLAP)) -+ free_sched_groups(sd->groups, 0); -+ kfree(*per_cpu_ptr(sdd->sd, j)); -+ } -+ -+ if (sdd->sg) -+ kfree(*per_cpu_ptr(sdd->sg, j)); -+ if (sdd->sgc) -+ kfree(*per_cpu_ptr(sdd->sgc, j)); -+ } -+ free_percpu(sdd->sd); -+ sdd->sd = NULL; -+ free_percpu(sdd->sg); -+ sdd->sg = NULL; -+ free_percpu(sdd->sgc); -+ sdd->sgc = NULL; -+ } -+} -+ -+struct sched_domain *build_sched_domain(struct sched_domain_topology_level *tl, -+ const struct cpumask *cpu_map, struct sched_domain_attr *attr, -+ struct sched_domain *child, int cpu) -+{ -+ struct sched_domain *sd = sd_init(tl, cpu); -+ if (!sd) -+ return child; -+ -+ cpumask_and(sched_domain_span(sd), cpu_map, tl->mask(cpu)); -+ if (child) { -+ sd->level = child->level + 1; -+ sched_domain_level_max = max(sched_domain_level_max, sd->level); -+ child->parent = sd; -+ sd->child = child; -+ -+ if (!cpumask_subset(sched_domain_span(child), -+ sched_domain_span(sd))) { -+ pr_err("BUG: arch topology borken\n"); -+#ifdef CONFIG_SCHED_DEBUG -+ pr_err(" the %s domain not a subset of the %s domain\n", -+ child->name, sd->name); -+#endif -+ /* Fixup, ensure @sd has at least @child cpus. */ -+ cpumask_or(sched_domain_span(sd), -+ sched_domain_span(sd), -+ sched_domain_span(child)); -+ } -+ -+ } -+ set_domain_attribute(sd, attr); -+ -+ return sd; -+} -+ -+/* -+ * Build sched domains for a given set of cpus and attach the sched domains -+ * to the individual cpus -+ */ -+static int build_sched_domains(const struct cpumask *cpu_map, -+ struct sched_domain_attr *attr) -+{ -+ enum s_alloc alloc_state; -+ struct sched_domain *sd; -+ struct s_data d; -+ int i, ret = -ENOMEM; -+ -+ alloc_state = __visit_domain_allocation_hell(&d, cpu_map); -+ if (alloc_state != sa_rootdomain) -+ goto error; -+ -+ /* Set up domains for cpus specified by the cpu_map. */ -+ for_each_cpu(i, cpu_map) { -+ struct sched_domain_topology_level *tl; -+ -+ sd = NULL; -+ for_each_sd_topology(tl) { -+ sd = build_sched_domain(tl, cpu_map, attr, sd, i); -+ if (tl == sched_domain_topology) -+ *per_cpu_ptr(d.sd, i) = sd; -+ if (tl->flags & SDTL_OVERLAP || sched_feat(FORCE_SD_OVERLAP)) -+ sd->flags |= SD_OVERLAP; -+ if (cpumask_equal(cpu_map, sched_domain_span(sd))) -+ break; -+ } -+ } -+ -+ /* Build the groups for the domains */ -+ for_each_cpu(i, cpu_map) { -+ for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) { -+ sd->span_weight = cpumask_weight(sched_domain_span(sd)); -+ if (sd->flags & SD_OVERLAP) { -+ if (build_overlap_sched_groups(sd, i)) -+ goto error; -+ } else { -+ if (build_sched_groups(sd, i)) -+ goto error; -+ } -+ } -+ } -+ -+ /* Calculate CPU capacity for physical packages and nodes */ -+ for (i = nr_cpumask_bits-1; i >= 0; i--) { -+ if (!cpumask_test_cpu(i, cpu_map)) -+ continue; -+ -+ for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) { -+ claim_allocations(i, sd); -+ init_sched_groups_capacity(i, sd); -+ } -+ } -+ -+ /* Attach the domains */ -+ rcu_read_lock(); -+ for_each_cpu(i, cpu_map) { -+ sd = *per_cpu_ptr(d.sd, i); -+ cpu_attach_domain(sd, d.rd, i); -+ } -+ rcu_read_unlock(); -+ -+ ret = 0; -+error: -+ __free_domain_allocs(&d, alloc_state, cpu_map); -+ return ret; -+} -+ -+static cpumask_var_t *doms_cur; /* current sched domains */ -+static int ndoms_cur; /* number of sched domains in 'doms_cur' */ -+static struct sched_domain_attr *dattr_cur; -+ /* attribues of custom domains in 'doms_cur' */ -+ -+/* -+ * Special case: If a kmalloc of a doms_cur partition (array of -+ * cpumask) fails, then fallback to a single sched domain, -+ * as determined by the single cpumask fallback_doms. -+ */ -+static cpumask_var_t fallback_doms; -+ -+/* -+ * arch_update_cpu_topology lets virtualized architectures update the -+ * cpu core maps. It is supposed to return 1 if the topology changed -+ * or 0 if it stayed the same. -+ */ -+int __weak arch_update_cpu_topology(void) -+{ -+ return 0; -+} -+ -+cpumask_var_t *alloc_sched_domains(unsigned int ndoms) -+{ -+ int i; -+ cpumask_var_t *doms; -+ -+ doms = kmalloc(sizeof(*doms) * ndoms, GFP_KERNEL); -+ if (!doms) -+ return NULL; -+ for (i = 0; i < ndoms; i++) { -+ if (!alloc_cpumask_var(&doms[i], GFP_KERNEL)) { -+ free_sched_domains(doms, i); -+ return NULL; -+ } -+ } -+ return doms; -+} -+ -+void free_sched_domains(cpumask_var_t doms[], unsigned int ndoms) -+{ -+ unsigned int i; -+ for (i = 0; i < ndoms; i++) -+ free_cpumask_var(doms[i]); -+ kfree(doms); -+} -+ -+/* -+ * Set up scheduler domains and groups. Callers must hold the hotplug lock. -+ * For now this just excludes isolated cpus, but could be used to -+ * exclude other special cases in the future. -+ */ -+static int init_sched_domains(const struct cpumask *cpu_map) -+{ -+ int err; -+ -+ arch_update_cpu_topology(); -+ ndoms_cur = 1; -+ doms_cur = alloc_sched_domains(ndoms_cur); -+ if (!doms_cur) -+ doms_cur = &fallback_doms; -+ cpumask_andnot(doms_cur[0], cpu_map, cpu_isolated_map); -+ err = build_sched_domains(doms_cur[0], NULL); -+ register_sched_domain_sysctl(); -+ -+ return err; -+} -+ -+/* -+ * Detach sched domains from a group of cpus specified in cpu_map -+ * These cpus will now be attached to the NULL domain -+ */ -+static void detach_destroy_domains(const struct cpumask *cpu_map) -+{ -+ int i; -+ -+ rcu_read_lock(); -+ for_each_cpu(i, cpu_map) -+ cpu_attach_domain(NULL, &def_root_domain, i); -+ rcu_read_unlock(); -+} -+ -+/* handle null as "default" */ -+static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur, -+ struct sched_domain_attr *new, int idx_new) -+{ -+ struct sched_domain_attr tmp; -+ -+ /* fast path */ -+ if (!new && !cur) -+ return 1; -+ -+ tmp = SD_ATTR_INIT; -+ return !memcmp(cur ? (cur + idx_cur) : &tmp, -+ new ? (new + idx_new) : &tmp, -+ sizeof(struct sched_domain_attr)); -+} -+ -+/* -+ * Partition sched domains as specified by the 'ndoms_new' -+ * cpumasks in the array doms_new[] of cpumasks. This compares -+ * doms_new[] to the current sched domain partitioning, doms_cur[]. -+ * It destroys each deleted domain and builds each new domain. -+ * -+ * 'doms_new' is an array of cpumask_var_t's of length 'ndoms_new'. -+ * The masks don't intersect (don't overlap.) We should setup one -+ * sched domain for each mask. CPUs not in any of the cpumasks will -+ * not be load balanced. If the same cpumask appears both in the -+ * current 'doms_cur' domains and in the new 'doms_new', we can leave -+ * it as it is. -+ * -+ * The passed in 'doms_new' should be allocated using -+ * alloc_sched_domains. This routine takes ownership of it and will -+ * free_sched_domains it when done with it. If the caller failed the -+ * alloc call, then it can pass in doms_new == NULL && ndoms_new == 1, -+ * and partition_sched_domains() will fallback to the single partition -+ * 'fallback_doms', it also forces the domains to be rebuilt. -+ * -+ * If doms_new == NULL it will be replaced with cpu_online_mask. -+ * ndoms_new == 0 is a special case for destroying existing domains, -+ * and it will not create the default domain. -+ * -+ * Call with hotplug lock held -+ */ -+void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[], -+ struct sched_domain_attr *dattr_new) -+{ -+ int i, j, n; -+ int new_topology; -+ -+ mutex_lock(&sched_domains_mutex); -+ -+ /* always unregister in case we don't destroy any domains */ -+ unregister_sched_domain_sysctl(); -+ -+ /* Let architecture update cpu core mappings. */ -+ new_topology = arch_update_cpu_topology(); -+ -+ n = doms_new ? ndoms_new : 0; -+ -+ /* Destroy deleted domains */ -+ for (i = 0; i < ndoms_cur; i++) { -+ for (j = 0; j < n && !new_topology; j++) { -+ if (cpumask_equal(doms_cur[i], doms_new[j]) -+ && dattrs_equal(dattr_cur, i, dattr_new, j)) -+ goto match1; -+ } -+ /* no match - a current sched domain not in new doms_new[] */ -+ detach_destroy_domains(doms_cur[i]); -+match1: -+ ; -+ } -+ -+ n = ndoms_cur; -+ if (doms_new == NULL) { -+ n = 0; -+ doms_new = &fallback_doms; -+ cpumask_andnot(doms_new[0], cpu_active_mask, cpu_isolated_map); -+ WARN_ON_ONCE(dattr_new); -+ } -+ -+ /* Build new domains */ -+ for (i = 0; i < ndoms_new; i++) { -+ for (j = 0; j < n && !new_topology; j++) { -+ if (cpumask_equal(doms_new[i], doms_cur[j]) -+ && dattrs_equal(dattr_new, i, dattr_cur, j)) -+ goto match2; -+ } -+ /* no match - add a new doms_new */ -+ build_sched_domains(doms_new[i], dattr_new ? dattr_new + i : NULL); -+match2: -+ ; -+ } -+ -+ /* Remember the new sched domains */ -+ if (doms_cur != &fallback_doms) -+ free_sched_domains(doms_cur, ndoms_cur); -+ kfree(dattr_cur); /* kfree(NULL) is safe */ -+ doms_cur = doms_new; -+ dattr_cur = dattr_new; -+ ndoms_cur = ndoms_new; -+ -+ register_sched_domain_sysctl(); -+ -+ mutex_unlock(&sched_domains_mutex); -+} -+ -+static int num_cpus_frozen; /* used to mark begin/end of suspend/resume */ -+ -+/* -+ * Update cpusets according to cpu_active mask. If cpusets are -+ * disabled, cpuset_update_active_cpus() becomes a simple wrapper -+ * around partition_sched_domains(). -+ * -+ * If we come here as part of a suspend/resume, don't touch cpusets because we -+ * want to restore it back to its original state upon resume anyway. -+ */ -+static int cpuset_cpu_active(struct notifier_block *nfb, unsigned long action, -+ void *hcpu) -+{ -+ switch (action) { -+ case CPU_ONLINE_FROZEN: -+ case CPU_DOWN_FAILED_FROZEN: -+ -+ /* -+ * num_cpus_frozen tracks how many CPUs are involved in suspend -+ * resume sequence. As long as this is not the last online -+ * operation in the resume sequence, just build a single sched -+ * domain, ignoring cpusets. -+ */ -+ num_cpus_frozen--; -+ if (likely(num_cpus_frozen)) { -+ partition_sched_domains(1, NULL, NULL); -+ break; -+ } -+ -+ /* -+ * This is the last CPU online operation. So fall through and -+ * restore the original sched domains by considering the -+ * cpuset configurations. -+ */ -+ -+ case CPU_ONLINE: -+ cpuset_update_active_cpus(true); -+ break; -+ default: -+ return NOTIFY_DONE; -+ } -+ return NOTIFY_OK; -+} -+ -+static int cpuset_cpu_inactive(struct notifier_block *nfb, unsigned long action, -+ void *hcpu) -+{ -+ unsigned long flags; -+ long cpu = (long)hcpu; -+ struct dl_bw *dl_b; -+ bool overflow; -+ int cpus; -+ -+ switch (action) { -+ case CPU_DOWN_PREPARE: -+ rcu_read_lock_sched(); -+ dl_b = dl_bw_of(cpu); -+ -+ raw_spin_lock_irqsave(&dl_b->lock, flags); -+ cpus = dl_bw_cpus(cpu); -+ overflow = __dl_overflow(dl_b, cpus, 0, 0); -+ raw_spin_unlock_irqrestore(&dl_b->lock, flags); -+ -+ rcu_read_unlock_sched(); -+ -+ if (overflow) -+ return notifier_from_errno(-EBUSY); -+ cpuset_update_active_cpus(false); -+ break; -+ case CPU_DOWN_PREPARE_FROZEN: -+ num_cpus_frozen++; -+ partition_sched_domains(1, NULL, NULL); -+ break; -+ default: -+ return NOTIFY_DONE; -+ } -+ return NOTIFY_OK; -+} -+ -+void __init sched_init_smp(void) -+{ -+ cpumask_var_t non_isolated_cpus; -+ -+ alloc_cpumask_var(&non_isolated_cpus, GFP_KERNEL); -+ alloc_cpumask_var(&fallback_doms, GFP_KERNEL); -+ -+ sched_init_numa(); -+ -+ /* -+ * There's no userspace yet to cause hotplug operations; hence all the -+ * cpu masks are stable and all blatant races in the below code cannot -+ * happen. -+ */ -+ mutex_lock(&sched_domains_mutex); -+ init_sched_domains(cpu_active_mask); -+ cpumask_andnot(non_isolated_cpus, cpu_possible_mask, cpu_isolated_map); -+ if (cpumask_empty(non_isolated_cpus)) -+ cpumask_set_cpu(smp_processor_id(), non_isolated_cpus); -+ mutex_unlock(&sched_domains_mutex); -+ -+ hotcpu_notifier(sched_domains_numa_masks_update, CPU_PRI_SCHED_ACTIVE); -+ hotcpu_notifier(cpuset_cpu_active, CPU_PRI_CPUSET_ACTIVE); -+ hotcpu_notifier(cpuset_cpu_inactive, CPU_PRI_CPUSET_INACTIVE); -+ -+ init_hrtick(); -+ -+ /* Move init over to a non-isolated CPU */ -+ if (set_cpus_allowed_ptr(current, non_isolated_cpus) < 0) -+ BUG(); -+ sched_init_granularity(); -+ free_cpumask_var(non_isolated_cpus); -+ -+ init_sched_rt_class(); -+ init_sched_dl_class(); -+} -+#else -+void __init sched_init_smp(void) -+{ -+ sched_init_granularity(); -+} -+#endif /* CONFIG_SMP */ -+ -+const_debug unsigned int sysctl_timer_migration = 1; -+ -+int in_sched_functions(unsigned long addr) -+{ -+ return in_lock_functions(addr) || -+ (addr >= (unsigned long)__sched_text_start -+ && addr < (unsigned long)__sched_text_end); -+} -+ -+#ifdef CONFIG_CGROUP_SCHED -+/* -+ * Default task group. -+ * Every task in system belongs to this group at bootup. -+ */ -+struct task_group root_task_group; -+LIST_HEAD(task_groups); -+#endif -+ -+DECLARE_PER_CPU(cpumask_var_t, load_balance_mask); -+ -+void __init sched_init(void) -+{ -+ int i, j; -+ unsigned long alloc_size = 0, ptr; -+ -+#ifdef CONFIG_FAIR_GROUP_SCHED -+ alloc_size += 2 * nr_cpu_ids * sizeof(void **); -+#endif -+#ifdef CONFIG_RT_GROUP_SCHED -+ alloc_size += 2 * nr_cpu_ids * sizeof(void **); -+#endif -+ if (alloc_size) { -+ ptr = (unsigned long)kzalloc(alloc_size, GFP_NOWAIT); -+ -+#ifdef CONFIG_FAIR_GROUP_SCHED -+ root_task_group.se = (struct sched_entity **)ptr; -+ ptr += nr_cpu_ids * sizeof(void **); -+ -+ root_task_group.cfs_rq = (struct cfs_rq **)ptr; -+ ptr += nr_cpu_ids * sizeof(void **); -+ -+#endif /* CONFIG_FAIR_GROUP_SCHED */ -+#ifdef CONFIG_RT_GROUP_SCHED -+ root_task_group.rt_se = (struct sched_rt_entity **)ptr; -+ ptr += nr_cpu_ids * sizeof(void **); -+ -+ root_task_group.rt_rq = (struct rt_rq **)ptr; -+ ptr += nr_cpu_ids * sizeof(void **); -+ -+#endif /* CONFIG_RT_GROUP_SCHED */ -+ } -+#ifdef CONFIG_CPUMASK_OFFSTACK -+ for_each_possible_cpu(i) { -+ per_cpu(load_balance_mask, i) = (cpumask_var_t)kzalloc_node( -+ cpumask_size(), GFP_KERNEL, cpu_to_node(i)); -+ } -+#endif /* CONFIG_CPUMASK_OFFSTACK */ -+ -+ init_rt_bandwidth(&def_rt_bandwidth, -+ global_rt_period(), global_rt_runtime()); -+ init_dl_bandwidth(&def_dl_bandwidth, -+ global_rt_period(), global_rt_runtime()); -+ -+#ifdef CONFIG_SMP -+ init_defrootdomain(); -+#endif -+ -+#ifdef CONFIG_RT_GROUP_SCHED -+ init_rt_bandwidth(&root_task_group.rt_bandwidth, -+ global_rt_period(), global_rt_runtime()); -+#endif /* CONFIG_RT_GROUP_SCHED */ -+ -+#ifdef CONFIG_CGROUP_SCHED -+ list_add(&root_task_group.list, &task_groups); -+ INIT_LIST_HEAD(&root_task_group.children); -+ INIT_LIST_HEAD(&root_task_group.siblings); -+ autogroup_init(&init_task); -+ -+#endif /* CONFIG_CGROUP_SCHED */ -+ -+ for_each_possible_cpu(i) { -+ struct rq *rq; -+ -+ rq = cpu_rq(i); -+ raw_spin_lock_init(&rq->lock); -+ rq->nr_running = 0; -+ rq->calc_load_active = 0; -+ rq->calc_load_update = jiffies + LOAD_FREQ; -+ init_cfs_rq(&rq->cfs); -+ init_rt_rq(&rq->rt); -+ init_dl_rq(&rq->dl); -+#ifdef CONFIG_FAIR_GROUP_SCHED -+ root_task_group.shares = ROOT_TASK_GROUP_LOAD; -+ INIT_LIST_HEAD(&rq->leaf_cfs_rq_list); -+ /* -+ * How much cpu bandwidth does root_task_group get? -+ * -+ * In case of task-groups formed thr' the cgroup filesystem, it -+ * gets 100% of the cpu resources in the system. This overall -+ * system cpu resource is divided among the tasks of -+ * root_task_group and its child task-groups in a fair manner, -+ * based on each entity's (task or task-group's) weight -+ * (se->load.weight). -+ * -+ * In other words, if root_task_group has 10 tasks of weight -+ * 1024) and two child groups A0 and A1 (of weight 1024 each), -+ * then A0's share of the cpu resource is: -+ * -+ * A0's bandwidth = 1024 / (10*1024 + 1024 + 1024) = 8.33% -+ * -+ * We achieve this by letting root_task_group's tasks sit -+ * directly in rq->cfs (i.e root_task_group->se[] = NULL). -+ */ -+ init_cfs_bandwidth(&root_task_group.cfs_bandwidth); -+ init_tg_cfs_entry(&root_task_group, &rq->cfs, NULL, i, NULL); -+#endif /* CONFIG_FAIR_GROUP_SCHED */ -+ -+ rq->rt.rt_runtime = def_rt_bandwidth.rt_runtime; -+#ifdef CONFIG_RT_GROUP_SCHED -+ init_tg_rt_entry(&root_task_group, &rq->rt, NULL, i, NULL); -+#endif -+ -+ for (j = 0; j < CPU_LOAD_IDX_MAX; j++) -+ rq->cpu_load[j] = 0; -+ -+ rq->last_load_update_tick = jiffies; -+ -+#ifdef CONFIG_SMP -+ rq->sd = NULL; -+ rq->rd = NULL; -+ rq->cpu_capacity = rq->cpu_capacity_orig = SCHED_CAPACITY_SCALE; -+ rq->post_schedule = 0; -+ rq->active_balance = 0; -+ rq->next_balance = jiffies; -+ rq->push_cpu = 0; -+ rq->cpu = i; -+ rq->online = 0; -+ rq->idle_stamp = 0; -+ rq->avg_idle = 2*sysctl_sched_migration_cost; -+ rq->max_idle_balance_cost = sysctl_sched_migration_cost; -+ -+ INIT_LIST_HEAD(&rq->cfs_tasks); -+ -+ rq_attach_root(rq, &def_root_domain); -+#ifdef CONFIG_NO_HZ_COMMON -+ rq->nohz_flags = 0; -+#endif -+#ifdef CONFIG_NO_HZ_FULL -+ rq->last_sched_tick = 0; -+#endif -+#endif -+ init_rq_hrtick(rq); -+ atomic_set(&rq->nr_iowait, 0); -+ } -+ -+ set_load_weight(&init_task); -+ -+#ifdef CONFIG_PREEMPT_NOTIFIERS -+ INIT_HLIST_HEAD(&init_task.preempt_notifiers); -+#endif -+ -+ /* -+ * The boot idle thread does lazy MMU switching as well: -+ */ -+ atomic_inc(&init_mm.mm_count); -+ enter_lazy_tlb(&init_mm, current); -+ -+ /* -+ * During early bootup we pretend to be a normal task: -+ */ -+ current->sched_class = &fair_sched_class; -+ -+ /* -+ * Make us the idle thread. Technically, schedule() should not be -+ * called from this thread, however somewhere below it might be, -+ * but because we are the idle thread, we just pick up running again -+ * when this runqueue becomes "idle". -+ */ -+ init_idle(current, smp_processor_id()); -+ -+ calc_load_update = jiffies + LOAD_FREQ; -+ -+#ifdef CONFIG_SMP -+ zalloc_cpumask_var(&sched_domains_tmpmask, GFP_NOWAIT); -+ /* May be allocated at isolcpus cmdline parse time */ -+ if (cpu_isolated_map == NULL) -+ zalloc_cpumask_var(&cpu_isolated_map, GFP_NOWAIT); -+ idle_thread_set_boot_cpu(); -+ set_cpu_rq_start_time(); -+#endif -+ init_sched_fair_class(); -+ -+ scheduler_running = 1; -+} -+ -+#ifdef CONFIG_DEBUG_ATOMIC_SLEEP -+static inline int preempt_count_equals(int preempt_offset) -+{ -+ int nested = (preempt_count() & ~PREEMPT_ACTIVE) + rcu_preempt_depth(); -+ -+ return (nested == preempt_offset); -+} -+ -+void __might_sleep(const char *file, int line, int preempt_offset) -+{ -+ /* -+ * Blocking primitives will set (and therefore destroy) current->state, -+ * since we will exit with TASK_RUNNING make sure we enter with it, -+ * otherwise we will destroy state. -+ */ -+ WARN_ONCE(current->state != TASK_RUNNING && current->task_state_change, -+ "do not call blocking ops when !TASK_RUNNING; " -+ "state=%lx set at [<%p>] %pS\n", -+ current->state, -+ (void *)current->task_state_change, -+ (void *)current->task_state_change); -+ -+ ___might_sleep(file, line, preempt_offset); -+} -+EXPORT_SYMBOL(__might_sleep); -+ -+void ___might_sleep(const char *file, int line, int preempt_offset) -+{ -+ static unsigned long prev_jiffy; /* ratelimiting */ -+ -+ rcu_sleep_check(); /* WARN_ON_ONCE() by default, no rate limit reqd. */ -+ if ((preempt_count_equals(preempt_offset) && !irqs_disabled() && -+ !is_idle_task(current)) || -+ system_state != SYSTEM_RUNNING || oops_in_progress) -+ return; -+ if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy) -+ return; -+ prev_jiffy = jiffies; -+ -+ printk(KERN_ERR -+ "BUG: sleeping function called from invalid context at %s:%d\n", -+ file, line); -+ printk(KERN_ERR -+ "in_atomic(): %d, irqs_disabled(): %d, pid: %d, name: %s\n", -+ in_atomic(), irqs_disabled(), -+ current->pid, current->comm); -+ -+ if (task_stack_end_corrupted(current)) -+ printk(KERN_EMERG "Thread overran stack, or stack corrupted\n"); -+ -+ debug_show_held_locks(current); -+ if (irqs_disabled()) -+ print_irqtrace_events(current); -+#ifdef CONFIG_DEBUG_PREEMPT -+ if (!preempt_count_equals(preempt_offset)) { -+ pr_err("Preemption disabled at:"); -+ print_ip_sym(current->preempt_disable_ip); -+ pr_cont("\n"); -+ } -+#endif -+ dump_stack(); -+} -+EXPORT_SYMBOL(___might_sleep); -+#endif -+ -+#ifdef CONFIG_MAGIC_SYSRQ -+static void normalize_task(struct rq *rq, struct task_struct *p) -+{ -+ const struct sched_class *prev_class = p->sched_class; -+ struct sched_attr attr = { -+ .sched_policy = SCHED_NORMAL, -+ }; -+ int old_prio = p->prio; -+ int queued; -+ -+ queued = task_on_rq_queued(p); -+ if (queued) -+ dequeue_task(rq, p, 0); -+ __setscheduler(rq, p, &attr, false); -+ if (queued) { -+ enqueue_task(rq, p, 0); -+ resched_curr(rq); -+ } -+ -+ check_class_changed(rq, p, prev_class, old_prio); -+} -+ -+void normalize_rt_tasks(void) -+{ -+ struct task_struct *g, *p; -+ unsigned long flags; -+ struct rq *rq; -+ -+ read_lock(&tasklist_lock); -+ for_each_process_thread(g, p) { -+ /* -+ * Only normalize user tasks: -+ */ -+ if (p->flags & PF_KTHREAD) -+ continue; -+ -+ p->se.exec_start = 0; -+#ifdef CONFIG_SCHEDSTATS -+ p->se.statistics.wait_start = 0; -+ p->se.statistics.sleep_start = 0; -+ p->se.statistics.block_start = 0; -+#endif -+ -+ if (!dl_task(p) && !rt_task(p)) { -+ /* -+ * Renice negative nice level userspace -+ * tasks back to 0: -+ */ -+ if (task_nice(p) < 0) -+ set_user_nice(p, 0); -+ continue; -+ } -+ -+ rq = task_rq_lock(p, &flags); -+ normalize_task(rq, p); -+ task_rq_unlock(rq, p, &flags); -+ } -+ read_unlock(&tasklist_lock); -+} -+ -+#endif /* CONFIG_MAGIC_SYSRQ */ -+ -+#if defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB) -+/* -+ * These functions are only useful for the IA64 MCA handling, or kdb. -+ * -+ * They can only be called when the whole system has been -+ * stopped - every CPU needs to be quiescent, and no scheduling -+ * activity can take place. Using them for anything else would -+ * be a serious bug, and as a result, they aren't even visible -+ * under any other configuration. -+ */ -+ -+/** -+ * curr_task - return the current task for a given cpu. -+ * @cpu: the processor in question. -+ * -+ * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED! -+ * -+ * Return: The current task for @cpu. -+ */ -+struct task_struct *curr_task(int cpu) -+{ -+ return cpu_curr(cpu); -+} -+ -+#endif /* defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB) */ -+ -+#ifdef CONFIG_IA64 -+/** -+ * set_curr_task - set the current task for a given cpu. -+ * @cpu: the processor in question. -+ * @p: the task pointer to set. -+ * -+ * Description: This function must only be used when non-maskable interrupts -+ * are serviced on a separate stack. It allows the architecture to switch the -+ * notion of the current task on a cpu in a non-blocking manner. This function -+ * must be called with all CPU's synchronized, and interrupts disabled, the -+ * and caller must save the original value of the current task (see -+ * curr_task() above) and restore that value before reenabling interrupts and -+ * re-starting the system. -+ * -+ * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED! -+ */ -+void set_curr_task(int cpu, struct task_struct *p) -+{ -+ cpu_curr(cpu) = p; -+} -+ -+#endif -+ -+#ifdef CONFIG_CGROUP_SCHED -+/* task_group_lock serializes the addition/removal of task groups */ -+static DEFINE_SPINLOCK(task_group_lock); -+ -+static void free_sched_group(struct task_group *tg) -+{ -+ free_fair_sched_group(tg); -+ free_rt_sched_group(tg); -+ autogroup_free(tg); -+ kfree(tg); -+} -+ -+/* allocate runqueue etc for a new task group */ -+struct task_group *sched_create_group(struct task_group *parent) -+{ -+ struct task_group *tg; -+ -+ tg = kzalloc(sizeof(*tg), GFP_KERNEL); -+ if (!tg) -+ return ERR_PTR(-ENOMEM); -+ -+ if (!alloc_fair_sched_group(tg, parent)) -+ goto err; -+ -+ if (!alloc_rt_sched_group(tg, parent)) -+ goto err; -+ -+ return tg; -+ -+err: -+ free_sched_group(tg); -+ return ERR_PTR(-ENOMEM); -+} -+ -+void sched_online_group(struct task_group *tg, struct task_group *parent) -+{ -+ unsigned long flags; -+ -+ spin_lock_irqsave(&task_group_lock, flags); -+ list_add_rcu(&tg->list, &task_groups); -+ -+ WARN_ON(!parent); /* root should already exist */ -+ -+ tg->parent = parent; -+ INIT_LIST_HEAD(&tg->children); -+ list_add_rcu(&tg->siblings, &parent->children); -+ spin_unlock_irqrestore(&task_group_lock, flags); -+} -+ -+/* rcu callback to free various structures associated with a task group */ -+static void free_sched_group_rcu(struct rcu_head *rhp) -+{ -+ /* now it should be safe to free those cfs_rqs */ -+ free_sched_group(container_of(rhp, struct task_group, rcu)); -+} -+ -+/* Destroy runqueue etc associated with a task group */ -+void sched_destroy_group(struct task_group *tg) -+{ -+ /* wait for possible concurrent references to cfs_rqs complete */ -+ call_rcu(&tg->rcu, free_sched_group_rcu); -+} -+ -+void sched_offline_group(struct task_group *tg) -+{ -+ unsigned long flags; -+ int i; -+ -+ /* end participation in shares distribution */ -+ for_each_possible_cpu(i) -+ unregister_fair_sched_group(tg, i); -+ -+ spin_lock_irqsave(&task_group_lock, flags); -+ list_del_rcu(&tg->list); -+ list_del_rcu(&tg->siblings); -+ spin_unlock_irqrestore(&task_group_lock, flags); -+} -+ -+/* change task's runqueue when it moves between groups. -+ * The caller of this function should have put the task in its new group -+ * by now. This function just updates tsk->se.cfs_rq and tsk->se.parent to -+ * reflect its new group. -+ */ -+void sched_move_task(struct task_struct *tsk) -+{ -+ struct task_group *tg; -+ int queued, running; -+ unsigned long flags; -+ struct rq *rq; -+ -+ rq = task_rq_lock(tsk, &flags); -+ -+ running = task_current(rq, tsk); -+ queued = task_on_rq_queued(tsk); -+ -+ if (queued) -+ dequeue_task(rq, tsk, 0); -+ if (unlikely(running)) -+ put_prev_task(rq, tsk); -+ -+ /* -+ * All callers are synchronized by task_rq_lock(); we do not use RCU -+ * which is pointless here. Thus, we pass "true" to task_css_check() -+ * to prevent lockdep warnings. -+ */ -+ tg = container_of(task_css_check(tsk, cpu_cgrp_id, true), -+ struct task_group, css); -+ tg = autogroup_task_group(tsk, tg); -+ tsk->sched_task_group = tg; -+ -+#ifdef CONFIG_FAIR_GROUP_SCHED -+ if (tsk->sched_class->task_move_group) -+ tsk->sched_class->task_move_group(tsk, queued); -+ else -+#endif -+ set_task_rq(tsk, task_cpu(tsk)); -+ -+ if (unlikely(running)) -+ tsk->sched_class->set_curr_task(rq); -+ if (queued) -+ enqueue_task(rq, tsk, 0); -+ -+ task_rq_unlock(rq, tsk, &flags); -+} -+#endif /* CONFIG_CGROUP_SCHED */ -+ -+#ifdef CONFIG_RT_GROUP_SCHED -+/* -+ * Ensure that the real time constraints are schedulable. -+ */ -+static DEFINE_MUTEX(rt_constraints_mutex); -+ -+/* Must be called with tasklist_lock held */ -+static inline int tg_has_rt_tasks(struct task_group *tg) -+{ -+ struct task_struct *g, *p; -+ -+ /* -+ * Autogroups do not have RT tasks; see autogroup_create(). -+ */ -+ if (task_group_is_autogroup(tg)) -+ return 0; -+ -+ for_each_process_thread(g, p) { -+ if (rt_task(p) && task_group(p) == tg) -+ return 1; -+ } -+ -+ return 0; -+} -+ -+struct rt_schedulable_data { -+ struct task_group *tg; -+ u64 rt_period; -+ u64 rt_runtime; -+}; -+ -+static int tg_rt_schedulable(struct task_group *tg, void *data) -+{ -+ struct rt_schedulable_data *d = data; -+ struct task_group *child; -+ unsigned long total, sum = 0; -+ u64 period, runtime; -+ -+ period = ktime_to_ns(tg->rt_bandwidth.rt_period); -+ runtime = tg->rt_bandwidth.rt_runtime; -+ -+ if (tg == d->tg) { -+ period = d->rt_period; -+ runtime = d->rt_runtime; -+ } -+ -+ /* -+ * Cannot have more runtime than the period. -+ */ -+ if (runtime > period && runtime != RUNTIME_INF) -+ return -EINVAL; -+ -+ /* -+ * Ensure we don't starve existing RT tasks. -+ */ -+ if (rt_bandwidth_enabled() && !runtime && tg_has_rt_tasks(tg)) -+ return -EBUSY; -+ -+ total = to_ratio(period, runtime); -+ -+ /* -+ * Nobody can have more than the global setting allows. -+ */ -+ if (total > to_ratio(global_rt_period(), global_rt_runtime())) -+ return -EINVAL; -+ -+ /* -+ * The sum of our children's runtime should not exceed our own. -+ */ -+ list_for_each_entry_rcu(child, &tg->children, siblings) { -+ period = ktime_to_ns(child->rt_bandwidth.rt_period); -+ runtime = child->rt_bandwidth.rt_runtime; -+ -+ if (child == d->tg) { -+ period = d->rt_period; -+ runtime = d->rt_runtime; -+ } -+ -+ sum += to_ratio(period, runtime); -+ } -+ -+ if (sum > total) -+ return -EINVAL; -+ -+ return 0; -+} -+ -+static int __rt_schedulable(struct task_group *tg, u64 period, u64 runtime) -+{ -+ int ret; -+ -+ struct rt_schedulable_data data = { -+ .tg = tg, -+ .rt_period = period, -+ .rt_runtime = runtime, -+ }; -+ -+ rcu_read_lock(); -+ ret = walk_tg_tree(tg_rt_schedulable, tg_nop, &data); -+ rcu_read_unlock(); -+ -+ return ret; -+} -+ -+static int tg_set_rt_bandwidth(struct task_group *tg, -+ u64 rt_period, u64 rt_runtime) -+{ -+ int i, err = 0; -+ -+ /* -+ * Disallowing the root group RT runtime is BAD, it would disallow the -+ * kernel creating (and or operating) RT threads. -+ */ -+ if (tg == &root_task_group && rt_runtime == 0) -+ return -EINVAL; -+ -+ /* No period doesn't make any sense. */ -+ if (rt_period == 0) -+ return -EINVAL; -+ -+ mutex_lock(&rt_constraints_mutex); -+ read_lock(&tasklist_lock); -+ err = __rt_schedulable(tg, rt_period, rt_runtime); -+ if (err) -+ goto unlock; -+ -+ raw_spin_lock_irq(&tg->rt_bandwidth.rt_runtime_lock); -+ tg->rt_bandwidth.rt_period = ns_to_ktime(rt_period); -+ tg->rt_bandwidth.rt_runtime = rt_runtime; -+ -+ for_each_possible_cpu(i) { -+ struct rt_rq *rt_rq = tg->rt_rq[i]; -+ -+ raw_spin_lock(&rt_rq->rt_runtime_lock); -+ rt_rq->rt_runtime = rt_runtime; -+ raw_spin_unlock(&rt_rq->rt_runtime_lock); -+ } -+ raw_spin_unlock_irq(&tg->rt_bandwidth.rt_runtime_lock); -+unlock: -+ read_unlock(&tasklist_lock); -+ mutex_unlock(&rt_constraints_mutex); -+ -+ return err; -+} -+ -+static int sched_group_set_rt_runtime(struct task_group *tg, long rt_runtime_us) -+{ -+ u64 rt_runtime, rt_period; -+ -+ rt_period = ktime_to_ns(tg->rt_bandwidth.rt_period); -+ rt_runtime = (u64)rt_runtime_us * NSEC_PER_USEC; -+ if (rt_runtime_us < 0) -+ rt_runtime = RUNTIME_INF; -+ -+ return tg_set_rt_bandwidth(tg, rt_period, rt_runtime); -+} -+ -+static long sched_group_rt_runtime(struct task_group *tg) -+{ -+ u64 rt_runtime_us; -+ -+ if (tg->rt_bandwidth.rt_runtime == RUNTIME_INF) -+ return -1; -+ -+ rt_runtime_us = tg->rt_bandwidth.rt_runtime; -+ do_div(rt_runtime_us, NSEC_PER_USEC); -+ return rt_runtime_us; -+} -+ -+static int sched_group_set_rt_period(struct task_group *tg, long rt_period_us) -+{ -+ u64 rt_runtime, rt_period; -+ -+ rt_period = (u64)rt_period_us * NSEC_PER_USEC; -+ rt_runtime = tg->rt_bandwidth.rt_runtime; -+ -+ return tg_set_rt_bandwidth(tg, rt_period, rt_runtime); -+} -+ -+static long sched_group_rt_period(struct task_group *tg) -+{ -+ u64 rt_period_us; -+ -+ rt_period_us = ktime_to_ns(tg->rt_bandwidth.rt_period); -+ do_div(rt_period_us, NSEC_PER_USEC); -+ return rt_period_us; -+} -+#endif /* CONFIG_RT_GROUP_SCHED */ -+ -+#ifdef CONFIG_RT_GROUP_SCHED -+static int sched_rt_global_constraints(void) -+{ -+ int ret = 0; -+ -+ mutex_lock(&rt_constraints_mutex); -+ read_lock(&tasklist_lock); -+ ret = __rt_schedulable(NULL, 0, 0); -+ read_unlock(&tasklist_lock); -+ mutex_unlock(&rt_constraints_mutex); -+ -+ return ret; -+} -+ -+static int sched_rt_can_attach(struct task_group *tg, struct task_struct *tsk) -+{ -+ /* Don't accept realtime tasks when there is no way for them to run */ -+ if (rt_task(tsk) && tg->rt_bandwidth.rt_runtime == 0) -+ return 0; -+ -+ return 1; -+} -+ -+#else /* !CONFIG_RT_GROUP_SCHED */ -+static int sched_rt_global_constraints(void) -+{ -+ unsigned long flags; -+ int i, ret = 0; -+ -+ raw_spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags); -+ for_each_possible_cpu(i) { -+ struct rt_rq *rt_rq = &cpu_rq(i)->rt; -+ -+ raw_spin_lock(&rt_rq->rt_runtime_lock); -+ rt_rq->rt_runtime = global_rt_runtime(); -+ raw_spin_unlock(&rt_rq->rt_runtime_lock); -+ } -+ raw_spin_unlock_irqrestore(&def_rt_bandwidth.rt_runtime_lock, flags); -+ -+ return ret; -+} -+#endif /* CONFIG_RT_GROUP_SCHED */ -+ -+static int sched_dl_global_validate(void) -+{ -+ u64 runtime = global_rt_runtime(); -+ u64 period = global_rt_period(); -+ u64 new_bw = to_ratio(period, runtime); -+ struct dl_bw *dl_b; -+ int cpu, ret = 0; -+ unsigned long flags; -+ -+ /* -+ * Here we want to check the bandwidth not being set to some -+ * value smaller than the currently allocated bandwidth in -+ * any of the root_domains. -+ * -+ * FIXME: Cycling on all the CPUs is overdoing, but simpler than -+ * cycling on root_domains... Discussion on different/better -+ * solutions is welcome! -+ */ -+ for_each_possible_cpu(cpu) { -+ rcu_read_lock_sched(); -+ dl_b = dl_bw_of(cpu); -+ -+ raw_spin_lock_irqsave(&dl_b->lock, flags); -+ if (new_bw < dl_b->total_bw) -+ ret = -EBUSY; -+ raw_spin_unlock_irqrestore(&dl_b->lock, flags); -+ -+ rcu_read_unlock_sched(); -+ -+ if (ret) -+ break; -+ } -+ -+ return ret; -+} -+ -+static void sched_dl_do_global(void) -+{ -+ u64 new_bw = -1; -+ struct dl_bw *dl_b; -+ int cpu; -+ unsigned long flags; -+ -+ def_dl_bandwidth.dl_period = global_rt_period(); -+ def_dl_bandwidth.dl_runtime = global_rt_runtime(); -+ -+ if (global_rt_runtime() != RUNTIME_INF) -+ new_bw = to_ratio(global_rt_period(), global_rt_runtime()); -+ -+ /* -+ * FIXME: As above... -+ */ -+ for_each_possible_cpu(cpu) { -+ rcu_read_lock_sched(); -+ dl_b = dl_bw_of(cpu); -+ -+ raw_spin_lock_irqsave(&dl_b->lock, flags); -+ dl_b->bw = new_bw; -+ raw_spin_unlock_irqrestore(&dl_b->lock, flags); -+ -+ rcu_read_unlock_sched(); -+ } -+} -+ -+static int sched_rt_global_validate(void) -+{ -+ if (sysctl_sched_rt_period <= 0) -+ return -EINVAL; -+ -+ if ((sysctl_sched_rt_runtime != RUNTIME_INF) && -+ (sysctl_sched_rt_runtime > sysctl_sched_rt_period)) -+ return -EINVAL; -+ -+ return 0; -+} -+ -+static void sched_rt_do_global(void) -+{ -+ def_rt_bandwidth.rt_runtime = global_rt_runtime(); -+ def_rt_bandwidth.rt_period = ns_to_ktime(global_rt_period()); -+} -+ -+int sched_rt_handler(struct ctl_table *table, int write, -+ void __user *buffer, size_t *lenp, -+ loff_t *ppos) -+{ -+ int old_period, old_runtime; -+ static DEFINE_MUTEX(mutex); -+ int ret; -+ -+ mutex_lock(&mutex); -+ old_period = sysctl_sched_rt_period; -+ old_runtime = sysctl_sched_rt_runtime; -+ -+ ret = proc_dointvec(table, write, buffer, lenp, ppos); -+ -+ if (!ret && write) { -+ ret = sched_rt_global_validate(); -+ if (ret) -+ goto undo; -+ -+ ret = sched_dl_global_validate(); -+ if (ret) -+ goto undo; -+ -+ ret = sched_rt_global_constraints(); -+ if (ret) -+ goto undo; -+ -+ sched_rt_do_global(); -+ sched_dl_do_global(); -+ } -+ if (0) { -+undo: -+ sysctl_sched_rt_period = old_period; -+ sysctl_sched_rt_runtime = old_runtime; -+ } -+ mutex_unlock(&mutex); -+ -+ return ret; -+} -+ -+int sched_rr_handler(struct ctl_table *table, int write, -+ void __user *buffer, size_t *lenp, -+ loff_t *ppos) -+{ -+ int ret; -+ static DEFINE_MUTEX(mutex); -+ -+ mutex_lock(&mutex); -+ ret = proc_dointvec(table, write, buffer, lenp, ppos); -+ /* make sure that internally we keep jiffies */ -+ /* also, writing zero resets timeslice to default */ -+ if (!ret && write) { -+ sched_rr_timeslice = sched_rr_timeslice <= 0 ? -+ RR_TIMESLICE : msecs_to_jiffies(sched_rr_timeslice); -+ } -+ mutex_unlock(&mutex); -+ return ret; -+} -+ -+#ifdef CONFIG_CGROUP_SCHED -+ -+static inline struct task_group *css_tg(struct cgroup_subsys_state *css) -+{ -+ return css ? container_of(css, struct task_group, css) : NULL; -+} -+ -+static struct cgroup_subsys_state * -+cpu_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) -+{ -+ struct task_group *parent = css_tg(parent_css); -+ struct task_group *tg; -+ -+ if (!parent) { -+ /* This is early initialization for the top cgroup */ -+ return &root_task_group.css; -+ } -+ -+ tg = sched_create_group(parent); -+ if (IS_ERR(tg)) -+ return ERR_PTR(-ENOMEM); -+ -+ return &tg->css; -+} -+ -+static int cpu_cgroup_css_online(struct cgroup_subsys_state *css) -+{ -+ struct task_group *tg = css_tg(css); -+ struct task_group *parent = css_tg(css->parent); -+ -+ if (parent) -+ sched_online_group(tg, parent); -+ return 0; -+} -+ -+static void cpu_cgroup_css_free(struct cgroup_subsys_state *css) -+{ -+ struct task_group *tg = css_tg(css); -+ -+ sched_destroy_group(tg); -+} -+ -+static void cpu_cgroup_css_offline(struct cgroup_subsys_state *css) -+{ -+ struct task_group *tg = css_tg(css); -+ -+ sched_offline_group(tg); -+} -+ -+static void cpu_cgroup_fork(struct task_struct *task) -+{ -+ sched_move_task(task); -+} -+ -+static int cpu_cgroup_can_attach(struct cgroup_subsys_state *css, -+ struct cgroup_taskset *tset) -+{ -+ struct task_struct *task; -+ -+ cgroup_taskset_for_each(task, tset) { -+#ifdef CONFIG_RT_GROUP_SCHED -+ if (!sched_rt_can_attach(css_tg(css), task)) -+ return -EINVAL; -+#else -+ /* We don't support RT-tasks being in separate groups */ -+ if (task->sched_class != &fair_sched_class) -+ return -EINVAL; -+#endif -+ } -+ return 0; -+} -+ -+static void cpu_cgroup_attach(struct cgroup_subsys_state *css, -+ struct cgroup_taskset *tset) -+{ -+ struct task_struct *task; -+ -+ cgroup_taskset_for_each(task, tset) -+ sched_move_task(task); -+} -+ -+static void cpu_cgroup_exit(struct cgroup_subsys_state *css, -+ struct cgroup_subsys_state *old_css, -+ struct task_struct *task) -+{ -+ /* -+ * cgroup_exit() is called in the copy_process() failure path. -+ * Ignore this case since the task hasn't ran yet, this avoids -+ * trying to poke a half freed task state from generic code. -+ */ -+ if (!(task->flags & PF_EXITING)) -+ return; -+ -+ sched_move_task(task); -+} -+ -+#ifdef CONFIG_FAIR_GROUP_SCHED -+static int cpu_shares_write_u64(struct cgroup_subsys_state *css, -+ struct cftype *cftype, u64 shareval) -+{ -+ return sched_group_set_shares(css_tg(css), scale_load(shareval)); -+} -+ -+static u64 cpu_shares_read_u64(struct cgroup_subsys_state *css, -+ struct cftype *cft) -+{ -+ struct task_group *tg = css_tg(css); -+ -+ return (u64) scale_load_down(tg->shares); -+} -+ -+#ifdef CONFIG_CFS_BANDWIDTH -+static DEFINE_MUTEX(cfs_constraints_mutex); -+ -+const u64 max_cfs_quota_period = 1 * NSEC_PER_SEC; /* 1s */ -+const u64 min_cfs_quota_period = 1 * NSEC_PER_MSEC; /* 1ms */ -+ -+static int __cfs_schedulable(struct task_group *tg, u64 period, u64 runtime); -+ -+static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota) -+{ -+ int i, ret = 0, runtime_enabled, runtime_was_enabled; -+ struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; -+ -+ if (tg == &root_task_group) -+ return -EINVAL; -+ -+ /* -+ * Ensure we have at some amount of bandwidth every period. This is -+ * to prevent reaching a state of large arrears when throttled via -+ * entity_tick() resulting in prolonged exit starvation. -+ */ -+ if (quota < min_cfs_quota_period || period < min_cfs_quota_period) -+ return -EINVAL; -+ -+ /* -+ * Likewise, bound things on the otherside by preventing insane quota -+ * periods. This also allows us to normalize in computing quota -+ * feasibility. -+ */ -+ if (period > max_cfs_quota_period) -+ return -EINVAL; -+ -+ /* -+ * Prevent race between setting of cfs_rq->runtime_enabled and -+ * unthrottle_offline_cfs_rqs(). -+ */ -+ get_online_cpus(); -+ mutex_lock(&cfs_constraints_mutex); -+ ret = __cfs_schedulable(tg, period, quota); -+ if (ret) -+ goto out_unlock; -+ -+ runtime_enabled = quota != RUNTIME_INF; -+ runtime_was_enabled = cfs_b->quota != RUNTIME_INF; -+ /* -+ * If we need to toggle cfs_bandwidth_used, off->on must occur -+ * before making related changes, and on->off must occur afterwards -+ */ -+ if (runtime_enabled && !runtime_was_enabled) -+ cfs_bandwidth_usage_inc(); -+ raw_spin_lock_irq(&cfs_b->lock); -+ cfs_b->period = ns_to_ktime(period); -+ cfs_b->quota = quota; -+ -+ __refill_cfs_bandwidth_runtime(cfs_b); -+ /* restart the period timer (if active) to handle new period expiry */ -+ if (runtime_enabled && cfs_b->timer_active) { -+ /* force a reprogram */ -+ __start_cfs_bandwidth(cfs_b, true); -+ } -+ raw_spin_unlock_irq(&cfs_b->lock); -+ -+ for_each_online_cpu(i) { -+ struct cfs_rq *cfs_rq = tg->cfs_rq[i]; -+ struct rq *rq = cfs_rq->rq; -+ -+ raw_spin_lock_irq(&rq->lock); -+ cfs_rq->runtime_enabled = runtime_enabled; -+ cfs_rq->runtime_remaining = 0; -+ -+ if (cfs_rq->throttled) -+ unthrottle_cfs_rq(cfs_rq); -+ raw_spin_unlock_irq(&rq->lock); -+ } -+ if (runtime_was_enabled && !runtime_enabled) -+ cfs_bandwidth_usage_dec(); -+out_unlock: -+ mutex_unlock(&cfs_constraints_mutex); -+ put_online_cpus(); -+ -+ return ret; -+} -+ -+int tg_set_cfs_quota(struct task_group *tg, long cfs_quota_us) -+{ -+ u64 quota, period; -+ -+ period = ktime_to_ns(tg->cfs_bandwidth.period); -+ if (cfs_quota_us < 0) -+ quota = RUNTIME_INF; -+ else -+ quota = (u64)cfs_quota_us * NSEC_PER_USEC; -+ -+ return tg_set_cfs_bandwidth(tg, period, quota); -+} -+ -+long tg_get_cfs_quota(struct task_group *tg) -+{ -+ u64 quota_us; -+ -+ if (tg->cfs_bandwidth.quota == RUNTIME_INF) -+ return -1; -+ -+ quota_us = tg->cfs_bandwidth.quota; -+ do_div(quota_us, NSEC_PER_USEC); -+ -+ return quota_us; -+} -+ -+int tg_set_cfs_period(struct task_group *tg, long cfs_period_us) -+{ -+ u64 quota, period; -+ -+ period = (u64)cfs_period_us * NSEC_PER_USEC; -+ quota = tg->cfs_bandwidth.quota; -+ -+ return tg_set_cfs_bandwidth(tg, period, quota); -+} -+ -+long tg_get_cfs_period(struct task_group *tg) -+{ -+ u64 cfs_period_us; -+ -+ cfs_period_us = ktime_to_ns(tg->cfs_bandwidth.period); -+ do_div(cfs_period_us, NSEC_PER_USEC); -+ -+ return cfs_period_us; -+} -+ -+static s64 cpu_cfs_quota_read_s64(struct cgroup_subsys_state *css, -+ struct cftype *cft) -+{ -+ return tg_get_cfs_quota(css_tg(css)); -+} -+ -+static int cpu_cfs_quota_write_s64(struct cgroup_subsys_state *css, -+ struct cftype *cftype, s64 cfs_quota_us) -+{ -+ return tg_set_cfs_quota(css_tg(css), cfs_quota_us); -+} -+ -+static u64 cpu_cfs_period_read_u64(struct cgroup_subsys_state *css, -+ struct cftype *cft) -+{ -+ return tg_get_cfs_period(css_tg(css)); -+} -+ -+static int cpu_cfs_period_write_u64(struct cgroup_subsys_state *css, -+ struct cftype *cftype, u64 cfs_period_us) -+{ -+ return tg_set_cfs_period(css_tg(css), cfs_period_us); -+} -+ -+struct cfs_schedulable_data { -+ struct task_group *tg; -+ u64 period, quota; -+}; -+ -+/* -+ * normalize group quota/period to be quota/max_period -+ * note: units are usecs -+ */ -+static u64 normalize_cfs_quota(struct task_group *tg, -+ struct cfs_schedulable_data *d) -+{ -+ u64 quota, period; -+ -+ if (tg == d->tg) { -+ period = d->period; -+ quota = d->quota; -+ } else { -+ period = tg_get_cfs_period(tg); -+ quota = tg_get_cfs_quota(tg); -+ } -+ -+ /* note: these should typically be equivalent */ -+ if (quota == RUNTIME_INF || quota == -1) -+ return RUNTIME_INF; -+ -+ return to_ratio(period, quota); -+} -+ -+static int tg_cfs_schedulable_down(struct task_group *tg, void *data) -+{ -+ struct cfs_schedulable_data *d = data; -+ struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; -+ s64 quota = 0, parent_quota = -1; -+ -+ if (!tg->parent) { -+ quota = RUNTIME_INF; -+ } else { -+ struct cfs_bandwidth *parent_b = &tg->parent->cfs_bandwidth; -+ -+ quota = normalize_cfs_quota(tg, d); -+ parent_quota = parent_b->hierarchical_quota; -+ -+ /* -+ * ensure max(child_quota) <= parent_quota, inherit when no -+ * limit is set -+ */ -+ if (quota == RUNTIME_INF) -+ quota = parent_quota; -+ else if (parent_quota != RUNTIME_INF && quota > parent_quota) -+ return -EINVAL; -+ } -+ cfs_b->hierarchical_quota = quota; -+ -+ return 0; -+} -+ -+static int __cfs_schedulable(struct task_group *tg, u64 period, u64 quota) -+{ -+ int ret; -+ struct cfs_schedulable_data data = { -+ .tg = tg, -+ .period = period, -+ .quota = quota, -+ }; -+ -+ if (quota != RUNTIME_INF) { -+ do_div(data.period, NSEC_PER_USEC); -+ do_div(data.quota, NSEC_PER_USEC); -+ } -+ -+ rcu_read_lock(); -+ ret = walk_tg_tree(tg_cfs_schedulable_down, tg_nop, &data); -+ rcu_read_unlock(); -+ -+ return ret; -+} -+ -+static int cpu_stats_show(struct seq_file *sf, void *v) -+{ -+ struct task_group *tg = css_tg(seq_css(sf)); -+ struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; -+ -+ seq_printf(sf, "nr_periods %d\n", cfs_b->nr_periods); -+ seq_printf(sf, "nr_throttled %d\n", cfs_b->nr_throttled); -+ seq_printf(sf, "throttled_time %llu\n", cfs_b->throttled_time); -+ -+ return 0; -+} -+#endif /* CONFIG_CFS_BANDWIDTH */ -+#endif /* CONFIG_FAIR_GROUP_SCHED */ -+ -+#ifdef CONFIG_RT_GROUP_SCHED -+static int cpu_rt_runtime_write(struct cgroup_subsys_state *css, -+ struct cftype *cft, s64 val) -+{ -+ return sched_group_set_rt_runtime(css_tg(css), val); -+} -+ -+static s64 cpu_rt_runtime_read(struct cgroup_subsys_state *css, -+ struct cftype *cft) -+{ -+ return sched_group_rt_runtime(css_tg(css)); -+} -+ -+static int cpu_rt_period_write_uint(struct cgroup_subsys_state *css, -+ struct cftype *cftype, u64 rt_period_us) -+{ -+ return sched_group_set_rt_period(css_tg(css), rt_period_us); -+} -+ -+static u64 cpu_rt_period_read_uint(struct cgroup_subsys_state *css, -+ struct cftype *cft) -+{ -+ return sched_group_rt_period(css_tg(css)); -+} -+#endif /* CONFIG_RT_GROUP_SCHED */ -+ -+static struct cftype cpu_files[] = { -+#ifdef CONFIG_FAIR_GROUP_SCHED -+ { -+ .name = "shares", -+ .read_u64 = cpu_shares_read_u64, -+ .write_u64 = cpu_shares_write_u64, -+ }, -+#endif -+#ifdef CONFIG_CFS_BANDWIDTH -+ { -+ .name = "cfs_quota_us", -+ .read_s64 = cpu_cfs_quota_read_s64, -+ .write_s64 = cpu_cfs_quota_write_s64, -+ }, -+ { -+ .name = "cfs_period_us", -+ .read_u64 = cpu_cfs_period_read_u64, -+ .write_u64 = cpu_cfs_period_write_u64, -+ }, -+ { -+ .name = "stat", -+ .seq_show = cpu_stats_show, -+ }, -+#endif -+#ifdef CONFIG_RT_GROUP_SCHED -+ { -+ .name = "rt_runtime_us", -+ .read_s64 = cpu_rt_runtime_read, -+ .write_s64 = cpu_rt_runtime_write, -+ }, -+ { -+ .name = "rt_period_us", -+ .read_u64 = cpu_rt_period_read_uint, -+ .write_u64 = cpu_rt_period_write_uint, -+ }, -+#endif -+ { } /* terminate */ -+}; -+ -+struct cgroup_subsys cpu_cgrp_subsys = { -+ .css_alloc = cpu_cgroup_css_alloc, -+ .css_free = cpu_cgroup_css_free, -+ .css_online = cpu_cgroup_css_online, -+ .css_offline = cpu_cgroup_css_offline, -+ .fork = cpu_cgroup_fork, -+ .can_attach = cpu_cgroup_can_attach, -+ .attach = cpu_cgroup_attach, -+ .exit = cpu_cgroup_exit, -+ .legacy_cftypes = cpu_files, -+ .early_init = 1, -+}; -+ -+#endif /* CONFIG_CGROUP_SCHED */ -+ -+void dump_cpu_task(int cpu) -+{ -+ pr_info("Task dump for CPU %d:\n", cpu); -+ sched_show_task(cpu_curr(cpu)); -+} -diff -Nur linux-4.1.10.orig/kernel/sched/cputime.c linux-4.1.10/kernel/sched/cputime.c ---- linux-4.1.10.orig/kernel/sched/cputime.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/sched/cputime.c 2015-10-07 18:00:08.000000000 +0200 -@@ -675,37 +675,45 @@ - - void vtime_account_system(struct task_struct *tsk) - { -- write_seqlock(&tsk->vtime_seqlock); -+ raw_spin_lock(&tsk->vtime_lock); -+ write_seqcount_begin(&tsk->vtime_seq); - __vtime_account_system(tsk); -- write_sequnlock(&tsk->vtime_seqlock); -+ write_seqcount_end(&tsk->vtime_seq); -+ raw_spin_unlock(&tsk->vtime_lock); - } - - void vtime_gen_account_irq_exit(struct task_struct *tsk) - { -- write_seqlock(&tsk->vtime_seqlock); -+ raw_spin_lock(&tsk->vtime_lock); -+ write_seqcount_begin(&tsk->vtime_seq); - __vtime_account_system(tsk); - if (context_tracking_in_user()) - tsk->vtime_snap_whence = VTIME_USER; -- write_sequnlock(&tsk->vtime_seqlock); -+ write_seqcount_end(&tsk->vtime_seq); -+ raw_spin_unlock(&tsk->vtime_lock); - } - - void vtime_account_user(struct task_struct *tsk) - { - cputime_t delta_cpu; - -- write_seqlock(&tsk->vtime_seqlock); -+ raw_spin_lock(&tsk->vtime_lock); -+ write_seqcount_begin(&tsk->vtime_seq); - delta_cpu = get_vtime_delta(tsk); - tsk->vtime_snap_whence = VTIME_SYS; - account_user_time(tsk, delta_cpu, cputime_to_scaled(delta_cpu)); -- write_sequnlock(&tsk->vtime_seqlock); -+ write_seqcount_end(&tsk->vtime_seq); -+ raw_spin_unlock(&tsk->vtime_lock); - } - - void vtime_user_enter(struct task_struct *tsk) - { -- write_seqlock(&tsk->vtime_seqlock); -+ raw_spin_lock(&tsk->vtime_lock); -+ write_seqcount_begin(&tsk->vtime_seq); - __vtime_account_system(tsk); - tsk->vtime_snap_whence = VTIME_USER; -- write_sequnlock(&tsk->vtime_seqlock); -+ write_seqcount_end(&tsk->vtime_seq); -+ raw_spin_unlock(&tsk->vtime_lock); - } - - void vtime_guest_enter(struct task_struct *tsk) -@@ -717,19 +725,23 @@ - * synchronization against the reader (task_gtime()) - * that can thus safely catch up with a tickless delta. - */ -- write_seqlock(&tsk->vtime_seqlock); -+ raw_spin_lock(&tsk->vtime_lock); -+ write_seqcount_begin(&tsk->vtime_seq); - __vtime_account_system(tsk); - current->flags |= PF_VCPU; -- write_sequnlock(&tsk->vtime_seqlock); -+ write_seqcount_end(&tsk->vtime_seq); -+ raw_spin_unlock(&tsk->vtime_lock); - } - EXPORT_SYMBOL_GPL(vtime_guest_enter); - - void vtime_guest_exit(struct task_struct *tsk) - { -- write_seqlock(&tsk->vtime_seqlock); -+ raw_spin_lock(&tsk->vtime_lock); -+ write_seqcount_begin(&tsk->vtime_seq); - __vtime_account_system(tsk); - current->flags &= ~PF_VCPU; -- write_sequnlock(&tsk->vtime_seqlock); -+ write_seqcount_end(&tsk->vtime_seq); -+ raw_spin_unlock(&tsk->vtime_lock); - } - EXPORT_SYMBOL_GPL(vtime_guest_exit); - -@@ -742,24 +754,30 @@ - - void arch_vtime_task_switch(struct task_struct *prev) - { -- write_seqlock(&prev->vtime_seqlock); -+ raw_spin_lock(&prev->vtime_lock); -+ write_seqcount_begin(&prev->vtime_seq); - prev->vtime_snap_whence = VTIME_SLEEPING; -- write_sequnlock(&prev->vtime_seqlock); -+ write_seqcount_end(&prev->vtime_seq); -+ raw_spin_unlock(&prev->vtime_lock); - -- write_seqlock(¤t->vtime_seqlock); -+ raw_spin_lock(¤t->vtime_lock); -+ write_seqcount_begin(¤t->vtime_seq); - current->vtime_snap_whence = VTIME_SYS; - current->vtime_snap = sched_clock_cpu(smp_processor_id()); -- write_sequnlock(¤t->vtime_seqlock); -+ write_seqcount_end(¤t->vtime_seq); -+ raw_spin_unlock(¤t->vtime_lock); - } - - void vtime_init_idle(struct task_struct *t, int cpu) - { - unsigned long flags; - -- write_seqlock_irqsave(&t->vtime_seqlock, flags); -+ raw_spin_lock_irqsave(&t->vtime_lock, flags); -+ write_seqcount_begin(&t->vtime_seq); - t->vtime_snap_whence = VTIME_SYS; - t->vtime_snap = sched_clock_cpu(cpu); -- write_sequnlock_irqrestore(&t->vtime_seqlock, flags); -+ write_seqcount_end(&t->vtime_seq); -+ raw_spin_unlock_irqrestore(&t->vtime_lock, flags); - } - - cputime_t task_gtime(struct task_struct *t) -@@ -768,13 +786,13 @@ - cputime_t gtime; - - do { -- seq = read_seqbegin(&t->vtime_seqlock); -+ seq = read_seqcount_begin(&t->vtime_seq); - - gtime = t->gtime; - if (t->flags & PF_VCPU) - gtime += vtime_delta(t); - -- } while (read_seqretry(&t->vtime_seqlock, seq)); -+ } while (read_seqcount_retry(&t->vtime_seq, seq)); - - return gtime; - } -@@ -797,7 +815,7 @@ - *udelta = 0; - *sdelta = 0; - -- seq = read_seqbegin(&t->vtime_seqlock); -+ seq = read_seqcount_begin(&t->vtime_seq); - - if (u_dst) - *u_dst = *u_src; -@@ -821,7 +839,7 @@ - if (t->vtime_snap_whence == VTIME_SYS) - *sdelta = delta; - } -- } while (read_seqretry(&t->vtime_seqlock, seq)); -+ } while (read_seqcount_retry(&t->vtime_seq, seq)); - } - - -diff -Nur linux-4.1.10.orig/kernel/sched/deadline.c linux-4.1.10/kernel/sched/deadline.c ---- linux-4.1.10.orig/kernel/sched/deadline.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/sched/deadline.c 2015-10-07 18:00:08.000000000 +0200 -@@ -637,6 +637,7 @@ - - hrtimer_init(timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); - timer->function = dl_task_timer; -+ timer->irqsafe = 1; - } - - static -diff -Nur linux-4.1.10.orig/kernel/sched/debug.c linux-4.1.10/kernel/sched/debug.c ---- linux-4.1.10.orig/kernel/sched/debug.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/sched/debug.c 2015-10-07 18:00:08.000000000 +0200 -@@ -260,6 +260,9 @@ - P(rt_throttled); - PN(rt_time); - PN(rt_runtime); -+#ifdef CONFIG_SMP -+ P(rt_nr_migratory); -+#endif - - #undef PN - #undef P -@@ -648,6 +651,10 @@ - #endif - P(policy); - P(prio); -+#ifdef CONFIG_PREEMPT_RT_FULL -+ P(migrate_disable); -+#endif -+ P(nr_cpus_allowed); - #undef PN - #undef __PN - #undef P -diff -Nur linux-4.1.10.orig/kernel/sched/fair.c linux-4.1.10/kernel/sched/fair.c ---- linux-4.1.10.orig/kernel/sched/fair.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/sched/fair.c 2015-10-07 18:00:08.000000000 +0200 -@@ -3201,7 +3201,7 @@ - ideal_runtime = sched_slice(cfs_rq, curr); - delta_exec = curr->sum_exec_runtime - curr->prev_sum_exec_runtime; - if (delta_exec > ideal_runtime) { -- resched_curr(rq_of(cfs_rq)); -+ resched_curr_lazy(rq_of(cfs_rq)); - /* - * The current task ran long enough, ensure it doesn't get - * re-elected due to buddy favours. -@@ -3225,7 +3225,7 @@ - return; - - if (delta > ideal_runtime) -- resched_curr(rq_of(cfs_rq)); -+ resched_curr_lazy(rq_of(cfs_rq)); - } - - static void -@@ -3366,7 +3366,7 @@ - * validating it and just reschedule. - */ - if (queued) { -- resched_curr(rq_of(cfs_rq)); -+ resched_curr_lazy(rq_of(cfs_rq)); - return; - } - /* -@@ -3557,7 +3557,7 @@ - * hierarchy can be throttled - */ - if (!assign_cfs_rq_runtime(cfs_rq) && likely(cfs_rq->curr)) -- resched_curr(rq_of(cfs_rq)); -+ resched_curr_lazy(rq_of(cfs_rq)); - } - - static __always_inline -@@ -4180,7 +4180,7 @@ - - if (delta < 0) { - if (rq->curr == p) -- resched_curr(rq); -+ resched_curr_lazy(rq); - return; - } - hrtick_start(rq, delta); -@@ -5076,7 +5076,7 @@ - return; - - preempt: -- resched_curr(rq); -+ resched_curr_lazy(rq); - /* - * Only set the backward buddy when the current task is still - * on the rq. This can happen when a wakeup gets interleaved -@@ -7866,7 +7866,7 @@ - * 'current' within the tree based on its new key value. - */ - swap(curr->vruntime, se->vruntime); -- resched_curr(rq); -+ resched_curr_lazy(rq); - } - - se->vruntime -= cfs_rq->min_vruntime; -@@ -7891,7 +7891,7 @@ - */ - if (rq->curr == p) { - if (p->prio > oldprio) -- resched_curr(rq); -+ resched_curr_lazy(rq); - } else - check_preempt_curr(rq, p, 0); - } -diff -Nur linux-4.1.10.orig/kernel/sched/features.h linux-4.1.10/kernel/sched/features.h ---- linux-4.1.10.orig/kernel/sched/features.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/sched/features.h 2015-10-07 18:00:08.000000000 +0200 -@@ -50,11 +50,19 @@ - */ - SCHED_FEAT(NONTASK_CAPACITY, true) - -+#ifdef CONFIG_PREEMPT_RT_FULL -+SCHED_FEAT(TTWU_QUEUE, false) -+# ifdef CONFIG_PREEMPT_LAZY -+SCHED_FEAT(PREEMPT_LAZY, true) -+# endif -+#else -+ - /* - * Queue remote wakeups on the target CPU and process them - * using the scheduler IPI. Reduces rq->lock contention/bounces. - */ - SCHED_FEAT(TTWU_QUEUE, true) -+#endif - - #ifdef HAVE_RT_PUSH_IPI - /* -diff -Nur linux-4.1.10.orig/kernel/sched/Makefile linux-4.1.10/kernel/sched/Makefile ---- linux-4.1.10.orig/kernel/sched/Makefile 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/sched/Makefile 2015-10-07 18:00:08.000000000 +0200 -@@ -13,7 +13,7 @@ - - obj-y += core.o proc.o clock.o cputime.o - obj-y += idle_task.o fair.o rt.o deadline.o stop_task.o --obj-y += wait.o completion.o idle.o -+obj-y += wait.o wait-simple.o work-simple.o completion.o idle.o - obj-$(CONFIG_SMP) += cpupri.o cpudeadline.o - obj-$(CONFIG_SCHED_AUTOGROUP) += auto_group.o - obj-$(CONFIG_SCHEDSTATS) += stats.o -diff -Nur linux-4.1.10.orig/kernel/sched/rt.c linux-4.1.10/kernel/sched/rt.c ---- linux-4.1.10.orig/kernel/sched/rt.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/sched/rt.c 2015-10-07 18:00:08.000000000 +0200 -@@ -44,6 +44,7 @@ - - hrtimer_init(&rt_b->rt_period_timer, - CLOCK_MONOTONIC, HRTIMER_MODE_REL); -+ rt_b->rt_period_timer.irqsafe = 1; - rt_b->rt_period_timer.function = sched_rt_period_timer; - } - -@@ -89,6 +90,7 @@ - rt_rq->push_cpu = nr_cpu_ids; - raw_spin_lock_init(&rt_rq->push_lock); - init_irq_work(&rt_rq->push_work, push_irq_work_func); -+ rt_rq->push_work.flags |= IRQ_WORK_HARD_IRQ; - #endif - #endif /* CONFIG_SMP */ - /* We start is dequeued state, because no RT tasks are queued */ -diff -Nur linux-4.1.10.orig/kernel/sched/sched.h linux-4.1.10/kernel/sched/sched.h ---- linux-4.1.10.orig/kernel/sched/sched.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/sched/sched.h 2015-10-07 18:00:08.000000000 +0200 -@@ -1092,6 +1092,7 @@ - #define WF_SYNC 0x01 /* waker goes to sleep after wakeup */ - #define WF_FORK 0x02 /* child wakeup after fork */ - #define WF_MIGRATED 0x4 /* internal use, task got migrated */ -+#define WF_LOCK_SLEEPER 0x08 /* wakeup spinlock "sleeper" */ - - /* - * To aid in avoiding the subversion of "niceness" due to uneven distribution -@@ -1289,6 +1290,15 @@ - extern void resched_curr(struct rq *rq); - extern void resched_cpu(int cpu); - -+#ifdef CONFIG_PREEMPT_LAZY -+extern void resched_curr_lazy(struct rq *rq); -+#else -+static inline void resched_curr_lazy(struct rq *rq) -+{ -+ resched_curr(rq); -+} -+#endif -+ - extern struct rt_bandwidth def_rt_bandwidth; - extern void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime); - -diff -Nur linux-4.1.10.orig/kernel/sched/wait-simple.c linux-4.1.10/kernel/sched/wait-simple.c ---- linux-4.1.10.orig/kernel/sched/wait-simple.c 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.10/kernel/sched/wait-simple.c 2015-10-07 18:00:08.000000000 +0200 -@@ -0,0 +1,115 @@ -+/* -+ * Simple waitqueues without fancy flags and callbacks -+ * -+ * (C) 2011 Thomas Gleixner -+ * -+ * Based on kernel/wait.c -+ * -+ * For licencing details see kernel-base/COPYING -+ */ -+#include -+#include -+#include -+#include -+ -+/* Adds w to head->list. Must be called with head->lock locked. */ -+static inline void __swait_enqueue(struct swait_head *head, struct swaiter *w) -+{ -+ list_add(&w->node, &head->list); -+ /* We can't let the condition leak before the setting of head */ -+ smp_mb(); -+} -+ -+/* Removes w from head->list. Must be called with head->lock locked. */ -+static inline void __swait_dequeue(struct swaiter *w) -+{ -+ list_del_init(&w->node); -+} -+ -+void __init_swait_head(struct swait_head *head, struct lock_class_key *key) -+{ -+ raw_spin_lock_init(&head->lock); -+ lockdep_set_class(&head->lock, key); -+ INIT_LIST_HEAD(&head->list); -+} -+EXPORT_SYMBOL(__init_swait_head); -+ -+void swait_prepare_locked(struct swait_head *head, struct swaiter *w) -+{ -+ w->task = current; -+ if (list_empty(&w->node)) -+ __swait_enqueue(head, w); -+} -+ -+void swait_prepare(struct swait_head *head, struct swaiter *w, int state) -+{ -+ unsigned long flags; -+ -+ raw_spin_lock_irqsave(&head->lock, flags); -+ swait_prepare_locked(head, w); -+ __set_current_state(state); -+ raw_spin_unlock_irqrestore(&head->lock, flags); -+} -+EXPORT_SYMBOL(swait_prepare); -+ -+void swait_finish_locked(struct swait_head *head, struct swaiter *w) -+{ -+ __set_current_state(TASK_RUNNING); -+ if (w->task) -+ __swait_dequeue(w); -+} -+ -+void swait_finish(struct swait_head *head, struct swaiter *w) -+{ -+ unsigned long flags; -+ -+ __set_current_state(TASK_RUNNING); -+ if (w->task) { -+ raw_spin_lock_irqsave(&head->lock, flags); -+ __swait_dequeue(w); -+ raw_spin_unlock_irqrestore(&head->lock, flags); -+ } -+} -+EXPORT_SYMBOL(swait_finish); -+ -+unsigned int -+__swait_wake_locked(struct swait_head *head, unsigned int state, unsigned int num) -+{ -+ struct swaiter *curr, *next; -+ int woken = 0; -+ -+ list_for_each_entry_safe(curr, next, &head->list, node) { -+ if (wake_up_state(curr->task, state)) { -+ __swait_dequeue(curr); -+ /* -+ * The waiting task can free the waiter as -+ * soon as curr->task = NULL is written, -+ * without taking any locks. A memory barrier -+ * is required here to prevent the following -+ * store to curr->task from getting ahead of -+ * the dequeue operation. -+ */ -+ smp_wmb(); -+ curr->task = NULL; -+ if (++woken == num) -+ break; -+ } -+ } -+ return woken; -+} -+ -+unsigned int -+__swait_wake(struct swait_head *head, unsigned int state, unsigned int num) -+{ -+ unsigned long flags; -+ int woken; -+ -+ if (!swaitqueue_active(head)) -+ return 0; -+ -+ raw_spin_lock_irqsave(&head->lock, flags); -+ woken = __swait_wake_locked(head, state, num); -+ raw_spin_unlock_irqrestore(&head->lock, flags); -+ return woken; -+} -+EXPORT_SYMBOL(__swait_wake); -diff -Nur linux-4.1.10.orig/kernel/sched/work-simple.c linux-4.1.10/kernel/sched/work-simple.c ---- linux-4.1.10.orig/kernel/sched/work-simple.c 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.10/kernel/sched/work-simple.c 2015-10-07 18:00:08.000000000 +0200 -@@ -0,0 +1,172 @@ -+/* -+ * Copyright (C) 2014 BMW Car IT GmbH, Daniel Wagner daniel.wagner@bmw-carit.de -+ * -+ * Provides a framework for enqueuing callbacks from irq context -+ * PREEMPT_RT_FULL safe. The callbacks are executed in kthread context. -+ */ -+ -+#include -+#include -+#include -+#include -+#include -+ -+#define SWORK_EVENT_PENDING (1 << 0) -+ -+static DEFINE_MUTEX(worker_mutex); -+static struct sworker *glob_worker; -+ -+struct sworker { -+ struct list_head events; -+ struct swait_head wq; -+ -+ raw_spinlock_t lock; -+ -+ struct task_struct *task; -+ int refs; -+}; -+ -+static bool swork_readable(struct sworker *worker) -+{ -+ bool r; -+ -+ if (kthread_should_stop()) -+ return true; -+ -+ raw_spin_lock_irq(&worker->lock); -+ r = !list_empty(&worker->events); -+ raw_spin_unlock_irq(&worker->lock); -+ -+ return r; -+} -+ -+static int swork_kthread(void *arg) -+{ -+ struct sworker *worker = arg; -+ -+ for (;;) { -+ swait_event_interruptible(worker->wq, -+ swork_readable(worker)); -+ if (kthread_should_stop()) -+ break; -+ -+ raw_spin_lock_irq(&worker->lock); -+ while (!list_empty(&worker->events)) { -+ struct swork_event *sev; -+ -+ sev = list_first_entry(&worker->events, -+ struct swork_event, item); -+ list_del(&sev->item); -+ raw_spin_unlock_irq(&worker->lock); -+ -+ WARN_ON_ONCE(!test_and_clear_bit(SWORK_EVENT_PENDING, -+ &sev->flags)); -+ sev->func(sev); -+ raw_spin_lock_irq(&worker->lock); -+ } -+ raw_spin_unlock_irq(&worker->lock); -+ } -+ return 0; -+} -+ -+static struct sworker *swork_create(void) -+{ -+ struct sworker *worker; -+ -+ worker = kzalloc(sizeof(*worker), GFP_KERNEL); -+ if (!worker) -+ return ERR_PTR(-ENOMEM); -+ -+ INIT_LIST_HEAD(&worker->events); -+ raw_spin_lock_init(&worker->lock); -+ init_swait_head(&worker->wq); -+ -+ worker->task = kthread_run(swork_kthread, worker, "kswork"); -+ if (IS_ERR(worker->task)) { -+ kfree(worker); -+ return ERR_PTR(-ENOMEM); -+ } -+ -+ return worker; -+} -+ -+static void swork_destroy(struct sworker *worker) -+{ -+ kthread_stop(worker->task); -+ -+ WARN_ON(!list_empty(&worker->events)); -+ kfree(worker); -+} -+ -+/** -+ * swork_queue - queue swork -+ * -+ * Returns %false if @work was already on a queue, %true otherwise. -+ * -+ * The work is queued and processed on a random CPU -+ */ -+bool swork_queue(struct swork_event *sev) -+{ -+ unsigned long flags; -+ -+ if (test_and_set_bit(SWORK_EVENT_PENDING, &sev->flags)) -+ return false; -+ -+ raw_spin_lock_irqsave(&glob_worker->lock, flags); -+ list_add_tail(&sev->item, &glob_worker->events); -+ raw_spin_unlock_irqrestore(&glob_worker->lock, flags); -+ -+ swait_wake(&glob_worker->wq); -+ return true; -+} -+EXPORT_SYMBOL_GPL(swork_queue); -+ -+/** -+ * swork_get - get an instance of the sworker -+ * -+ * Returns an negative error code if the initialization if the worker did not -+ * work, %0 otherwise. -+ * -+ */ -+int swork_get(void) -+{ -+ struct sworker *worker; -+ -+ mutex_lock(&worker_mutex); -+ if (!glob_worker) { -+ worker = swork_create(); -+ if (IS_ERR(worker)) { -+ mutex_unlock(&worker_mutex); -+ return -ENOMEM; -+ } -+ -+ glob_worker = worker; -+ } -+ -+ glob_worker->refs++; -+ mutex_unlock(&worker_mutex); -+ -+ return 0; -+} -+EXPORT_SYMBOL_GPL(swork_get); -+ -+/** -+ * swork_put - puts an instance of the sworker -+ * -+ * Will destroy the sworker thread. This function must not be called until all -+ * queued events have been completed. -+ */ -+void swork_put(void) -+{ -+ mutex_lock(&worker_mutex); -+ -+ glob_worker->refs--; -+ if (glob_worker->refs > 0) -+ goto out; -+ -+ swork_destroy(glob_worker); -+ glob_worker = NULL; -+out: -+ mutex_unlock(&worker_mutex); -+} -+EXPORT_SYMBOL_GPL(swork_put); -diff -Nur linux-4.1.10.orig/kernel/signal.c linux-4.1.10/kernel/signal.c ---- linux-4.1.10.orig/kernel/signal.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/signal.c 2015-10-07 18:00:08.000000000 +0200 -@@ -14,6 +14,7 @@ - #include - #include - #include -+#include - #include - #include - #include -@@ -352,13 +353,45 @@ - return false; - } - -+#ifdef __HAVE_ARCH_CMPXCHG -+static inline struct sigqueue *get_task_cache(struct task_struct *t) -+{ -+ struct sigqueue *q = t->sigqueue_cache; -+ -+ if (cmpxchg(&t->sigqueue_cache, q, NULL) != q) -+ return NULL; -+ return q; -+} -+ -+static inline int put_task_cache(struct task_struct *t, struct sigqueue *q) -+{ -+ if (cmpxchg(&t->sigqueue_cache, NULL, q) == NULL) -+ return 0; -+ return 1; -+} -+ -+#else -+ -+static inline struct sigqueue *get_task_cache(struct task_struct *t) -+{ -+ return NULL; -+} -+ -+static inline int put_task_cache(struct task_struct *t, struct sigqueue *q) -+{ -+ return 1; -+} -+ -+#endif -+ - /* - * allocate a new signal queue record - * - this may be called without locks if and only if t == current, otherwise an - * appropriate lock must be held to stop the target task from exiting - */ - static struct sigqueue * --__sigqueue_alloc(int sig, struct task_struct *t, gfp_t flags, int override_rlimit) -+__sigqueue_do_alloc(int sig, struct task_struct *t, gfp_t flags, -+ int override_rlimit, int fromslab) - { - struct sigqueue *q = NULL; - struct user_struct *user; -@@ -375,7 +408,10 @@ - if (override_rlimit || - atomic_read(&user->sigpending) <= - task_rlimit(t, RLIMIT_SIGPENDING)) { -- q = kmem_cache_alloc(sigqueue_cachep, flags); -+ if (!fromslab) -+ q = get_task_cache(t); -+ if (!q) -+ q = kmem_cache_alloc(sigqueue_cachep, flags); - } else { - print_dropped_signal(sig); - } -@@ -392,6 +428,13 @@ - return q; - } - -+static struct sigqueue * -+__sigqueue_alloc(int sig, struct task_struct *t, gfp_t flags, -+ int override_rlimit) -+{ -+ return __sigqueue_do_alloc(sig, t, flags, override_rlimit, 0); -+} -+ - static void __sigqueue_free(struct sigqueue *q) - { - if (q->flags & SIGQUEUE_PREALLOC) -@@ -401,6 +444,21 @@ - kmem_cache_free(sigqueue_cachep, q); - } - -+static void sigqueue_free_current(struct sigqueue *q) -+{ -+ struct user_struct *up; -+ -+ if (q->flags & SIGQUEUE_PREALLOC) -+ return; -+ -+ up = q->user; -+ if (rt_prio(current->normal_prio) && !put_task_cache(current, q)) { -+ atomic_dec(&up->sigpending); -+ free_uid(up); -+ } else -+ __sigqueue_free(q); -+} -+ - void flush_sigqueue(struct sigpending *queue) - { - struct sigqueue *q; -@@ -414,6 +472,21 @@ - } - - /* -+ * Called from __exit_signal. Flush tsk->pending and -+ * tsk->sigqueue_cache -+ */ -+void flush_task_sigqueue(struct task_struct *tsk) -+{ -+ struct sigqueue *q; -+ -+ flush_sigqueue(&tsk->pending); -+ -+ q = get_task_cache(tsk); -+ if (q) -+ kmem_cache_free(sigqueue_cachep, q); -+} -+ -+/* - * Flush all pending signals for a task. - */ - void __flush_signals(struct task_struct *t) -@@ -565,7 +638,7 @@ - still_pending: - list_del_init(&first->list); - copy_siginfo(info, &first->info); -- __sigqueue_free(first); -+ sigqueue_free_current(first); - } else { - /* - * Ok, it wasn't in the queue. This must be -@@ -611,6 +684,8 @@ - { - int signr; - -+ WARN_ON_ONCE(tsk != current); -+ - /* We only dequeue private signals from ourselves, we don't let - * signalfd steal them - */ -@@ -1207,8 +1282,8 @@ - * We don't want to have recursive SIGSEGV's etc, for example, - * that is why we also clear SIGNAL_UNKILLABLE. - */ --int --force_sig_info(int sig, struct siginfo *info, struct task_struct *t) -+static int -+do_force_sig_info(int sig, struct siginfo *info, struct task_struct *t) - { - unsigned long int flags; - int ret, blocked, ignored; -@@ -1233,6 +1308,39 @@ - return ret; - } - -+int force_sig_info(int sig, struct siginfo *info, struct task_struct *t) -+{ -+/* -+ * On some archs, PREEMPT_RT has to delay sending a signal from a trap -+ * since it can not enable preemption, and the signal code's spin_locks -+ * turn into mutexes. Instead, it must set TIF_NOTIFY_RESUME which will -+ * send the signal on exit of the trap. -+ */ -+#ifdef ARCH_RT_DELAYS_SIGNAL_SEND -+ if (in_atomic()) { -+ if (WARN_ON_ONCE(t != current)) -+ return 0; -+ if (WARN_ON_ONCE(t->forced_info.si_signo)) -+ return 0; -+ -+ if (is_si_special(info)) { -+ WARN_ON_ONCE(info != SEND_SIG_PRIV); -+ t->forced_info.si_signo = sig; -+ t->forced_info.si_errno = 0; -+ t->forced_info.si_code = SI_KERNEL; -+ t->forced_info.si_pid = 0; -+ t->forced_info.si_uid = 0; -+ } else { -+ t->forced_info = *info; -+ } -+ -+ set_tsk_thread_flag(t, TIF_NOTIFY_RESUME); -+ return 0; -+ } -+#endif -+ return do_force_sig_info(sig, info, t); -+} -+ - /* - * Nuke all other threads in the group. - */ -@@ -1267,12 +1375,12 @@ - * Disable interrupts early to avoid deadlocks. - * See rcu_read_unlock() comment header for details. - */ -- local_irq_save(*flags); -+ local_irq_save_nort(*flags); - rcu_read_lock(); - sighand = rcu_dereference(tsk->sighand); - if (unlikely(sighand == NULL)) { - rcu_read_unlock(); -- local_irq_restore(*flags); -+ local_irq_restore_nort(*flags); - break; - } - /* -@@ -1293,7 +1401,7 @@ - } - spin_unlock(&sighand->siglock); - rcu_read_unlock(); -- local_irq_restore(*flags); -+ local_irq_restore_nort(*flags); - } - - return sighand; -@@ -1536,7 +1644,8 @@ - */ - struct sigqueue *sigqueue_alloc(void) - { -- struct sigqueue *q = __sigqueue_alloc(-1, current, GFP_KERNEL, 0); -+ /* Preallocated sigqueue objects always from the slabcache ! */ -+ struct sigqueue *q = __sigqueue_do_alloc(-1, current, GFP_KERNEL, 0, 1); - - if (q) - q->flags |= SIGQUEUE_PREALLOC; -@@ -1897,15 +2006,7 @@ - if (gstop_done && ptrace_reparented(current)) - do_notify_parent_cldstop(current, false, why); - -- /* -- * Don't want to allow preemption here, because -- * sys_ptrace() needs this task to be inactive. -- * -- * XXX: implement read_unlock_no_resched(). -- */ -- preempt_disable(); - read_unlock(&tasklist_lock); -- preempt_enable_no_resched(); - freezable_schedule(); - } else { - /* -diff -Nur linux-4.1.10.orig/kernel/softirq.c linux-4.1.10/kernel/softirq.c ---- linux-4.1.10.orig/kernel/softirq.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/softirq.c 2015-10-07 18:00:08.000000000 +0200 -@@ -21,10 +21,12 @@ - #include - #include - #include -+#include - #include - #include - #include - #include -+#include - #include - - #define CREATE_TRACE_POINTS -@@ -62,6 +64,98 @@ - "TASKLET", "SCHED", "HRTIMER", "RCU" - }; - -+#ifdef CONFIG_NO_HZ_COMMON -+# ifdef CONFIG_PREEMPT_RT_FULL -+ -+struct softirq_runner { -+ struct task_struct *runner[NR_SOFTIRQS]; -+}; -+ -+static DEFINE_PER_CPU(struct softirq_runner, softirq_runners); -+ -+static inline void softirq_set_runner(unsigned int sirq) -+{ -+ struct softirq_runner *sr = this_cpu_ptr(&softirq_runners); -+ -+ sr->runner[sirq] = current; -+} -+ -+static inline void softirq_clr_runner(unsigned int sirq) -+{ -+ struct softirq_runner *sr = this_cpu_ptr(&softirq_runners); -+ -+ sr->runner[sirq] = NULL; -+} -+ -+/* -+ * On preempt-rt a softirq running context might be blocked on a -+ * lock. There might be no other runnable task on this CPU because the -+ * lock owner runs on some other CPU. So we have to go into idle with -+ * the pending bit set. Therefor we need to check this otherwise we -+ * warn about false positives which confuses users and defeats the -+ * whole purpose of this test. -+ * -+ * This code is called with interrupts disabled. -+ */ -+void softirq_check_pending_idle(void) -+{ -+ static int rate_limit; -+ struct softirq_runner *sr = this_cpu_ptr(&softirq_runners); -+ u32 warnpending; -+ int i; -+ -+ if (rate_limit >= 10) -+ return; -+ -+ warnpending = local_softirq_pending() & SOFTIRQ_STOP_IDLE_MASK; -+ for (i = 0; i < NR_SOFTIRQS; i++) { -+ struct task_struct *tsk = sr->runner[i]; -+ -+ /* -+ * The wakeup code in rtmutex.c wakes up the task -+ * _before_ it sets pi_blocked_on to NULL under -+ * tsk->pi_lock. So we need to check for both: state -+ * and pi_blocked_on. -+ */ -+ if (tsk) { -+ raw_spin_lock(&tsk->pi_lock); -+ if (tsk->pi_blocked_on || tsk->state == TASK_RUNNING) { -+ /* Clear all bits pending in that task */ -+ warnpending &= ~(tsk->softirqs_raised); -+ warnpending &= ~(1 << i); -+ } -+ raw_spin_unlock(&tsk->pi_lock); -+ } -+ } -+ -+ if (warnpending) { -+ printk(KERN_ERR "NOHZ: local_softirq_pending %02x\n", -+ warnpending); -+ rate_limit++; -+ } -+} -+# else -+/* -+ * On !PREEMPT_RT we just printk rate limited: -+ */ -+void softirq_check_pending_idle(void) -+{ -+ static int rate_limit; -+ -+ if (rate_limit < 10 && -+ (local_softirq_pending() & SOFTIRQ_STOP_IDLE_MASK)) { -+ printk(KERN_ERR "NOHZ: local_softirq_pending %02x\n", -+ local_softirq_pending()); -+ rate_limit++; -+ } -+} -+# endif -+ -+#else /* !CONFIG_NO_HZ_COMMON */ -+static inline void softirq_set_runner(unsigned int sirq) { } -+static inline void softirq_clr_runner(unsigned int sirq) { } -+#endif -+ - /* - * we cannot loop indefinitely here to avoid userspace starvation, - * but we also don't want to introduce a worst case 1/HZ latency -@@ -77,6 +171,68 @@ - wake_up_process(tsk); - } - -+static void handle_softirq(unsigned int vec_nr) -+{ -+ struct softirq_action *h = softirq_vec + vec_nr; -+ int prev_count; -+ -+ prev_count = preempt_count(); -+ -+ kstat_incr_softirqs_this_cpu(vec_nr); -+ -+ trace_softirq_entry(vec_nr); -+ h->action(h); -+ trace_softirq_exit(vec_nr); -+ if (unlikely(prev_count != preempt_count())) { -+ pr_err("huh, entered softirq %u %s %p with preempt_count %08x, exited with %08x?\n", -+ vec_nr, softirq_to_name[vec_nr], h->action, -+ prev_count, preempt_count()); -+ preempt_count_set(prev_count); -+ } -+} -+ -+#ifndef CONFIG_PREEMPT_RT_FULL -+static inline int ksoftirqd_softirq_pending(void) -+{ -+ return local_softirq_pending(); -+} -+ -+static void handle_pending_softirqs(u32 pending) -+{ -+ struct softirq_action *h = softirq_vec; -+ int softirq_bit; -+ -+ local_irq_enable(); -+ -+ h = softirq_vec; -+ -+ while ((softirq_bit = ffs(pending))) { -+ unsigned int vec_nr; -+ -+ h += softirq_bit - 1; -+ vec_nr = h - softirq_vec; -+ handle_softirq(vec_nr); -+ -+ h++; -+ pending >>= softirq_bit; -+ } -+ -+ rcu_bh_qs(); -+ local_irq_disable(); -+} -+ -+static void run_ksoftirqd(unsigned int cpu) -+{ -+ local_irq_disable(); -+ if (ksoftirqd_softirq_pending()) { -+ __do_softirq(); -+ local_irq_enable(); -+ cond_resched_rcu_qs(); -+ return; -+ } -+ local_irq_enable(); -+} -+ - /* - * preempt_count and SOFTIRQ_OFFSET usage: - * - preempt_count is changed by SOFTIRQ_OFFSET on entering or leaving -@@ -232,10 +388,8 @@ - unsigned long end = jiffies + MAX_SOFTIRQ_TIME; - unsigned long old_flags = current->flags; - int max_restart = MAX_SOFTIRQ_RESTART; -- struct softirq_action *h; - bool in_hardirq; - __u32 pending; -- int softirq_bit; - - /* - * Mask out PF_MEMALLOC s current task context is borrowed for the -@@ -254,36 +408,7 @@ - /* Reset the pending bitmask before enabling irqs */ - set_softirq_pending(0); - -- local_irq_enable(); -- -- h = softirq_vec; -- -- while ((softirq_bit = ffs(pending))) { -- unsigned int vec_nr; -- int prev_count; -- -- h += softirq_bit - 1; -- -- vec_nr = h - softirq_vec; -- prev_count = preempt_count(); -- -- kstat_incr_softirqs_this_cpu(vec_nr); -- -- trace_softirq_entry(vec_nr); -- h->action(h); -- trace_softirq_exit(vec_nr); -- if (unlikely(prev_count != preempt_count())) { -- pr_err("huh, entered softirq %u %s %p with preempt_count %08x, exited with %08x?\n", -- vec_nr, softirq_to_name[vec_nr], h->action, -- prev_count, preempt_count()); -- preempt_count_set(prev_count); -- } -- h++; -- pending >>= softirq_bit; -- } -- -- rcu_bh_qs(); -- local_irq_disable(); -+ handle_pending_softirqs(pending); - - pending = local_softirq_pending(); - if (pending) { -@@ -320,6 +445,285 @@ - } - - /* -+ * This function must run with irqs disabled! -+ */ -+void raise_softirq_irqoff(unsigned int nr) -+{ -+ __raise_softirq_irqoff(nr); -+ -+ /* -+ * If we're in an interrupt or softirq, we're done -+ * (this also catches softirq-disabled code). We will -+ * actually run the softirq once we return from -+ * the irq or softirq. -+ * -+ * Otherwise we wake up ksoftirqd to make sure we -+ * schedule the softirq soon. -+ */ -+ if (!in_interrupt()) -+ wakeup_softirqd(); -+} -+ -+void __raise_softirq_irqoff(unsigned int nr) -+{ -+ trace_softirq_raise(nr); -+ or_softirq_pending(1UL << nr); -+} -+ -+static inline void local_bh_disable_nort(void) { local_bh_disable(); } -+static inline void _local_bh_enable_nort(void) { _local_bh_enable(); } -+static void ksoftirqd_set_sched_params(unsigned int cpu) { } -+static void ksoftirqd_clr_sched_params(unsigned int cpu, bool online) { } -+ -+#else /* !PREEMPT_RT_FULL */ -+ -+/* -+ * On RT we serialize softirq execution with a cpu local lock per softirq -+ */ -+static DEFINE_PER_CPU(struct local_irq_lock [NR_SOFTIRQS], local_softirq_locks); -+ -+void __init softirq_early_init(void) -+{ -+ int i; -+ -+ for (i = 0; i < NR_SOFTIRQS; i++) -+ local_irq_lock_init(local_softirq_locks[i]); -+} -+ -+static void lock_softirq(int which) -+{ -+ local_lock(local_softirq_locks[which]); -+} -+ -+static void unlock_softirq(int which) -+{ -+ local_unlock(local_softirq_locks[which]); -+} -+ -+static void do_single_softirq(int which) -+{ -+ unsigned long old_flags = current->flags; -+ -+ current->flags &= ~PF_MEMALLOC; -+ vtime_account_irq_enter(current); -+ current->flags |= PF_IN_SOFTIRQ; -+ lockdep_softirq_enter(); -+ local_irq_enable(); -+ handle_softirq(which); -+ local_irq_disable(); -+ lockdep_softirq_exit(); -+ current->flags &= ~PF_IN_SOFTIRQ; -+ vtime_account_irq_enter(current); -+ tsk_restore_flags(current, old_flags, PF_MEMALLOC); -+} -+ -+/* -+ * Called with interrupts disabled. Process softirqs which were raised -+ * in current context (or on behalf of ksoftirqd). -+ */ -+static void do_current_softirqs(void) -+{ -+ while (current->softirqs_raised) { -+ int i = __ffs(current->softirqs_raised); -+ unsigned int pending, mask = (1U << i); -+ -+ current->softirqs_raised &= ~mask; -+ local_irq_enable(); -+ -+ /* -+ * If the lock is contended, we boost the owner to -+ * process the softirq or leave the critical section -+ * now. -+ */ -+ lock_softirq(i); -+ local_irq_disable(); -+ softirq_set_runner(i); -+ /* -+ * Check with the local_softirq_pending() bits, -+ * whether we need to process this still or if someone -+ * else took care of it. -+ */ -+ pending = local_softirq_pending(); -+ if (pending & mask) { -+ set_softirq_pending(pending & ~mask); -+ do_single_softirq(i); -+ } -+ softirq_clr_runner(i); -+ unlock_softirq(i); -+ WARN_ON(current->softirq_nestcnt != 1); -+ } -+} -+ -+static void __local_bh_disable(void) -+{ -+ if (++current->softirq_nestcnt == 1) -+ migrate_disable(); -+} -+ -+void local_bh_disable(void) -+{ -+ __local_bh_disable(); -+} -+EXPORT_SYMBOL(local_bh_disable); -+ -+void __local_bh_disable_ip(unsigned long ip, unsigned int cnt) -+{ -+ __local_bh_disable(); -+ if (cnt & PREEMPT_CHECK_OFFSET) -+ preempt_disable(); -+} -+ -+static void __local_bh_enable(void) -+{ -+ if (WARN_ON(current->softirq_nestcnt == 0)) -+ return; -+ -+ local_irq_disable(); -+ if (current->softirq_nestcnt == 1 && current->softirqs_raised) -+ do_current_softirqs(); -+ local_irq_enable(); -+ -+ if (--current->softirq_nestcnt == 0) -+ migrate_enable(); -+} -+ -+void local_bh_enable(void) -+{ -+ __local_bh_enable(); -+} -+EXPORT_SYMBOL(local_bh_enable); -+ -+extern void __local_bh_enable_ip(unsigned long ip, unsigned int cnt) -+{ -+ __local_bh_enable(); -+ if (cnt & PREEMPT_CHECK_OFFSET) -+ preempt_enable(); -+} -+ -+void local_bh_enable_ip(unsigned long ip) -+{ -+ local_bh_enable(); -+} -+EXPORT_SYMBOL(local_bh_enable_ip); -+ -+void _local_bh_enable(void) -+{ -+ if (WARN_ON(current->softirq_nestcnt == 0)) -+ return; -+ if (--current->softirq_nestcnt == 0) -+ migrate_enable(); -+} -+EXPORT_SYMBOL(_local_bh_enable); -+ -+int in_serving_softirq(void) -+{ -+ return current->flags & PF_IN_SOFTIRQ; -+} -+EXPORT_SYMBOL(in_serving_softirq); -+ -+/* Called with preemption disabled */ -+static void run_ksoftirqd(unsigned int cpu) -+{ -+ local_irq_disable(); -+ current->softirq_nestcnt++; -+ -+ do_current_softirqs(); -+ current->softirq_nestcnt--; -+ rcu_note_context_switch(); -+ local_irq_enable(); -+} -+ -+/* -+ * Called from netif_rx_ni(). Preemption enabled, but migration -+ * disabled. So the cpu can't go away under us. -+ */ -+void thread_do_softirq(void) -+{ -+ if (!in_serving_softirq() && current->softirqs_raised) { -+ current->softirq_nestcnt++; -+ do_current_softirqs(); -+ current->softirq_nestcnt--; -+ } -+} -+ -+static void do_raise_softirq_irqoff(unsigned int nr) -+{ -+ trace_softirq_raise(nr); -+ or_softirq_pending(1UL << nr); -+ -+ /* -+ * If we are not in a hard interrupt and inside a bh disabled -+ * region, we simply raise the flag on current. local_bh_enable() -+ * will make sure that the softirq is executed. Otherwise we -+ * delegate it to ksoftirqd. -+ */ -+ if (!in_irq() && current->softirq_nestcnt) -+ current->softirqs_raised |= (1U << nr); -+ else if (__this_cpu_read(ksoftirqd)) -+ __this_cpu_read(ksoftirqd)->softirqs_raised |= (1U << nr); -+} -+ -+void __raise_softirq_irqoff(unsigned int nr) -+{ -+ do_raise_softirq_irqoff(nr); -+ if (!in_irq() && !current->softirq_nestcnt) -+ wakeup_softirqd(); -+} -+ -+/* -+ * This function must run with irqs disabled! -+ */ -+void raise_softirq_irqoff(unsigned int nr) -+{ -+ do_raise_softirq_irqoff(nr); -+ -+ /* -+ * If we're in an hard interrupt we let irq return code deal -+ * with the wakeup of ksoftirqd. -+ */ -+ if (in_irq()) -+ return; -+ /* -+ * If we are in thread context but outside of a bh disabled -+ * region, we need to wake ksoftirqd as well. -+ * -+ * CHECKME: Some of the places which do that could be wrapped -+ * into local_bh_disable/enable pairs. Though it's unclear -+ * whether this is worth the effort. To find those places just -+ * raise a WARN() if the condition is met. -+ */ -+ if (!current->softirq_nestcnt) -+ wakeup_softirqd(); -+} -+ -+static inline int ksoftirqd_softirq_pending(void) -+{ -+ return current->softirqs_raised; -+} -+ -+static inline void local_bh_disable_nort(void) { } -+static inline void _local_bh_enable_nort(void) { } -+ -+static inline void ksoftirqd_set_sched_params(unsigned int cpu) -+{ -+ struct sched_param param = { .sched_priority = 1 }; -+ -+ sched_setscheduler(current, SCHED_FIFO, ¶m); -+ /* Take over all pending softirqs when starting */ -+ local_irq_disable(); -+ current->softirqs_raised = local_softirq_pending(); -+ local_irq_enable(); -+} -+ -+static inline void ksoftirqd_clr_sched_params(unsigned int cpu, bool online) -+{ -+ struct sched_param param = { .sched_priority = 0 }; -+ -+ sched_setscheduler(current, SCHED_NORMAL, ¶m); -+} -+ -+#endif /* PREEMPT_RT_FULL */ -+/* - * Enter an interrupt context. - */ - void irq_enter(void) -@@ -330,9 +734,9 @@ - * Prevent raise_softirq from needlessly waking up ksoftirqd - * here, as softirq will be serviced on return from interrupt. - */ -- local_bh_disable(); -+ local_bh_disable_nort(); - tick_irq_enter(); -- _local_bh_enable(); -+ _local_bh_enable_nort(); - } - - __irq_enter(); -@@ -340,6 +744,7 @@ - - static inline void invoke_softirq(void) - { -+#ifndef CONFIG_PREEMPT_RT_FULL - if (!force_irqthreads) { - #ifdef CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK - /* -@@ -359,6 +764,15 @@ - } else { - wakeup_softirqd(); - } -+#else /* PREEMPT_RT_FULL */ -+ unsigned long flags; -+ -+ local_irq_save(flags); -+ if (__this_cpu_read(ksoftirqd) && -+ __this_cpu_read(ksoftirqd)->softirqs_raised) -+ wakeup_softirqd(); -+ local_irq_restore(flags); -+#endif - } - - static inline void tick_irq_exit(void) -@@ -395,26 +809,6 @@ - trace_hardirq_exit(); /* must be last! */ - } - --/* -- * This function must run with irqs disabled! -- */ --inline void raise_softirq_irqoff(unsigned int nr) --{ -- __raise_softirq_irqoff(nr); -- -- /* -- * If we're in an interrupt or softirq, we're done -- * (this also catches softirq-disabled code). We will -- * actually run the softirq once we return from -- * the irq or softirq. -- * -- * Otherwise we wake up ksoftirqd to make sure we -- * schedule the softirq soon. -- */ -- if (!in_interrupt()) -- wakeup_softirqd(); --} -- - void raise_softirq(unsigned int nr) - { - unsigned long flags; -@@ -424,12 +818,6 @@ - local_irq_restore(flags); - } - --void __raise_softirq_irqoff(unsigned int nr) --{ -- trace_softirq_raise(nr); -- or_softirq_pending(1UL << nr); --} -- - void open_softirq(int nr, void (*action)(struct softirq_action *)) - { - softirq_vec[nr].action = action; -@@ -446,15 +834,45 @@ - static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec); - static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec); - -+static void inline -+__tasklet_common_schedule(struct tasklet_struct *t, struct tasklet_head *head, unsigned int nr) -+{ -+ if (tasklet_trylock(t)) { -+again: -+ /* We may have been preempted before tasklet_trylock -+ * and __tasklet_action may have already run. -+ * So double check the sched bit while the takslet -+ * is locked before adding it to the list. -+ */ -+ if (test_bit(TASKLET_STATE_SCHED, &t->state)) { -+ t->next = NULL; -+ *head->tail = t; -+ head->tail = &(t->next); -+ raise_softirq_irqoff(nr); -+ tasklet_unlock(t); -+ } else { -+ /* This is subtle. If we hit the corner case above -+ * It is possible that we get preempted right here, -+ * and another task has successfully called -+ * tasklet_schedule(), then this function, and -+ * failed on the trylock. Thus we must be sure -+ * before releasing the tasklet lock, that the -+ * SCHED_BIT is clear. Otherwise the tasklet -+ * may get its SCHED_BIT set, but not added to the -+ * list -+ */ -+ if (!tasklet_tryunlock(t)) -+ goto again; -+ } -+ } -+} -+ - void __tasklet_schedule(struct tasklet_struct *t) - { - unsigned long flags; - - local_irq_save(flags); -- t->next = NULL; -- *__this_cpu_read(tasklet_vec.tail) = t; -- __this_cpu_write(tasklet_vec.tail, &(t->next)); -- raise_softirq_irqoff(TASKLET_SOFTIRQ); -+ __tasklet_common_schedule(t, this_cpu_ptr(&tasklet_vec), TASKLET_SOFTIRQ); - local_irq_restore(flags); - } - EXPORT_SYMBOL(__tasklet_schedule); -@@ -464,10 +882,7 @@ - unsigned long flags; - - local_irq_save(flags); -- t->next = NULL; -- *__this_cpu_read(tasklet_hi_vec.tail) = t; -- __this_cpu_write(tasklet_hi_vec.tail, &(t->next)); -- raise_softirq_irqoff(HI_SOFTIRQ); -+ __tasklet_common_schedule(t, this_cpu_ptr(&tasklet_hi_vec), HI_SOFTIRQ); - local_irq_restore(flags); - } - EXPORT_SYMBOL(__tasklet_hi_schedule); -@@ -476,82 +891,122 @@ - { - BUG_ON(!irqs_disabled()); - -- t->next = __this_cpu_read(tasklet_hi_vec.head); -- __this_cpu_write(tasklet_hi_vec.head, t); -- __raise_softirq_irqoff(HI_SOFTIRQ); -+ __tasklet_hi_schedule(t); - } - EXPORT_SYMBOL(__tasklet_hi_schedule_first); - --static void tasklet_action(struct softirq_action *a) -+void tasklet_enable(struct tasklet_struct *t) - { -- struct tasklet_struct *list; -+ if (!atomic_dec_and_test(&t->count)) -+ return; -+ if (test_and_clear_bit(TASKLET_STATE_PENDING, &t->state)) -+ tasklet_schedule(t); -+} -+EXPORT_SYMBOL(tasklet_enable); - -- local_irq_disable(); -- list = __this_cpu_read(tasklet_vec.head); -- __this_cpu_write(tasklet_vec.head, NULL); -- __this_cpu_write(tasklet_vec.tail, this_cpu_ptr(&tasklet_vec.head)); -- local_irq_enable(); -+static void __tasklet_action(struct softirq_action *a, -+ struct tasklet_struct *list) -+{ -+ int loops = 1000000; - - while (list) { - struct tasklet_struct *t = list; - - list = list->next; - -- if (tasklet_trylock(t)) { -- if (!atomic_read(&t->count)) { -- if (!test_and_clear_bit(TASKLET_STATE_SCHED, -- &t->state)) -- BUG(); -- t->func(t->data); -- tasklet_unlock(t); -- continue; -- } -- tasklet_unlock(t); -+ /* -+ * Should always succeed - after a tasklist got on the -+ * list (after getting the SCHED bit set from 0 to 1), -+ * nothing but the tasklet softirq it got queued to can -+ * lock it: -+ */ -+ if (!tasklet_trylock(t)) { -+ WARN_ON(1); -+ continue; - } - -- local_irq_disable(); - t->next = NULL; -- *__this_cpu_read(tasklet_vec.tail) = t; -- __this_cpu_write(tasklet_vec.tail, &(t->next)); -- __raise_softirq_irqoff(TASKLET_SOFTIRQ); -- local_irq_enable(); -+ -+ /* -+ * If we cannot handle the tasklet because it's disabled, -+ * mark it as pending. tasklet_enable() will later -+ * re-schedule the tasklet. -+ */ -+ if (unlikely(atomic_read(&t->count))) { -+out_disabled: -+ /* implicit unlock: */ -+ wmb(); -+ t->state = TASKLET_STATEF_PENDING; -+ continue; -+ } -+ -+ /* -+ * After this point on the tasklet might be rescheduled -+ * on another CPU, but it can only be added to another -+ * CPU's tasklet list if we unlock the tasklet (which we -+ * dont do yet). -+ */ -+ if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state)) -+ WARN_ON(1); -+ -+again: -+ t->func(t->data); -+ -+ /* -+ * Try to unlock the tasklet. We must use cmpxchg, because -+ * another CPU might have scheduled or disabled the tasklet. -+ * We only allow the STATE_RUN -> 0 transition here. -+ */ -+ while (!tasklet_tryunlock(t)) { -+ /* -+ * If it got disabled meanwhile, bail out: -+ */ -+ if (atomic_read(&t->count)) -+ goto out_disabled; -+ /* -+ * If it got scheduled meanwhile, re-execute -+ * the tasklet function: -+ */ -+ if (test_and_clear_bit(TASKLET_STATE_SCHED, &t->state)) -+ goto again; -+ if (!--loops) { -+ printk("hm, tasklet state: %08lx\n", t->state); -+ WARN_ON(1); -+ tasklet_unlock(t); -+ break; -+ } -+ } - } - } - -+static void tasklet_action(struct softirq_action *a) -+{ -+ struct tasklet_struct *list; -+ -+ local_irq_disable(); -+ -+ list = __this_cpu_read(tasklet_vec.head); -+ __this_cpu_write(tasklet_vec.head, NULL); -+ __this_cpu_write(tasklet_vec.tail, this_cpu_ptr(&tasklet_vec.head)); -+ -+ local_irq_enable(); -+ -+ __tasklet_action(a, list); -+} -+ - static void tasklet_hi_action(struct softirq_action *a) - { - struct tasklet_struct *list; - - local_irq_disable(); -+ - list = __this_cpu_read(tasklet_hi_vec.head); - __this_cpu_write(tasklet_hi_vec.head, NULL); - __this_cpu_write(tasklet_hi_vec.tail, this_cpu_ptr(&tasklet_hi_vec.head)); -- local_irq_enable(); -- -- while (list) { -- struct tasklet_struct *t = list; -- -- list = list->next; - -- if (tasklet_trylock(t)) { -- if (!atomic_read(&t->count)) { -- if (!test_and_clear_bit(TASKLET_STATE_SCHED, -- &t->state)) -- BUG(); -- t->func(t->data); -- tasklet_unlock(t); -- continue; -- } -- tasklet_unlock(t); -- } -+ local_irq_enable(); - -- local_irq_disable(); -- t->next = NULL; -- *__this_cpu_read(tasklet_hi_vec.tail) = t; -- __this_cpu_write(tasklet_hi_vec.tail, &(t->next)); -- __raise_softirq_irqoff(HI_SOFTIRQ); -- local_irq_enable(); -- } -+ __tasklet_action(a, list); - } - - void tasklet_init(struct tasklet_struct *t, -@@ -572,7 +1027,7 @@ - - while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) { - do { -- yield(); -+ msleep(1); - } while (test_bit(TASKLET_STATE_SCHED, &t->state)); - } - tasklet_unlock_wait(t); -@@ -646,25 +1101,26 @@ - open_softirq(HI_SOFTIRQ, tasklet_hi_action); - } - --static int ksoftirqd_should_run(unsigned int cpu) --{ -- return local_softirq_pending(); --} -- --static void run_ksoftirqd(unsigned int cpu) -+#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT_FULL) -+void tasklet_unlock_wait(struct tasklet_struct *t) - { -- local_irq_disable(); -- if (local_softirq_pending()) { -+ while (test_bit(TASKLET_STATE_RUN, &(t)->state)) { - /* -- * We can safely run softirq on inline stack, as we are not deep -- * in the task stack here. -+ * Hack for now to avoid this busy-loop: - */ -- __do_softirq(); -- local_irq_enable(); -- cond_resched_rcu_qs(); -- return; -+#ifdef CONFIG_PREEMPT_RT_FULL -+ msleep(1); -+#else -+ barrier(); -+#endif - } -- local_irq_enable(); -+} -+EXPORT_SYMBOL(tasklet_unlock_wait); -+#endif -+ -+static int ksoftirqd_should_run(unsigned int cpu) -+{ -+ return ksoftirqd_softirq_pending(); - } - - #ifdef CONFIG_HOTPLUG_CPU -@@ -746,6 +1202,8 @@ - - static struct smp_hotplug_thread softirq_threads = { - .store = &ksoftirqd, -+ .setup = ksoftirqd_set_sched_params, -+ .cleanup = ksoftirqd_clr_sched_params, - .thread_should_run = ksoftirqd_should_run, - .thread_fn = run_ksoftirqd, - .thread_comm = "ksoftirqd/%u", -diff -Nur linux-4.1.10.orig/kernel/stop_machine.c linux-4.1.10/kernel/stop_machine.c ---- linux-4.1.10.orig/kernel/stop_machine.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/stop_machine.c 2015-10-07 18:00:08.000000000 +0200 -@@ -30,12 +30,12 @@ - atomic_t nr_todo; /* nr left to execute */ - bool executed; /* actually executed? */ - int ret; /* collected return value */ -- struct completion completion; /* fired if nr_todo reaches 0 */ -+ struct task_struct *waiter; /* woken when nr_todo reaches 0 */ - }; - - /* the actual stopper, one per every possible cpu, enabled on online cpus */ - struct cpu_stopper { -- spinlock_t lock; -+ raw_spinlock_t lock; - bool enabled; /* is this stopper enabled? */ - struct list_head works; /* list of pending works */ - }; -@@ -56,7 +56,7 @@ - { - memset(done, 0, sizeof(*done)); - atomic_set(&done->nr_todo, nr_todo); -- init_completion(&done->completion); -+ done->waiter = current; - } - - /* signal completion unless @done is NULL */ -@@ -65,8 +65,10 @@ - if (done) { - if (executed) - done->executed = true; -- if (atomic_dec_and_test(&done->nr_todo)) -- complete(&done->completion); -+ if (atomic_dec_and_test(&done->nr_todo)) { -+ wake_up_process(done->waiter); -+ done->waiter = NULL; -+ } - } - } - -@@ -78,7 +80,7 @@ - - unsigned long flags; - -- spin_lock_irqsave(&stopper->lock, flags); -+ raw_spin_lock_irqsave(&stopper->lock, flags); - - if (stopper->enabled) { - list_add_tail(&work->list, &stopper->works); -@@ -86,7 +88,23 @@ - } else - cpu_stop_signal_done(work->done, false); - -- spin_unlock_irqrestore(&stopper->lock, flags); -+ raw_spin_unlock_irqrestore(&stopper->lock, flags); -+} -+ -+static void wait_for_stop_done(struct cpu_stop_done *done) -+{ -+ set_current_state(TASK_UNINTERRUPTIBLE); -+ while (atomic_read(&done->nr_todo)) { -+ schedule(); -+ set_current_state(TASK_UNINTERRUPTIBLE); -+ } -+ /* -+ * We need to wait until cpu_stop_signal_done() has cleared -+ * done->waiter. -+ */ -+ while (done->waiter) -+ cpu_relax(); -+ set_current_state(TASK_RUNNING); - } - - /** -@@ -120,7 +138,7 @@ - - cpu_stop_init_done(&done, 1); - cpu_stop_queue_work(cpu, &work); -- wait_for_completion(&done.completion); -+ wait_for_stop_done(&done); - return done.executed ? done.ret : -ENOENT; - } - -@@ -248,7 +266,7 @@ - struct irq_cpu_stop_queue_work_info call_args; - struct multi_stop_data msdata; - -- preempt_disable(); -+ preempt_disable_nort(); - msdata = (struct multi_stop_data){ - .fn = fn, - .data = arg, -@@ -281,7 +299,7 @@ - * This relies on the stopper workqueues to be FIFO. - */ - if (!cpu_active(cpu1) || !cpu_active(cpu2)) { -- preempt_enable(); -+ preempt_enable_nort(); - return -ENOENT; - } - -@@ -295,9 +313,9 @@ - &irq_cpu_stop_queue_work, - &call_args, 1); - lg_local_unlock(&stop_cpus_lock); -- preempt_enable(); -+ preempt_enable_nort(); - -- wait_for_completion(&done.completion); -+ wait_for_stop_done(&done); - - return done.executed ? done.ret : -ENOENT; - } -@@ -329,7 +347,7 @@ - - static void queue_stop_cpus_work(const struct cpumask *cpumask, - cpu_stop_fn_t fn, void *arg, -- struct cpu_stop_done *done) -+ struct cpu_stop_done *done, bool inactive) - { - struct cpu_stop_work *work; - unsigned int cpu; -@@ -343,11 +361,13 @@ - } - - /* -- * Disable preemption while queueing to avoid getting -- * preempted by a stopper which might wait for other stoppers -- * to enter @fn which can lead to deadlock. -+ * Make sure that all work is queued on all cpus before -+ * any of the cpus can execute it. - */ -- lg_global_lock(&stop_cpus_lock); -+ if (!inactive) -+ lg_global_lock(&stop_cpus_lock); -+ else -+ lg_global_trylock_relax(&stop_cpus_lock); - for_each_cpu(cpu, cpumask) - cpu_stop_queue_work(cpu, &per_cpu(stop_cpus_work, cpu)); - lg_global_unlock(&stop_cpus_lock); -@@ -359,8 +379,8 @@ - struct cpu_stop_done done; - - cpu_stop_init_done(&done, cpumask_weight(cpumask)); -- queue_stop_cpus_work(cpumask, fn, arg, &done); -- wait_for_completion(&done.completion); -+ queue_stop_cpus_work(cpumask, fn, arg, &done, false); -+ wait_for_stop_done(&done); - return done.executed ? done.ret : -ENOENT; - } - -@@ -439,9 +459,9 @@ - unsigned long flags; - int run; - -- spin_lock_irqsave(&stopper->lock, flags); -+ raw_spin_lock_irqsave(&stopper->lock, flags); - run = !list_empty(&stopper->works); -- spin_unlock_irqrestore(&stopper->lock, flags); -+ raw_spin_unlock_irqrestore(&stopper->lock, flags); - return run; - } - -@@ -453,13 +473,13 @@ - - repeat: - work = NULL; -- spin_lock_irq(&stopper->lock); -+ raw_spin_lock_irq(&stopper->lock); - if (!list_empty(&stopper->works)) { - work = list_first_entry(&stopper->works, - struct cpu_stop_work, list); - list_del_init(&work->list); - } -- spin_unlock_irq(&stopper->lock); -+ raw_spin_unlock_irq(&stopper->lock); - - if (work) { - cpu_stop_fn_t fn = work->fn; -@@ -467,6 +487,16 @@ - struct cpu_stop_done *done = work->done; - char ksym_buf[KSYM_NAME_LEN] __maybe_unused; - -+ /* -+ * Wait until the stopper finished scheduling on all -+ * cpus -+ */ -+ lg_global_lock(&stop_cpus_lock); -+ /* -+ * Let other cpu threads continue as well -+ */ -+ lg_global_unlock(&stop_cpus_lock); -+ - /* cpu stop callbacks are not allowed to sleep */ - preempt_disable(); - -@@ -481,7 +511,13 @@ - kallsyms_lookup((unsigned long)fn, NULL, NULL, NULL, - ksym_buf), arg); - -+ /* -+ * Make sure that the wakeup and setting done->waiter -+ * to NULL is atomic. -+ */ -+ local_irq_disable(); - cpu_stop_signal_done(done, true); -+ local_irq_enable(); - goto repeat; - } - } -@@ -500,20 +536,20 @@ - unsigned long flags; - - /* drain remaining works */ -- spin_lock_irqsave(&stopper->lock, flags); -+ raw_spin_lock_irqsave(&stopper->lock, flags); - list_for_each_entry(work, &stopper->works, list) - cpu_stop_signal_done(work->done, false); - stopper->enabled = false; -- spin_unlock_irqrestore(&stopper->lock, flags); -+ raw_spin_unlock_irqrestore(&stopper->lock, flags); - } - - static void cpu_stop_unpark(unsigned int cpu) - { - struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu); - -- spin_lock_irq(&stopper->lock); -+ raw_spin_lock_irq(&stopper->lock); - stopper->enabled = true; -- spin_unlock_irq(&stopper->lock); -+ raw_spin_unlock_irq(&stopper->lock); - } - - static struct smp_hotplug_thread cpu_stop_threads = { -@@ -535,10 +571,12 @@ - for_each_possible_cpu(cpu) { - struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu); - -- spin_lock_init(&stopper->lock); -+ raw_spin_lock_init(&stopper->lock); - INIT_LIST_HEAD(&stopper->works); - } - -+ lg_lock_init(&stop_cpus_lock, "stop_cpus_lock"); -+ - BUG_ON(smpboot_register_percpu_thread(&cpu_stop_threads)); - stop_machine_initialized = true; - return 0; -@@ -634,11 +672,11 @@ - set_state(&msdata, MULTI_STOP_PREPARE); - cpu_stop_init_done(&done, num_active_cpus()); - queue_stop_cpus_work(cpu_active_mask, multi_cpu_stop, &msdata, -- &done); -+ &done, true); - ret = multi_cpu_stop(&msdata); - - /* Busy wait for completion. */ -- while (!completion_done(&done.completion)) -+ while (atomic_read(&done.nr_todo)) - cpu_relax(); - - mutex_unlock(&stop_cpus_mutex); -diff -Nur linux-4.1.10.orig/kernel/time/hrtimer.c linux-4.1.10/kernel/time/hrtimer.c ---- linux-4.1.10.orig/kernel/time/hrtimer.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/time/hrtimer.c 2015-10-07 18:00:08.000000000 +0200 -@@ -48,11 +48,13 @@ - #include - #include - #include -+#include - #include - - #include - - #include -+#include - - #include "tick-internal.h" - -@@ -576,8 +578,7 @@ - * When the callback is running, we do not reprogram the clock event - * device. The timer callback is either running on a different CPU or - * the callback is executed in the hrtimer_interrupt context. The -- * reprogramming is handled either by the softirq, which called the -- * callback or at the end of the hrtimer_interrupt. -+ * reprogramming is handled at the end of the hrtimer_interrupt. - */ - if (hrtimer_callback_running(timer)) - return 0; -@@ -621,6 +622,9 @@ - return res; - } - -+static void __run_hrtimer(struct hrtimer *timer, ktime_t *now); -+static int hrtimer_rt_defer(struct hrtimer *timer); -+ - /* - * Initialize the high resolution related parts of cpu_base - */ -@@ -630,6 +634,21 @@ - base->hres_active = 0; - } - -+static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer, -+ struct hrtimer_clock_base *base, -+ int wakeup) -+{ -+ if (!hrtimer_reprogram(timer, base)) -+ return 0; -+ if (!wakeup) -+ return -ETIME; -+#ifdef CONFIG_PREEMPT_RT_BASE -+ if (!hrtimer_rt_defer(timer)) -+ return -ETIME; -+#endif -+ return 1; -+} -+ - static inline ktime_t hrtimer_update_base(struct hrtimer_cpu_base *base) - { - ktime_t *offs_real = &base->clock_base[HRTIMER_BASE_REALTIME].offset; -@@ -695,6 +714,44 @@ - - static DECLARE_WORK(hrtimer_work, clock_was_set_work); - -+#ifdef CONFIG_PREEMPT_RT_FULL -+/* -+ * RT can not call schedule_work from real interrupt context. -+ * Need to make a thread to do the real work. -+ */ -+static struct task_struct *clock_set_delay_thread; -+static bool do_clock_set_delay; -+ -+static int run_clock_set_delay(void *ignore) -+{ -+ while (!kthread_should_stop()) { -+ set_current_state(TASK_INTERRUPTIBLE); -+ if (do_clock_set_delay) { -+ do_clock_set_delay = false; -+ schedule_work(&hrtimer_work); -+ } -+ schedule(); -+ } -+ __set_current_state(TASK_RUNNING); -+ return 0; -+} -+ -+void clock_was_set_delayed(void) -+{ -+ do_clock_set_delay = true; -+ /* Make visible before waking up process */ -+ smp_wmb(); -+ wake_up_process(clock_set_delay_thread); -+} -+ -+static __init int create_clock_set_delay_thread(void) -+{ -+ clock_set_delay_thread = kthread_run(run_clock_set_delay, NULL, "kclksetdelayd"); -+ BUG_ON(!clock_set_delay_thread); -+ return 0; -+} -+early_initcall(create_clock_set_delay_thread); -+#else /* PREEMPT_RT_FULL */ - /* - * Called from timekeeping and resume code to reprogramm the hrtimer - * interrupt device on all cpus. -@@ -703,6 +760,7 @@ - { - schedule_work(&hrtimer_work); - } -+#endif - - #else - -@@ -711,6 +769,13 @@ - static inline int hrtimer_switch_to_hres(void) { return 0; } - static inline void - hrtimer_force_reprogram(struct hrtimer_cpu_base *base, int skip_equal) { } -+static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer, -+ struct hrtimer_clock_base *base, -+ int wakeup) -+{ -+ return 0; -+} -+ - static inline int hrtimer_reprogram(struct hrtimer *timer, - struct hrtimer_clock_base *base) - { -@@ -718,7 +783,6 @@ - } - static inline void hrtimer_init_hres(struct hrtimer_cpu_base *base) { } - static inline void retrigger_next_event(void *arg) { } -- - #endif /* CONFIG_HIGH_RES_TIMERS */ - - /* -@@ -836,6 +900,32 @@ - } - EXPORT_SYMBOL_GPL(hrtimer_forward); - -+#ifdef CONFIG_PREEMPT_RT_BASE -+# define wake_up_timer_waiters(b) wake_up(&(b)->wait) -+ -+/** -+ * hrtimer_wait_for_timer - Wait for a running timer -+ * -+ * @timer: timer to wait for -+ * -+ * The function waits in case the timers callback function is -+ * currently executed on the waitqueue of the timer base. The -+ * waitqueue is woken up after the timer callback function has -+ * finished execution. -+ */ -+void hrtimer_wait_for_timer(const struct hrtimer *timer) -+{ -+ struct hrtimer_clock_base *base = timer->base; -+ -+ if (base && base->cpu_base && !timer->irqsafe) -+ wait_event(base->cpu_base->wait, -+ !(timer->state & HRTIMER_STATE_CALLBACK)); -+} -+ -+#else -+# define wake_up_timer_waiters(b) do { } while (0) -+#endif -+ - /* - * enqueue_hrtimer - internal function to (re)start a timer - * -@@ -879,6 +969,11 @@ - if (!(timer->state & HRTIMER_STATE_ENQUEUED)) - goto out; - -+ if (unlikely(!list_empty(&timer->cb_entry))) { -+ list_del_init(&timer->cb_entry); -+ goto out; -+ } -+ - next_timer = timerqueue_getnext(&base->active); - timerqueue_del(&base->active, &timer->node); - if (&timer->node == next_timer) { -@@ -966,7 +1061,16 @@ - new_base = switch_hrtimer_base(timer, base, mode & HRTIMER_MODE_PINNED); - - timer_stats_hrtimer_set_start_info(timer); -+#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST -+ { -+ ktime_t now = new_base->get_time(); - -+ if (ktime_to_ns(tim) < ktime_to_ns(now)) -+ timer->praecox = now; -+ else -+ timer->praecox = ktime_set(0, 0); -+ } -+#endif - leftmost = enqueue_hrtimer(timer, new_base); - - if (!leftmost) { -@@ -980,15 +1084,26 @@ - * on dynticks target. - */ - wake_up_nohz_cpu(new_base->cpu_base->cpu); -- } else if (new_base->cpu_base == this_cpu_ptr(&hrtimer_bases) && -- hrtimer_reprogram(timer, new_base)) { -+ } else if (new_base->cpu_base == this_cpu_ptr(&hrtimer_bases)) { -+ -+ ret = hrtimer_enqueue_reprogram(timer, new_base, wakeup); -+ if (ret < 0) { -+ /* -+ * In case we failed to reprogram the timer (mostly -+ * because out current timer is already elapsed), -+ * remove it again and report a failure. This avoids -+ * stale base->first entries. -+ */ -+ debug_deactivate(timer); -+ __remove_hrtimer(timer, new_base, -+ timer->state & HRTIMER_STATE_CALLBACK, 0); -+ } else if (ret > 0) { - /* - * Only allow reprogramming if the new base is on this CPU. - * (it might still be on another CPU if the timer was pending) - * - * XXX send_remote_softirq() ? - */ -- if (wakeup) { - /* - * We need to drop cpu_base->lock to avoid a - * lock ordering issue vs. rq->lock. -@@ -996,9 +1111,7 @@ - raw_spin_unlock(&new_base->cpu_base->lock); - raise_softirq_irqoff(HRTIMER_SOFTIRQ); - local_irq_restore(flags); -- return ret; -- } else { -- __raise_softirq_irqoff(HRTIMER_SOFTIRQ); -+ return 0; - } - } - -@@ -1089,7 +1202,7 @@ - - if (ret >= 0) - return ret; -- cpu_relax(); -+ hrtimer_wait_for_timer(timer); - } - } - EXPORT_SYMBOL_GPL(hrtimer_cancel); -@@ -1153,6 +1266,7 @@ - - base = hrtimer_clockid_to_base(clock_id); - timer->base = &cpu_base->clock_base[base]; -+ INIT_LIST_HEAD(&timer->cb_entry); - timerqueue_init(&timer->node); - - #ifdef CONFIG_TIMER_STATS -@@ -1236,6 +1350,126 @@ - timer->state &= ~HRTIMER_STATE_CALLBACK; - } - -+static enum hrtimer_restart hrtimer_wakeup(struct hrtimer *timer); -+ -+#ifdef CONFIG_PREEMPT_RT_BASE -+static void hrtimer_rt_reprogram(int restart, struct hrtimer *timer, -+ struct hrtimer_clock_base *base) -+{ -+ /* -+ * Note, we clear the callback flag before we requeue the -+ * timer otherwise we trigger the callback_running() check -+ * in hrtimer_reprogram(). -+ */ -+ timer->state &= ~HRTIMER_STATE_CALLBACK; -+ -+ if (restart != HRTIMER_NORESTART) { -+ BUG_ON(hrtimer_active(timer)); -+ /* -+ * Enqueue the timer, if it's the leftmost timer then -+ * we need to reprogram it. -+ */ -+ if (!enqueue_hrtimer(timer, base)) -+ return; -+ -+#ifndef CONFIG_HIGH_RES_TIMERS -+ } -+#else -+ if (base->cpu_base->hres_active && -+ hrtimer_reprogram(timer, base)) -+ goto requeue; -+ -+ } else if (hrtimer_active(timer)) { -+ /* -+ * If the timer was rearmed on another CPU, reprogram -+ * the event device. -+ */ -+ if (&timer->node == base->active.next && -+ base->cpu_base->hres_active && -+ hrtimer_reprogram(timer, base)) -+ goto requeue; -+ } -+ return; -+ -+requeue: -+ /* -+ * Timer is expired. Thus move it from tree to pending list -+ * again. -+ */ -+ __remove_hrtimer(timer, base, timer->state, 0); -+ list_add_tail(&timer->cb_entry, &base->expired); -+#endif -+} -+ -+/* -+ * The changes in mainline which removed the callback modes from -+ * hrtimer are not yet working with -rt. The non wakeup_process() -+ * based callbacks which involve sleeping locks need to be treated -+ * seperately. -+ */ -+static void hrtimer_rt_run_pending(void) -+{ -+ enum hrtimer_restart (*fn)(struct hrtimer *); -+ struct hrtimer_cpu_base *cpu_base; -+ struct hrtimer_clock_base *base; -+ struct hrtimer *timer; -+ int index, restart; -+ -+ local_irq_disable(); -+ cpu_base = &per_cpu(hrtimer_bases, smp_processor_id()); -+ -+ raw_spin_lock(&cpu_base->lock); -+ -+ for (index = 0; index < HRTIMER_MAX_CLOCK_BASES; index++) { -+ base = &cpu_base->clock_base[index]; -+ -+ while (!list_empty(&base->expired)) { -+ timer = list_first_entry(&base->expired, -+ struct hrtimer, cb_entry); -+ -+ /* -+ * Same as the above __run_hrtimer function -+ * just we run with interrupts enabled. -+ */ -+ debug_hrtimer_deactivate(timer); -+ __remove_hrtimer(timer, base, HRTIMER_STATE_CALLBACK, 0); -+ timer_stats_account_hrtimer(timer); -+ fn = timer->function; -+ -+ raw_spin_unlock_irq(&cpu_base->lock); -+ restart = fn(timer); -+ raw_spin_lock_irq(&cpu_base->lock); -+ -+ hrtimer_rt_reprogram(restart, timer, base); -+ } -+ } -+ -+ raw_spin_unlock_irq(&cpu_base->lock); -+ -+ wake_up_timer_waiters(cpu_base); -+} -+ -+static int hrtimer_rt_defer(struct hrtimer *timer) -+{ -+ if (timer->irqsafe) -+ return 0; -+ -+ __remove_hrtimer(timer, timer->base, timer->state, 0); -+ list_add_tail(&timer->cb_entry, &timer->base->expired); -+ return 1; -+} -+ -+#else -+ -+static inline void hrtimer_rt_run_pending(void) -+{ -+ hrtimer_peek_ahead_timers(); -+} -+ -+static inline int hrtimer_rt_defer(struct hrtimer *timer) { return 0; } -+ -+#endif -+ - #ifdef CONFIG_HIGH_RES_TIMERS - - /* -@@ -1246,7 +1480,7 @@ - { - struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases); - ktime_t expires_next, now, entry_time, delta; -- int i, retries = 0; -+ int i, retries = 0, raise = 0; - - BUG_ON(!cpu_base->hres_active); - cpu_base->nr_events++; -@@ -1281,6 +1515,15 @@ - - timer = container_of(node, struct hrtimer, node); - -+ trace_hrtimer_interrupt(raw_smp_processor_id(), -+ ktime_to_ns(ktime_sub(ktime_to_ns(timer->praecox) ? -+ timer->praecox : hrtimer_get_expires(timer), -+ basenow)), -+ current, -+ timer->function == hrtimer_wakeup ? -+ container_of(timer, struct hrtimer_sleeper, -+ timer)->task : NULL); -+ - /* - * The immediate goal for using the softexpires is - * minimizing wakeups, not running timers at the -@@ -1296,7 +1539,10 @@ - if (basenow.tv64 < hrtimer_get_softexpires_tv64(timer)) - break; - -- __run_hrtimer(timer, &basenow); -+ if (!hrtimer_rt_defer(timer)) -+ __run_hrtimer(timer, &basenow); -+ else -+ raise = 1; - } - } - /* Reevaluate the clock bases for the next expiry */ -@@ -1313,7 +1559,7 @@ - if (expires_next.tv64 == KTIME_MAX || - !tick_program_event(expires_next, 0)) { - cpu_base->hang_detected = 0; -- return; -+ goto out; - } - - /* -@@ -1357,6 +1603,9 @@ - tick_program_event(expires_next, 1); - printk_once(KERN_WARNING "hrtimer: interrupt took %llu ns\n", - ktime_to_ns(delta)); -+out: -+ if (raise) -+ raise_softirq_irqoff(HRTIMER_SOFTIRQ); - } - - /* -@@ -1392,18 +1641,18 @@ - __hrtimer_peek_ahead_timers(); - local_irq_restore(flags); - } -- --static void run_hrtimer_softirq(struct softirq_action *h) --{ -- hrtimer_peek_ahead_timers(); --} -- - #else /* CONFIG_HIGH_RES_TIMERS */ - - static inline void __hrtimer_peek_ahead_timers(void) { } - - #endif /* !CONFIG_HIGH_RES_TIMERS */ - -+ -+static void run_hrtimer_softirq(struct softirq_action *h) -+{ -+ hrtimer_rt_run_pending(); -+} -+ - /* - * Called from timer softirq every jiffy, expire hrtimers: - * -@@ -1436,7 +1685,7 @@ - struct timerqueue_node *node; - struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases); - struct hrtimer_clock_base *base; -- int index, gettime = 1; -+ int index, gettime = 1, raise = 0; - - if (hrtimer_hres_active()) - return; -@@ -1461,10 +1710,16 @@ - hrtimer_get_expires_tv64(timer)) - break; - -- __run_hrtimer(timer, &base->softirq_time); -+ if (!hrtimer_rt_defer(timer)) -+ __run_hrtimer(timer, &base->softirq_time); -+ else -+ raise = 1; - } - raw_spin_unlock(&cpu_base->lock); - } -+ -+ if (raise) -+ raise_softirq_irqoff(HRTIMER_SOFTIRQ); - } - - /* -@@ -1486,16 +1741,18 @@ - void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, struct task_struct *task) - { - sl->timer.function = hrtimer_wakeup; -+ sl->timer.irqsafe = 1; - sl->task = task; - } - EXPORT_SYMBOL_GPL(hrtimer_init_sleeper); - --static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mode) -+static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mode, -+ unsigned long state) - { - hrtimer_init_sleeper(t, current); - - do { -- set_current_state(TASK_INTERRUPTIBLE); -+ set_current_state(state); - hrtimer_start_expires(&t->timer, mode); - if (!hrtimer_active(&t->timer)) - t->task = NULL; -@@ -1539,7 +1796,8 @@ - HRTIMER_MODE_ABS); - hrtimer_set_expires_tv64(&t.timer, restart->nanosleep.expires); - -- if (do_nanosleep(&t, HRTIMER_MODE_ABS)) -+ /* cpu_chill() does not care about restart state. */ -+ if (do_nanosleep(&t, HRTIMER_MODE_ABS, TASK_INTERRUPTIBLE)) - goto out; - - rmtp = restart->nanosleep.rmtp; -@@ -1556,8 +1814,10 @@ - return ret; - } - --long hrtimer_nanosleep(struct timespec *rqtp, struct timespec __user *rmtp, -- const enum hrtimer_mode mode, const clockid_t clockid) -+static long -+__hrtimer_nanosleep(struct timespec *rqtp, struct timespec __user *rmtp, -+ const enum hrtimer_mode mode, const clockid_t clockid, -+ unsigned long state) - { - struct restart_block *restart; - struct hrtimer_sleeper t; -@@ -1570,7 +1830,7 @@ - - hrtimer_init_on_stack(&t.timer, clockid, mode); - hrtimer_set_expires_range_ns(&t.timer, timespec_to_ktime(*rqtp), slack); -- if (do_nanosleep(&t, mode)) -+ if (do_nanosleep(&t, mode, state)) - goto out; - - /* Absolute timers do not update the rmtp value and restart: */ -@@ -1597,6 +1857,12 @@ - return ret; - } - -+long hrtimer_nanosleep(struct timespec *rqtp, struct timespec __user *rmtp, -+ const enum hrtimer_mode mode, const clockid_t clockid) -+{ -+ return __hrtimer_nanosleep(rqtp, rmtp, mode, clockid, TASK_INTERRUPTIBLE); -+} -+ - SYSCALL_DEFINE2(nanosleep, struct timespec __user *, rqtp, - struct timespec __user *, rmtp) - { -@@ -1611,6 +1877,26 @@ - return hrtimer_nanosleep(&tu, rmtp, HRTIMER_MODE_REL, CLOCK_MONOTONIC); - } - -+#ifdef CONFIG_PREEMPT_RT_FULL -+/* -+ * Sleep for 1 ms in hope whoever holds what we want will let it go. -+ */ -+void cpu_chill(void) -+{ -+ struct timespec tu = { -+ .tv_nsec = NSEC_PER_MSEC, -+ }; -+ unsigned int freeze_flag = current->flags & PF_NOFREEZE; -+ -+ current->flags |= PF_NOFREEZE; -+ __hrtimer_nanosleep(&tu, NULL, HRTIMER_MODE_REL, CLOCK_MONOTONIC, -+ TASK_UNINTERRUPTIBLE); -+ if (!freeze_flag) -+ current->flags &= ~PF_NOFREEZE; -+} -+EXPORT_SYMBOL(cpu_chill); -+#endif -+ - /* - * Functions related to boot-time initialization: - */ -@@ -1622,10 +1908,14 @@ - for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) { - cpu_base->clock_base[i].cpu_base = cpu_base; - timerqueue_init_head(&cpu_base->clock_base[i].active); -+ INIT_LIST_HEAD(&cpu_base->clock_base[i].expired); - } - - cpu_base->cpu = cpu; - hrtimer_init_hres(cpu_base); -+#ifdef CONFIG_PREEMPT_RT_BASE -+ init_waitqueue_head(&cpu_base->wait); -+#endif - } - - #ifdef CONFIG_HOTPLUG_CPU -@@ -1731,9 +2021,7 @@ - hrtimer_cpu_notify(&hrtimers_nb, (unsigned long)CPU_UP_PREPARE, - (void *)(long)smp_processor_id()); - register_cpu_notifier(&hrtimers_nb); --#ifdef CONFIG_HIGH_RES_TIMERS - open_softirq(HRTIMER_SOFTIRQ, run_hrtimer_softirq); --#endif - } - - /** -diff -Nur linux-4.1.10.orig/kernel/time/itimer.c linux-4.1.10/kernel/time/itimer.c ---- linux-4.1.10.orig/kernel/time/itimer.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/time/itimer.c 2015-10-07 18:00:08.000000000 +0200 -@@ -213,6 +213,7 @@ - /* We are sharing ->siglock with it_real_fn() */ - if (hrtimer_try_to_cancel(timer) < 0) { - spin_unlock_irq(&tsk->sighand->siglock); -+ hrtimer_wait_for_timer(&tsk->signal->real_timer); - goto again; - } - expires = timeval_to_ktime(value->it_value); -diff -Nur linux-4.1.10.orig/kernel/time/jiffies.c linux-4.1.10/kernel/time/jiffies.c ---- linux-4.1.10.orig/kernel/time/jiffies.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/time/jiffies.c 2015-10-07 18:00:08.000000000 +0200 -@@ -74,7 +74,8 @@ - .max_cycles = 10, - }; - --__cacheline_aligned_in_smp DEFINE_SEQLOCK(jiffies_lock); -+__cacheline_aligned_in_smp DEFINE_RAW_SPINLOCK(jiffies_lock); -+__cacheline_aligned_in_smp seqcount_t jiffies_seq; - - #if (BITS_PER_LONG < 64) - u64 get_jiffies_64(void) -@@ -83,9 +84,9 @@ - u64 ret; - - do { -- seq = read_seqbegin(&jiffies_lock); -+ seq = read_seqcount_begin(&jiffies_seq); - ret = jiffies_64; -- } while (read_seqretry(&jiffies_lock, seq)); -+ } while (read_seqcount_retry(&jiffies_seq, seq)); - return ret; - } - EXPORT_SYMBOL(get_jiffies_64); -diff -Nur linux-4.1.10.orig/kernel/time/ntp.c linux-4.1.10/kernel/time/ntp.c ---- linux-4.1.10.orig/kernel/time/ntp.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/time/ntp.c 2015-10-07 18:00:08.000000000 +0200 -@@ -10,6 +10,7 @@ - #include - #include - #include -+#include - #include - #include - #include -@@ -529,10 +530,52 @@ - &sync_cmos_work, timespec_to_jiffies(&next)); - } - -+#ifdef CONFIG_PREEMPT_RT_FULL -+/* -+ * RT can not call schedule_delayed_work from real interrupt context. -+ * Need to make a thread to do the real work. -+ */ -+static struct task_struct *cmos_delay_thread; -+static bool do_cmos_delay; -+ -+static int run_cmos_delay(void *ignore) -+{ -+ while (!kthread_should_stop()) { -+ set_current_state(TASK_INTERRUPTIBLE); -+ if (do_cmos_delay) { -+ do_cmos_delay = false; -+ queue_delayed_work(system_power_efficient_wq, -+ &sync_cmos_work, 0); -+ } -+ schedule(); -+ } -+ __set_current_state(TASK_RUNNING); -+ return 0; -+} -+ -+void ntp_notify_cmos_timer(void) -+{ -+ do_cmos_delay = true; -+ /* Make visible before waking up process */ -+ smp_wmb(); -+ wake_up_process(cmos_delay_thread); -+} -+ -+static __init int create_cmos_delay_thread(void) -+{ -+ cmos_delay_thread = kthread_run(run_cmos_delay, NULL, "kcmosdelayd"); -+ BUG_ON(!cmos_delay_thread); -+ return 0; -+} -+early_initcall(create_cmos_delay_thread); -+ -+#else -+ - void ntp_notify_cmos_timer(void) - { - queue_delayed_work(system_power_efficient_wq, &sync_cmos_work, 0); - } -+#endif /* CONFIG_PREEMPT_RT_FULL */ - - #else - void ntp_notify_cmos_timer(void) { } -diff -Nur linux-4.1.10.orig/kernel/time/posix-cpu-timers.c linux-4.1.10/kernel/time/posix-cpu-timers.c ---- linux-4.1.10.orig/kernel/time/posix-cpu-timers.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/time/posix-cpu-timers.c 2015-10-07 18:00:08.000000000 +0200 -@@ -3,6 +3,7 @@ - */ - - #include -+#include - #include - #include - #include -@@ -626,7 +627,7 @@ - /* - * Disarm any old timer after extracting its expiry time. - */ -- WARN_ON_ONCE(!irqs_disabled()); -+ WARN_ON_ONCE_NONRT(!irqs_disabled()); - - ret = 0; - old_incr = timer->it.cpu.incr; -@@ -1047,7 +1048,7 @@ - /* - * Now re-arm for the new expiry time. - */ -- WARN_ON_ONCE(!irqs_disabled()); -+ WARN_ON_ONCE_NONRT(!irqs_disabled()); - arm_timer(timer); - unlock_task_sighand(p, &flags); - -@@ -1113,10 +1114,11 @@ - sig = tsk->signal; - if (sig->cputimer.running) { - struct task_cputime group_sample; -+ unsigned long flags; - -- raw_spin_lock(&sig->cputimer.lock); -+ raw_spin_lock_irqsave(&sig->cputimer.lock, flags); - group_sample = sig->cputimer.cputime; -- raw_spin_unlock(&sig->cputimer.lock); -+ raw_spin_unlock_irqrestore(&sig->cputimer.lock, flags); - - if (task_cputime_expired(&group_sample, &sig->cputime_expires)) - return 1; -@@ -1130,13 +1132,13 @@ - * already updated our counts. We need to check if any timers fire now. - * Interrupts are disabled. - */ --void run_posix_cpu_timers(struct task_struct *tsk) -+static void __run_posix_cpu_timers(struct task_struct *tsk) - { - LIST_HEAD(firing); - struct k_itimer *timer, *next; - unsigned long flags; - -- WARN_ON_ONCE(!irqs_disabled()); -+ WARN_ON_ONCE_NONRT(!irqs_disabled()); - - /* - * The fast path checks that there are no expired thread or thread -@@ -1194,6 +1196,190 @@ - } - } - -+#ifdef CONFIG_PREEMPT_RT_BASE -+#include -+#include -+DEFINE_PER_CPU(struct task_struct *, posix_timer_task); -+DEFINE_PER_CPU(struct task_struct *, posix_timer_tasklist); -+ -+static int posix_cpu_timers_thread(void *data) -+{ -+ int cpu = (long)data; -+ -+ BUG_ON(per_cpu(posix_timer_task,cpu) != current); -+ -+ while (!kthread_should_stop()) { -+ struct task_struct *tsk = NULL; -+ struct task_struct *next = NULL; -+ -+ if (cpu_is_offline(cpu)) -+ goto wait_to_die; -+ -+ /* grab task list */ -+ raw_local_irq_disable(); -+ tsk = per_cpu(posix_timer_tasklist, cpu); -+ per_cpu(posix_timer_tasklist, cpu) = NULL; -+ raw_local_irq_enable(); -+ -+ /* its possible the list is empty, just return */ -+ if (!tsk) { -+ set_current_state(TASK_INTERRUPTIBLE); -+ schedule(); -+ __set_current_state(TASK_RUNNING); -+ continue; -+ } -+ -+ /* Process task list */ -+ while (1) { -+ /* save next */ -+ next = tsk->posix_timer_list; -+ -+ /* run the task timers, clear its ptr and -+ * unreference it -+ */ -+ __run_posix_cpu_timers(tsk); -+ tsk->posix_timer_list = NULL; -+ put_task_struct(tsk); -+ -+ /* check if this is the last on the list */ -+ if (next == tsk) -+ break; -+ tsk = next; -+ } -+ } -+ return 0; -+ -+wait_to_die: -+ /* Wait for kthread_stop */ -+ set_current_state(TASK_INTERRUPTIBLE); -+ while (!kthread_should_stop()) { -+ schedule(); -+ set_current_state(TASK_INTERRUPTIBLE); -+ } -+ __set_current_state(TASK_RUNNING); -+ return 0; -+} -+ -+static inline int __fastpath_timer_check(struct task_struct *tsk) -+{ -+ /* tsk == current, ensure it is safe to use ->signal/sighand */ -+ if (unlikely(tsk->exit_state)) -+ return 0; -+ -+ if (!task_cputime_zero(&tsk->cputime_expires)) -+ return 1; -+ -+ if (!task_cputime_zero(&tsk->signal->cputime_expires)) -+ return 1; -+ -+ return 0; -+} -+ -+void run_posix_cpu_timers(struct task_struct *tsk) -+{ -+ unsigned long cpu = smp_processor_id(); -+ struct task_struct *tasklist; -+ -+ BUG_ON(!irqs_disabled()); -+ if(!per_cpu(posix_timer_task, cpu)) -+ return; -+ /* get per-cpu references */ -+ tasklist = per_cpu(posix_timer_tasklist, cpu); -+ -+ /* check to see if we're already queued */ -+ if (!tsk->posix_timer_list && __fastpath_timer_check(tsk)) { -+ get_task_struct(tsk); -+ if (tasklist) { -+ tsk->posix_timer_list = tasklist; -+ } else { -+ /* -+ * The list is terminated by a self-pointing -+ * task_struct -+ */ -+ tsk->posix_timer_list = tsk; -+ } -+ per_cpu(posix_timer_tasklist, cpu) = tsk; -+ -+ wake_up_process(per_cpu(posix_timer_task, cpu)); -+ } -+} -+ -+/* -+ * posix_cpu_thread_call - callback that gets triggered when a CPU is added. -+ * Here we can start up the necessary migration thread for the new CPU. -+ */ -+static int posix_cpu_thread_call(struct notifier_block *nfb, -+ unsigned long action, void *hcpu) -+{ -+ int cpu = (long)hcpu; -+ struct task_struct *p; -+ struct sched_param param; -+ -+ switch (action) { -+ case CPU_UP_PREPARE: -+ p = kthread_create(posix_cpu_timers_thread, hcpu, -+ "posixcputmr/%d",cpu); -+ if (IS_ERR(p)) -+ return NOTIFY_BAD; -+ p->flags |= PF_NOFREEZE; -+ kthread_bind(p, cpu); -+ /* Must be high prio to avoid getting starved */ -+ param.sched_priority = MAX_RT_PRIO-1; -+ sched_setscheduler(p, SCHED_FIFO, ¶m); -+ per_cpu(posix_timer_task,cpu) = p; -+ break; -+ case CPU_ONLINE: -+ /* Strictly unneccessary, as first user will wake it. */ -+ wake_up_process(per_cpu(posix_timer_task,cpu)); -+ break; -+#ifdef CONFIG_HOTPLUG_CPU -+ case CPU_UP_CANCELED: -+ /* Unbind it from offline cpu so it can run. Fall thru. */ -+ kthread_bind(per_cpu(posix_timer_task, cpu), -+ cpumask_any(cpu_online_mask)); -+ kthread_stop(per_cpu(posix_timer_task,cpu)); -+ per_cpu(posix_timer_task,cpu) = NULL; -+ break; -+ case CPU_DEAD: -+ kthread_stop(per_cpu(posix_timer_task,cpu)); -+ per_cpu(posix_timer_task,cpu) = NULL; -+ break; -+#endif -+ } -+ return NOTIFY_OK; -+} -+ -+/* Register at highest priority so that task migration (migrate_all_tasks) -+ * happens before everything else. -+ */ -+static struct notifier_block posix_cpu_thread_notifier = { -+ .notifier_call = posix_cpu_thread_call, -+ .priority = 10 -+}; -+ -+static int __init posix_cpu_thread_init(void) -+{ -+ void *hcpu = (void *)(long)smp_processor_id(); -+ /* Start one for boot CPU. */ -+ unsigned long cpu; -+ -+ /* init the per-cpu posix_timer_tasklets */ -+ for_each_possible_cpu(cpu) -+ per_cpu(posix_timer_tasklist, cpu) = NULL; -+ -+ posix_cpu_thread_call(&posix_cpu_thread_notifier, CPU_UP_PREPARE, hcpu); -+ posix_cpu_thread_call(&posix_cpu_thread_notifier, CPU_ONLINE, hcpu); -+ register_cpu_notifier(&posix_cpu_thread_notifier); -+ return 0; -+} -+early_initcall(posix_cpu_thread_init); -+#else /* CONFIG_PREEMPT_RT_BASE */ -+void run_posix_cpu_timers(struct task_struct *tsk) -+{ -+ __run_posix_cpu_timers(tsk); -+} -+#endif /* CONFIG_PREEMPT_RT_BASE */ -+ - /* - * Set one of the process-wide special case CPU timers or RLIMIT_CPU. - * The tsk->sighand->siglock must be held by the caller. -diff -Nur linux-4.1.10.orig/kernel/time/posix-timers.c linux-4.1.10/kernel/time/posix-timers.c ---- linux-4.1.10.orig/kernel/time/posix-timers.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/time/posix-timers.c 2015-10-07 18:00:08.000000000 +0200 -@@ -499,6 +499,7 @@ - static struct pid *good_sigevent(sigevent_t * event) - { - struct task_struct *rtn = current->group_leader; -+ int sig = event->sigev_signo; - - if ((event->sigev_notify & SIGEV_THREAD_ID ) && - (!(rtn = find_task_by_vpid(event->sigev_notify_thread_id)) || -@@ -507,7 +508,8 @@ - return NULL; - - if (((event->sigev_notify & ~SIGEV_THREAD_ID) != SIGEV_NONE) && -- ((event->sigev_signo <= 0) || (event->sigev_signo > SIGRTMAX))) -+ (sig <= 0 || sig > SIGRTMAX || sig_kernel_only(sig) || -+ sig_kernel_coredump(sig))) - return NULL; - - return task_pid(rtn); -@@ -819,6 +821,20 @@ - return overrun; - } - -+/* -+ * Protected by RCU! -+ */ -+static void timer_wait_for_callback(struct k_clock *kc, struct k_itimer *timr) -+{ -+#ifdef CONFIG_PREEMPT_RT_FULL -+ if (kc->timer_set == common_timer_set) -+ hrtimer_wait_for_timer(&timr->it.real.timer); -+ else -+ /* FIXME: Whacky hack for posix-cpu-timers */ -+ schedule_timeout(1); -+#endif -+} -+ - /* Set a POSIX.1b interval timer. */ - /* timr->it_lock is taken. */ - static int -@@ -896,6 +912,7 @@ - if (!timr) - return -EINVAL; - -+ rcu_read_lock(); - kc = clockid_to_kclock(timr->it_clock); - if (WARN_ON_ONCE(!kc || !kc->timer_set)) - error = -EINVAL; -@@ -904,9 +921,12 @@ - - unlock_timer(timr, flag); - if (error == TIMER_RETRY) { -+ timer_wait_for_callback(kc, timr); - rtn = NULL; // We already got the old time... -+ rcu_read_unlock(); - goto retry; - } -+ rcu_read_unlock(); - - if (old_setting && !error && - copy_to_user(old_setting, &old_spec, sizeof (old_spec))) -@@ -944,10 +964,15 @@ - if (!timer) - return -EINVAL; - -+ rcu_read_lock(); - if (timer_delete_hook(timer) == TIMER_RETRY) { - unlock_timer(timer, flags); -+ timer_wait_for_callback(clockid_to_kclock(timer->it_clock), -+ timer); -+ rcu_read_unlock(); - goto retry_delete; - } -+ rcu_read_unlock(); - - spin_lock(¤t->sighand->siglock); - list_del(&timer->list); -@@ -973,8 +998,18 @@ - retry_delete: - spin_lock_irqsave(&timer->it_lock, flags); - -+ /* On RT we can race with a deletion */ -+ if (!timer->it_signal) { -+ unlock_timer(timer, flags); -+ return; -+ } -+ - if (timer_delete_hook(timer) == TIMER_RETRY) { -+ rcu_read_lock(); - unlock_timer(timer, flags); -+ timer_wait_for_callback(clockid_to_kclock(timer->it_clock), -+ timer); -+ rcu_read_unlock(); - goto retry_delete; - } - list_del(&timer->list); -diff -Nur linux-4.1.10.orig/kernel/time/tick-common.c linux-4.1.10/kernel/time/tick-common.c ---- linux-4.1.10.orig/kernel/time/tick-common.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/time/tick-common.c 2015-10-07 18:00:08.000000000 +0200 -@@ -78,13 +78,15 @@ - static void tick_periodic(int cpu) - { - if (tick_do_timer_cpu == cpu) { -- write_seqlock(&jiffies_lock); -+ raw_spin_lock(&jiffies_lock); -+ write_seqcount_begin(&jiffies_seq); - - /* Keep track of the next tick event */ - tick_next_period = ktime_add(tick_next_period, tick_period); - - do_timer(1); -- write_sequnlock(&jiffies_lock); -+ write_seqcount_end(&jiffies_seq); -+ raw_spin_unlock(&jiffies_lock); - update_wall_time(); - } - -@@ -146,9 +148,9 @@ - ktime_t next; - - do { -- seq = read_seqbegin(&jiffies_lock); -+ seq = read_seqcount_begin(&jiffies_seq); - next = tick_next_period; -- } while (read_seqretry(&jiffies_lock, seq)); -+ } while (read_seqcount_retry(&jiffies_seq, seq)); - - clockevents_set_state(dev, CLOCK_EVT_STATE_ONESHOT); - -diff -Nur linux-4.1.10.orig/kernel/time/tick-sched.c linux-4.1.10/kernel/time/tick-sched.c ---- linux-4.1.10.orig/kernel/time/tick-sched.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/time/tick-sched.c 2015-10-07 18:00:08.000000000 +0200 -@@ -62,7 +62,8 @@ - return; - - /* Reevalute with jiffies_lock held */ -- write_seqlock(&jiffies_lock); -+ raw_spin_lock(&jiffies_lock); -+ write_seqcount_begin(&jiffies_seq); - - delta = ktime_sub(now, last_jiffies_update); - if (delta.tv64 >= tick_period.tv64) { -@@ -85,10 +86,12 @@ - /* Keep the tick_next_period variable up to date */ - tick_next_period = ktime_add(last_jiffies_update, tick_period); - } else { -- write_sequnlock(&jiffies_lock); -+ write_seqcount_end(&jiffies_seq); -+ raw_spin_unlock(&jiffies_lock); - return; - } -- write_sequnlock(&jiffies_lock); -+ write_seqcount_end(&jiffies_seq); -+ raw_spin_unlock(&jiffies_lock); - update_wall_time(); - } - -@@ -99,12 +102,14 @@ - { - ktime_t period; - -- write_seqlock(&jiffies_lock); -+ raw_spin_lock(&jiffies_lock); -+ write_seqcount_begin(&jiffies_seq); - /* Did we start the jiffies update yet ? */ - if (last_jiffies_update.tv64 == 0) - last_jiffies_update = tick_next_period; - period = last_jiffies_update; -- write_sequnlock(&jiffies_lock); -+ write_seqcount_end(&jiffies_seq); -+ raw_spin_unlock(&jiffies_lock); - return period; - } - -@@ -176,6 +181,11 @@ - return false; - } - -+ if (!arch_irq_work_has_interrupt()) { -+ trace_tick_stop(0, "missing irq work interrupt\n"); -+ return false; -+ } -+ - /* sched_clock_tick() needs us? */ - #ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK - /* -@@ -222,6 +232,7 @@ - - static DEFINE_PER_CPU(struct irq_work, nohz_full_kick_work) = { - .func = nohz_full_kick_work_func, -+ .flags = IRQ_WORK_HARD_IRQ, - }; - - /* -@@ -578,10 +589,10 @@ - - /* Read jiffies and the time when jiffies were updated last */ - do { -- seq = read_seqbegin(&jiffies_lock); -+ seq = read_seqcount_begin(&jiffies_seq); - last_update = last_jiffies_update; - last_jiffies = jiffies; -- } while (read_seqretry(&jiffies_lock, seq)); -+ } while (read_seqcount_retry(&jiffies_seq, seq)); - - if (rcu_needs_cpu(&rcu_delta_jiffies) || - arch_needs_cpu() || irq_work_needs_cpu()) { -@@ -759,14 +770,7 @@ - return false; - - if (unlikely(local_softirq_pending() && cpu_online(cpu))) { -- static int ratelimit; -- -- if (ratelimit < 10 && -- (local_softirq_pending() & SOFTIRQ_STOP_IDLE_MASK)) { -- pr_warn("NOHZ: local_softirq_pending %02x\n", -- (unsigned int) local_softirq_pending()); -- ratelimit++; -- } -+ softirq_check_pending_idle(); - return false; - } - -@@ -1154,6 +1158,7 @@ - * Emulate tick processing via per-CPU hrtimers: - */ - hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); -+ ts->sched_timer.irqsafe = 1; - ts->sched_timer.function = tick_sched_timer; - - /* Get the next period (per cpu) */ -diff -Nur linux-4.1.10.orig/kernel/time/timekeeping.c linux-4.1.10/kernel/time/timekeeping.c ---- linux-4.1.10.orig/kernel/time/timekeeping.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/time/timekeeping.c 2015-10-07 18:00:08.000000000 +0200 -@@ -2065,8 +2065,10 @@ - */ - void xtime_update(unsigned long ticks) - { -- write_seqlock(&jiffies_lock); -+ raw_spin_lock(&jiffies_lock); -+ write_seqcount_begin(&jiffies_seq); - do_timer(ticks); -- write_sequnlock(&jiffies_lock); -+ write_seqcount_end(&jiffies_seq); -+ raw_spin_unlock(&jiffies_lock); - update_wall_time(); - } -diff -Nur linux-4.1.10.orig/kernel/time/timekeeping.h linux-4.1.10/kernel/time/timekeeping.h ---- linux-4.1.10.orig/kernel/time/timekeeping.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/time/timekeeping.h 2015-10-07 18:00:08.000000000 +0200 -@@ -22,7 +22,8 @@ - extern void do_timer(unsigned long ticks); - extern void update_wall_time(void); - --extern seqlock_t jiffies_lock; -+extern raw_spinlock_t jiffies_lock; -+extern seqcount_t jiffies_seq; - - #define CS_NAME_LEN 32 - -diff -Nur linux-4.1.10.orig/kernel/time/timer.c linux-4.1.10/kernel/time/timer.c ---- linux-4.1.10.orig/kernel/time/timer.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/time/timer.c 2015-10-07 18:00:08.000000000 +0200 -@@ -78,6 +78,9 @@ - struct tvec_base { - spinlock_t lock; - struct timer_list *running_timer; -+#ifdef CONFIG_PREEMPT_RT_FULL -+ wait_queue_head_t wait_for_running_timer; -+#endif - unsigned long timer_jiffies; - unsigned long next_timer; - unsigned long active_timers; -@@ -768,6 +771,36 @@ - } - } - -+#ifndef CONFIG_PREEMPT_RT_FULL -+static inline struct tvec_base *switch_timer_base(struct timer_list *timer, -+ struct tvec_base *old, -+ struct tvec_base *new) -+{ -+ /* See the comment in lock_timer_base() */ -+ timer_set_base(timer, NULL); -+ spin_unlock(&old->lock); -+ spin_lock(&new->lock); -+ timer_set_base(timer, new); -+ return new; -+} -+#else -+static inline struct tvec_base *switch_timer_base(struct timer_list *timer, -+ struct tvec_base *old, -+ struct tvec_base *new) -+{ -+ /* -+ * We cannot do the above because we might be preempted and -+ * then the preempter would see NULL and loop forever. -+ */ -+ if (spin_trylock(&new->lock)) { -+ timer_set_base(timer, new); -+ spin_unlock(&old->lock); -+ return new; -+ } -+ return old; -+} -+#endif -+ - static inline int - __mod_timer(struct timer_list *timer, unsigned long expires, - bool pending_only, int pinned) -@@ -798,14 +831,8 @@ - * handler yet has not finished. This also guarantees that - * the timer is serialized wrt itself. - */ -- if (likely(base->running_timer != timer)) { -- /* See the comment in lock_timer_base() */ -- timer_set_base(timer, NULL); -- spin_unlock(&base->lock); -- base = new_base; -- spin_lock(&base->lock); -- timer_set_base(timer, base); -- } -+ if (likely(base->running_timer != timer)) -+ base = switch_timer_base(timer, base, new_base); - } - - timer->expires = expires; -@@ -979,6 +1006,29 @@ - } - EXPORT_SYMBOL_GPL(add_timer_on); - -+#ifdef CONFIG_PREEMPT_RT_FULL -+/* -+ * Wait for a running timer -+ */ -+static void wait_for_running_timer(struct timer_list *timer) -+{ -+ struct tvec_base *base = timer->base; -+ -+ if (base->running_timer == timer) -+ wait_event(base->wait_for_running_timer, -+ base->running_timer != timer); -+} -+ -+# define wakeup_timer_waiters(b) wake_up(&(b)->wait_for_running_timer) -+#else -+static inline void wait_for_running_timer(struct timer_list *timer) -+{ -+ cpu_relax(); -+} -+ -+# define wakeup_timer_waiters(b) do { } while (0) -+#endif -+ - /** - * del_timer - deactive a timer. - * @timer: the timer to be deactivated -@@ -1036,7 +1086,7 @@ - } - EXPORT_SYMBOL(try_to_del_timer_sync); - --#ifdef CONFIG_SMP -+#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT_FULL) - static DEFINE_PER_CPU(struct tvec_base, __tvec_bases); - - /** -@@ -1098,7 +1148,7 @@ - int ret = try_to_del_timer_sync(timer); - if (ret >= 0) - return ret; -- cpu_relax(); -+ wait_for_running_timer(timer); - } - } - EXPORT_SYMBOL(del_timer_sync); -@@ -1219,15 +1269,17 @@ - if (irqsafe) { - spin_unlock(&base->lock); - call_timer_fn(timer, fn, data); -+ base->running_timer = NULL; - spin_lock(&base->lock); - } else { - spin_unlock_irq(&base->lock); - call_timer_fn(timer, fn, data); -+ base->running_timer = NULL; - spin_lock_irq(&base->lock); - } - } - } -- base->running_timer = NULL; -+ wakeup_timer_waiters(base); - spin_unlock_irq(&base->lock); - } - -@@ -1367,6 +1419,14 @@ - if (cpu_is_offline(smp_processor_id())) - return expires; - -+#ifdef CONFIG_PREEMPT_RT_FULL -+ /* -+ * On PREEMPT_RT we cannot sleep here. As a result we can't take -+ * the base lock to check when the next timer is pending and so -+ * we assume the next jiffy. -+ */ -+ return now + 1; -+#endif - spin_lock(&base->lock); - if (base->active_timers) { - if (time_before_eq(base->next_timer, base->timer_jiffies)) -@@ -1392,13 +1452,13 @@ - - /* Note: this timer irq context must be accounted for as well. */ - account_process_tick(p, user_tick); -+ scheduler_tick(); - run_local_timers(); - rcu_check_callbacks(user_tick); --#ifdef CONFIG_IRQ_WORK -+#if defined(CONFIG_IRQ_WORK) && !defined(CONFIG_PREEMPT_RT_FULL) - if (in_irq()) - irq_work_tick(); - #endif -- scheduler_tick(); - run_posix_cpu_timers(p); - } - -@@ -1411,6 +1471,10 @@ - - hrtimer_run_pending(); - -+#if defined(CONFIG_IRQ_WORK) && defined(CONFIG_PREEMPT_RT_FULL) -+ irq_work_tick(); -+#endif -+ - if (time_after_eq(jiffies, base->timer_jiffies)) - __run_timers(base); - } -@@ -1566,7 +1630,7 @@ - - BUG_ON(cpu_online(cpu)); - old_base = per_cpu(tvec_bases, cpu); -- new_base = get_cpu_var(tvec_bases); -+ new_base = get_local_var(tvec_bases); - /* - * The caller is globally serialized and nobody else - * takes two locks at once, deadlock is not possible. -@@ -1590,7 +1654,7 @@ - - spin_unlock(&old_base->lock); - spin_unlock_irq(&new_base->lock); -- put_cpu_var(tvec_bases); -+ put_local_var(tvec_bases); - } - - static int timer_cpu_notify(struct notifier_block *self, -@@ -1625,6 +1689,9 @@ - base->cpu = cpu; - per_cpu(tvec_bases, cpu) = base; - spin_lock_init(&base->lock); -+#ifdef CONFIG_PREEMPT_RT_FULL -+ init_waitqueue_head(&base->wait_for_running_timer); -+#endif - - for (j = 0; j < TVN_SIZE; j++) { - INIT_LIST_HEAD(base->tv5.vec + j); -diff -Nur linux-4.1.10.orig/kernel/trace/Kconfig linux-4.1.10/kernel/trace/Kconfig ---- linux-4.1.10.orig/kernel/trace/Kconfig 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/trace/Kconfig 2015-10-07 18:00:08.000000000 +0200 -@@ -187,6 +187,24 @@ - enabled. This option and the preempt-off timing option can be - used together or separately.) - -+config INTERRUPT_OFF_HIST -+ bool "Interrupts-off Latency Histogram" -+ depends on IRQSOFF_TRACER -+ help -+ This option generates continuously updated histograms (one per cpu) -+ of the duration of time periods with interrupts disabled. The -+ histograms are disabled by default. To enable them, write a non-zero -+ number to -+ -+ /sys/kernel/debug/tracing/latency_hist/enable/preemptirqsoff -+ -+ If PREEMPT_OFF_HIST is also selected, additional histograms (one -+ per cpu) are generated that accumulate the duration of time periods -+ when both interrupts and preemption are disabled. The histogram data -+ will be located in the debug file system at -+ -+ /sys/kernel/debug/tracing/latency_hist/irqsoff -+ - config PREEMPT_TRACER - bool "Preemption-off Latency Tracer" - default n -@@ -211,6 +229,24 @@ - enabled. This option and the irqs-off timing option can be - used together or separately.) - -+config PREEMPT_OFF_HIST -+ bool "Preemption-off Latency Histogram" -+ depends on PREEMPT_TRACER -+ help -+ This option generates continuously updated histograms (one per cpu) -+ of the duration of time periods with preemption disabled. The -+ histograms are disabled by default. To enable them, write a non-zero -+ number to -+ -+ /sys/kernel/debug/tracing/latency_hist/enable/preemptirqsoff -+ -+ If INTERRUPT_OFF_HIST is also selected, additional histograms (one -+ per cpu) are generated that accumulate the duration of time periods -+ when both interrupts and preemption are disabled. The histogram data -+ will be located in the debug file system at -+ -+ /sys/kernel/debug/tracing/latency_hist/preemptoff -+ - config SCHED_TRACER - bool "Scheduling Latency Tracer" - select GENERIC_TRACER -@@ -221,6 +257,74 @@ - This tracer tracks the latency of the highest priority task - to be scheduled in, starting from the point it has woken up. - -+config WAKEUP_LATENCY_HIST -+ bool "Scheduling Latency Histogram" -+ depends on SCHED_TRACER -+ help -+ This option generates continuously updated histograms (one per cpu) -+ of the scheduling latency of the highest priority task. -+ The histograms are disabled by default. To enable them, write a -+ non-zero number to -+ -+ /sys/kernel/debug/tracing/latency_hist/enable/wakeup -+ -+ Two different algorithms are used, one to determine the latency of -+ processes that exclusively use the highest priority of the system and -+ another one to determine the latency of processes that share the -+ highest system priority with other processes. The former is used to -+ improve hardware and system software, the latter to optimize the -+ priority design of a given system. The histogram data will be -+ located in the debug file system at -+ -+ /sys/kernel/debug/tracing/latency_hist/wakeup -+ -+ and -+ -+ /sys/kernel/debug/tracing/latency_hist/wakeup/sharedprio -+ -+ If both Scheduling Latency Histogram and Missed Timer Offsets -+ Histogram are selected, additional histogram data will be collected -+ that contain, in addition to the wakeup latency, the timer latency, in -+ case the wakeup was triggered by an expired timer. These histograms -+ are available in the -+ -+ /sys/kernel/debug/tracing/latency_hist/timerandwakeup -+ -+ directory. They reflect the apparent interrupt and scheduling latency -+ and are best suitable to determine the worst-case latency of a given -+ system. To enable these histograms, write a non-zero number to -+ -+ /sys/kernel/debug/tracing/latency_hist/enable/timerandwakeup -+ -+config MISSED_TIMER_OFFSETS_HIST -+ depends on HIGH_RES_TIMERS -+ select GENERIC_TRACER -+ bool "Missed Timer Offsets Histogram" -+ help -+ Generate a histogram of missed timer offsets in microseconds. The -+ histograms are disabled by default. To enable them, write a non-zero -+ number to -+ -+ /sys/kernel/debug/tracing/latency_hist/enable/missed_timer_offsets -+ -+ The histogram data will be located in the debug file system at -+ -+ /sys/kernel/debug/tracing/latency_hist/missed_timer_offsets -+ -+ If both Scheduling Latency Histogram and Missed Timer Offsets -+ Histogram are selected, additional histogram data will be collected -+ that contain, in addition to the wakeup latency, the timer latency, in -+ case the wakeup was triggered by an expired timer. These histograms -+ are available in the -+ -+ /sys/kernel/debug/tracing/latency_hist/timerandwakeup -+ -+ directory. They reflect the apparent interrupt and scheduling latency -+ and are best suitable to determine the worst-case latency of a given -+ system. To enable these histograms, write a non-zero number to -+ -+ /sys/kernel/debug/tracing/latency_hist/enable/timerandwakeup -+ - config ENABLE_DEFAULT_TRACERS - bool "Trace process context switches and events" - depends on !GENERIC_TRACER -diff -Nur linux-4.1.10.orig/kernel/trace/latency_hist.c linux-4.1.10/kernel/trace/latency_hist.c ---- linux-4.1.10.orig/kernel/trace/latency_hist.c 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.10/kernel/trace/latency_hist.c 2015-10-07 18:00:08.000000000 +0200 -@@ -0,0 +1,1178 @@ -+/* -+ * kernel/trace/latency_hist.c -+ * -+ * Add support for histograms of preemption-off latency and -+ * interrupt-off latency and wakeup latency, it depends on -+ * Real-Time Preemption Support. -+ * -+ * Copyright (C) 2005 MontaVista Software, Inc. -+ * Yi Yang -+ * -+ * Converted to work with the new latency tracer. -+ * Copyright (C) 2008 Red Hat, Inc. -+ * Steven Rostedt -+ * -+ */ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include "trace.h" -+#include -+ -+#define NSECS_PER_USECS 1000L -+ -+#define CREATE_TRACE_POINTS -+#include -+ -+enum { -+ IRQSOFF_LATENCY = 0, -+ PREEMPTOFF_LATENCY, -+ PREEMPTIRQSOFF_LATENCY, -+ WAKEUP_LATENCY, -+ WAKEUP_LATENCY_SHAREDPRIO, -+ MISSED_TIMER_OFFSETS, -+ TIMERANDWAKEUP_LATENCY, -+ MAX_LATENCY_TYPE, -+}; -+ -+#define MAX_ENTRY_NUM 10240 -+ -+struct hist_data { -+ atomic_t hist_mode; /* 0 log, 1 don't log */ -+ long offset; /* set it to MAX_ENTRY_NUM/2 for a bipolar scale */ -+ long min_lat; -+ long max_lat; -+ unsigned long long below_hist_bound_samples; -+ unsigned long long above_hist_bound_samples; -+ long long accumulate_lat; -+ unsigned long long total_samples; -+ unsigned long long hist_array[MAX_ENTRY_NUM]; -+}; -+ -+struct enable_data { -+ int latency_type; -+ int enabled; -+}; -+ -+static char *latency_hist_dir_root = "latency_hist"; -+ -+#ifdef CONFIG_INTERRUPT_OFF_HIST -+static DEFINE_PER_CPU(struct hist_data, irqsoff_hist); -+static char *irqsoff_hist_dir = "irqsoff"; -+static DEFINE_PER_CPU(cycles_t, hist_irqsoff_start); -+static DEFINE_PER_CPU(int, hist_irqsoff_counting); -+#endif -+ -+#ifdef CONFIG_PREEMPT_OFF_HIST -+static DEFINE_PER_CPU(struct hist_data, preemptoff_hist); -+static char *preemptoff_hist_dir = "preemptoff"; -+static DEFINE_PER_CPU(cycles_t, hist_preemptoff_start); -+static DEFINE_PER_CPU(int, hist_preemptoff_counting); -+#endif -+ -+#if defined(CONFIG_PREEMPT_OFF_HIST) && defined(CONFIG_INTERRUPT_OFF_HIST) -+static DEFINE_PER_CPU(struct hist_data, preemptirqsoff_hist); -+static char *preemptirqsoff_hist_dir = "preemptirqsoff"; -+static DEFINE_PER_CPU(cycles_t, hist_preemptirqsoff_start); -+static DEFINE_PER_CPU(int, hist_preemptirqsoff_counting); -+#endif -+ -+#if defined(CONFIG_PREEMPT_OFF_HIST) || defined(CONFIG_INTERRUPT_OFF_HIST) -+static notrace void probe_preemptirqsoff_hist(void *v, int reason, int start); -+static struct enable_data preemptirqsoff_enabled_data = { -+ .latency_type = PREEMPTIRQSOFF_LATENCY, -+ .enabled = 0, -+}; -+#endif -+ -+#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \ -+ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) -+struct maxlatproc_data { -+ char comm[FIELD_SIZEOF(struct task_struct, comm)]; -+ char current_comm[FIELD_SIZEOF(struct task_struct, comm)]; -+ int pid; -+ int current_pid; -+ int prio; -+ int current_prio; -+ long latency; -+ long timeroffset; -+ cycle_t timestamp; -+}; -+#endif -+ -+#ifdef CONFIG_WAKEUP_LATENCY_HIST -+static DEFINE_PER_CPU(struct hist_data, wakeup_latency_hist); -+static DEFINE_PER_CPU(struct hist_data, wakeup_latency_hist_sharedprio); -+static char *wakeup_latency_hist_dir = "wakeup"; -+static char *wakeup_latency_hist_dir_sharedprio = "sharedprio"; -+static notrace void probe_wakeup_latency_hist_start(void *v, -+ struct task_struct *p, int success); -+static notrace void probe_wakeup_latency_hist_stop(void *v, -+ struct task_struct *prev, struct task_struct *next); -+static notrace void probe_sched_migrate_task(void *, -+ struct task_struct *task, int cpu); -+static struct enable_data wakeup_latency_enabled_data = { -+ .latency_type = WAKEUP_LATENCY, -+ .enabled = 0, -+}; -+static DEFINE_PER_CPU(struct maxlatproc_data, wakeup_maxlatproc); -+static DEFINE_PER_CPU(struct maxlatproc_data, wakeup_maxlatproc_sharedprio); -+static DEFINE_PER_CPU(struct task_struct *, wakeup_task); -+static DEFINE_PER_CPU(int, wakeup_sharedprio); -+static unsigned long wakeup_pid; -+#endif -+ -+#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST -+static DEFINE_PER_CPU(struct hist_data, missed_timer_offsets); -+static char *missed_timer_offsets_dir = "missed_timer_offsets"; -+static notrace void probe_hrtimer_interrupt(void *v, int cpu, -+ long long offset, struct task_struct *curr, struct task_struct *task); -+static struct enable_data missed_timer_offsets_enabled_data = { -+ .latency_type = MISSED_TIMER_OFFSETS, -+ .enabled = 0, -+}; -+static DEFINE_PER_CPU(struct maxlatproc_data, missed_timer_offsets_maxlatproc); -+static unsigned long missed_timer_offsets_pid; -+#endif -+ -+#if defined(CONFIG_WAKEUP_LATENCY_HIST) && \ -+ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) -+static DEFINE_PER_CPU(struct hist_data, timerandwakeup_latency_hist); -+static char *timerandwakeup_latency_hist_dir = "timerandwakeup"; -+static struct enable_data timerandwakeup_enabled_data = { -+ .latency_type = TIMERANDWAKEUP_LATENCY, -+ .enabled = 0, -+}; -+static DEFINE_PER_CPU(struct maxlatproc_data, timerandwakeup_maxlatproc); -+#endif -+ -+void notrace latency_hist(int latency_type, int cpu, long latency, -+ long timeroffset, cycle_t stop, -+ struct task_struct *p) -+{ -+ struct hist_data *my_hist; -+#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \ -+ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) -+ struct maxlatproc_data *mp = NULL; -+#endif -+ -+ if (!cpu_possible(cpu) || latency_type < 0 || -+ latency_type >= MAX_LATENCY_TYPE) -+ return; -+ -+ switch (latency_type) { -+#ifdef CONFIG_INTERRUPT_OFF_HIST -+ case IRQSOFF_LATENCY: -+ my_hist = &per_cpu(irqsoff_hist, cpu); -+ break; -+#endif -+#ifdef CONFIG_PREEMPT_OFF_HIST -+ case PREEMPTOFF_LATENCY: -+ my_hist = &per_cpu(preemptoff_hist, cpu); -+ break; -+#endif -+#if defined(CONFIG_PREEMPT_OFF_HIST) && defined(CONFIG_INTERRUPT_OFF_HIST) -+ case PREEMPTIRQSOFF_LATENCY: -+ my_hist = &per_cpu(preemptirqsoff_hist, cpu); -+ break; -+#endif -+#ifdef CONFIG_WAKEUP_LATENCY_HIST -+ case WAKEUP_LATENCY: -+ my_hist = &per_cpu(wakeup_latency_hist, cpu); -+ mp = &per_cpu(wakeup_maxlatproc, cpu); -+ break; -+ case WAKEUP_LATENCY_SHAREDPRIO: -+ my_hist = &per_cpu(wakeup_latency_hist_sharedprio, cpu); -+ mp = &per_cpu(wakeup_maxlatproc_sharedprio, cpu); -+ break; -+#endif -+#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST -+ case MISSED_TIMER_OFFSETS: -+ my_hist = &per_cpu(missed_timer_offsets, cpu); -+ mp = &per_cpu(missed_timer_offsets_maxlatproc, cpu); -+ break; -+#endif -+#if defined(CONFIG_WAKEUP_LATENCY_HIST) && \ -+ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) -+ case TIMERANDWAKEUP_LATENCY: -+ my_hist = &per_cpu(timerandwakeup_latency_hist, cpu); -+ mp = &per_cpu(timerandwakeup_maxlatproc, cpu); -+ break; -+#endif -+ -+ default: -+ return; -+ } -+ -+ latency += my_hist->offset; -+ -+ if (atomic_read(&my_hist->hist_mode) == 0) -+ return; -+ -+ if (latency < 0 || latency >= MAX_ENTRY_NUM) { -+ if (latency < 0) -+ my_hist->below_hist_bound_samples++; -+ else -+ my_hist->above_hist_bound_samples++; -+ } else -+ my_hist->hist_array[latency]++; -+ -+ if (unlikely(latency > my_hist->max_lat || -+ my_hist->min_lat == LONG_MAX)) { -+#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \ -+ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) -+ if (latency_type == WAKEUP_LATENCY || -+ latency_type == WAKEUP_LATENCY_SHAREDPRIO || -+ latency_type == MISSED_TIMER_OFFSETS || -+ latency_type == TIMERANDWAKEUP_LATENCY) { -+ strncpy(mp->comm, p->comm, sizeof(mp->comm)); -+ strncpy(mp->current_comm, current->comm, -+ sizeof(mp->current_comm)); -+ mp->pid = task_pid_nr(p); -+ mp->current_pid = task_pid_nr(current); -+ mp->prio = p->prio; -+ mp->current_prio = current->prio; -+ mp->latency = latency; -+ mp->timeroffset = timeroffset; -+ mp->timestamp = stop; -+ } -+#endif -+ my_hist->max_lat = latency; -+ } -+ if (unlikely(latency < my_hist->min_lat)) -+ my_hist->min_lat = latency; -+ my_hist->total_samples++; -+ my_hist->accumulate_lat += latency; -+} -+ -+static void *l_start(struct seq_file *m, loff_t *pos) -+{ -+ loff_t *index_ptr = NULL; -+ loff_t index = *pos; -+ struct hist_data *my_hist = m->private; -+ -+ if (index == 0) { -+ char minstr[32], avgstr[32], maxstr[32]; -+ -+ atomic_dec(&my_hist->hist_mode); -+ -+ if (likely(my_hist->total_samples)) { -+ long avg = (long) div64_s64(my_hist->accumulate_lat, -+ my_hist->total_samples); -+ snprintf(minstr, sizeof(minstr), "%ld", -+ my_hist->min_lat - my_hist->offset); -+ snprintf(avgstr, sizeof(avgstr), "%ld", -+ avg - my_hist->offset); -+ snprintf(maxstr, sizeof(maxstr), "%ld", -+ my_hist->max_lat - my_hist->offset); -+ } else { -+ strcpy(minstr, ""); -+ strcpy(avgstr, minstr); -+ strcpy(maxstr, minstr); -+ } -+ -+ seq_printf(m, "#Minimum latency: %s microseconds\n" -+ "#Average latency: %s microseconds\n" -+ "#Maximum latency: %s microseconds\n" -+ "#Total samples: %llu\n" -+ "#There are %llu samples lower than %ld" -+ " microseconds.\n" -+ "#There are %llu samples greater or equal" -+ " than %ld microseconds.\n" -+ "#usecs\t%16s\n", -+ minstr, avgstr, maxstr, -+ my_hist->total_samples, -+ my_hist->below_hist_bound_samples, -+ -my_hist->offset, -+ my_hist->above_hist_bound_samples, -+ MAX_ENTRY_NUM - my_hist->offset, -+ "samples"); -+ } -+ if (index < MAX_ENTRY_NUM) { -+ index_ptr = kmalloc(sizeof(loff_t), GFP_KERNEL); -+ if (index_ptr) -+ *index_ptr = index; -+ } -+ -+ return index_ptr; -+} -+ -+static void *l_next(struct seq_file *m, void *p, loff_t *pos) -+{ -+ loff_t *index_ptr = p; -+ struct hist_data *my_hist = m->private; -+ -+ if (++*pos >= MAX_ENTRY_NUM) { -+ atomic_inc(&my_hist->hist_mode); -+ return NULL; -+ } -+ *index_ptr = *pos; -+ return index_ptr; -+} -+ -+static void l_stop(struct seq_file *m, void *p) -+{ -+ kfree(p); -+} -+ -+static int l_show(struct seq_file *m, void *p) -+{ -+ int index = *(loff_t *) p; -+ struct hist_data *my_hist = m->private; -+ -+ seq_printf(m, "%6ld\t%16llu\n", index - my_hist->offset, -+ my_hist->hist_array[index]); -+ return 0; -+} -+ -+static const struct seq_operations latency_hist_seq_op = { -+ .start = l_start, -+ .next = l_next, -+ .stop = l_stop, -+ .show = l_show -+}; -+ -+static int latency_hist_open(struct inode *inode, struct file *file) -+{ -+ int ret; -+ -+ ret = seq_open(file, &latency_hist_seq_op); -+ if (!ret) { -+ struct seq_file *seq = file->private_data; -+ seq->private = inode->i_private; -+ } -+ return ret; -+} -+ -+static const struct file_operations latency_hist_fops = { -+ .open = latency_hist_open, -+ .read = seq_read, -+ .llseek = seq_lseek, -+ .release = seq_release, -+}; -+ -+#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \ -+ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) -+static void clear_maxlatprocdata(struct maxlatproc_data *mp) -+{ -+ mp->comm[0] = mp->current_comm[0] = '\0'; -+ mp->prio = mp->current_prio = mp->pid = mp->current_pid = -+ mp->latency = mp->timeroffset = -1; -+ mp->timestamp = 0; -+} -+#endif -+ -+static void hist_reset(struct hist_data *hist) -+{ -+ atomic_dec(&hist->hist_mode); -+ -+ memset(hist->hist_array, 0, sizeof(hist->hist_array)); -+ hist->below_hist_bound_samples = 0ULL; -+ hist->above_hist_bound_samples = 0ULL; -+ hist->min_lat = LONG_MAX; -+ hist->max_lat = LONG_MIN; -+ hist->total_samples = 0ULL; -+ hist->accumulate_lat = 0LL; -+ -+ atomic_inc(&hist->hist_mode); -+} -+ -+static ssize_t -+latency_hist_reset(struct file *file, const char __user *a, -+ size_t size, loff_t *off) -+{ -+ int cpu; -+ struct hist_data *hist = NULL; -+#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \ -+ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) -+ struct maxlatproc_data *mp = NULL; -+#endif -+ off_t latency_type = (off_t) file->private_data; -+ -+ for_each_online_cpu(cpu) { -+ -+ switch (latency_type) { -+#ifdef CONFIG_PREEMPT_OFF_HIST -+ case PREEMPTOFF_LATENCY: -+ hist = &per_cpu(preemptoff_hist, cpu); -+ break; -+#endif -+#ifdef CONFIG_INTERRUPT_OFF_HIST -+ case IRQSOFF_LATENCY: -+ hist = &per_cpu(irqsoff_hist, cpu); -+ break; -+#endif -+#if defined(CONFIG_INTERRUPT_OFF_HIST) && defined(CONFIG_PREEMPT_OFF_HIST) -+ case PREEMPTIRQSOFF_LATENCY: -+ hist = &per_cpu(preemptirqsoff_hist, cpu); -+ break; -+#endif -+#ifdef CONFIG_WAKEUP_LATENCY_HIST -+ case WAKEUP_LATENCY: -+ hist = &per_cpu(wakeup_latency_hist, cpu); -+ mp = &per_cpu(wakeup_maxlatproc, cpu); -+ break; -+ case WAKEUP_LATENCY_SHAREDPRIO: -+ hist = &per_cpu(wakeup_latency_hist_sharedprio, cpu); -+ mp = &per_cpu(wakeup_maxlatproc_sharedprio, cpu); -+ break; -+#endif -+#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST -+ case MISSED_TIMER_OFFSETS: -+ hist = &per_cpu(missed_timer_offsets, cpu); -+ mp = &per_cpu(missed_timer_offsets_maxlatproc, cpu); -+ break; -+#endif -+#if defined(CONFIG_WAKEUP_LATENCY_HIST) && \ -+ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) -+ case TIMERANDWAKEUP_LATENCY: -+ hist = &per_cpu(timerandwakeup_latency_hist, cpu); -+ mp = &per_cpu(timerandwakeup_maxlatproc, cpu); -+ break; -+#endif -+ } -+ -+ hist_reset(hist); -+#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \ -+ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) -+ if (latency_type == WAKEUP_LATENCY || -+ latency_type == WAKEUP_LATENCY_SHAREDPRIO || -+ latency_type == MISSED_TIMER_OFFSETS || -+ latency_type == TIMERANDWAKEUP_LATENCY) -+ clear_maxlatprocdata(mp); -+#endif -+ } -+ -+ return size; -+} -+ -+#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \ -+ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) -+static ssize_t -+show_pid(struct file *file, char __user *ubuf, size_t cnt, loff_t *ppos) -+{ -+ char buf[64]; -+ int r; -+ unsigned long *this_pid = file->private_data; -+ -+ r = snprintf(buf, sizeof(buf), "%lu\n", *this_pid); -+ return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); -+} -+ -+static ssize_t do_pid(struct file *file, const char __user *ubuf, -+ size_t cnt, loff_t *ppos) -+{ -+ char buf[64]; -+ unsigned long pid; -+ unsigned long *this_pid = file->private_data; -+ -+ if (cnt >= sizeof(buf)) -+ return -EINVAL; -+ -+ if (copy_from_user(&buf, ubuf, cnt)) -+ return -EFAULT; -+ -+ buf[cnt] = '\0'; -+ -+ if (kstrtoul(buf, 10, &pid)) -+ return -EINVAL; -+ -+ *this_pid = pid; -+ -+ return cnt; -+} -+#endif -+ -+#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \ -+ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) -+static ssize_t -+show_maxlatproc(struct file *file, char __user *ubuf, size_t cnt, loff_t *ppos) -+{ -+ int r; -+ struct maxlatproc_data *mp = file->private_data; -+ int strmaxlen = (TASK_COMM_LEN * 2) + (8 * 8); -+ unsigned long long t; -+ unsigned long usecs, secs; -+ char *buf; -+ -+ if (mp->pid == -1 || mp->current_pid == -1) { -+ buf = "(none)\n"; -+ return simple_read_from_buffer(ubuf, cnt, ppos, buf, -+ strlen(buf)); -+ } -+ -+ buf = kmalloc(strmaxlen, GFP_KERNEL); -+ if (buf == NULL) -+ return -ENOMEM; -+ -+ t = ns2usecs(mp->timestamp); -+ usecs = do_div(t, USEC_PER_SEC); -+ secs = (unsigned long) t; -+ r = snprintf(buf, strmaxlen, -+ "%d %d %ld (%ld) %s <- %d %d %s %lu.%06lu\n", mp->pid, -+ MAX_RT_PRIO-1 - mp->prio, mp->latency, mp->timeroffset, mp->comm, -+ mp->current_pid, MAX_RT_PRIO-1 - mp->current_prio, mp->current_comm, -+ secs, usecs); -+ r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r); -+ kfree(buf); -+ return r; -+} -+#endif -+ -+static ssize_t -+show_enable(struct file *file, char __user *ubuf, size_t cnt, loff_t *ppos) -+{ -+ char buf[64]; -+ struct enable_data *ed = file->private_data; -+ int r; -+ -+ r = snprintf(buf, sizeof(buf), "%d\n", ed->enabled); -+ return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); -+} -+ -+static ssize_t -+do_enable(struct file *file, const char __user *ubuf, size_t cnt, loff_t *ppos) -+{ -+ char buf[64]; -+ long enable; -+ struct enable_data *ed = file->private_data; -+ -+ if (cnt >= sizeof(buf)) -+ return -EINVAL; -+ -+ if (copy_from_user(&buf, ubuf, cnt)) -+ return -EFAULT; -+ -+ buf[cnt] = 0; -+ -+ if (kstrtoul(buf, 10, &enable)) -+ return -EINVAL; -+ -+ if ((enable && ed->enabled) || (!enable && !ed->enabled)) -+ return cnt; -+ -+ if (enable) { -+ int ret; -+ -+ switch (ed->latency_type) { -+#if defined(CONFIG_INTERRUPT_OFF_HIST) || defined(CONFIG_PREEMPT_OFF_HIST) -+ case PREEMPTIRQSOFF_LATENCY: -+ ret = register_trace_preemptirqsoff_hist( -+ probe_preemptirqsoff_hist, NULL); -+ if (ret) { -+ pr_info("wakeup trace: Couldn't assign " -+ "probe_preemptirqsoff_hist " -+ "to trace_preemptirqsoff_hist\n"); -+ return ret; -+ } -+ break; -+#endif -+#ifdef CONFIG_WAKEUP_LATENCY_HIST -+ case WAKEUP_LATENCY: -+ ret = register_trace_sched_wakeup( -+ probe_wakeup_latency_hist_start, NULL); -+ if (ret) { -+ pr_info("wakeup trace: Couldn't assign " -+ "probe_wakeup_latency_hist_start " -+ "to trace_sched_wakeup\n"); -+ return ret; -+ } -+ ret = register_trace_sched_wakeup_new( -+ probe_wakeup_latency_hist_start, NULL); -+ if (ret) { -+ pr_info("wakeup trace: Couldn't assign " -+ "probe_wakeup_latency_hist_start " -+ "to trace_sched_wakeup_new\n"); -+ unregister_trace_sched_wakeup( -+ probe_wakeup_latency_hist_start, NULL); -+ return ret; -+ } -+ ret = register_trace_sched_switch( -+ probe_wakeup_latency_hist_stop, NULL); -+ if (ret) { -+ pr_info("wakeup trace: Couldn't assign " -+ "probe_wakeup_latency_hist_stop " -+ "to trace_sched_switch\n"); -+ unregister_trace_sched_wakeup( -+ probe_wakeup_latency_hist_start, NULL); -+ unregister_trace_sched_wakeup_new( -+ probe_wakeup_latency_hist_start, NULL); -+ return ret; -+ } -+ ret = register_trace_sched_migrate_task( -+ probe_sched_migrate_task, NULL); -+ if (ret) { -+ pr_info("wakeup trace: Couldn't assign " -+ "probe_sched_migrate_task " -+ "to trace_sched_migrate_task\n"); -+ unregister_trace_sched_wakeup( -+ probe_wakeup_latency_hist_start, NULL); -+ unregister_trace_sched_wakeup_new( -+ probe_wakeup_latency_hist_start, NULL); -+ unregister_trace_sched_switch( -+ probe_wakeup_latency_hist_stop, NULL); -+ return ret; -+ } -+ break; -+#endif -+#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST -+ case MISSED_TIMER_OFFSETS: -+ ret = register_trace_hrtimer_interrupt( -+ probe_hrtimer_interrupt, NULL); -+ if (ret) { -+ pr_info("wakeup trace: Couldn't assign " -+ "probe_hrtimer_interrupt " -+ "to trace_hrtimer_interrupt\n"); -+ return ret; -+ } -+ break; -+#endif -+#if defined(CONFIG_WAKEUP_LATENCY_HIST) && \ -+ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) -+ case TIMERANDWAKEUP_LATENCY: -+ if (!wakeup_latency_enabled_data.enabled || -+ !missed_timer_offsets_enabled_data.enabled) -+ return -EINVAL; -+ break; -+#endif -+ default: -+ break; -+ } -+ } else { -+ switch (ed->latency_type) { -+#if defined(CONFIG_INTERRUPT_OFF_HIST) || defined(CONFIG_PREEMPT_OFF_HIST) -+ case PREEMPTIRQSOFF_LATENCY: -+ { -+ int cpu; -+ -+ unregister_trace_preemptirqsoff_hist( -+ probe_preemptirqsoff_hist, NULL); -+ for_each_online_cpu(cpu) { -+#ifdef CONFIG_INTERRUPT_OFF_HIST -+ per_cpu(hist_irqsoff_counting, -+ cpu) = 0; -+#endif -+#ifdef CONFIG_PREEMPT_OFF_HIST -+ per_cpu(hist_preemptoff_counting, -+ cpu) = 0; -+#endif -+#if defined(CONFIG_INTERRUPT_OFF_HIST) && defined(CONFIG_PREEMPT_OFF_HIST) -+ per_cpu(hist_preemptirqsoff_counting, -+ cpu) = 0; -+#endif -+ } -+ } -+ break; -+#endif -+#ifdef CONFIG_WAKEUP_LATENCY_HIST -+ case WAKEUP_LATENCY: -+ { -+ int cpu; -+ -+ unregister_trace_sched_wakeup( -+ probe_wakeup_latency_hist_start, NULL); -+ unregister_trace_sched_wakeup_new( -+ probe_wakeup_latency_hist_start, NULL); -+ unregister_trace_sched_switch( -+ probe_wakeup_latency_hist_stop, NULL); -+ unregister_trace_sched_migrate_task( -+ probe_sched_migrate_task, NULL); -+ -+ for_each_online_cpu(cpu) { -+ per_cpu(wakeup_task, cpu) = NULL; -+ per_cpu(wakeup_sharedprio, cpu) = 0; -+ } -+ } -+#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST -+ timerandwakeup_enabled_data.enabled = 0; -+#endif -+ break; -+#endif -+#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST -+ case MISSED_TIMER_OFFSETS: -+ unregister_trace_hrtimer_interrupt( -+ probe_hrtimer_interrupt, NULL); -+#ifdef CONFIG_WAKEUP_LATENCY_HIST -+ timerandwakeup_enabled_data.enabled = 0; -+#endif -+ break; -+#endif -+ default: -+ break; -+ } -+ } -+ ed->enabled = enable; -+ return cnt; -+} -+ -+static const struct file_operations latency_hist_reset_fops = { -+ .open = tracing_open_generic, -+ .write = latency_hist_reset, -+}; -+ -+static const struct file_operations enable_fops = { -+ .open = tracing_open_generic, -+ .read = show_enable, -+ .write = do_enable, -+}; -+ -+#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \ -+ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) -+static const struct file_operations pid_fops = { -+ .open = tracing_open_generic, -+ .read = show_pid, -+ .write = do_pid, -+}; -+ -+static const struct file_operations maxlatproc_fops = { -+ .open = tracing_open_generic, -+ .read = show_maxlatproc, -+}; -+#endif -+ -+#if defined(CONFIG_INTERRUPT_OFF_HIST) || defined(CONFIG_PREEMPT_OFF_HIST) -+static notrace void probe_preemptirqsoff_hist(void *v, int reason, -+ int starthist) -+{ -+ int cpu = raw_smp_processor_id(); -+ int time_set = 0; -+ -+ if (starthist) { -+ cycle_t uninitialized_var(start); -+ -+ if (!preempt_count() && !irqs_disabled()) -+ return; -+ -+#ifdef CONFIG_INTERRUPT_OFF_HIST -+ if ((reason == IRQS_OFF || reason == TRACE_START) && -+ !per_cpu(hist_irqsoff_counting, cpu)) { -+ per_cpu(hist_irqsoff_counting, cpu) = 1; -+ start = ftrace_now(cpu); -+ time_set++; -+ per_cpu(hist_irqsoff_start, cpu) = start; -+ } -+#endif -+ -+#ifdef CONFIG_PREEMPT_OFF_HIST -+ if ((reason == PREEMPT_OFF || reason == TRACE_START) && -+ !per_cpu(hist_preemptoff_counting, cpu)) { -+ per_cpu(hist_preemptoff_counting, cpu) = 1; -+ if (!(time_set++)) -+ start = ftrace_now(cpu); -+ per_cpu(hist_preemptoff_start, cpu) = start; -+ } -+#endif -+ -+#if defined(CONFIG_INTERRUPT_OFF_HIST) && defined(CONFIG_PREEMPT_OFF_HIST) -+ if (per_cpu(hist_irqsoff_counting, cpu) && -+ per_cpu(hist_preemptoff_counting, cpu) && -+ !per_cpu(hist_preemptirqsoff_counting, cpu)) { -+ per_cpu(hist_preemptirqsoff_counting, cpu) = 1; -+ if (!time_set) -+ start = ftrace_now(cpu); -+ per_cpu(hist_preemptirqsoff_start, cpu) = start; -+ } -+#endif -+ } else { -+ cycle_t uninitialized_var(stop); -+ -+#ifdef CONFIG_INTERRUPT_OFF_HIST -+ if ((reason == IRQS_ON || reason == TRACE_STOP) && -+ per_cpu(hist_irqsoff_counting, cpu)) { -+ cycle_t start = per_cpu(hist_irqsoff_start, cpu); -+ -+ stop = ftrace_now(cpu); -+ time_set++; -+ if (start) { -+ long latency = ((long) (stop - start)) / -+ NSECS_PER_USECS; -+ -+ latency_hist(IRQSOFF_LATENCY, cpu, latency, 0, -+ stop, NULL); -+ } -+ per_cpu(hist_irqsoff_counting, cpu) = 0; -+ } -+#endif -+ -+#ifdef CONFIG_PREEMPT_OFF_HIST -+ if ((reason == PREEMPT_ON || reason == TRACE_STOP) && -+ per_cpu(hist_preemptoff_counting, cpu)) { -+ cycle_t start = per_cpu(hist_preemptoff_start, cpu); -+ -+ if (!(time_set++)) -+ stop = ftrace_now(cpu); -+ if (start) { -+ long latency = ((long) (stop - start)) / -+ NSECS_PER_USECS; -+ -+ latency_hist(PREEMPTOFF_LATENCY, cpu, latency, -+ 0, stop, NULL); -+ } -+ per_cpu(hist_preemptoff_counting, cpu) = 0; -+ } -+#endif -+ -+#if defined(CONFIG_INTERRUPT_OFF_HIST) && defined(CONFIG_PREEMPT_OFF_HIST) -+ if ((!per_cpu(hist_irqsoff_counting, cpu) || -+ !per_cpu(hist_preemptoff_counting, cpu)) && -+ per_cpu(hist_preemptirqsoff_counting, cpu)) { -+ cycle_t start = per_cpu(hist_preemptirqsoff_start, cpu); -+ -+ if (!time_set) -+ stop = ftrace_now(cpu); -+ if (start) { -+ long latency = ((long) (stop - start)) / -+ NSECS_PER_USECS; -+ -+ latency_hist(PREEMPTIRQSOFF_LATENCY, cpu, -+ latency, 0, stop, NULL); -+ } -+ per_cpu(hist_preemptirqsoff_counting, cpu) = 0; -+ } -+#endif -+ } -+} -+#endif -+ -+#ifdef CONFIG_WAKEUP_LATENCY_HIST -+static DEFINE_RAW_SPINLOCK(wakeup_lock); -+static notrace void probe_sched_migrate_task(void *v, struct task_struct *task, -+ int cpu) -+{ -+ int old_cpu = task_cpu(task); -+ -+ if (cpu != old_cpu) { -+ unsigned long flags; -+ struct task_struct *cpu_wakeup_task; -+ -+ raw_spin_lock_irqsave(&wakeup_lock, flags); -+ -+ cpu_wakeup_task = per_cpu(wakeup_task, old_cpu); -+ if (task == cpu_wakeup_task) { -+ put_task_struct(cpu_wakeup_task); -+ per_cpu(wakeup_task, old_cpu) = NULL; -+ cpu_wakeup_task = per_cpu(wakeup_task, cpu) = task; -+ get_task_struct(cpu_wakeup_task); -+ } -+ -+ raw_spin_unlock_irqrestore(&wakeup_lock, flags); -+ } -+} -+ -+static notrace void probe_wakeup_latency_hist_start(void *v, -+ struct task_struct *p, int success) -+{ -+ unsigned long flags; -+ struct task_struct *curr = current; -+ int cpu = task_cpu(p); -+ struct task_struct *cpu_wakeup_task; -+ -+ raw_spin_lock_irqsave(&wakeup_lock, flags); -+ -+ cpu_wakeup_task = per_cpu(wakeup_task, cpu); -+ -+ if (wakeup_pid) { -+ if ((cpu_wakeup_task && p->prio == cpu_wakeup_task->prio) || -+ p->prio == curr->prio) -+ per_cpu(wakeup_sharedprio, cpu) = 1; -+ if (likely(wakeup_pid != task_pid_nr(p))) -+ goto out; -+ } else { -+ if (likely(!rt_task(p)) || -+ (cpu_wakeup_task && p->prio > cpu_wakeup_task->prio) || -+ p->prio > curr->prio) -+ goto out; -+ if ((cpu_wakeup_task && p->prio == cpu_wakeup_task->prio) || -+ p->prio == curr->prio) -+ per_cpu(wakeup_sharedprio, cpu) = 1; -+ } -+ -+ if (cpu_wakeup_task) -+ put_task_struct(cpu_wakeup_task); -+ cpu_wakeup_task = per_cpu(wakeup_task, cpu) = p; -+ get_task_struct(cpu_wakeup_task); -+ cpu_wakeup_task->preempt_timestamp_hist = -+ ftrace_now(raw_smp_processor_id()); -+out: -+ raw_spin_unlock_irqrestore(&wakeup_lock, flags); -+} -+ -+static notrace void probe_wakeup_latency_hist_stop(void *v, -+ struct task_struct *prev, struct task_struct *next) -+{ -+ unsigned long flags; -+ int cpu = task_cpu(next); -+ long latency; -+ cycle_t stop; -+ struct task_struct *cpu_wakeup_task; -+ -+ raw_spin_lock_irqsave(&wakeup_lock, flags); -+ -+ cpu_wakeup_task = per_cpu(wakeup_task, cpu); -+ -+ if (cpu_wakeup_task == NULL) -+ goto out; -+ -+ /* Already running? */ -+ if (unlikely(current == cpu_wakeup_task)) -+ goto out_reset; -+ -+ if (next != cpu_wakeup_task) { -+ if (next->prio < cpu_wakeup_task->prio) -+ goto out_reset; -+ -+ if (next->prio == cpu_wakeup_task->prio) -+ per_cpu(wakeup_sharedprio, cpu) = 1; -+ -+ goto out; -+ } -+ -+ if (current->prio == cpu_wakeup_task->prio) -+ per_cpu(wakeup_sharedprio, cpu) = 1; -+ -+ /* -+ * The task we are waiting for is about to be switched to. -+ * Calculate latency and store it in histogram. -+ */ -+ stop = ftrace_now(raw_smp_processor_id()); -+ -+ latency = ((long) (stop - next->preempt_timestamp_hist)) / -+ NSECS_PER_USECS; -+ -+ if (per_cpu(wakeup_sharedprio, cpu)) { -+ latency_hist(WAKEUP_LATENCY_SHAREDPRIO, cpu, latency, 0, stop, -+ next); -+ per_cpu(wakeup_sharedprio, cpu) = 0; -+ } else { -+ latency_hist(WAKEUP_LATENCY, cpu, latency, 0, stop, next); -+#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST -+ if (timerandwakeup_enabled_data.enabled) { -+ latency_hist(TIMERANDWAKEUP_LATENCY, cpu, -+ next->timer_offset + latency, next->timer_offset, -+ stop, next); -+ } -+#endif -+ } -+ -+out_reset: -+#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST -+ next->timer_offset = 0; -+#endif -+ put_task_struct(cpu_wakeup_task); -+ per_cpu(wakeup_task, cpu) = NULL; -+out: -+ raw_spin_unlock_irqrestore(&wakeup_lock, flags); -+} -+#endif -+ -+#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST -+static notrace void probe_hrtimer_interrupt(void *v, int cpu, -+ long long latency_ns, struct task_struct *curr, -+ struct task_struct *task) -+{ -+ if (latency_ns <= 0 && task != NULL && rt_task(task) && -+ (task->prio < curr->prio || -+ (task->prio == curr->prio && -+ !cpumask_test_cpu(cpu, &task->cpus_allowed)))) { -+ long latency; -+ cycle_t now; -+ -+ if (missed_timer_offsets_pid) { -+ if (likely(missed_timer_offsets_pid != -+ task_pid_nr(task))) -+ return; -+ } -+ -+ now = ftrace_now(cpu); -+ latency = (long) div_s64(-latency_ns, NSECS_PER_USECS); -+ latency_hist(MISSED_TIMER_OFFSETS, cpu, latency, latency, now, -+ task); -+#ifdef CONFIG_WAKEUP_LATENCY_HIST -+ task->timer_offset = latency; -+#endif -+ } -+} -+#endif -+ -+static __init int latency_hist_init(void) -+{ -+ struct dentry *latency_hist_root = NULL; -+ struct dentry *dentry; -+#ifdef CONFIG_WAKEUP_LATENCY_HIST -+ struct dentry *dentry_sharedprio; -+#endif -+ struct dentry *entry; -+ struct dentry *enable_root; -+ int i = 0; -+ struct hist_data *my_hist; -+ char name[64]; -+ char *cpufmt = "CPU%d"; -+#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \ -+ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) -+ char *cpufmt_maxlatproc = "max_latency-CPU%d"; -+ struct maxlatproc_data *mp = NULL; -+#endif -+ -+ dentry = tracing_init_dentry(); -+ latency_hist_root = debugfs_create_dir(latency_hist_dir_root, dentry); -+ enable_root = debugfs_create_dir("enable", latency_hist_root); -+ -+#ifdef CONFIG_INTERRUPT_OFF_HIST -+ dentry = debugfs_create_dir(irqsoff_hist_dir, latency_hist_root); -+ for_each_possible_cpu(i) { -+ sprintf(name, cpufmt, i); -+ entry = debugfs_create_file(name, 0444, dentry, -+ &per_cpu(irqsoff_hist, i), &latency_hist_fops); -+ my_hist = &per_cpu(irqsoff_hist, i); -+ atomic_set(&my_hist->hist_mode, 1); -+ my_hist->min_lat = LONG_MAX; -+ } -+ entry = debugfs_create_file("reset", 0644, dentry, -+ (void *)IRQSOFF_LATENCY, &latency_hist_reset_fops); -+#endif -+ -+#ifdef CONFIG_PREEMPT_OFF_HIST -+ dentry = debugfs_create_dir(preemptoff_hist_dir, -+ latency_hist_root); -+ for_each_possible_cpu(i) { -+ sprintf(name, cpufmt, i); -+ entry = debugfs_create_file(name, 0444, dentry, -+ &per_cpu(preemptoff_hist, i), &latency_hist_fops); -+ my_hist = &per_cpu(preemptoff_hist, i); -+ atomic_set(&my_hist->hist_mode, 1); -+ my_hist->min_lat = LONG_MAX; -+ } -+ entry = debugfs_create_file("reset", 0644, dentry, -+ (void *)PREEMPTOFF_LATENCY, &latency_hist_reset_fops); -+#endif -+ -+#if defined(CONFIG_INTERRUPT_OFF_HIST) && defined(CONFIG_PREEMPT_OFF_HIST) -+ dentry = debugfs_create_dir(preemptirqsoff_hist_dir, -+ latency_hist_root); -+ for_each_possible_cpu(i) { -+ sprintf(name, cpufmt, i); -+ entry = debugfs_create_file(name, 0444, dentry, -+ &per_cpu(preemptirqsoff_hist, i), &latency_hist_fops); -+ my_hist = &per_cpu(preemptirqsoff_hist, i); -+ atomic_set(&my_hist->hist_mode, 1); -+ my_hist->min_lat = LONG_MAX; -+ } -+ entry = debugfs_create_file("reset", 0644, dentry, -+ (void *)PREEMPTIRQSOFF_LATENCY, &latency_hist_reset_fops); -+#endif -+ -+#if defined(CONFIG_INTERRUPT_OFF_HIST) || defined(CONFIG_PREEMPT_OFF_HIST) -+ entry = debugfs_create_file("preemptirqsoff", 0644, -+ enable_root, (void *)&preemptirqsoff_enabled_data, -+ &enable_fops); -+#endif -+ -+#ifdef CONFIG_WAKEUP_LATENCY_HIST -+ dentry = debugfs_create_dir(wakeup_latency_hist_dir, -+ latency_hist_root); -+ dentry_sharedprio = debugfs_create_dir( -+ wakeup_latency_hist_dir_sharedprio, dentry); -+ for_each_possible_cpu(i) { -+ sprintf(name, cpufmt, i); -+ -+ entry = debugfs_create_file(name, 0444, dentry, -+ &per_cpu(wakeup_latency_hist, i), -+ &latency_hist_fops); -+ my_hist = &per_cpu(wakeup_latency_hist, i); -+ atomic_set(&my_hist->hist_mode, 1); -+ my_hist->min_lat = LONG_MAX; -+ -+ entry = debugfs_create_file(name, 0444, dentry_sharedprio, -+ &per_cpu(wakeup_latency_hist_sharedprio, i), -+ &latency_hist_fops); -+ my_hist = &per_cpu(wakeup_latency_hist_sharedprio, i); -+ atomic_set(&my_hist->hist_mode, 1); -+ my_hist->min_lat = LONG_MAX; -+ -+ sprintf(name, cpufmt_maxlatproc, i); -+ -+ mp = &per_cpu(wakeup_maxlatproc, i); -+ entry = debugfs_create_file(name, 0444, dentry, mp, -+ &maxlatproc_fops); -+ clear_maxlatprocdata(mp); -+ -+ mp = &per_cpu(wakeup_maxlatproc_sharedprio, i); -+ entry = debugfs_create_file(name, 0444, dentry_sharedprio, mp, -+ &maxlatproc_fops); -+ clear_maxlatprocdata(mp); -+ } -+ entry = debugfs_create_file("pid", 0644, dentry, -+ (void *)&wakeup_pid, &pid_fops); -+ entry = debugfs_create_file("reset", 0644, dentry, -+ (void *)WAKEUP_LATENCY, &latency_hist_reset_fops); -+ entry = debugfs_create_file("reset", 0644, dentry_sharedprio, -+ (void *)WAKEUP_LATENCY_SHAREDPRIO, &latency_hist_reset_fops); -+ entry = debugfs_create_file("wakeup", 0644, -+ enable_root, (void *)&wakeup_latency_enabled_data, -+ &enable_fops); -+#endif -+ -+#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST -+ dentry = debugfs_create_dir(missed_timer_offsets_dir, -+ latency_hist_root); -+ for_each_possible_cpu(i) { -+ sprintf(name, cpufmt, i); -+ entry = debugfs_create_file(name, 0444, dentry, -+ &per_cpu(missed_timer_offsets, i), &latency_hist_fops); -+ my_hist = &per_cpu(missed_timer_offsets, i); -+ atomic_set(&my_hist->hist_mode, 1); -+ my_hist->min_lat = LONG_MAX; -+ -+ sprintf(name, cpufmt_maxlatproc, i); -+ mp = &per_cpu(missed_timer_offsets_maxlatproc, i); -+ entry = debugfs_create_file(name, 0444, dentry, mp, -+ &maxlatproc_fops); -+ clear_maxlatprocdata(mp); -+ } -+ entry = debugfs_create_file("pid", 0644, dentry, -+ (void *)&missed_timer_offsets_pid, &pid_fops); -+ entry = debugfs_create_file("reset", 0644, dentry, -+ (void *)MISSED_TIMER_OFFSETS, &latency_hist_reset_fops); -+ entry = debugfs_create_file("missed_timer_offsets", 0644, -+ enable_root, (void *)&missed_timer_offsets_enabled_data, -+ &enable_fops); -+#endif -+ -+#if defined(CONFIG_WAKEUP_LATENCY_HIST) && \ -+ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) -+ dentry = debugfs_create_dir(timerandwakeup_latency_hist_dir, -+ latency_hist_root); -+ for_each_possible_cpu(i) { -+ sprintf(name, cpufmt, i); -+ entry = debugfs_create_file(name, 0444, dentry, -+ &per_cpu(timerandwakeup_latency_hist, i), -+ &latency_hist_fops); -+ my_hist = &per_cpu(timerandwakeup_latency_hist, i); -+ atomic_set(&my_hist->hist_mode, 1); -+ my_hist->min_lat = LONG_MAX; -+ -+ sprintf(name, cpufmt_maxlatproc, i); -+ mp = &per_cpu(timerandwakeup_maxlatproc, i); -+ entry = debugfs_create_file(name, 0444, dentry, mp, -+ &maxlatproc_fops); -+ clear_maxlatprocdata(mp); -+ } -+ entry = debugfs_create_file("reset", 0644, dentry, -+ (void *)TIMERANDWAKEUP_LATENCY, &latency_hist_reset_fops); -+ entry = debugfs_create_file("timerandwakeup", 0644, -+ enable_root, (void *)&timerandwakeup_enabled_data, -+ &enable_fops); -+#endif -+ return 0; -+} -+ -+device_initcall(latency_hist_init); -diff -Nur linux-4.1.10.orig/kernel/trace/Makefile linux-4.1.10/kernel/trace/Makefile ---- linux-4.1.10.orig/kernel/trace/Makefile 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/trace/Makefile 2015-10-07 18:00:08.000000000 +0200 -@@ -36,6 +36,10 @@ - obj-$(CONFIG_IRQSOFF_TRACER) += trace_irqsoff.o - obj-$(CONFIG_PREEMPT_TRACER) += trace_irqsoff.o - obj-$(CONFIG_SCHED_TRACER) += trace_sched_wakeup.o -+obj-$(CONFIG_INTERRUPT_OFF_HIST) += latency_hist.o -+obj-$(CONFIG_PREEMPT_OFF_HIST) += latency_hist.o -+obj-$(CONFIG_WAKEUP_LATENCY_HIST) += latency_hist.o -+obj-$(CONFIG_MISSED_TIMER_OFFSETS_HIST) += latency_hist.o - obj-$(CONFIG_NOP_TRACER) += trace_nop.o - obj-$(CONFIG_STACK_TRACER) += trace_stack.o - obj-$(CONFIG_MMIOTRACE) += trace_mmiotrace.o -diff -Nur linux-4.1.10.orig/kernel/trace/trace.c linux-4.1.10/kernel/trace/trace.c ---- linux-4.1.10.orig/kernel/trace/trace.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/trace/trace.c 2015-10-07 18:00:08.000000000 +0200 -@@ -1630,6 +1630,7 @@ - struct task_struct *tsk = current; - - entry->preempt_count = pc & 0xff; -+ entry->preempt_lazy_count = preempt_lazy_count(); - entry->pid = (tsk) ? tsk->pid : 0; - entry->flags = - #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT -@@ -1639,8 +1640,11 @@ - #endif - ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) | - ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) | -- (tif_need_resched() ? TRACE_FLAG_NEED_RESCHED : 0) | -+ (tif_need_resched_now() ? TRACE_FLAG_NEED_RESCHED : 0) | -+ (need_resched_lazy() ? TRACE_FLAG_NEED_RESCHED_LAZY : 0) | - (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED : 0); -+ -+ entry->migrate_disable = (tsk) ? __migrate_disabled(tsk) & 0xFF : 0; - } - EXPORT_SYMBOL_GPL(tracing_generic_entry_update); - -@@ -2558,14 +2562,17 @@ - - static void print_lat_help_header(struct seq_file *m) - { -- seq_puts(m, "# _------=> CPU# \n" -- "# / _-----=> irqs-off \n" -- "# | / _----=> need-resched \n" -- "# || / _---=> hardirq/softirq \n" -- "# ||| / _--=> preempt-depth \n" -- "# |||| / delay \n" -- "# cmd pid ||||| time | caller \n" -- "# \\ / ||||| \\ | / \n"); -+ seq_puts(m, "# _--------=> CPU# \n" -+ "# / _-------=> irqs-off \n" -+ "# | / _------=> need-resched \n" -+ "# || / _-----=> need-resched_lazy \n" -+ "# ||| / _----=> hardirq/softirq \n" -+ "# |||| / _---=> preempt-depth \n" -+ "# ||||| / _--=> preempt-lazy-depth\n" -+ "# |||||| / _-=> migrate-disable \n" -+ "# ||||||| / delay \n" -+ "# cmd pid |||||||| time | caller \n" -+ "# \\ / |||||||| \\ | / \n"); - } - - static void print_event_info(struct trace_buffer *buf, struct seq_file *m) -@@ -2591,11 +2598,14 @@ - print_event_info(buf, m); - seq_puts(m, "# _-----=> irqs-off\n" - "# / _----=> need-resched\n" -- "# | / _---=> hardirq/softirq\n" -- "# || / _--=> preempt-depth\n" -- "# ||| / delay\n" -- "# TASK-PID CPU# |||| TIMESTAMP FUNCTION\n" -- "# | | | |||| | |\n"); -+ "# |/ _-----=> need-resched_lazy\n" -+ "# || / _---=> hardirq/softirq\n" -+ "# ||| / _--=> preempt-depth\n" -+ "# |||| /_--=> preempt-lazy-depth\n" -+ "# ||||| _-=> migrate-disable \n" -+ "# ||||| / delay\n" -+ "# TASK-PID CPU# |||||| TIMESTAMP FUNCTION\n" -+ "# | | | |||||| | |\n"); - } - - void -diff -Nur linux-4.1.10.orig/kernel/trace/trace_events.c linux-4.1.10/kernel/trace/trace_events.c ---- linux-4.1.10.orig/kernel/trace/trace_events.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/trace/trace_events.c 2015-10-07 18:00:08.000000000 +0200 -@@ -162,6 +162,8 @@ - __common_field(unsigned char, flags); - __common_field(unsigned char, preempt_count); - __common_field(int, pid); -+ __common_field(unsigned short, migrate_disable); -+ __common_field(unsigned short, padding); - - return ret; - } -diff -Nur linux-4.1.10.orig/kernel/trace/trace.h linux-4.1.10/kernel/trace/trace.h ---- linux-4.1.10.orig/kernel/trace/trace.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/trace/trace.h 2015-10-07 18:00:08.000000000 +0200 -@@ -120,6 +120,7 @@ - * NEED_RESCHED - reschedule is requested - * HARDIRQ - inside an interrupt handler - * SOFTIRQ - inside a softirq handler -+ * NEED_RESCHED_LAZY - lazy reschedule is requested - */ - enum trace_flag_type { - TRACE_FLAG_IRQS_OFF = 0x01, -@@ -128,6 +129,7 @@ - TRACE_FLAG_HARDIRQ = 0x08, - TRACE_FLAG_SOFTIRQ = 0x10, - TRACE_FLAG_PREEMPT_RESCHED = 0x20, -+ TRACE_FLAG_NEED_RESCHED_LAZY = 0x40, - }; - - #define TRACE_BUF_SIZE 1024 -diff -Nur linux-4.1.10.orig/kernel/trace/trace_irqsoff.c linux-4.1.10/kernel/trace/trace_irqsoff.c ---- linux-4.1.10.orig/kernel/trace/trace_irqsoff.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/trace/trace_irqsoff.c 2015-10-07 18:00:08.000000000 +0200 -@@ -13,6 +13,7 @@ - #include - #include - #include -+#include - - #include "trace.h" - -@@ -433,11 +434,13 @@ - { - if (preempt_trace() || irq_trace()) - start_critical_timing(CALLER_ADDR0, CALLER_ADDR1); -+ trace_preemptirqsoff_hist(TRACE_START, 1); - } - EXPORT_SYMBOL_GPL(start_critical_timings); - - void stop_critical_timings(void) - { -+ trace_preemptirqsoff_hist(TRACE_STOP, 0); - if (preempt_trace() || irq_trace()) - stop_critical_timing(CALLER_ADDR0, CALLER_ADDR1); - } -@@ -447,6 +450,7 @@ - #ifdef CONFIG_PROVE_LOCKING - void time_hardirqs_on(unsigned long a0, unsigned long a1) - { -+ trace_preemptirqsoff_hist_rcuidle(IRQS_ON, 0); - if (!preempt_trace() && irq_trace()) - stop_critical_timing(a0, a1); - } -@@ -455,6 +459,7 @@ - { - if (!preempt_trace() && irq_trace()) - start_critical_timing(a0, a1); -+ trace_preemptirqsoff_hist_rcuidle(IRQS_OFF, 1); - } - - #else /* !CONFIG_PROVE_LOCKING */ -@@ -480,6 +485,7 @@ - */ - void trace_hardirqs_on(void) - { -+ trace_preemptirqsoff_hist(IRQS_ON, 0); - if (!preempt_trace() && irq_trace()) - stop_critical_timing(CALLER_ADDR0, CALLER_ADDR1); - } -@@ -489,11 +495,13 @@ - { - if (!preempt_trace() && irq_trace()) - start_critical_timing(CALLER_ADDR0, CALLER_ADDR1); -+ trace_preemptirqsoff_hist(IRQS_OFF, 1); - } - EXPORT_SYMBOL(trace_hardirqs_off); - - __visible void trace_hardirqs_on_caller(unsigned long caller_addr) - { -+ trace_preemptirqsoff_hist(IRQS_ON, 0); - if (!preempt_trace() && irq_trace()) - stop_critical_timing(CALLER_ADDR0, caller_addr); - } -@@ -503,6 +511,7 @@ - { - if (!preempt_trace() && irq_trace()) - start_critical_timing(CALLER_ADDR0, caller_addr); -+ trace_preemptirqsoff_hist(IRQS_OFF, 1); - } - EXPORT_SYMBOL(trace_hardirqs_off_caller); - -@@ -512,12 +521,14 @@ - #ifdef CONFIG_PREEMPT_TRACER - void trace_preempt_on(unsigned long a0, unsigned long a1) - { -+ trace_preemptirqsoff_hist(PREEMPT_ON, 0); - if (preempt_trace() && !irq_trace()) - stop_critical_timing(a0, a1); - } - - void trace_preempt_off(unsigned long a0, unsigned long a1) - { -+ trace_preemptirqsoff_hist(PREEMPT_ON, 1); - if (preempt_trace() && !irq_trace()) - start_critical_timing(a0, a1); - } -diff -Nur linux-4.1.10.orig/kernel/trace/trace_output.c linux-4.1.10/kernel/trace/trace_output.c ---- linux-4.1.10.orig/kernel/trace/trace_output.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/trace/trace_output.c 2015-10-07 18:00:08.000000000 +0200 -@@ -430,6 +430,7 @@ - { - char hardsoft_irq; - char need_resched; -+ char need_resched_lazy; - char irqs_off; - int hardirq; - int softirq; -@@ -457,6 +458,8 @@ - need_resched = '.'; - break; - } -+ need_resched_lazy = -+ (entry->flags & TRACE_FLAG_NEED_RESCHED_LAZY) ? 'L' : '.'; - - hardsoft_irq = - (hardirq && softirq) ? 'H' : -@@ -464,14 +467,25 @@ - softirq ? 's' : - '.'; - -- trace_seq_printf(s, "%c%c%c", -- irqs_off, need_resched, hardsoft_irq); -+ trace_seq_printf(s, "%c%c%c%c", -+ irqs_off, need_resched, need_resched_lazy, -+ hardsoft_irq); - - if (entry->preempt_count) - trace_seq_printf(s, "%x", entry->preempt_count); - else - trace_seq_putc(s, '.'); - -+ if (entry->preempt_lazy_count) -+ trace_seq_printf(s, "%x", entry->preempt_lazy_count); -+ else -+ trace_seq_putc(s, '.'); -+ -+ if (entry->migrate_disable) -+ trace_seq_printf(s, "%x", entry->migrate_disable); -+ else -+ trace_seq_putc(s, '.'); -+ - return !trace_seq_has_overflowed(s); - } - -diff -Nur linux-4.1.10.orig/kernel/user.c linux-4.1.10/kernel/user.c ---- linux-4.1.10.orig/kernel/user.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/user.c 2015-10-07 18:00:08.000000000 +0200 -@@ -161,11 +161,11 @@ - if (!up) - return; - -- local_irq_save(flags); -+ local_irq_save_nort(flags); - if (atomic_dec_and_lock(&up->__count, &uidhash_lock)) - free_user(up, flags); - else -- local_irq_restore(flags); -+ local_irq_restore_nort(flags); - } - - struct user_struct *alloc_uid(kuid_t uid) -diff -Nur linux-4.1.10.orig/kernel/watchdog.c linux-4.1.10/kernel/watchdog.c ---- linux-4.1.10.orig/kernel/watchdog.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/watchdog.c 2015-10-07 18:00:08.000000000 +0200 -@@ -262,6 +262,8 @@ - - #ifdef CONFIG_HARDLOCKUP_DETECTOR - -+static DEFINE_RAW_SPINLOCK(watchdog_output_lock); -+ - static struct perf_event_attr wd_hw_attr = { - .type = PERF_TYPE_HARDWARE, - .config = PERF_COUNT_HW_CPU_CYCLES, -@@ -295,13 +297,21 @@ - /* only print hardlockups once */ - if (__this_cpu_read(hard_watchdog_warn) == true) - return; -+ /* -+ * If early-printk is enabled then make sure we do not -+ * lock up in printk() and kill console logging: -+ */ -+ printk_kill(); - -- if (hardlockup_panic) -+ if (hardlockup_panic) { - panic("Watchdog detected hard LOCKUP on cpu %d", - this_cpu); -- else -+ } else { -+ raw_spin_lock(&watchdog_output_lock); - WARN(1, "Watchdog detected hard LOCKUP on cpu %d", - this_cpu); -+ raw_spin_unlock(&watchdog_output_lock); -+ } - - __this_cpu_write(hard_watchdog_warn, true); - return; -@@ -444,6 +454,7 @@ - /* kick off the timer for the hardlockup detector */ - hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); - hrtimer->function = watchdog_timer_fn; -+ hrtimer->irqsafe = 1; - - /* Enable the perf event */ - watchdog_nmi_enable(cpu); -diff -Nur linux-4.1.10.orig/kernel/workqueue.c linux-4.1.10/kernel/workqueue.c ---- linux-4.1.10.orig/kernel/workqueue.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/workqueue.c 2015-10-07 18:00:08.000000000 +0200 -@@ -48,6 +48,8 @@ - #include - #include - #include -+#include -+#include - - #include "workqueue_internal.h" - -@@ -121,15 +123,20 @@ - * cpu or grabbing pool->lock is enough for read access. If - * POOL_DISASSOCIATED is set, it's identical to L. - * -+ * On RT we need the extra protection via rt_lock_idle_list() for -+ * the list manipulations against read access from -+ * wq_worker_sleeping(). All other places are nicely serialized via -+ * pool->lock. -+ * - * A: pool->attach_mutex protected. - * - * PL: wq_pool_mutex protected. - * -- * PR: wq_pool_mutex protected for writes. Sched-RCU protected for reads. -+ * PR: wq_pool_mutex protected for writes. RCU protected for reads. - * - * WQ: wq->mutex protected. - * -- * WR: wq->mutex protected for writes. Sched-RCU protected for reads. -+ * WR: wq->mutex protected for writes. RCU protected for reads. - * - * MD: wq_mayday_lock protected. - */ -@@ -178,7 +185,7 @@ - atomic_t nr_running ____cacheline_aligned_in_smp; - - /* -- * Destruction of pool is sched-RCU protected to allow dereferences -+ * Destruction of pool is RCU protected to allow dereferences - * from get_work_pool(). - */ - struct rcu_head rcu; -@@ -207,7 +214,7 @@ - /* - * Release of unbound pwq is punted to system_wq. See put_pwq() - * and pwq_unbound_release_workfn() for details. pool_workqueue -- * itself is also sched-RCU protected so that the first pwq can be -+ * itself is also RCU protected so that the first pwq can be - * determined without grabbing wq->mutex. - */ - struct work_struct unbound_release_work; -@@ -329,6 +336,8 @@ - struct workqueue_struct *system_freezable_power_efficient_wq __read_mostly; - EXPORT_SYMBOL_GPL(system_freezable_power_efficient_wq); - -+static DEFINE_LOCAL_IRQ_LOCK(pendingb_lock); -+ - static int worker_thread(void *__worker); - static void copy_workqueue_attrs(struct workqueue_attrs *to, - const struct workqueue_attrs *from); -@@ -338,14 +347,14 @@ - #include - - #define assert_rcu_or_pool_mutex() \ -- rcu_lockdep_assert(rcu_read_lock_sched_held() || \ -+ rcu_lockdep_assert(rcu_read_lock_held() || \ - lockdep_is_held(&wq_pool_mutex), \ -- "sched RCU or wq_pool_mutex should be held") -+ "RCU or wq_pool_mutex should be held") - - #define assert_rcu_or_wq_mutex(wq) \ -- rcu_lockdep_assert(rcu_read_lock_sched_held() || \ -+ rcu_lockdep_assert(rcu_read_lock_held() || \ - lockdep_is_held(&wq->mutex), \ -- "sched RCU or wq->mutex should be held") -+ "RCU or wq->mutex should be held") - - #define for_each_cpu_worker_pool(pool, cpu) \ - for ((pool) = &per_cpu(cpu_worker_pools, cpu)[0]; \ -@@ -357,7 +366,7 @@ - * @pool: iteration cursor - * @pi: integer used for iteration - * -- * This must be called either with wq_pool_mutex held or sched RCU read -+ * This must be called either with wq_pool_mutex held or RCU read - * locked. If the pool needs to be used beyond the locking in effect, the - * caller is responsible for guaranteeing that the pool stays online. - * -@@ -389,7 +398,7 @@ - * @pwq: iteration cursor - * @wq: the target workqueue - * -- * This must be called either with wq->mutex held or sched RCU read locked. -+ * This must be called either with wq->mutex held or RCU read locked. - * If the pwq needs to be used beyond the locking in effect, the caller is - * responsible for guaranteeing that the pwq stays online. - * -@@ -401,6 +410,31 @@ - if (({ assert_rcu_or_wq_mutex(wq); false; })) { } \ - else - -+#ifdef CONFIG_PREEMPT_RT_BASE -+static inline void rt_lock_idle_list(struct worker_pool *pool) -+{ -+ preempt_disable(); -+} -+static inline void rt_unlock_idle_list(struct worker_pool *pool) -+{ -+ preempt_enable(); -+} -+static inline void sched_lock_idle_list(struct worker_pool *pool) { } -+static inline void sched_unlock_idle_list(struct worker_pool *pool) { } -+#else -+static inline void rt_lock_idle_list(struct worker_pool *pool) { } -+static inline void rt_unlock_idle_list(struct worker_pool *pool) { } -+static inline void sched_lock_idle_list(struct worker_pool *pool) -+{ -+ spin_lock_irq(&pool->lock); -+} -+static inline void sched_unlock_idle_list(struct worker_pool *pool) -+{ -+ spin_unlock_irq(&pool->lock); -+} -+#endif -+ -+ - #ifdef CONFIG_DEBUG_OBJECTS_WORK - - static struct debug_obj_descr work_debug_descr; -@@ -551,7 +585,7 @@ - * @wq: the target workqueue - * @node: the node ID - * -- * This must be called either with pwq_lock held or sched RCU read locked. -+ * This must be called either with pwq_lock held or RCU read locked. - * If the pwq needs to be used beyond the locking in effect, the caller is - * responsible for guaranteeing that the pwq stays online. - * -@@ -655,8 +689,8 @@ - * @work: the work item of interest - * - * Pools are created and destroyed under wq_pool_mutex, and allows read -- * access under sched-RCU read lock. As such, this function should be -- * called under wq_pool_mutex or with preemption disabled. -+ * access under RCU read lock. As such, this function should be -+ * called under wq_pool_mutex or inside of a rcu_read_lock() region. - * - * All fields of the returned pool are accessible as long as the above - * mentioned locking is in effect. If the returned pool needs to be used -@@ -793,51 +827,44 @@ - */ - static void wake_up_worker(struct worker_pool *pool) - { -- struct worker *worker = first_idle_worker(pool); -+ struct worker *worker; -+ -+ rt_lock_idle_list(pool); -+ -+ worker = first_idle_worker(pool); - - if (likely(worker)) - wake_up_process(worker->task); -+ -+ rt_unlock_idle_list(pool); - } - - /** -- * wq_worker_waking_up - a worker is waking up -- * @task: task waking up -- * @cpu: CPU @task is waking up to -+ * wq_worker_running - a worker is running again -+ * @task: task returning from sleep - * -- * This function is called during try_to_wake_up() when a worker is -- * being awoken. -- * -- * CONTEXT: -- * spin_lock_irq(rq->lock) -+ * This function is called when a worker returns from schedule() - */ --void wq_worker_waking_up(struct task_struct *task, int cpu) -+void wq_worker_running(struct task_struct *task) - { - struct worker *worker = kthread_data(task); - -- if (!(worker->flags & WORKER_NOT_RUNNING)) { -- WARN_ON_ONCE(worker->pool->cpu != cpu); -+ if (!worker->sleeping) -+ return; -+ if (!(worker->flags & WORKER_NOT_RUNNING)) - atomic_inc(&worker->pool->nr_running); -- } -+ worker->sleeping = 0; - } - - /** - * wq_worker_sleeping - a worker is going to sleep - * @task: task going to sleep -- * @cpu: CPU in question, must be the current CPU number -- * -- * This function is called during schedule() when a busy worker is -- * going to sleep. Worker on the same cpu can be woken up by -- * returning pointer to its task. -- * -- * CONTEXT: -- * spin_lock_irq(rq->lock) -- * -- * Return: -- * Worker task on @cpu to wake up, %NULL if none. -+ * This function is called from schedule() when a busy worker is -+ * going to sleep. - */ --struct task_struct *wq_worker_sleeping(struct task_struct *task, int cpu) -+void wq_worker_sleeping(struct task_struct *task) - { -- struct worker *worker = kthread_data(task), *to_wakeup = NULL; -+ struct worker *worker = kthread_data(task); - struct worker_pool *pool; - - /* -@@ -846,29 +873,26 @@ - * checking NOT_RUNNING. - */ - if (worker->flags & WORKER_NOT_RUNNING) -- return NULL; -+ return; - - pool = worker->pool; - -- /* this can only happen on the local cpu */ -- if (WARN_ON_ONCE(cpu != raw_smp_processor_id() || pool->cpu != cpu)) -- return NULL; -+ if (WARN_ON_ONCE(worker->sleeping)) -+ return; -+ -+ worker->sleeping = 1; - - /* - * The counterpart of the following dec_and_test, implied mb, - * worklist not empty test sequence is in insert_work(). - * Please read comment there. -- * -- * NOT_RUNNING is clear. This means that we're bound to and -- * running on the local cpu w/ rq lock held and preemption -- * disabled, which in turn means that none else could be -- * manipulating idle_list, so dereferencing idle_list without pool -- * lock is safe. - */ - if (atomic_dec_and_test(&pool->nr_running) && -- !list_empty(&pool->worklist)) -- to_wakeup = first_idle_worker(pool); -- return to_wakeup ? to_wakeup->task : NULL; -+ !list_empty(&pool->worklist)) { -+ sched_lock_idle_list(pool); -+ wake_up_worker(pool); -+ sched_unlock_idle_list(pool); -+ } - } - - /** -@@ -1062,12 +1086,12 @@ - { - if (pwq) { - /* -- * As both pwqs and pools are sched-RCU protected, the -+ * As both pwqs and pools are RCU protected, the - * following lock operations are safe. - */ -- spin_lock_irq(&pwq->pool->lock); -+ local_spin_lock_irq(pendingb_lock, &pwq->pool->lock); - put_pwq(pwq); -- spin_unlock_irq(&pwq->pool->lock); -+ local_spin_unlock_irq(pendingb_lock, &pwq->pool->lock); - } - } - -@@ -1169,7 +1193,7 @@ - struct worker_pool *pool; - struct pool_workqueue *pwq; - -- local_irq_save(*flags); -+ local_lock_irqsave(pendingb_lock, *flags); - - /* try to steal the timer if it exists */ - if (is_dwork) { -@@ -1188,6 +1212,7 @@ - if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) - return 0; - -+ rcu_read_lock(); - /* - * The queueing is in progress, or it is already queued. Try to - * steal it from ->worklist without clearing WORK_STRUCT_PENDING. -@@ -1226,14 +1251,16 @@ - set_work_pool_and_keep_pending(work, pool->id); - - spin_unlock(&pool->lock); -+ rcu_read_unlock(); - return 1; - } - spin_unlock(&pool->lock); - fail: -- local_irq_restore(*flags); -+ rcu_read_unlock(); -+ local_unlock_irqrestore(pendingb_lock, *flags); - if (work_is_canceling(work)) - return -ENOENT; -- cpu_relax(); -+ cpu_chill(); - return -EAGAIN; - } - -@@ -1302,7 +1329,7 @@ - * queued or lose PENDING. Grabbing PENDING and queueing should - * happen with IRQ disabled. - */ -- WARN_ON_ONCE(!irqs_disabled()); -+ WARN_ON_ONCE_NONRT(!irqs_disabled()); - - debug_work_activate(work); - -@@ -1310,6 +1337,8 @@ - if (unlikely(wq->flags & __WQ_DRAINING) && - WARN_ON_ONCE(!is_chained_work(wq))) - return; -+ -+ rcu_read_lock(); - retry: - if (req_cpu == WORK_CPU_UNBOUND) - cpu = raw_smp_processor_id(); -@@ -1366,10 +1395,8 @@ - /* pwq determined, queue */ - trace_workqueue_queue_work(req_cpu, pwq, work); - -- if (WARN_ON(!list_empty(&work->entry))) { -- spin_unlock(&pwq->pool->lock); -- return; -- } -+ if (WARN_ON(!list_empty(&work->entry))) -+ goto out; - - pwq->nr_in_flight[pwq->work_color]++; - work_flags = work_color_to_flags(pwq->work_color); -@@ -1385,7 +1412,9 @@ - - insert_work(pwq, work, worklist, work_flags); - -+out: - spin_unlock(&pwq->pool->lock); -+ rcu_read_unlock(); - } - - /** -@@ -1405,14 +1434,14 @@ - bool ret = false; - unsigned long flags; - -- local_irq_save(flags); -+ local_lock_irqsave(pendingb_lock,flags); - - if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) { - __queue_work(cpu, wq, work); - ret = true; - } - -- local_irq_restore(flags); -+ local_unlock_irqrestore(pendingb_lock, flags); - return ret; - } - EXPORT_SYMBOL(queue_work_on); -@@ -1479,14 +1508,14 @@ - unsigned long flags; - - /* read the comment in __queue_work() */ -- local_irq_save(flags); -+ local_lock_irqsave(pendingb_lock, flags); - - if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) { - __queue_delayed_work(cpu, wq, dwork, delay); - ret = true; - } - -- local_irq_restore(flags); -+ local_unlock_irqrestore(pendingb_lock, flags); - return ret; - } - EXPORT_SYMBOL(queue_delayed_work_on); -@@ -1521,7 +1550,7 @@ - - if (likely(ret >= 0)) { - __queue_delayed_work(cpu, wq, dwork, delay); -- local_irq_restore(flags); -+ local_unlock_irqrestore(pendingb_lock, flags); - } - - /* -ENOENT from try_to_grab_pending() becomes %true */ -@@ -1554,7 +1583,9 @@ - worker->last_active = jiffies; - - /* idle_list is LIFO */ -+ rt_lock_idle_list(pool); - list_add(&worker->entry, &pool->idle_list); -+ rt_unlock_idle_list(pool); - - if (too_many_workers(pool) && !timer_pending(&pool->idle_timer)) - mod_timer(&pool->idle_timer, jiffies + IDLE_WORKER_TIMEOUT); -@@ -1587,7 +1618,9 @@ - return; - worker_clr_flags(worker, WORKER_IDLE); - pool->nr_idle--; -+ rt_lock_idle_list(pool); - list_del_init(&worker->entry); -+ rt_unlock_idle_list(pool); - } - - static struct worker *alloc_worker(int node) -@@ -1755,7 +1788,9 @@ - pool->nr_workers--; - pool->nr_idle--; - -+ rt_lock_idle_list(pool); - list_del_init(&worker->entry); -+ rt_unlock_idle_list(pool); - worker->flags |= WORKER_DIE; - wake_up_process(worker->task); - } -@@ -2672,14 +2707,14 @@ - - might_sleep(); - -- local_irq_disable(); -+ rcu_read_lock(); - pool = get_work_pool(work); - if (!pool) { -- local_irq_enable(); -+ rcu_read_unlock(); - return false; - } - -- spin_lock(&pool->lock); -+ spin_lock_irq(&pool->lock); - /* see the comment in try_to_grab_pending() with the same code */ - pwq = get_work_pwq(work); - if (pwq) { -@@ -2706,10 +2741,11 @@ - else - lock_map_acquire_read(&pwq->wq->lockdep_map); - lock_map_release(&pwq->wq->lockdep_map); -- -+ rcu_read_unlock(); - return true; - already_gone: - spin_unlock_irq(&pool->lock); -+ rcu_read_unlock(); - return false; - } - -@@ -2796,7 +2832,7 @@ - - /* tell other tasks trying to grab @work to back off */ - mark_work_canceling(work); -- local_irq_restore(flags); -+ local_unlock_irqrestore(pendingb_lock, flags); - - flush_work(work); - clear_work_data(work); -@@ -2851,10 +2887,10 @@ - */ - bool flush_delayed_work(struct delayed_work *dwork) - { -- local_irq_disable(); -+ local_lock_irq(pendingb_lock); - if (del_timer_sync(&dwork->timer)) - __queue_work(dwork->cpu, dwork->wq, &dwork->work); -- local_irq_enable(); -+ local_unlock_irq(pendingb_lock); - return flush_work(&dwork->work); - } - EXPORT_SYMBOL(flush_delayed_work); -@@ -2889,7 +2925,7 @@ - - set_work_pool_and_clear_pending(&dwork->work, - get_work_pool_id(&dwork->work)); -- local_irq_restore(flags); -+ local_unlock_irqrestore(pendingb_lock, flags); - return ret; - } - EXPORT_SYMBOL(cancel_delayed_work); -@@ -3147,7 +3183,7 @@ - * put_unbound_pool - put a worker_pool - * @pool: worker_pool to put - * -- * Put @pool. If its refcnt reaches zero, it gets destroyed in sched-RCU -+ * Put @pool. If its refcnt reaches zero, it gets destroyed in RCU - * safe manner. get_unbound_pool() calls this function on its failure path - * and this function should be able to release pools which went through, - * successfully or not, init_worker_pool(). -@@ -3201,8 +3237,8 @@ - del_timer_sync(&pool->idle_timer); - del_timer_sync(&pool->mayday_timer); - -- /* sched-RCU protected to allow dereferences from get_work_pool() */ -- call_rcu_sched(&pool->rcu, rcu_free_pool); -+ /* RCU protected to allow dereferences from get_work_pool() */ -+ call_rcu(&pool->rcu, rcu_free_pool); - } - - /** -@@ -3307,14 +3343,14 @@ - put_unbound_pool(pool); - mutex_unlock(&wq_pool_mutex); - -- call_rcu_sched(&pwq->rcu, rcu_free_pwq); -+ call_rcu(&pwq->rcu, rcu_free_pwq); - - /* - * If we're the last pwq going away, @wq is already dead and no one - * is gonna access it anymore. Schedule RCU free. - */ - if (is_last) -- call_rcu_sched(&wq->rcu, rcu_free_wq); -+ call_rcu(&wq->rcu, rcu_free_wq); - } - - /** -@@ -3920,7 +3956,7 @@ - * The base ref is never dropped on per-cpu pwqs. Directly - * schedule RCU free. - */ -- call_rcu_sched(&wq->rcu, rcu_free_wq); -+ call_rcu(&wq->rcu, rcu_free_wq); - } else { - /* - * We're the sole accessor of @wq at this point. Directly -@@ -4013,7 +4049,8 @@ - struct pool_workqueue *pwq; - bool ret; - -- rcu_read_lock_sched(); -+ rcu_read_lock(); -+ preempt_disable(); - - if (cpu == WORK_CPU_UNBOUND) - cpu = smp_processor_id(); -@@ -4024,7 +4061,8 @@ - pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu)); - - ret = !list_empty(&pwq->delayed_works); -- rcu_read_unlock_sched(); -+ preempt_enable(); -+ rcu_read_unlock(); - - return ret; - } -@@ -4050,15 +4088,15 @@ - if (work_pending(work)) - ret |= WORK_BUSY_PENDING; - -- local_irq_save(flags); -+ rcu_read_lock(); - pool = get_work_pool(work); - if (pool) { -- spin_lock(&pool->lock); -+ spin_lock_irqsave(&pool->lock, flags); - if (find_worker_executing_work(pool, work)) - ret |= WORK_BUSY_RUNNING; -- spin_unlock(&pool->lock); -+ spin_unlock_irqrestore(&pool->lock, flags); - } -- local_irq_restore(flags); -+ rcu_read_unlock(); - - return ret; - } -@@ -4247,7 +4285,7 @@ - unsigned long flags; - int pi; - -- rcu_read_lock_sched(); -+ rcu_read_lock(); - - pr_info("Showing busy workqueues and worker pools:\n"); - -@@ -4298,7 +4336,7 @@ - spin_unlock_irqrestore(&pool->lock, flags); - } - -- rcu_read_unlock_sched(); -+ rcu_read_unlock(); - } - - /* -@@ -4648,16 +4686,16 @@ - * nr_active is monotonically decreasing. It's safe - * to peek without lock. - */ -- rcu_read_lock_sched(); -+ rcu_read_lock(); - for_each_pwq(pwq, wq) { - WARN_ON_ONCE(pwq->nr_active < 0); - if (pwq->nr_active) { - busy = true; -- rcu_read_unlock_sched(); -+ rcu_read_unlock(); - goto out_unlock; - } - } -- rcu_read_unlock_sched(); -+ rcu_read_unlock(); - } - out_unlock: - mutex_unlock(&wq_pool_mutex); -@@ -4771,7 +4809,8 @@ - const char *delim = ""; - int node, written = 0; - -- rcu_read_lock_sched(); -+ get_online_cpus(); -+ rcu_read_lock(); - for_each_node(node) { - written += scnprintf(buf + written, PAGE_SIZE - written, - "%s%d:%d", delim, node, -@@ -4779,7 +4818,8 @@ - delim = " "; - } - written += scnprintf(buf + written, PAGE_SIZE - written, "\n"); -- rcu_read_unlock_sched(); -+ rcu_read_unlock(); -+ put_online_cpus(); - - return written; - } -diff -Nur linux-4.1.10.orig/kernel/workqueue_internal.h linux-4.1.10/kernel/workqueue_internal.h ---- linux-4.1.10.orig/kernel/workqueue_internal.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/kernel/workqueue_internal.h 2015-10-07 18:00:08.000000000 +0200 -@@ -43,6 +43,7 @@ - unsigned long last_active; /* L: last active timestamp */ - unsigned int flags; /* X: flags */ - int id; /* I: worker id */ -+ int sleeping; /* None */ - - /* - * Opaque string set with work_set_desc(). Printed out with task -@@ -68,7 +69,7 @@ - * Scheduler hooks for concurrency managed workqueue. Only to be used from - * sched/core.c and workqueue.c. - */ --void wq_worker_waking_up(struct task_struct *task, int cpu); --struct task_struct *wq_worker_sleeping(struct task_struct *task, int cpu); -+void wq_worker_running(struct task_struct *task); -+void wq_worker_sleeping(struct task_struct *task); - - #endif /* _KERNEL_WORKQUEUE_INTERNAL_H */ -diff -Nur linux-4.1.10.orig/lib/debugobjects.c linux-4.1.10/lib/debugobjects.c ---- linux-4.1.10.orig/lib/debugobjects.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/lib/debugobjects.c 2015-10-07 18:00:08.000000000 +0200 -@@ -309,7 +309,10 @@ - struct debug_obj *obj; - unsigned long flags; - -- fill_pool(); -+#ifdef CONFIG_PREEMPT_RT_FULL -+ if (preempt_count() == 0 && !irqs_disabled()) -+#endif -+ fill_pool(); - - db = get_bucket((unsigned long) addr); - -diff -Nur linux-4.1.10.orig/lib/dump_stack.c linux-4.1.10/lib/dump_stack.c ---- linux-4.1.10.orig/lib/dump_stack.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/lib/dump_stack.c 2015-10-07 18:00:08.000000000 +0200 -@@ -33,7 +33,7 @@ - * Permit this cpu to perform nested stack dumps while serialising - * against other CPUs - */ -- preempt_disable(); -+ migrate_disable(); - - retry: - cpu = smp_processor_id(); -@@ -52,7 +52,7 @@ - if (!was_locked) - atomic_set(&dump_lock, -1); - -- preempt_enable(); -+ migrate_enable(); - } - #else - asmlinkage __visible void dump_stack(void) -diff -Nur linux-4.1.10.orig/lib/idr.c linux-4.1.10/lib/idr.c ---- linux-4.1.10.orig/lib/idr.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/lib/idr.c 2015-10-07 18:00:08.000000000 +0200 -@@ -30,6 +30,7 @@ - #include - #include - #include -+#include - - #define MAX_IDR_SHIFT (sizeof(int) * 8 - 1) - #define MAX_IDR_BIT (1U << MAX_IDR_SHIFT) -@@ -366,6 +367,35 @@ - idr_mark_full(pa, id); - } - -+#ifdef CONFIG_PREEMPT_RT_FULL -+static DEFINE_LOCAL_IRQ_LOCK(idr_lock); -+ -+static inline void idr_preload_lock(void) -+{ -+ local_lock(idr_lock); -+} -+ -+static inline void idr_preload_unlock(void) -+{ -+ local_unlock(idr_lock); -+} -+ -+void idr_preload_end(void) -+{ -+ idr_preload_unlock(); -+} -+EXPORT_SYMBOL(idr_preload_end); -+#else -+static inline void idr_preload_lock(void) -+{ -+ preempt_disable(); -+} -+ -+static inline void idr_preload_unlock(void) -+{ -+ preempt_enable(); -+} -+#endif - - /** - * idr_preload - preload for idr_alloc() -@@ -401,7 +431,7 @@ - WARN_ON_ONCE(in_interrupt()); - might_sleep_if(gfp_mask & __GFP_WAIT); - -- preempt_disable(); -+ idr_preload_lock(); - - /* - * idr_alloc() is likely to succeed w/o full idr_layer buffer and -@@ -413,9 +443,9 @@ - while (__this_cpu_read(idr_preload_cnt) < MAX_IDR_FREE) { - struct idr_layer *new; - -- preempt_enable(); -+ idr_preload_unlock(); - new = kmem_cache_zalloc(idr_layer_cache, gfp_mask); -- preempt_disable(); -+ idr_preload_lock(); - if (!new) - break; - -diff -Nur linux-4.1.10.orig/lib/Kconfig linux-4.1.10/lib/Kconfig ---- linux-4.1.10.orig/lib/Kconfig 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/lib/Kconfig 2015-10-07 18:00:08.000000000 +0200 -@@ -391,6 +391,7 @@ - - config CPUMASK_OFFSTACK - bool "Force CPU masks off stack" if DEBUG_PER_CPU_MAPS -+ depends on !PREEMPT_RT_FULL - help - Use dynamic allocation for cpumask_var_t, instead of putting - them on the stack. This is a bit more expensive, but avoids -diff -Nur linux-4.1.10.orig/lib/locking-selftest.c linux-4.1.10/lib/locking-selftest.c ---- linux-4.1.10.orig/lib/locking-selftest.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/lib/locking-selftest.c 2015-10-07 18:00:08.000000000 +0200 -@@ -590,6 +590,8 @@ - #include "locking-selftest-spin-hardirq.h" - GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_hard_spin) - -+#ifndef CONFIG_PREEMPT_RT_FULL -+ - #include "locking-selftest-rlock-hardirq.h" - GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_hard_rlock) - -@@ -605,9 +607,12 @@ - #include "locking-selftest-wlock-softirq.h" - GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_soft_wlock) - -+#endif -+ - #undef E1 - #undef E2 - -+#ifndef CONFIG_PREEMPT_RT_FULL - /* - * Enabling hardirqs with a softirq-safe lock held: - */ -@@ -640,6 +645,8 @@ - #undef E1 - #undef E2 - -+#endif -+ - /* - * Enabling irqs with an irq-safe lock held: - */ -@@ -663,6 +670,8 @@ - #include "locking-selftest-spin-hardirq.h" - GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_hard_spin) - -+#ifndef CONFIG_PREEMPT_RT_FULL -+ - #include "locking-selftest-rlock-hardirq.h" - GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_hard_rlock) - -@@ -678,6 +687,8 @@ - #include "locking-selftest-wlock-softirq.h" - GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_soft_wlock) - -+#endif -+ - #undef E1 - #undef E2 - -@@ -709,6 +720,8 @@ - #include "locking-selftest-spin-hardirq.h" - GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_hard_spin) - -+#ifndef CONFIG_PREEMPT_RT_FULL -+ - #include "locking-selftest-rlock-hardirq.h" - GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_hard_rlock) - -@@ -724,6 +737,8 @@ - #include "locking-selftest-wlock-softirq.h" - GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_soft_wlock) - -+#endif -+ - #undef E1 - #undef E2 - #undef E3 -@@ -757,6 +772,8 @@ - #include "locking-selftest-spin-hardirq.h" - GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_hard_spin) - -+#ifndef CONFIG_PREEMPT_RT_FULL -+ - #include "locking-selftest-rlock-hardirq.h" - GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_hard_rlock) - -@@ -772,10 +789,14 @@ - #include "locking-selftest-wlock-softirq.h" - GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_soft_wlock) - -+#endif -+ - #undef E1 - #undef E2 - #undef E3 - -+#ifndef CONFIG_PREEMPT_RT_FULL -+ - /* - * read-lock / write-lock irq inversion. - * -@@ -838,6 +859,10 @@ - #undef E2 - #undef E3 - -+#endif -+ -+#ifndef CONFIG_PREEMPT_RT_FULL -+ - /* - * read-lock / write-lock recursion that is actually safe. - */ -@@ -876,6 +901,8 @@ - #undef E2 - #undef E3 - -+#endif -+ - /* - * read-lock / write-lock recursion that is unsafe. - */ -@@ -1858,6 +1885,7 @@ - - printk(" --------------------------------------------------------------------------\n"); - -+#ifndef CONFIG_PREEMPT_RT_FULL - /* - * irq-context testcases: - */ -@@ -1870,6 +1898,28 @@ - - DO_TESTCASE_6x2("irq read-recursion", irq_read_recursion); - // DO_TESTCASE_6x2B("irq read-recursion #2", irq_read_recursion2); -+#else -+ /* On -rt, we only do hardirq context test for raw spinlock */ -+ DO_TESTCASE_1B("hard-irqs-on + irq-safe-A", irqsafe1_hard_spin, 12); -+ DO_TESTCASE_1B("hard-irqs-on + irq-safe-A", irqsafe1_hard_spin, 21); -+ -+ DO_TESTCASE_1B("hard-safe-A + irqs-on", irqsafe2B_hard_spin, 12); -+ DO_TESTCASE_1B("hard-safe-A + irqs-on", irqsafe2B_hard_spin, 21); -+ -+ DO_TESTCASE_1B("hard-safe-A + unsafe-B #1", irqsafe3_hard_spin, 123); -+ DO_TESTCASE_1B("hard-safe-A + unsafe-B #1", irqsafe3_hard_spin, 132); -+ DO_TESTCASE_1B("hard-safe-A + unsafe-B #1", irqsafe3_hard_spin, 213); -+ DO_TESTCASE_1B("hard-safe-A + unsafe-B #1", irqsafe3_hard_spin, 231); -+ DO_TESTCASE_1B("hard-safe-A + unsafe-B #1", irqsafe3_hard_spin, 312); -+ DO_TESTCASE_1B("hard-safe-A + unsafe-B #1", irqsafe3_hard_spin, 321); -+ -+ DO_TESTCASE_1B("hard-safe-A + unsafe-B #2", irqsafe4_hard_spin, 123); -+ DO_TESTCASE_1B("hard-safe-A + unsafe-B #2", irqsafe4_hard_spin, 132); -+ DO_TESTCASE_1B("hard-safe-A + unsafe-B #2", irqsafe4_hard_spin, 213); -+ DO_TESTCASE_1B("hard-safe-A + unsafe-B #2", irqsafe4_hard_spin, 231); -+ DO_TESTCASE_1B("hard-safe-A + unsafe-B #2", irqsafe4_hard_spin, 312); -+ DO_TESTCASE_1B("hard-safe-A + unsafe-B #2", irqsafe4_hard_spin, 321); -+#endif - - ww_tests(); - -diff -Nur linux-4.1.10.orig/lib/percpu_ida.c linux-4.1.10/lib/percpu_ida.c ---- linux-4.1.10.orig/lib/percpu_ida.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/lib/percpu_ida.c 2015-10-07 18:00:08.000000000 +0200 -@@ -26,6 +26,9 @@ - #include - #include - #include -+#include -+ -+static DEFINE_LOCAL_IRQ_LOCK(irq_off_lock); - - struct percpu_ida_cpu { - /* -@@ -148,13 +151,13 @@ - unsigned long flags; - int tag; - -- local_irq_save(flags); -+ local_lock_irqsave(irq_off_lock, flags); - tags = this_cpu_ptr(pool->tag_cpu); - - /* Fastpath */ - tag = alloc_local_tag(tags); - if (likely(tag >= 0)) { -- local_irq_restore(flags); -+ local_unlock_irqrestore(irq_off_lock, flags); - return tag; - } - -@@ -173,6 +176,7 @@ - - if (!tags->nr_free) - alloc_global_tags(pool, tags); -+ - if (!tags->nr_free) - steal_tags(pool, tags); - -@@ -184,7 +188,7 @@ - } - - spin_unlock(&pool->lock); -- local_irq_restore(flags); -+ local_unlock_irqrestore(irq_off_lock, flags); - - if (tag >= 0 || state == TASK_RUNNING) - break; -@@ -196,7 +200,7 @@ - - schedule(); - -- local_irq_save(flags); -+ local_lock_irqsave(irq_off_lock, flags); - tags = this_cpu_ptr(pool->tag_cpu); - } - if (state != TASK_RUNNING) -@@ -221,7 +225,7 @@ - - BUG_ON(tag >= pool->nr_tags); - -- local_irq_save(flags); -+ local_lock_irqsave(irq_off_lock, flags); - tags = this_cpu_ptr(pool->tag_cpu); - - spin_lock(&tags->lock); -@@ -253,7 +257,7 @@ - spin_unlock(&pool->lock); - } - -- local_irq_restore(flags); -+ local_unlock_irqrestore(irq_off_lock, flags); - } - EXPORT_SYMBOL_GPL(percpu_ida_free); - -@@ -345,7 +349,7 @@ - struct percpu_ida_cpu *remote; - unsigned cpu, i, err = 0; - -- local_irq_save(flags); -+ local_lock_irqsave(irq_off_lock, flags); - for_each_possible_cpu(cpu) { - remote = per_cpu_ptr(pool->tag_cpu, cpu); - spin_lock(&remote->lock); -@@ -367,7 +371,7 @@ - } - spin_unlock(&pool->lock); - out: -- local_irq_restore(flags); -+ local_unlock_irqrestore(irq_off_lock, flags); - return err; - } - EXPORT_SYMBOL_GPL(percpu_ida_for_each_free); -diff -Nur linux-4.1.10.orig/lib/radix-tree.c linux-4.1.10/lib/radix-tree.c ---- linux-4.1.10.orig/lib/radix-tree.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/lib/radix-tree.c 2015-10-07 18:00:08.000000000 +0200 -@@ -195,12 +195,13 @@ - * succeed in getting a node here (and never reach - * kmem_cache_alloc) - */ -- rtp = this_cpu_ptr(&radix_tree_preloads); -+ rtp = &get_cpu_var(radix_tree_preloads); - if (rtp->nr) { - ret = rtp->nodes[rtp->nr - 1]; - rtp->nodes[rtp->nr - 1] = NULL; - rtp->nr--; - } -+ put_cpu_var(radix_tree_preloads); - /* - * Update the allocation stack trace as this is more useful - * for debugging. -@@ -240,6 +241,7 @@ - call_rcu(&node->rcu_head, radix_tree_node_rcu_free); - } - -+#ifndef CONFIG_PREEMPT_RT_FULL - /* - * Load up this CPU's radix_tree_node buffer with sufficient objects to - * ensure that the addition of a single element in the tree cannot fail. On -@@ -305,6 +307,7 @@ - return 0; - } - EXPORT_SYMBOL(radix_tree_maybe_preload); -+#endif - - /* - * Return the maximum key which can be store into a -diff -Nur linux-4.1.10.orig/lib/scatterlist.c linux-4.1.10/lib/scatterlist.c ---- linux-4.1.10.orig/lib/scatterlist.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/lib/scatterlist.c 2015-10-07 18:00:08.000000000 +0200 -@@ -592,7 +592,7 @@ - flush_kernel_dcache_page(miter->page); - - if (miter->__flags & SG_MITER_ATOMIC) { -- WARN_ON_ONCE(preemptible()); -+ WARN_ON_ONCE(!pagefault_disabled()); - kunmap_atomic(miter->addr); - } else - kunmap(miter->page); -@@ -637,7 +637,7 @@ - if (!sg_miter_skip(&miter, skip)) - return false; - -- local_irq_save(flags); -+ local_irq_save_nort(flags); - - while (sg_miter_next(&miter) && offset < buflen) { - unsigned int len; -@@ -654,7 +654,7 @@ - - sg_miter_stop(&miter); - -- local_irq_restore(flags); -+ local_irq_restore_nort(flags); - return offset; - } - -diff -Nur linux-4.1.10.orig/lib/smp_processor_id.c linux-4.1.10/lib/smp_processor_id.c ---- linux-4.1.10.orig/lib/smp_processor_id.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/lib/smp_processor_id.c 2015-10-07 18:00:08.000000000 +0200 -@@ -39,8 +39,9 @@ - if (!printk_ratelimit()) - goto out_enable; - -- printk(KERN_ERR "BUG: using %s%s() in preemptible [%08x] code: %s/%d\n", -- what1, what2, preempt_count() - 1, current->comm, current->pid); -+ printk(KERN_ERR "BUG: using %s%s() in preemptible [%08x %08x] code: %s/%d\n", -+ what1, what2, preempt_count() - 1, __migrate_disabled(current), -+ current->comm, current->pid); - - print_symbol("caller is %s\n", (long)__builtin_return_address(0)); - dump_stack(); -diff -Nur linux-4.1.10.orig/lib/strnlen_user.c linux-4.1.10/lib/strnlen_user.c ---- linux-4.1.10.orig/lib/strnlen_user.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/lib/strnlen_user.c 2015-10-07 18:00:08.000000000 +0200 -@@ -85,7 +85,8 @@ - * @str: The string to measure. - * @count: Maximum count (including NUL character) - * -- * Context: User context only. This function may sleep. -+ * Context: User context only. This function may sleep if pagefaults are -+ * enabled. - * - * Get the size of a NUL-terminated string in user space. - * -@@ -121,7 +122,8 @@ - * strlen_user: - Get the size of a user string INCLUDING final NUL. - * @str: The string to measure. - * -- * Context: User context only. This function may sleep. -+ * Context: User context only. This function may sleep if pagefaults are -+ * enabled. - * - * Get the size of a NUL-terminated string in user space. - * -diff -Nur linux-4.1.10.orig/mm/compaction.c linux-4.1.10/mm/compaction.c ---- linux-4.1.10.orig/mm/compaction.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/mm/compaction.c 2015-10-07 18:00:08.000000000 +0200 -@@ -1406,10 +1406,12 @@ - cc->migrate_pfn & ~((1UL << cc->order) - 1); - - if (last_migrated_pfn < current_block_start) { -- cpu = get_cpu(); -+ cpu = get_cpu_light(); -+ local_lock_irq(swapvec_lock); - lru_add_drain_cpu(cpu); -+ local_unlock_irq(swapvec_lock); - drain_local_pages(zone); -- put_cpu(); -+ put_cpu_light(); - /* No more flushing until we migrate again */ - last_migrated_pfn = 0; - } -diff -Nur linux-4.1.10.orig/mm/filemap.c linux-4.1.10/mm/filemap.c ---- linux-4.1.10.orig/mm/filemap.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/mm/filemap.c 2015-10-07 18:00:08.000000000 +0200 -@@ -167,7 +167,9 @@ - if (!workingset_node_pages(node) && - list_empty(&node->private_list)) { - node->private_data = mapping; -- list_lru_add(&workingset_shadow_nodes, &node->private_list); -+ local_lock(workingset_shadow_lock); -+ list_lru_add(&__workingset_shadow_nodes, &node->private_list); -+ local_unlock(workingset_shadow_lock); - } - } - -@@ -533,9 +535,12 @@ - * node->private_list is protected by - * mapping->tree_lock. - */ -- if (!list_empty(&node->private_list)) -- list_lru_del(&workingset_shadow_nodes, -+ if (!list_empty(&node->private_list)) { -+ local_lock(workingset_shadow_lock); -+ list_lru_del(&__workingset_shadow_nodes, - &node->private_list); -+ local_unlock(workingset_shadow_lock); -+ } - } - return 0; - } -diff -Nur linux-4.1.10.orig/mm/highmem.c linux-4.1.10/mm/highmem.c ---- linux-4.1.10.orig/mm/highmem.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/mm/highmem.c 2015-10-07 18:00:08.000000000 +0200 -@@ -29,10 +29,11 @@ - #include - #include - -- -+#ifndef CONFIG_PREEMPT_RT_FULL - #if defined(CONFIG_HIGHMEM) || defined(CONFIG_X86_32) - DEFINE_PER_CPU(int, __kmap_atomic_idx); - #endif -+#endif - - /* - * Virtual_count is not a pure "count". -@@ -107,8 +108,9 @@ - unsigned long totalhigh_pages __read_mostly; - EXPORT_SYMBOL(totalhigh_pages); - -- -+#ifndef CONFIG_PREEMPT_RT_FULL - EXPORT_PER_CPU_SYMBOL(__kmap_atomic_idx); -+#endif - - unsigned int nr_free_highpages (void) - { -diff -Nur linux-4.1.10.orig/mm/Kconfig linux-4.1.10/mm/Kconfig ---- linux-4.1.10.orig/mm/Kconfig 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/mm/Kconfig 2015-10-07 18:00:08.000000000 +0200 -@@ -409,7 +409,7 @@ - - config TRANSPARENT_HUGEPAGE - bool "Transparent Hugepage Support" -- depends on HAVE_ARCH_TRANSPARENT_HUGEPAGE -+ depends on HAVE_ARCH_TRANSPARENT_HUGEPAGE && !PREEMPT_RT_FULL - select COMPACTION - help - Transparent Hugepages allows the kernel to use huge pages and -diff -Nur linux-4.1.10.orig/mm/memcontrol.c linux-4.1.10/mm/memcontrol.c ---- linux-4.1.10.orig/mm/memcontrol.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/mm/memcontrol.c 2015-10-07 18:00:08.000000000 +0200 -@@ -66,6 +66,8 @@ - #include - #include - #include -+#include -+ - #include "slab.h" - - #include -@@ -85,6 +87,7 @@ - #define do_swap_account 0 - #endif - -+static DEFINE_LOCAL_IRQ_LOCK(event_lock); - static const char * const mem_cgroup_stat_names[] = { - "cache", - "rss", -@@ -2124,14 +2127,17 @@ - */ - static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages) - { -- struct memcg_stock_pcp *stock = &get_cpu_var(memcg_stock); -+ struct memcg_stock_pcp *stock; -+ int cpu = get_cpu_light(); -+ -+ stock = &per_cpu(memcg_stock, cpu); - - if (stock->cached != memcg) { /* reset if necessary */ - drain_stock(stock); - stock->cached = memcg; - } - stock->nr_pages += nr_pages; -- put_cpu_var(memcg_stock); -+ put_cpu_light(); - } - - /* -@@ -2147,7 +2153,7 @@ - return; - /* Notify other cpus that system-wide "drain" is running */ - get_online_cpus(); -- curcpu = get_cpu(); -+ curcpu = get_cpu_light(); - for_each_online_cpu(cpu) { - struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu); - struct mem_cgroup *memcg; -@@ -2164,7 +2170,7 @@ - schedule_work_on(cpu, &stock->work); - } - } -- put_cpu(); -+ put_cpu_light(); - put_online_cpus(); - mutex_unlock(&percpu_charge_mutex); - } -@@ -4801,12 +4807,12 @@ - - ret = 0; - -- local_irq_disable(); -+ local_lock_irq(event_lock); - mem_cgroup_charge_statistics(to, page, nr_pages); - memcg_check_events(to, page); - mem_cgroup_charge_statistics(from, page, -nr_pages); - memcg_check_events(from, page); -- local_irq_enable(); -+ local_unlock_irq(event_lock); - out_unlock: - unlock_page(page); - out: -@@ -5543,10 +5549,10 @@ - VM_BUG_ON_PAGE(!PageTransHuge(page), page); - } - -- local_irq_disable(); -+ local_lock_irq(event_lock); - mem_cgroup_charge_statistics(memcg, page, nr_pages); - memcg_check_events(memcg, page); -- local_irq_enable(); -+ local_unlock_irq(event_lock); - - if (do_swap_account && PageSwapCache(page)) { - swp_entry_t entry = { .val = page_private(page) }; -@@ -5602,14 +5608,14 @@ - memcg_oom_recover(memcg); - } - -- local_irq_save(flags); -+ local_lock_irqsave(event_lock, flags); - __this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_RSS], nr_anon); - __this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_CACHE], nr_file); - __this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_RSS_HUGE], nr_huge); - __this_cpu_add(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGOUT], pgpgout); - __this_cpu_add(memcg->stat->nr_page_events, nr_pages); - memcg_check_events(memcg, dummy_page); -- local_irq_restore(flags); -+ local_unlock_irqrestore(event_lock, flags); - - if (!mem_cgroup_is_root(memcg)) - css_put_many(&memcg->css, nr_pages); -@@ -5813,6 +5819,7 @@ - { - struct mem_cgroup *memcg; - unsigned short oldid; -+ unsigned long flags; - - VM_BUG_ON_PAGE(PageLRU(page), page); - VM_BUG_ON_PAGE(page_count(page), page); -@@ -5835,9 +5842,11 @@ - if (!mem_cgroup_is_root(memcg)) - page_counter_uncharge(&memcg->memory, 1); - -+ local_lock_irqsave(event_lock, flags); - /* Caller disabled preemption with mapping->tree_lock */ - mem_cgroup_charge_statistics(memcg, page, -1); - memcg_check_events(memcg, page); -+ local_unlock_irqrestore(event_lock, flags); - } - - /** -diff -Nur linux-4.1.10.orig/mm/memory.c linux-4.1.10/mm/memory.c ---- linux-4.1.10.orig/mm/memory.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/mm/memory.c 2015-10-07 18:00:08.000000000 +0200 -@@ -3743,7 +3743,7 @@ - } - - #if defined(CONFIG_PROVE_LOCKING) || defined(CONFIG_DEBUG_ATOMIC_SLEEP) --void might_fault(void) -+void __might_fault(const char *file, int line) - { - /* - * Some code (nfs/sunrpc) uses socket ops on kernel memory while -@@ -3753,21 +3753,15 @@ - */ - if (segment_eq(get_fs(), KERNEL_DS)) - return; -- -- /* -- * it would be nicer only to annotate paths which are not under -- * pagefault_disable, however that requires a larger audit and -- * providing helpers like get_user_atomic. -- */ -- if (in_atomic()) -+ if (pagefault_disabled()) - return; -- -- __might_sleep(__FILE__, __LINE__, 0); -- -+ __might_sleep(file, line, 0); -+#if defined(CONFIG_DEBUG_ATOMIC_SLEEP) - if (current->mm) - might_lock_read(¤t->mm->mmap_sem); -+#endif - } --EXPORT_SYMBOL(might_fault); -+EXPORT_SYMBOL(__might_fault); - #endif - - #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS) -diff -Nur linux-4.1.10.orig/mm/mmu_context.c linux-4.1.10/mm/mmu_context.c ---- linux-4.1.10.orig/mm/mmu_context.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/mm/mmu_context.c 2015-10-07 18:00:08.000000000 +0200 -@@ -23,6 +23,7 @@ - struct task_struct *tsk = current; - - task_lock(tsk); -+ preempt_disable_rt(); - active_mm = tsk->active_mm; - if (active_mm != mm) { - atomic_inc(&mm->mm_count); -@@ -30,6 +31,7 @@ - } - tsk->mm = mm; - switch_mm(active_mm, mm, tsk); -+ preempt_enable_rt(); - task_unlock(tsk); - #ifdef finish_arch_post_lock_switch - finish_arch_post_lock_switch(); -diff -Nur linux-4.1.10.orig/mm/page_alloc.c linux-4.1.10/mm/page_alloc.c ---- linux-4.1.10.orig/mm/page_alloc.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/mm/page_alloc.c 2015-10-07 18:00:08.000000000 +0200 -@@ -60,6 +60,7 @@ - #include - #include - #include -+#include - #include - - #include -@@ -233,6 +234,18 @@ - EXPORT_SYMBOL(nr_online_nodes); - #endif - -+static DEFINE_LOCAL_IRQ_LOCK(pa_lock); -+ -+#ifdef CONFIG_PREEMPT_RT_BASE -+# define cpu_lock_irqsave(cpu, flags) \ -+ local_lock_irqsave_on(pa_lock, flags, cpu) -+# define cpu_unlock_irqrestore(cpu, flags) \ -+ local_unlock_irqrestore_on(pa_lock, flags, cpu) -+#else -+# define cpu_lock_irqsave(cpu, flags) local_irq_save(flags) -+# define cpu_unlock_irqrestore(cpu, flags) local_irq_restore(flags) -+#endif -+ - int page_group_by_mobility_disabled __read_mostly; - - void set_pageblock_migratetype(struct page *page, int migratetype) -@@ -681,7 +694,7 @@ - } - - /* -- * Frees a number of pages from the PCP lists -+ * Frees a number of pages which have been collected from the pcp lists. - * Assumes all pages on list are in same zone, and of same order. - * count is the number of pages to free. - * -@@ -692,18 +705,51 @@ - * pinned" detection logic. - */ - static void free_pcppages_bulk(struct zone *zone, int count, -- struct per_cpu_pages *pcp) -+ struct list_head *list) - { -- int migratetype = 0; -- int batch_free = 0; - int to_free = count; - unsigned long nr_scanned; -+ unsigned long flags; -+ -+ spin_lock_irqsave(&zone->lock, flags); - -- spin_lock(&zone->lock); - nr_scanned = zone_page_state(zone, NR_PAGES_SCANNED); - if (nr_scanned) - __mod_zone_page_state(zone, NR_PAGES_SCANNED, -nr_scanned); - -+ while (!list_empty(list)) { -+ struct page *page = list_first_entry(list, struct page, lru); -+ int mt; /* migratetype of the to-be-freed page */ -+ -+ /* must delete as __free_one_page list manipulates */ -+ list_del(&page->lru); -+ -+ mt = get_freepage_migratetype(page); -+ if (unlikely(has_isolate_pageblock(zone))) -+ mt = get_pageblock_migratetype(page); -+ -+ /* MIGRATE_MOVABLE list may include MIGRATE_RESERVEs */ -+ __free_one_page(page, page_to_pfn(page), zone, 0, mt); -+ trace_mm_page_pcpu_drain(page, 0, mt); -+ to_free--; -+ } -+ WARN_ON(to_free != 0); -+ spin_unlock_irqrestore(&zone->lock, flags); -+} -+ -+/* -+ * Moves a number of pages from the PCP lists to free list which -+ * is freed outside of the locked region. -+ * -+ * Assumes all pages on list are in same zone, and of same order. -+ * count is the number of pages to free. -+ */ -+static void isolate_pcp_pages(int to_free, struct per_cpu_pages *src, -+ struct list_head *dst) -+{ -+ int migratetype = 0; -+ int batch_free = 0; -+ - while (to_free) { - struct page *page; - struct list_head *list; -@@ -719,7 +765,7 @@ - batch_free++; - if (++migratetype == MIGRATE_PCPTYPES) - migratetype = 0; -- list = &pcp->lists[migratetype]; -+ list = &src->lists[migratetype]; - } while (list_empty(list)); - - /* This is the only non-empty list. Free them all. */ -@@ -727,21 +773,11 @@ - batch_free = to_free; - - do { -- int mt; /* migratetype of the to-be-freed page */ -- -- page = list_entry(list->prev, struct page, lru); -- /* must delete as __free_one_page list manipulates */ -+ page = list_last_entry(list, struct page, lru); - list_del(&page->lru); -- mt = get_freepage_migratetype(page); -- if (unlikely(has_isolate_pageblock(zone))) -- mt = get_pageblock_migratetype(page); -- -- /* MIGRATE_MOVABLE list may include MIGRATE_RESERVEs */ -- __free_one_page(page, page_to_pfn(page), zone, 0, mt); -- trace_mm_page_pcpu_drain(page, 0, mt); -+ list_add(&page->lru, dst); - } while (--to_free && --batch_free && !list_empty(list)); - } -- spin_unlock(&zone->lock); - } - - static void free_one_page(struct zone *zone, -@@ -750,7 +786,9 @@ - int migratetype) - { - unsigned long nr_scanned; -- spin_lock(&zone->lock); -+ unsigned long flags; -+ -+ spin_lock_irqsave(&zone->lock, flags); - nr_scanned = zone_page_state(zone, NR_PAGES_SCANNED); - if (nr_scanned) - __mod_zone_page_state(zone, NR_PAGES_SCANNED, -nr_scanned); -@@ -760,7 +798,7 @@ - migratetype = get_pfnblock_migratetype(page, pfn); - } - __free_one_page(page, pfn, zone, order, migratetype); -- spin_unlock(&zone->lock); -+ spin_unlock_irqrestore(&zone->lock, flags); - } - - static int free_tail_pages_check(struct page *head_page, struct page *page) -@@ -825,11 +863,11 @@ - return; - - migratetype = get_pfnblock_migratetype(page, pfn); -- local_irq_save(flags); -+ local_lock_irqsave(pa_lock, flags); - __count_vm_events(PGFREE, 1 << order); - set_freepage_migratetype(page, migratetype); - free_one_page(page_zone(page), page, pfn, order, migratetype); -- local_irq_restore(flags); -+ local_unlock_irqrestore(pa_lock, flags); - } - - void __init __free_pages_bootmem(struct page *page, unsigned int order) -@@ -1371,16 +1409,18 @@ - void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp) - { - unsigned long flags; -+ LIST_HEAD(dst); - int to_drain, batch; - -- local_irq_save(flags); -+ local_lock_irqsave(pa_lock, flags); - batch = READ_ONCE(pcp->batch); - to_drain = min(pcp->count, batch); - if (to_drain > 0) { -- free_pcppages_bulk(zone, to_drain, pcp); -+ isolate_pcp_pages(to_drain, pcp, &dst); - pcp->count -= to_drain; - } -- local_irq_restore(flags); -+ local_unlock_irqrestore(pa_lock, flags); -+ free_pcppages_bulk(zone, to_drain, &dst); - } - #endif - -@@ -1396,16 +1436,21 @@ - unsigned long flags; - struct per_cpu_pageset *pset; - struct per_cpu_pages *pcp; -+ LIST_HEAD(dst); -+ int count; - -- local_irq_save(flags); -+ cpu_lock_irqsave(cpu, flags); - pset = per_cpu_ptr(zone->pageset, cpu); - - pcp = &pset->pcp; -- if (pcp->count) { -- free_pcppages_bulk(zone, pcp->count, pcp); -+ count = pcp->count; -+ if (count) { -+ isolate_pcp_pages(count, pcp, &dst); - pcp->count = 0; - } -- local_irq_restore(flags); -+ cpu_unlock_irqrestore(cpu, flags); -+ if (count) -+ free_pcppages_bulk(zone, count, &dst); - } - - /* -@@ -1491,8 +1536,17 @@ - else - cpumask_clear_cpu(cpu, &cpus_with_pcps); - } -+#ifndef CONFIG_PREEMPT_RT_BASE - on_each_cpu_mask(&cpus_with_pcps, (smp_call_func_t) drain_local_pages, - zone, 1); -+#else -+ for_each_cpu(cpu, &cpus_with_pcps) { -+ if (zone) -+ drain_pages_zone(cpu, zone); -+ else -+ drain_pages(cpu); -+ } -+#endif - } - - #ifdef CONFIG_HIBERNATION -@@ -1548,7 +1602,7 @@ - - migratetype = get_pfnblock_migratetype(page, pfn); - set_freepage_migratetype(page, migratetype); -- local_irq_save(flags); -+ local_lock_irqsave(pa_lock, flags); - __count_vm_event(PGFREE); - - /* -@@ -1574,12 +1628,17 @@ - pcp->count++; - if (pcp->count >= pcp->high) { - unsigned long batch = READ_ONCE(pcp->batch); -- free_pcppages_bulk(zone, batch, pcp); -+ LIST_HEAD(dst); -+ -+ isolate_pcp_pages(batch, pcp, &dst); - pcp->count -= batch; -+ local_unlock_irqrestore(pa_lock, flags); -+ free_pcppages_bulk(zone, batch, &dst); -+ return; - } - - out: -- local_irq_restore(flags); -+ local_unlock_irqrestore(pa_lock, flags); - } - - /* -@@ -1710,7 +1769,7 @@ - struct per_cpu_pages *pcp; - struct list_head *list; - -- local_irq_save(flags); -+ local_lock_irqsave(pa_lock, flags); - pcp = &this_cpu_ptr(zone->pageset)->pcp; - list = &pcp->lists[migratetype]; - if (list_empty(list)) { -@@ -1742,13 +1801,15 @@ - */ - WARN_ON_ONCE(order > 1); - } -- spin_lock_irqsave(&zone->lock, flags); -+ local_spin_lock_irqsave(pa_lock, &zone->lock, flags); - page = __rmqueue(zone, order, migratetype); -- spin_unlock(&zone->lock); -- if (!page) -+ if (!page) { -+ spin_unlock(&zone->lock); - goto failed; -+ } - __mod_zone_freepage_state(zone, -(1 << order), - get_freepage_migratetype(page)); -+ spin_unlock(&zone->lock); - } - - __mod_zone_page_state(zone, NR_ALLOC_BATCH, -(1 << order)); -@@ -1758,13 +1819,13 @@ - - __count_zone_vm_events(PGALLOC, zone, 1 << order); - zone_statistics(preferred_zone, zone, gfp_flags); -- local_irq_restore(flags); -+ local_unlock_irqrestore(pa_lock, flags); - - VM_BUG_ON_PAGE(bad_range(zone, page), page); - return page; - - failed: -- local_irq_restore(flags); -+ local_unlock_irqrestore(pa_lock, flags); - return NULL; - } - -@@ -5653,6 +5714,7 @@ - void __init page_alloc_init(void) - { - hotcpu_notifier(page_alloc_cpu_notify, 0); -+ local_irq_lock_init(pa_lock); - } - - /* -@@ -6547,7 +6609,7 @@ - struct per_cpu_pageset *pset; - - /* avoid races with drain_pages() */ -- local_irq_save(flags); -+ local_lock_irqsave(pa_lock, flags); - if (zone->pageset != &boot_pageset) { - for_each_online_cpu(cpu) { - pset = per_cpu_ptr(zone->pageset, cpu); -@@ -6556,7 +6618,7 @@ - free_percpu(zone->pageset); - zone->pageset = &boot_pageset; - } -- local_irq_restore(flags); -+ local_unlock_irqrestore(pa_lock, flags); - } - - #ifdef CONFIG_MEMORY_HOTREMOVE -diff -Nur linux-4.1.10.orig/mm/page_alloc.c.orig linux-4.1.10/mm/page_alloc.c.orig ---- linux-4.1.10.orig/mm/page_alloc.c.orig 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.10/mm/page_alloc.c.orig 2015-10-03 13:49:38.000000000 +0200 -@@ -0,0 +1,6636 @@ -+/* -+ * linux/mm/page_alloc.c -+ * -+ * Manages the free list, the system allocates free pages here. -+ * Note that kmalloc() lives in slab.c -+ * -+ * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds -+ * Swap reorganised 29.12.95, Stephen Tweedie -+ * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999 -+ * Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999 -+ * Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999 -+ * Zone balancing, Kanoj Sarcar, SGI, Jan 2000 -+ * Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002 -+ * (lots of bits borrowed from Ingo Molnar & Andrew Morton) -+ */ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include -+#include -+#include -+#include "internal.h" -+ -+/* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */ -+static DEFINE_MUTEX(pcp_batch_high_lock); -+#define MIN_PERCPU_PAGELIST_FRACTION (8) -+ -+#ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID -+DEFINE_PER_CPU(int, numa_node); -+EXPORT_PER_CPU_SYMBOL(numa_node); -+#endif -+ -+#ifdef CONFIG_HAVE_MEMORYLESS_NODES -+/* -+ * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly. -+ * It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined. -+ * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem() -+ * defined in . -+ */ -+DEFINE_PER_CPU(int, _numa_mem_); /* Kernel "local memory" node */ -+EXPORT_PER_CPU_SYMBOL(_numa_mem_); -+int _node_numa_mem_[MAX_NUMNODES]; -+#endif -+ -+/* -+ * Array of node states. -+ */ -+nodemask_t node_states[NR_NODE_STATES] __read_mostly = { -+ [N_POSSIBLE] = NODE_MASK_ALL, -+ [N_ONLINE] = { { [0] = 1UL } }, -+#ifndef CONFIG_NUMA -+ [N_NORMAL_MEMORY] = { { [0] = 1UL } }, -+#ifdef CONFIG_HIGHMEM -+ [N_HIGH_MEMORY] = { { [0] = 1UL } }, -+#endif -+#ifdef CONFIG_MOVABLE_NODE -+ [N_MEMORY] = { { [0] = 1UL } }, -+#endif -+ [N_CPU] = { { [0] = 1UL } }, -+#endif /* NUMA */ -+}; -+EXPORT_SYMBOL(node_states); -+ -+/* Protect totalram_pages and zone->managed_pages */ -+static DEFINE_SPINLOCK(managed_page_count_lock); -+ -+unsigned long totalram_pages __read_mostly; -+unsigned long totalreserve_pages __read_mostly; -+unsigned long totalcma_pages __read_mostly; -+/* -+ * When calculating the number of globally allowed dirty pages, there -+ * is a certain number of per-zone reserves that should not be -+ * considered dirtyable memory. This is the sum of those reserves -+ * over all existing zones that contribute dirtyable memory. -+ */ -+unsigned long dirty_balance_reserve __read_mostly; -+ -+int percpu_pagelist_fraction; -+gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK; -+ -+#ifdef CONFIG_PM_SLEEP -+/* -+ * The following functions are used by the suspend/hibernate code to temporarily -+ * change gfp_allowed_mask in order to avoid using I/O during memory allocations -+ * while devices are suspended. To avoid races with the suspend/hibernate code, -+ * they should always be called with pm_mutex held (gfp_allowed_mask also should -+ * only be modified with pm_mutex held, unless the suspend/hibernate code is -+ * guaranteed not to run in parallel with that modification). -+ */ -+ -+static gfp_t saved_gfp_mask; -+ -+void pm_restore_gfp_mask(void) -+{ -+ WARN_ON(!mutex_is_locked(&pm_mutex)); -+ if (saved_gfp_mask) { -+ gfp_allowed_mask = saved_gfp_mask; -+ saved_gfp_mask = 0; -+ } -+} -+ -+void pm_restrict_gfp_mask(void) -+{ -+ WARN_ON(!mutex_is_locked(&pm_mutex)); -+ WARN_ON(saved_gfp_mask); -+ saved_gfp_mask = gfp_allowed_mask; -+ gfp_allowed_mask &= ~GFP_IOFS; -+} -+ -+bool pm_suspended_storage(void) -+{ -+ if ((gfp_allowed_mask & GFP_IOFS) == GFP_IOFS) -+ return false; -+ return true; -+} -+#endif /* CONFIG_PM_SLEEP */ -+ -+#ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE -+int pageblock_order __read_mostly; -+#endif -+ -+static void __free_pages_ok(struct page *page, unsigned int order); -+ -+/* -+ * results with 256, 32 in the lowmem_reserve sysctl: -+ * 1G machine -> (16M dma, 800M-16M normal, 1G-800M high) -+ * 1G machine -> (16M dma, 784M normal, 224M high) -+ * NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA -+ * HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL -+ * HIGHMEM allocation will leave (224M+784M)/256 of ram reserved in ZONE_DMA -+ * -+ * TBD: should special case ZONE_DMA32 machines here - in those we normally -+ * don't need any ZONE_NORMAL reservation -+ */ -+int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES-1] = { -+#ifdef CONFIG_ZONE_DMA -+ 256, -+#endif -+#ifdef CONFIG_ZONE_DMA32 -+ 256, -+#endif -+#ifdef CONFIG_HIGHMEM -+ 32, -+#endif -+ 32, -+}; -+ -+EXPORT_SYMBOL(totalram_pages); -+ -+static char * const zone_names[MAX_NR_ZONES] = { -+#ifdef CONFIG_ZONE_DMA -+ "DMA", -+#endif -+#ifdef CONFIG_ZONE_DMA32 -+ "DMA32", -+#endif -+ "Normal", -+#ifdef CONFIG_HIGHMEM -+ "HighMem", -+#endif -+ "Movable", -+}; -+ -+int min_free_kbytes = 1024; -+int user_min_free_kbytes = -1; -+ -+static unsigned long __meminitdata nr_kernel_pages; -+static unsigned long __meminitdata nr_all_pages; -+static unsigned long __meminitdata dma_reserve; -+ -+#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP -+static unsigned long __meminitdata arch_zone_lowest_possible_pfn[MAX_NR_ZONES]; -+static unsigned long __meminitdata arch_zone_highest_possible_pfn[MAX_NR_ZONES]; -+static unsigned long __initdata required_kernelcore; -+static unsigned long __initdata required_movablecore; -+static unsigned long __meminitdata zone_movable_pfn[MAX_NUMNODES]; -+ -+/* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */ -+int movable_zone; -+EXPORT_SYMBOL(movable_zone); -+#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ -+ -+#if MAX_NUMNODES > 1 -+int nr_node_ids __read_mostly = MAX_NUMNODES; -+int nr_online_nodes __read_mostly = 1; -+EXPORT_SYMBOL(nr_node_ids); -+EXPORT_SYMBOL(nr_online_nodes); -+#endif -+ -+int page_group_by_mobility_disabled __read_mostly; -+ -+void set_pageblock_migratetype(struct page *page, int migratetype) -+{ -+ if (unlikely(page_group_by_mobility_disabled && -+ migratetype < MIGRATE_PCPTYPES)) -+ migratetype = MIGRATE_UNMOVABLE; -+ -+ set_pageblock_flags_group(page, (unsigned long)migratetype, -+ PB_migrate, PB_migrate_end); -+} -+ -+#ifdef CONFIG_DEBUG_VM -+static int page_outside_zone_boundaries(struct zone *zone, struct page *page) -+{ -+ int ret = 0; -+ unsigned seq; -+ unsigned long pfn = page_to_pfn(page); -+ unsigned long sp, start_pfn; -+ -+ do { -+ seq = zone_span_seqbegin(zone); -+ start_pfn = zone->zone_start_pfn; -+ sp = zone->spanned_pages; -+ if (!zone_spans_pfn(zone, pfn)) -+ ret = 1; -+ } while (zone_span_seqretry(zone, seq)); -+ -+ if (ret) -+ pr_err("page 0x%lx outside node %d zone %s [ 0x%lx - 0x%lx ]\n", -+ pfn, zone_to_nid(zone), zone->name, -+ start_pfn, start_pfn + sp); -+ -+ return ret; -+} -+ -+static int page_is_consistent(struct zone *zone, struct page *page) -+{ -+ if (!pfn_valid_within(page_to_pfn(page))) -+ return 0; -+ if (zone != page_zone(page)) -+ return 0; -+ -+ return 1; -+} -+/* -+ * Temporary debugging check for pages not lying within a given zone. -+ */ -+static int bad_range(struct zone *zone, struct page *page) -+{ -+ if (page_outside_zone_boundaries(zone, page)) -+ return 1; -+ if (!page_is_consistent(zone, page)) -+ return 1; -+ -+ return 0; -+} -+#else -+static inline int bad_range(struct zone *zone, struct page *page) -+{ -+ return 0; -+} -+#endif -+ -+static void bad_page(struct page *page, const char *reason, -+ unsigned long bad_flags) -+{ -+ static unsigned long resume; -+ static unsigned long nr_shown; -+ static unsigned long nr_unshown; -+ -+ /* Don't complain about poisoned pages */ -+ if (PageHWPoison(page)) { -+ page_mapcount_reset(page); /* remove PageBuddy */ -+ return; -+ } -+ -+ /* -+ * Allow a burst of 60 reports, then keep quiet for that minute; -+ * or allow a steady drip of one report per second. -+ */ -+ if (nr_shown == 60) { -+ if (time_before(jiffies, resume)) { -+ nr_unshown++; -+ goto out; -+ } -+ if (nr_unshown) { -+ printk(KERN_ALERT -+ "BUG: Bad page state: %lu messages suppressed\n", -+ nr_unshown); -+ nr_unshown = 0; -+ } -+ nr_shown = 0; -+ } -+ if (nr_shown++ == 0) -+ resume = jiffies + 60 * HZ; -+ -+ printk(KERN_ALERT "BUG: Bad page state in process %s pfn:%05lx\n", -+ current->comm, page_to_pfn(page)); -+ dump_page_badflags(page, reason, bad_flags); -+ -+ print_modules(); -+ dump_stack(); -+out: -+ /* Leave bad fields for debug, except PageBuddy could make trouble */ -+ page_mapcount_reset(page); /* remove PageBuddy */ -+ add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); -+} -+ -+/* -+ * Higher-order pages are called "compound pages". They are structured thusly: -+ * -+ * The first PAGE_SIZE page is called the "head page". -+ * -+ * The remaining PAGE_SIZE pages are called "tail pages". -+ * -+ * All pages have PG_compound set. All tail pages have their ->first_page -+ * pointing at the head page. -+ * -+ * The first tail page's ->lru.next holds the address of the compound page's -+ * put_page() function. Its ->lru.prev holds the order of allocation. -+ * This usage means that zero-order pages may not be compound. -+ */ -+ -+static void free_compound_page(struct page *page) -+{ -+ __free_pages_ok(page, compound_order(page)); -+} -+ -+void prep_compound_page(struct page *page, unsigned long order) -+{ -+ int i; -+ int nr_pages = 1 << order; -+ -+ set_compound_page_dtor(page, free_compound_page); -+ set_compound_order(page, order); -+ __SetPageHead(page); -+ for (i = 1; i < nr_pages; i++) { -+ struct page *p = page + i; -+ set_page_count(p, 0); -+ p->first_page = page; -+ /* Make sure p->first_page is always valid for PageTail() */ -+ smp_wmb(); -+ __SetPageTail(p); -+ } -+} -+ -+static inline void prep_zero_page(struct page *page, unsigned int order, -+ gfp_t gfp_flags) -+{ -+ int i; -+ -+ /* -+ * clear_highpage() will use KM_USER0, so it's a bug to use __GFP_ZERO -+ * and __GFP_HIGHMEM from hard or soft interrupt context. -+ */ -+ VM_BUG_ON((gfp_flags & __GFP_HIGHMEM) && in_interrupt()); -+ for (i = 0; i < (1 << order); i++) -+ clear_highpage(page + i); -+} -+ -+#ifdef CONFIG_DEBUG_PAGEALLOC -+unsigned int _debug_guardpage_minorder; -+bool _debug_pagealloc_enabled __read_mostly; -+bool _debug_guardpage_enabled __read_mostly; -+ -+static int __init early_debug_pagealloc(char *buf) -+{ -+ if (!buf) -+ return -EINVAL; -+ -+ if (strcmp(buf, "on") == 0) -+ _debug_pagealloc_enabled = true; -+ -+ return 0; -+} -+early_param("debug_pagealloc", early_debug_pagealloc); -+ -+static bool need_debug_guardpage(void) -+{ -+ /* If we don't use debug_pagealloc, we don't need guard page */ -+ if (!debug_pagealloc_enabled()) -+ return false; -+ -+ return true; -+} -+ -+static void init_debug_guardpage(void) -+{ -+ if (!debug_pagealloc_enabled()) -+ return; -+ -+ _debug_guardpage_enabled = true; -+} -+ -+struct page_ext_operations debug_guardpage_ops = { -+ .need = need_debug_guardpage, -+ .init = init_debug_guardpage, -+}; -+ -+static int __init debug_guardpage_minorder_setup(char *buf) -+{ -+ unsigned long res; -+ -+ if (kstrtoul(buf, 10, &res) < 0 || res > MAX_ORDER / 2) { -+ printk(KERN_ERR "Bad debug_guardpage_minorder value\n"); -+ return 0; -+ } -+ _debug_guardpage_minorder = res; -+ printk(KERN_INFO "Setting debug_guardpage_minorder to %lu\n", res); -+ return 0; -+} -+__setup("debug_guardpage_minorder=", debug_guardpage_minorder_setup); -+ -+static inline void set_page_guard(struct zone *zone, struct page *page, -+ unsigned int order, int migratetype) -+{ -+ struct page_ext *page_ext; -+ -+ if (!debug_guardpage_enabled()) -+ return; -+ -+ page_ext = lookup_page_ext(page); -+ __set_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags); -+ -+ INIT_LIST_HEAD(&page->lru); -+ set_page_private(page, order); -+ /* Guard pages are not available for any usage */ -+ __mod_zone_freepage_state(zone, -(1 << order), migratetype); -+} -+ -+static inline void clear_page_guard(struct zone *zone, struct page *page, -+ unsigned int order, int migratetype) -+{ -+ struct page_ext *page_ext; -+ -+ if (!debug_guardpage_enabled()) -+ return; -+ -+ page_ext = lookup_page_ext(page); -+ __clear_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags); -+ -+ set_page_private(page, 0); -+ if (!is_migrate_isolate(migratetype)) -+ __mod_zone_freepage_state(zone, (1 << order), migratetype); -+} -+#else -+struct page_ext_operations debug_guardpage_ops = { NULL, }; -+static inline void set_page_guard(struct zone *zone, struct page *page, -+ unsigned int order, int migratetype) {} -+static inline void clear_page_guard(struct zone *zone, struct page *page, -+ unsigned int order, int migratetype) {} -+#endif -+ -+static inline void set_page_order(struct page *page, unsigned int order) -+{ -+ set_page_private(page, order); -+ __SetPageBuddy(page); -+} -+ -+static inline void rmv_page_order(struct page *page) -+{ -+ __ClearPageBuddy(page); -+ set_page_private(page, 0); -+} -+ -+/* -+ * This function checks whether a page is free && is the buddy -+ * we can do coalesce a page and its buddy if -+ * (a) the buddy is not in a hole && -+ * (b) the buddy is in the buddy system && -+ * (c) a page and its buddy have the same order && -+ * (d) a page and its buddy are in the same zone. -+ * -+ * For recording whether a page is in the buddy system, we set ->_mapcount -+ * PAGE_BUDDY_MAPCOUNT_VALUE. -+ * Setting, clearing, and testing _mapcount PAGE_BUDDY_MAPCOUNT_VALUE is -+ * serialized by zone->lock. -+ * -+ * For recording page's order, we use page_private(page). -+ */ -+static inline int page_is_buddy(struct page *page, struct page *buddy, -+ unsigned int order) -+{ -+ if (!pfn_valid_within(page_to_pfn(buddy))) -+ return 0; -+ -+ if (page_is_guard(buddy) && page_order(buddy) == order) { -+ if (page_zone_id(page) != page_zone_id(buddy)) -+ return 0; -+ -+ VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy); -+ -+ return 1; -+ } -+ -+ if (PageBuddy(buddy) && page_order(buddy) == order) { -+ /* -+ * zone check is done late to avoid uselessly -+ * calculating zone/node ids for pages that could -+ * never merge. -+ */ -+ if (page_zone_id(page) != page_zone_id(buddy)) -+ return 0; -+ -+ VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy); -+ -+ return 1; -+ } -+ return 0; -+} -+ -+/* -+ * Freeing function for a buddy system allocator. -+ * -+ * The concept of a buddy system is to maintain direct-mapped table -+ * (containing bit values) for memory blocks of various "orders". -+ * The bottom level table contains the map for the smallest allocatable -+ * units of memory (here, pages), and each level above it describes -+ * pairs of units from the levels below, hence, "buddies". -+ * At a high level, all that happens here is marking the table entry -+ * at the bottom level available, and propagating the changes upward -+ * as necessary, plus some accounting needed to play nicely with other -+ * parts of the VM system. -+ * At each level, we keep a list of pages, which are heads of continuous -+ * free pages of length of (1 << order) and marked with _mapcount -+ * PAGE_BUDDY_MAPCOUNT_VALUE. Page's order is recorded in page_private(page) -+ * field. -+ * So when we are allocating or freeing one, we can derive the state of the -+ * other. That is, if we allocate a small block, and both were -+ * free, the remainder of the region must be split into blocks. -+ * If a block is freed, and its buddy is also free, then this -+ * triggers coalescing into a block of larger size. -+ * -+ * -- nyc -+ */ -+ -+static inline void __free_one_page(struct page *page, -+ unsigned long pfn, -+ struct zone *zone, unsigned int order, -+ int migratetype) -+{ -+ unsigned long page_idx; -+ unsigned long combined_idx; -+ unsigned long uninitialized_var(buddy_idx); -+ struct page *buddy; -+ int max_order = MAX_ORDER; -+ -+ VM_BUG_ON(!zone_is_initialized(zone)); -+ VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page); -+ -+ VM_BUG_ON(migratetype == -1); -+ if (is_migrate_isolate(migratetype)) { -+ /* -+ * We restrict max order of merging to prevent merge -+ * between freepages on isolate pageblock and normal -+ * pageblock. Without this, pageblock isolation -+ * could cause incorrect freepage accounting. -+ */ -+ max_order = min(MAX_ORDER, pageblock_order + 1); -+ } else { -+ __mod_zone_freepage_state(zone, 1 << order, migratetype); -+ } -+ -+ page_idx = pfn & ((1 << max_order) - 1); -+ -+ VM_BUG_ON_PAGE(page_idx & ((1 << order) - 1), page); -+ VM_BUG_ON_PAGE(bad_range(zone, page), page); -+ -+ while (order < max_order - 1) { -+ buddy_idx = __find_buddy_index(page_idx, order); -+ buddy = page + (buddy_idx - page_idx); -+ if (!page_is_buddy(page, buddy, order)) -+ break; -+ /* -+ * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page, -+ * merge with it and move up one order. -+ */ -+ if (page_is_guard(buddy)) { -+ clear_page_guard(zone, buddy, order, migratetype); -+ } else { -+ list_del(&buddy->lru); -+ zone->free_area[order].nr_free--; -+ rmv_page_order(buddy); -+ } -+ combined_idx = buddy_idx & page_idx; -+ page = page + (combined_idx - page_idx); -+ page_idx = combined_idx; -+ order++; -+ } -+ set_page_order(page, order); -+ -+ /* -+ * If this is not the largest possible page, check if the buddy -+ * of the next-highest order is free. If it is, it's possible -+ * that pages are being freed that will coalesce soon. In case, -+ * that is happening, add the free page to the tail of the list -+ * so it's less likely to be used soon and more likely to be merged -+ * as a higher order page -+ */ -+ if ((order < MAX_ORDER-2) && pfn_valid_within(page_to_pfn(buddy))) { -+ struct page *higher_page, *higher_buddy; -+ combined_idx = buddy_idx & page_idx; -+ higher_page = page + (combined_idx - page_idx); -+ buddy_idx = __find_buddy_index(combined_idx, order + 1); -+ higher_buddy = higher_page + (buddy_idx - combined_idx); -+ if (page_is_buddy(higher_page, higher_buddy, order + 1)) { -+ list_add_tail(&page->lru, -+ &zone->free_area[order].free_list[migratetype]); -+ goto out; -+ } -+ } -+ -+ list_add(&page->lru, &zone->free_area[order].free_list[migratetype]); -+out: -+ zone->free_area[order].nr_free++; -+} -+ -+static inline int free_pages_check(struct page *page) -+{ -+ const char *bad_reason = NULL; -+ unsigned long bad_flags = 0; -+ -+ if (unlikely(page_mapcount(page))) -+ bad_reason = "nonzero mapcount"; -+ if (unlikely(page->mapping != NULL)) -+ bad_reason = "non-NULL mapping"; -+ if (unlikely(atomic_read(&page->_count) != 0)) -+ bad_reason = "nonzero _count"; -+ if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_FREE)) { -+ bad_reason = "PAGE_FLAGS_CHECK_AT_FREE flag(s) set"; -+ bad_flags = PAGE_FLAGS_CHECK_AT_FREE; -+ } -+#ifdef CONFIG_MEMCG -+ if (unlikely(page->mem_cgroup)) -+ bad_reason = "page still charged to cgroup"; -+#endif -+ if (unlikely(bad_reason)) { -+ bad_page(page, bad_reason, bad_flags); -+ return 1; -+ } -+ page_cpupid_reset_last(page); -+ if (page->flags & PAGE_FLAGS_CHECK_AT_PREP) -+ page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; -+ return 0; -+} -+ -+/* -+ * Frees a number of pages from the PCP lists -+ * Assumes all pages on list are in same zone, and of same order. -+ * count is the number of pages to free. -+ * -+ * If the zone was previously in an "all pages pinned" state then look to -+ * see if this freeing clears that state. -+ * -+ * And clear the zone's pages_scanned counter, to hold off the "all pages are -+ * pinned" detection logic. -+ */ -+static void free_pcppages_bulk(struct zone *zone, int count, -+ struct per_cpu_pages *pcp) -+{ -+ int migratetype = 0; -+ int batch_free = 0; -+ int to_free = count; -+ unsigned long nr_scanned; -+ -+ spin_lock(&zone->lock); -+ nr_scanned = zone_page_state(zone, NR_PAGES_SCANNED); -+ if (nr_scanned) -+ __mod_zone_page_state(zone, NR_PAGES_SCANNED, -nr_scanned); -+ -+ while (to_free) { -+ struct page *page; -+ struct list_head *list; -+ -+ /* -+ * Remove pages from lists in a round-robin fashion. A -+ * batch_free count is maintained that is incremented when an -+ * empty list is encountered. This is so more pages are freed -+ * off fuller lists instead of spinning excessively around empty -+ * lists -+ */ -+ do { -+ batch_free++; -+ if (++migratetype == MIGRATE_PCPTYPES) -+ migratetype = 0; -+ list = &pcp->lists[migratetype]; -+ } while (list_empty(list)); -+ -+ /* This is the only non-empty list. Free them all. */ -+ if (batch_free == MIGRATE_PCPTYPES) -+ batch_free = to_free; -+ -+ do { -+ int mt; /* migratetype of the to-be-freed page */ -+ -+ page = list_entry(list->prev, struct page, lru); -+ /* must delete as __free_one_page list manipulates */ -+ list_del(&page->lru); -+ mt = get_freepage_migratetype(page); -+ if (unlikely(has_isolate_pageblock(zone))) -+ mt = get_pageblock_migratetype(page); -+ -+ /* MIGRATE_MOVABLE list may include MIGRATE_RESERVEs */ -+ __free_one_page(page, page_to_pfn(page), zone, 0, mt); -+ trace_mm_page_pcpu_drain(page, 0, mt); -+ } while (--to_free && --batch_free && !list_empty(list)); -+ } -+ spin_unlock(&zone->lock); -+} -+ -+static void free_one_page(struct zone *zone, -+ struct page *page, unsigned long pfn, -+ unsigned int order, -+ int migratetype) -+{ -+ unsigned long nr_scanned; -+ spin_lock(&zone->lock); -+ nr_scanned = zone_page_state(zone, NR_PAGES_SCANNED); -+ if (nr_scanned) -+ __mod_zone_page_state(zone, NR_PAGES_SCANNED, -nr_scanned); -+ -+ if (unlikely(has_isolate_pageblock(zone) || -+ is_migrate_isolate(migratetype))) { -+ migratetype = get_pfnblock_migratetype(page, pfn); -+ } -+ __free_one_page(page, pfn, zone, order, migratetype); -+ spin_unlock(&zone->lock); -+} -+ -+static int free_tail_pages_check(struct page *head_page, struct page *page) -+{ -+ if (!IS_ENABLED(CONFIG_DEBUG_VM)) -+ return 0; -+ if (unlikely(!PageTail(page))) { -+ bad_page(page, "PageTail not set", 0); -+ return 1; -+ } -+ if (unlikely(page->first_page != head_page)) { -+ bad_page(page, "first_page not consistent", 0); -+ return 1; -+ } -+ return 0; -+} -+ -+static bool free_pages_prepare(struct page *page, unsigned int order) -+{ -+ bool compound = PageCompound(page); -+ int i, bad = 0; -+ -+ VM_BUG_ON_PAGE(PageTail(page), page); -+ VM_BUG_ON_PAGE(compound && compound_order(page) != order, page); -+ -+ trace_mm_page_free(page, order); -+ kmemcheck_free_shadow(page, order); -+ kasan_free_pages(page, order); -+ -+ if (PageAnon(page)) -+ page->mapping = NULL; -+ bad += free_pages_check(page); -+ for (i = 1; i < (1 << order); i++) { -+ if (compound) -+ bad += free_tail_pages_check(page, page + i); -+ bad += free_pages_check(page + i); -+ } -+ if (bad) -+ return false; -+ -+ reset_page_owner(page, order); -+ -+ if (!PageHighMem(page)) { -+ debug_check_no_locks_freed(page_address(page), -+ PAGE_SIZE << order); -+ debug_check_no_obj_freed(page_address(page), -+ PAGE_SIZE << order); -+ } -+ arch_free_page(page, order); -+ kernel_map_pages(page, 1 << order, 0); -+ -+ return true; -+} -+ -+static void __free_pages_ok(struct page *page, unsigned int order) -+{ -+ unsigned long flags; -+ int migratetype; -+ unsigned long pfn = page_to_pfn(page); -+ -+ if (!free_pages_prepare(page, order)) -+ return; -+ -+ migratetype = get_pfnblock_migratetype(page, pfn); -+ local_irq_save(flags); -+ __count_vm_events(PGFREE, 1 << order); -+ set_freepage_migratetype(page, migratetype); -+ free_one_page(page_zone(page), page, pfn, order, migratetype); -+ local_irq_restore(flags); -+} -+ -+void __init __free_pages_bootmem(struct page *page, unsigned int order) -+{ -+ unsigned int nr_pages = 1 << order; -+ struct page *p = page; -+ unsigned int loop; -+ -+ prefetchw(p); -+ for (loop = 0; loop < (nr_pages - 1); loop++, p++) { -+ prefetchw(p + 1); -+ __ClearPageReserved(p); -+ set_page_count(p, 0); -+ } -+ __ClearPageReserved(p); -+ set_page_count(p, 0); -+ -+ page_zone(page)->managed_pages += nr_pages; -+ set_page_refcounted(page); -+ __free_pages(page, order); -+} -+ -+#ifdef CONFIG_CMA -+/* Free whole pageblock and set its migration type to MIGRATE_CMA. */ -+void __init init_cma_reserved_pageblock(struct page *page) -+{ -+ unsigned i = pageblock_nr_pages; -+ struct page *p = page; -+ -+ do { -+ __ClearPageReserved(p); -+ set_page_count(p, 0); -+ } while (++p, --i); -+ -+ set_pageblock_migratetype(page, MIGRATE_CMA); -+ -+ if (pageblock_order >= MAX_ORDER) { -+ i = pageblock_nr_pages; -+ p = page; -+ do { -+ set_page_refcounted(p); -+ __free_pages(p, MAX_ORDER - 1); -+ p += MAX_ORDER_NR_PAGES; -+ } while (i -= MAX_ORDER_NR_PAGES); -+ } else { -+ set_page_refcounted(page); -+ __free_pages(page, pageblock_order); -+ } -+ -+ adjust_managed_page_count(page, pageblock_nr_pages); -+} -+#endif -+ -+/* -+ * The order of subdivision here is critical for the IO subsystem. -+ * Please do not alter this order without good reasons and regression -+ * testing. Specifically, as large blocks of memory are subdivided, -+ * the order in which smaller blocks are delivered depends on the order -+ * they're subdivided in this function. This is the primary factor -+ * influencing the order in which pages are delivered to the IO -+ * subsystem according to empirical testing, and this is also justified -+ * by considering the behavior of a buddy system containing a single -+ * large block of memory acted on by a series of small allocations. -+ * This behavior is a critical factor in sglist merging's success. -+ * -+ * -- nyc -+ */ -+static inline void expand(struct zone *zone, struct page *page, -+ int low, int high, struct free_area *area, -+ int migratetype) -+{ -+ unsigned long size = 1 << high; -+ -+ while (high > low) { -+ area--; -+ high--; -+ size >>= 1; -+ VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]); -+ -+ if (IS_ENABLED(CONFIG_DEBUG_PAGEALLOC) && -+ debug_guardpage_enabled() && -+ high < debug_guardpage_minorder()) { -+ /* -+ * Mark as guard pages (or page), that will allow to -+ * merge back to allocator when buddy will be freed. -+ * Corresponding page table entries will not be touched, -+ * pages will stay not present in virtual address space -+ */ -+ set_page_guard(zone, &page[size], high, migratetype); -+ continue; -+ } -+ list_add(&page[size].lru, &area->free_list[migratetype]); -+ area->nr_free++; -+ set_page_order(&page[size], high); -+ } -+} -+ -+/* -+ * This page is about to be returned from the page allocator -+ */ -+static inline int check_new_page(struct page *page) -+{ -+ const char *bad_reason = NULL; -+ unsigned long bad_flags = 0; -+ -+ if (unlikely(page_mapcount(page))) -+ bad_reason = "nonzero mapcount"; -+ if (unlikely(page->mapping != NULL)) -+ bad_reason = "non-NULL mapping"; -+ if (unlikely(atomic_read(&page->_count) != 0)) -+ bad_reason = "nonzero _count"; -+ if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_PREP)) { -+ bad_reason = "PAGE_FLAGS_CHECK_AT_PREP flag set"; -+ bad_flags = PAGE_FLAGS_CHECK_AT_PREP; -+ } -+#ifdef CONFIG_MEMCG -+ if (unlikely(page->mem_cgroup)) -+ bad_reason = "page still charged to cgroup"; -+#endif -+ if (unlikely(bad_reason)) { -+ bad_page(page, bad_reason, bad_flags); -+ return 1; -+ } -+ return 0; -+} -+ -+static int prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags, -+ int alloc_flags) -+{ -+ int i; -+ -+ for (i = 0; i < (1 << order); i++) { -+ struct page *p = page + i; -+ if (unlikely(check_new_page(p))) -+ return 1; -+ } -+ -+ set_page_private(page, 0); -+ set_page_refcounted(page); -+ -+ arch_alloc_page(page, order); -+ kernel_map_pages(page, 1 << order, 1); -+ kasan_alloc_pages(page, order); -+ -+ if (gfp_flags & __GFP_ZERO) -+ prep_zero_page(page, order, gfp_flags); -+ -+ if (order && (gfp_flags & __GFP_COMP)) -+ prep_compound_page(page, order); -+ -+ set_page_owner(page, order, gfp_flags); -+ -+ /* -+ * page is set pfmemalloc when ALLOC_NO_WATERMARKS was necessary to -+ * allocate the page. The expectation is that the caller is taking -+ * steps that will free more memory. The caller should avoid the page -+ * being used for !PFMEMALLOC purposes. -+ */ -+ if (alloc_flags & ALLOC_NO_WATERMARKS) -+ set_page_pfmemalloc(page); -+ else -+ clear_page_pfmemalloc(page); -+ -+ return 0; -+} -+ -+/* -+ * Go through the free lists for the given migratetype and remove -+ * the smallest available page from the freelists -+ */ -+static inline -+struct page *__rmqueue_smallest(struct zone *zone, unsigned int order, -+ int migratetype) -+{ -+ unsigned int current_order; -+ struct free_area *area; -+ struct page *page; -+ -+ /* Find a page of the appropriate size in the preferred list */ -+ for (current_order = order; current_order < MAX_ORDER; ++current_order) { -+ area = &(zone->free_area[current_order]); -+ if (list_empty(&area->free_list[migratetype])) -+ continue; -+ -+ page = list_entry(area->free_list[migratetype].next, -+ struct page, lru); -+ list_del(&page->lru); -+ rmv_page_order(page); -+ area->nr_free--; -+ expand(zone, page, order, current_order, area, migratetype); -+ set_freepage_migratetype(page, migratetype); -+ return page; -+ } -+ -+ return NULL; -+} -+ -+ -+/* -+ * This array describes the order lists are fallen back to when -+ * the free lists for the desirable migrate type are depleted -+ */ -+static int fallbacks[MIGRATE_TYPES][4] = { -+ [MIGRATE_UNMOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE, MIGRATE_RESERVE }, -+ [MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, MIGRATE_RESERVE }, -+ [MIGRATE_MOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_RESERVE }, -+#ifdef CONFIG_CMA -+ [MIGRATE_CMA] = { MIGRATE_RESERVE }, /* Never used */ -+#endif -+ [MIGRATE_RESERVE] = { MIGRATE_RESERVE }, /* Never used */ -+#ifdef CONFIG_MEMORY_ISOLATION -+ [MIGRATE_ISOLATE] = { MIGRATE_RESERVE }, /* Never used */ -+#endif -+}; -+ -+#ifdef CONFIG_CMA -+static struct page *__rmqueue_cma_fallback(struct zone *zone, -+ unsigned int order) -+{ -+ return __rmqueue_smallest(zone, order, MIGRATE_CMA); -+} -+#else -+static inline struct page *__rmqueue_cma_fallback(struct zone *zone, -+ unsigned int order) { return NULL; } -+#endif -+ -+/* -+ * Move the free pages in a range to the free lists of the requested type. -+ * Note that start_page and end_pages are not aligned on a pageblock -+ * boundary. If alignment is required, use move_freepages_block() -+ */ -+int move_freepages(struct zone *zone, -+ struct page *start_page, struct page *end_page, -+ int migratetype) -+{ -+ struct page *page; -+ unsigned long order; -+ int pages_moved = 0; -+ -+#ifndef CONFIG_HOLES_IN_ZONE -+ /* -+ * page_zone is not safe to call in this context when -+ * CONFIG_HOLES_IN_ZONE is set. This bug check is probably redundant -+ * anyway as we check zone boundaries in move_freepages_block(). -+ * Remove at a later date when no bug reports exist related to -+ * grouping pages by mobility -+ */ -+ VM_BUG_ON(page_zone(start_page) != page_zone(end_page)); -+#endif -+ -+ for (page = start_page; page <= end_page;) { -+ /* Make sure we are not inadvertently changing nodes */ -+ VM_BUG_ON_PAGE(page_to_nid(page) != zone_to_nid(zone), page); -+ -+ if (!pfn_valid_within(page_to_pfn(page))) { -+ page++; -+ continue; -+ } -+ -+ if (!PageBuddy(page)) { -+ page++; -+ continue; -+ } -+ -+ order = page_order(page); -+ list_move(&page->lru, -+ &zone->free_area[order].free_list[migratetype]); -+ set_freepage_migratetype(page, migratetype); -+ page += 1 << order; -+ pages_moved += 1 << order; -+ } -+ -+ return pages_moved; -+} -+ -+int move_freepages_block(struct zone *zone, struct page *page, -+ int migratetype) -+{ -+ unsigned long start_pfn, end_pfn; -+ struct page *start_page, *end_page; -+ -+ start_pfn = page_to_pfn(page); -+ start_pfn = start_pfn & ~(pageblock_nr_pages-1); -+ start_page = pfn_to_page(start_pfn); -+ end_page = start_page + pageblock_nr_pages - 1; -+ end_pfn = start_pfn + pageblock_nr_pages - 1; -+ -+ /* Do not cross zone boundaries */ -+ if (!zone_spans_pfn(zone, start_pfn)) -+ start_page = page; -+ if (!zone_spans_pfn(zone, end_pfn)) -+ return 0; -+ -+ return move_freepages(zone, start_page, end_page, migratetype); -+} -+ -+static void change_pageblock_range(struct page *pageblock_page, -+ int start_order, int migratetype) -+{ -+ int nr_pageblocks = 1 << (start_order - pageblock_order); -+ -+ while (nr_pageblocks--) { -+ set_pageblock_migratetype(pageblock_page, migratetype); -+ pageblock_page += pageblock_nr_pages; -+ } -+} -+ -+/* -+ * When we are falling back to another migratetype during allocation, try to -+ * steal extra free pages from the same pageblocks to satisfy further -+ * allocations, instead of polluting multiple pageblocks. -+ * -+ * If we are stealing a relatively large buddy page, it is likely there will -+ * be more free pages in the pageblock, so try to steal them all. For -+ * reclaimable and unmovable allocations, we steal regardless of page size, -+ * as fragmentation caused by those allocations polluting movable pageblocks -+ * is worse than movable allocations stealing from unmovable and reclaimable -+ * pageblocks. -+ */ -+static bool can_steal_fallback(unsigned int order, int start_mt) -+{ -+ /* -+ * Leaving this order check is intended, although there is -+ * relaxed order check in next check. The reason is that -+ * we can actually steal whole pageblock if this condition met, -+ * but, below check doesn't guarantee it and that is just heuristic -+ * so could be changed anytime. -+ */ -+ if (order >= pageblock_order) -+ return true; -+ -+ if (order >= pageblock_order / 2 || -+ start_mt == MIGRATE_RECLAIMABLE || -+ start_mt == MIGRATE_UNMOVABLE || -+ page_group_by_mobility_disabled) -+ return true; -+ -+ return false; -+} -+ -+/* -+ * This function implements actual steal behaviour. If order is large enough, -+ * we can steal whole pageblock. If not, we first move freepages in this -+ * pageblock and check whether half of pages are moved or not. If half of -+ * pages are moved, we can change migratetype of pageblock and permanently -+ * use it's pages as requested migratetype in the future. -+ */ -+static void steal_suitable_fallback(struct zone *zone, struct page *page, -+ int start_type) -+{ -+ int current_order = page_order(page); -+ int pages; -+ -+ /* Take ownership for orders >= pageblock_order */ -+ if (current_order >= pageblock_order) { -+ change_pageblock_range(page, current_order, start_type); -+ return; -+ } -+ -+ pages = move_freepages_block(zone, page, start_type); -+ -+ /* Claim the whole block if over half of it is free */ -+ if (pages >= (1 << (pageblock_order-1)) || -+ page_group_by_mobility_disabled) -+ set_pageblock_migratetype(page, start_type); -+} -+ -+/* -+ * Check whether there is a suitable fallback freepage with requested order. -+ * If only_stealable is true, this function returns fallback_mt only if -+ * we can steal other freepages all together. This would help to reduce -+ * fragmentation due to mixed migratetype pages in one pageblock. -+ */ -+int find_suitable_fallback(struct free_area *area, unsigned int order, -+ int migratetype, bool only_stealable, bool *can_steal) -+{ -+ int i; -+ int fallback_mt; -+ -+ if (area->nr_free == 0) -+ return -1; -+ -+ *can_steal = false; -+ for (i = 0;; i++) { -+ fallback_mt = fallbacks[migratetype][i]; -+ if (fallback_mt == MIGRATE_RESERVE) -+ break; -+ -+ if (list_empty(&area->free_list[fallback_mt])) -+ continue; -+ -+ if (can_steal_fallback(order, migratetype)) -+ *can_steal = true; -+ -+ if (!only_stealable) -+ return fallback_mt; -+ -+ if (*can_steal) -+ return fallback_mt; -+ } -+ -+ return -1; -+} -+ -+/* Remove an element from the buddy allocator from the fallback list */ -+static inline struct page * -+__rmqueue_fallback(struct zone *zone, unsigned int order, int start_migratetype) -+{ -+ struct free_area *area; -+ unsigned int current_order; -+ struct page *page; -+ int fallback_mt; -+ bool can_steal; -+ -+ /* Find the largest possible block of pages in the other list */ -+ for (current_order = MAX_ORDER-1; -+ current_order >= order && current_order <= MAX_ORDER-1; -+ --current_order) { -+ area = &(zone->free_area[current_order]); -+ fallback_mt = find_suitable_fallback(area, current_order, -+ start_migratetype, false, &can_steal); -+ if (fallback_mt == -1) -+ continue; -+ -+ page = list_entry(area->free_list[fallback_mt].next, -+ struct page, lru); -+ if (can_steal) -+ steal_suitable_fallback(zone, page, start_migratetype); -+ -+ /* Remove the page from the freelists */ -+ area->nr_free--; -+ list_del(&page->lru); -+ rmv_page_order(page); -+ -+ expand(zone, page, order, current_order, area, -+ start_migratetype); -+ /* -+ * The freepage_migratetype may differ from pageblock's -+ * migratetype depending on the decisions in -+ * try_to_steal_freepages(). This is OK as long as it -+ * does not differ for MIGRATE_CMA pageblocks. For CMA -+ * we need to make sure unallocated pages flushed from -+ * pcp lists are returned to the correct freelist. -+ */ -+ set_freepage_migratetype(page, start_migratetype); -+ -+ trace_mm_page_alloc_extfrag(page, order, current_order, -+ start_migratetype, fallback_mt); -+ -+ return page; -+ } -+ -+ return NULL; -+} -+ -+/* -+ * Do the hard work of removing an element from the buddy allocator. -+ * Call me with the zone->lock already held. -+ */ -+static struct page *__rmqueue(struct zone *zone, unsigned int order, -+ int migratetype) -+{ -+ struct page *page; -+ -+retry_reserve: -+ page = __rmqueue_smallest(zone, order, migratetype); -+ -+ if (unlikely(!page) && migratetype != MIGRATE_RESERVE) { -+ if (migratetype == MIGRATE_MOVABLE) -+ page = __rmqueue_cma_fallback(zone, order); -+ -+ if (!page) -+ page = __rmqueue_fallback(zone, order, migratetype); -+ -+ /* -+ * Use MIGRATE_RESERVE rather than fail an allocation. goto -+ * is used because __rmqueue_smallest is an inline function -+ * and we want just one call site -+ */ -+ if (!page) { -+ migratetype = MIGRATE_RESERVE; -+ goto retry_reserve; -+ } -+ } -+ -+ trace_mm_page_alloc_zone_locked(page, order, migratetype); -+ return page; -+} -+ -+/* -+ * Obtain a specified number of elements from the buddy allocator, all under -+ * a single hold of the lock, for efficiency. Add them to the supplied list. -+ * Returns the number of new pages which were placed at *list. -+ */ -+static int rmqueue_bulk(struct zone *zone, unsigned int order, -+ unsigned long count, struct list_head *list, -+ int migratetype, bool cold) -+{ -+ int i; -+ -+ spin_lock(&zone->lock); -+ for (i = 0; i < count; ++i) { -+ struct page *page = __rmqueue(zone, order, migratetype); -+ if (unlikely(page == NULL)) -+ break; -+ -+ /* -+ * Split buddy pages returned by expand() are received here -+ * in physical page order. The page is added to the callers and -+ * list and the list head then moves forward. From the callers -+ * perspective, the linked list is ordered by page number in -+ * some conditions. This is useful for IO devices that can -+ * merge IO requests if the physical pages are ordered -+ * properly. -+ */ -+ if (likely(!cold)) -+ list_add(&page->lru, list); -+ else -+ list_add_tail(&page->lru, list); -+ list = &page->lru; -+ if (is_migrate_cma(get_freepage_migratetype(page))) -+ __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, -+ -(1 << order)); -+ } -+ __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order)); -+ spin_unlock(&zone->lock); -+ return i; -+} -+ -+#ifdef CONFIG_NUMA -+/* -+ * Called from the vmstat counter updater to drain pagesets of this -+ * currently executing processor on remote nodes after they have -+ * expired. -+ * -+ * Note that this function must be called with the thread pinned to -+ * a single processor. -+ */ -+void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp) -+{ -+ unsigned long flags; -+ int to_drain, batch; -+ -+ local_irq_save(flags); -+ batch = READ_ONCE(pcp->batch); -+ to_drain = min(pcp->count, batch); -+ if (to_drain > 0) { -+ free_pcppages_bulk(zone, to_drain, pcp); -+ pcp->count -= to_drain; -+ } -+ local_irq_restore(flags); -+} -+#endif -+ -+/* -+ * Drain pcplists of the indicated processor and zone. -+ * -+ * The processor must either be the current processor and the -+ * thread pinned to the current processor or a processor that -+ * is not online. -+ */ -+static void drain_pages_zone(unsigned int cpu, struct zone *zone) -+{ -+ unsigned long flags; -+ struct per_cpu_pageset *pset; -+ struct per_cpu_pages *pcp; -+ -+ local_irq_save(flags); -+ pset = per_cpu_ptr(zone->pageset, cpu); -+ -+ pcp = &pset->pcp; -+ if (pcp->count) { -+ free_pcppages_bulk(zone, pcp->count, pcp); -+ pcp->count = 0; -+ } -+ local_irq_restore(flags); -+} -+ -+/* -+ * Drain pcplists of all zones on the indicated processor. -+ * -+ * The processor must either be the current processor and the -+ * thread pinned to the current processor or a processor that -+ * is not online. -+ */ -+static void drain_pages(unsigned int cpu) -+{ -+ struct zone *zone; -+ -+ for_each_populated_zone(zone) { -+ drain_pages_zone(cpu, zone); -+ } -+} -+ -+/* -+ * Spill all of this CPU's per-cpu pages back into the buddy allocator. -+ * -+ * The CPU has to be pinned. When zone parameter is non-NULL, spill just -+ * the single zone's pages. -+ */ -+void drain_local_pages(struct zone *zone) -+{ -+ int cpu = smp_processor_id(); -+ -+ if (zone) -+ drain_pages_zone(cpu, zone); -+ else -+ drain_pages(cpu); -+} -+ -+/* -+ * Spill all the per-cpu pages from all CPUs back into the buddy allocator. -+ * -+ * When zone parameter is non-NULL, spill just the single zone's pages. -+ * -+ * Note that this code is protected against sending an IPI to an offline -+ * CPU but does not guarantee sending an IPI to newly hotplugged CPUs: -+ * on_each_cpu_mask() blocks hotplug and won't talk to offlined CPUs but -+ * nothing keeps CPUs from showing up after we populated the cpumask and -+ * before the call to on_each_cpu_mask(). -+ */ -+void drain_all_pages(struct zone *zone) -+{ -+ int cpu; -+ -+ /* -+ * Allocate in the BSS so we wont require allocation in -+ * direct reclaim path for CONFIG_CPUMASK_OFFSTACK=y -+ */ -+ static cpumask_t cpus_with_pcps; -+ -+ /* -+ * We don't care about racing with CPU hotplug event -+ * as offline notification will cause the notified -+ * cpu to drain that CPU pcps and on_each_cpu_mask -+ * disables preemption as part of its processing -+ */ -+ for_each_online_cpu(cpu) { -+ struct per_cpu_pageset *pcp; -+ struct zone *z; -+ bool has_pcps = false; -+ -+ if (zone) { -+ pcp = per_cpu_ptr(zone->pageset, cpu); -+ if (pcp->pcp.count) -+ has_pcps = true; -+ } else { -+ for_each_populated_zone(z) { -+ pcp = per_cpu_ptr(z->pageset, cpu); -+ if (pcp->pcp.count) { -+ has_pcps = true; -+ break; -+ } -+ } -+ } -+ -+ if (has_pcps) -+ cpumask_set_cpu(cpu, &cpus_with_pcps); -+ else -+ cpumask_clear_cpu(cpu, &cpus_with_pcps); -+ } -+ on_each_cpu_mask(&cpus_with_pcps, (smp_call_func_t) drain_local_pages, -+ zone, 1); -+} -+ -+#ifdef CONFIG_HIBERNATION -+ -+void mark_free_pages(struct zone *zone) -+{ -+ unsigned long pfn, max_zone_pfn; -+ unsigned long flags; -+ unsigned int order, t; -+ struct list_head *curr; -+ -+ if (zone_is_empty(zone)) -+ return; -+ -+ spin_lock_irqsave(&zone->lock, flags); -+ -+ max_zone_pfn = zone_end_pfn(zone); -+ for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) -+ if (pfn_valid(pfn)) { -+ struct page *page = pfn_to_page(pfn); -+ -+ if (!swsusp_page_is_forbidden(page)) -+ swsusp_unset_page_free(page); -+ } -+ -+ for_each_migratetype_order(order, t) { -+ list_for_each(curr, &zone->free_area[order].free_list[t]) { -+ unsigned long i; -+ -+ pfn = page_to_pfn(list_entry(curr, struct page, lru)); -+ for (i = 0; i < (1UL << order); i++) -+ swsusp_set_page_free(pfn_to_page(pfn + i)); -+ } -+ } -+ spin_unlock_irqrestore(&zone->lock, flags); -+} -+#endif /* CONFIG_PM */ -+ -+/* -+ * Free a 0-order page -+ * cold == true ? free a cold page : free a hot page -+ */ -+void free_hot_cold_page(struct page *page, bool cold) -+{ -+ struct zone *zone = page_zone(page); -+ struct per_cpu_pages *pcp; -+ unsigned long flags; -+ unsigned long pfn = page_to_pfn(page); -+ int migratetype; -+ -+ if (!free_pages_prepare(page, 0)) -+ return; -+ -+ migratetype = get_pfnblock_migratetype(page, pfn); -+ set_freepage_migratetype(page, migratetype); -+ local_irq_save(flags); -+ __count_vm_event(PGFREE); -+ -+ /* -+ * We only track unmovable, reclaimable and movable on pcp lists. -+ * Free ISOLATE pages back to the allocator because they are being -+ * offlined but treat RESERVE as movable pages so we can get those -+ * areas back if necessary. Otherwise, we may have to free -+ * excessively into the page allocator -+ */ -+ if (migratetype >= MIGRATE_PCPTYPES) { -+ if (unlikely(is_migrate_isolate(migratetype))) { -+ free_one_page(zone, page, pfn, 0, migratetype); -+ goto out; -+ } -+ migratetype = MIGRATE_MOVABLE; -+ } -+ -+ pcp = &this_cpu_ptr(zone->pageset)->pcp; -+ if (!cold) -+ list_add(&page->lru, &pcp->lists[migratetype]); -+ else -+ list_add_tail(&page->lru, &pcp->lists[migratetype]); -+ pcp->count++; -+ if (pcp->count >= pcp->high) { -+ unsigned long batch = READ_ONCE(pcp->batch); -+ free_pcppages_bulk(zone, batch, pcp); -+ pcp->count -= batch; -+ } -+ -+out: -+ local_irq_restore(flags); -+} -+ -+/* -+ * Free a list of 0-order pages -+ */ -+void free_hot_cold_page_list(struct list_head *list, bool cold) -+{ -+ struct page *page, *next; -+ -+ list_for_each_entry_safe(page, next, list, lru) { -+ trace_mm_page_free_batched(page, cold); -+ free_hot_cold_page(page, cold); -+ } -+} -+ -+/* -+ * split_page takes a non-compound higher-order page, and splits it into -+ * n (1<lru); -+ zone->free_area[order].nr_free--; -+ rmv_page_order(page); -+ -+ /* Set the pageblock if the isolated page is at least a pageblock */ -+ if (order >= pageblock_order - 1) { -+ struct page *endpage = page + (1 << order) - 1; -+ for (; page < endpage; page += pageblock_nr_pages) { -+ int mt = get_pageblock_migratetype(page); -+ if (!is_migrate_isolate(mt) && !is_migrate_cma(mt)) -+ set_pageblock_migratetype(page, -+ MIGRATE_MOVABLE); -+ } -+ } -+ -+ set_page_owner(page, order, 0); -+ return 1UL << order; -+} -+ -+/* -+ * Similar to split_page except the page is already free. As this is only -+ * being used for migration, the migratetype of the block also changes. -+ * As this is called with interrupts disabled, the caller is responsible -+ * for calling arch_alloc_page() and kernel_map_page() after interrupts -+ * are enabled. -+ * -+ * Note: this is probably too low level an operation for use in drivers. -+ * Please consult with lkml before using this in your driver. -+ */ -+int split_free_page(struct page *page) -+{ -+ unsigned int order; -+ int nr_pages; -+ -+ order = page_order(page); -+ -+ nr_pages = __isolate_free_page(page, order); -+ if (!nr_pages) -+ return 0; -+ -+ /* Split into individual pages */ -+ set_page_refcounted(page); -+ split_page(page, order); -+ return nr_pages; -+} -+ -+/* -+ * Allocate a page from the given zone. Use pcplists for order-0 allocations. -+ */ -+static inline -+struct page *buffered_rmqueue(struct zone *preferred_zone, -+ struct zone *zone, unsigned int order, -+ gfp_t gfp_flags, int migratetype) -+{ -+ unsigned long flags; -+ struct page *page; -+ bool cold = ((gfp_flags & __GFP_COLD) != 0); -+ -+ if (likely(order == 0)) { -+ struct per_cpu_pages *pcp; -+ struct list_head *list; -+ -+ local_irq_save(flags); -+ pcp = &this_cpu_ptr(zone->pageset)->pcp; -+ list = &pcp->lists[migratetype]; -+ if (list_empty(list)) { -+ pcp->count += rmqueue_bulk(zone, 0, -+ pcp->batch, list, -+ migratetype, cold); -+ if (unlikely(list_empty(list))) -+ goto failed; -+ } -+ -+ if (cold) -+ page = list_entry(list->prev, struct page, lru); -+ else -+ page = list_entry(list->next, struct page, lru); -+ -+ list_del(&page->lru); -+ pcp->count--; -+ } else { -+ if (unlikely(gfp_flags & __GFP_NOFAIL)) { -+ /* -+ * __GFP_NOFAIL is not to be used in new code. -+ * -+ * All __GFP_NOFAIL callers should be fixed so that they -+ * properly detect and handle allocation failures. -+ * -+ * We most definitely don't want callers attempting to -+ * allocate greater than order-1 page units with -+ * __GFP_NOFAIL. -+ */ -+ WARN_ON_ONCE(order > 1); -+ } -+ spin_lock_irqsave(&zone->lock, flags); -+ page = __rmqueue(zone, order, migratetype); -+ spin_unlock(&zone->lock); -+ if (!page) -+ goto failed; -+ __mod_zone_freepage_state(zone, -(1 << order), -+ get_freepage_migratetype(page)); -+ } -+ -+ __mod_zone_page_state(zone, NR_ALLOC_BATCH, -(1 << order)); -+ if (atomic_long_read(&zone->vm_stat[NR_ALLOC_BATCH]) <= 0 && -+ !test_bit(ZONE_FAIR_DEPLETED, &zone->flags)) -+ set_bit(ZONE_FAIR_DEPLETED, &zone->flags); -+ -+ __count_zone_vm_events(PGALLOC, zone, 1 << order); -+ zone_statistics(preferred_zone, zone, gfp_flags); -+ local_irq_restore(flags); -+ -+ VM_BUG_ON_PAGE(bad_range(zone, page), page); -+ return page; -+ -+failed: -+ local_irq_restore(flags); -+ return NULL; -+} -+ -+#ifdef CONFIG_FAIL_PAGE_ALLOC -+ -+static struct { -+ struct fault_attr attr; -+ -+ u32 ignore_gfp_highmem; -+ u32 ignore_gfp_wait; -+ u32 min_order; -+} fail_page_alloc = { -+ .attr = FAULT_ATTR_INITIALIZER, -+ .ignore_gfp_wait = 1, -+ .ignore_gfp_highmem = 1, -+ .min_order = 1, -+}; -+ -+static int __init setup_fail_page_alloc(char *str) -+{ -+ return setup_fault_attr(&fail_page_alloc.attr, str); -+} -+__setup("fail_page_alloc=", setup_fail_page_alloc); -+ -+static bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) -+{ -+ if (order < fail_page_alloc.min_order) -+ return false; -+ if (gfp_mask & __GFP_NOFAIL) -+ return false; -+ if (fail_page_alloc.ignore_gfp_highmem && (gfp_mask & __GFP_HIGHMEM)) -+ return false; -+ if (fail_page_alloc.ignore_gfp_wait && (gfp_mask & __GFP_WAIT)) -+ return false; -+ -+ return should_fail(&fail_page_alloc.attr, 1 << order); -+} -+ -+#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS -+ -+static int __init fail_page_alloc_debugfs(void) -+{ -+ umode_t mode = S_IFREG | S_IRUSR | S_IWUSR; -+ struct dentry *dir; -+ -+ dir = fault_create_debugfs_attr("fail_page_alloc", NULL, -+ &fail_page_alloc.attr); -+ if (IS_ERR(dir)) -+ return PTR_ERR(dir); -+ -+ if (!debugfs_create_bool("ignore-gfp-wait", mode, dir, -+ &fail_page_alloc.ignore_gfp_wait)) -+ goto fail; -+ if (!debugfs_create_bool("ignore-gfp-highmem", mode, dir, -+ &fail_page_alloc.ignore_gfp_highmem)) -+ goto fail; -+ if (!debugfs_create_u32("min-order", mode, dir, -+ &fail_page_alloc.min_order)) -+ goto fail; -+ -+ return 0; -+fail: -+ debugfs_remove_recursive(dir); -+ -+ return -ENOMEM; -+} -+ -+late_initcall(fail_page_alloc_debugfs); -+ -+#endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */ -+ -+#else /* CONFIG_FAIL_PAGE_ALLOC */ -+ -+static inline bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) -+{ -+ return false; -+} -+ -+#endif /* CONFIG_FAIL_PAGE_ALLOC */ -+ -+/* -+ * Return true if free pages are above 'mark'. This takes into account the order -+ * of the allocation. -+ */ -+static bool __zone_watermark_ok(struct zone *z, unsigned int order, -+ unsigned long mark, int classzone_idx, int alloc_flags, -+ long free_pages) -+{ -+ /* free_pages may go negative - that's OK */ -+ long min = mark; -+ int o; -+ long free_cma = 0; -+ -+ free_pages -= (1 << order) - 1; -+ if (alloc_flags & ALLOC_HIGH) -+ min -= min / 2; -+ if (alloc_flags & ALLOC_HARDER) -+ min -= min / 4; -+#ifdef CONFIG_CMA -+ /* If allocation can't use CMA areas don't use free CMA pages */ -+ if (!(alloc_flags & ALLOC_CMA)) -+ free_cma = zone_page_state(z, NR_FREE_CMA_PAGES); -+#endif -+ -+ if (free_pages - free_cma <= min + z->lowmem_reserve[classzone_idx]) -+ return false; -+ for (o = 0; o < order; o++) { -+ /* At the next order, this order's pages become unavailable */ -+ free_pages -= z->free_area[o].nr_free << o; -+ -+ /* Require fewer higher order pages to be free */ -+ min >>= 1; -+ -+ if (free_pages <= min) -+ return false; -+ } -+ return true; -+} -+ -+bool zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, -+ int classzone_idx, int alloc_flags) -+{ -+ return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags, -+ zone_page_state(z, NR_FREE_PAGES)); -+} -+ -+bool zone_watermark_ok_safe(struct zone *z, unsigned int order, -+ unsigned long mark, int classzone_idx, int alloc_flags) -+{ -+ long free_pages = zone_page_state(z, NR_FREE_PAGES); -+ -+ if (z->percpu_drift_mark && free_pages < z->percpu_drift_mark) -+ free_pages = zone_page_state_snapshot(z, NR_FREE_PAGES); -+ -+ return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags, -+ free_pages); -+} -+ -+#ifdef CONFIG_NUMA -+/* -+ * zlc_setup - Setup for "zonelist cache". Uses cached zone data to -+ * skip over zones that are not allowed by the cpuset, or that have -+ * been recently (in last second) found to be nearly full. See further -+ * comments in mmzone.h. Reduces cache footprint of zonelist scans -+ * that have to skip over a lot of full or unallowed zones. -+ * -+ * If the zonelist cache is present in the passed zonelist, then -+ * returns a pointer to the allowed node mask (either the current -+ * tasks mems_allowed, or node_states[N_MEMORY].) -+ * -+ * If the zonelist cache is not available for this zonelist, does -+ * nothing and returns NULL. -+ * -+ * If the fullzones BITMAP in the zonelist cache is stale (more than -+ * a second since last zap'd) then we zap it out (clear its bits.) -+ * -+ * We hold off even calling zlc_setup, until after we've checked the -+ * first zone in the zonelist, on the theory that most allocations will -+ * be satisfied from that first zone, so best to examine that zone as -+ * quickly as we can. -+ */ -+static nodemask_t *zlc_setup(struct zonelist *zonelist, int alloc_flags) -+{ -+ struct zonelist_cache *zlc; /* cached zonelist speedup info */ -+ nodemask_t *allowednodes; /* zonelist_cache approximation */ -+ -+ zlc = zonelist->zlcache_ptr; -+ if (!zlc) -+ return NULL; -+ -+ if (time_after(jiffies, zlc->last_full_zap + HZ)) { -+ bitmap_zero(zlc->fullzones, MAX_ZONES_PER_ZONELIST); -+ zlc->last_full_zap = jiffies; -+ } -+ -+ allowednodes = !in_interrupt() && (alloc_flags & ALLOC_CPUSET) ? -+ &cpuset_current_mems_allowed : -+ &node_states[N_MEMORY]; -+ return allowednodes; -+} -+ -+/* -+ * Given 'z' scanning a zonelist, run a couple of quick checks to see -+ * if it is worth looking at further for free memory: -+ * 1) Check that the zone isn't thought to be full (doesn't have its -+ * bit set in the zonelist_cache fullzones BITMAP). -+ * 2) Check that the zones node (obtained from the zonelist_cache -+ * z_to_n[] mapping) is allowed in the passed in allowednodes mask. -+ * Return true (non-zero) if zone is worth looking at further, or -+ * else return false (zero) if it is not. -+ * -+ * This check -ignores- the distinction between various watermarks, -+ * such as GFP_HIGH, GFP_ATOMIC, PF_MEMALLOC, ... If a zone is -+ * found to be full for any variation of these watermarks, it will -+ * be considered full for up to one second by all requests, unless -+ * we are so low on memory on all allowed nodes that we are forced -+ * into the second scan of the zonelist. -+ * -+ * In the second scan we ignore this zonelist cache and exactly -+ * apply the watermarks to all zones, even it is slower to do so. -+ * We are low on memory in the second scan, and should leave no stone -+ * unturned looking for a free page. -+ */ -+static int zlc_zone_worth_trying(struct zonelist *zonelist, struct zoneref *z, -+ nodemask_t *allowednodes) -+{ -+ struct zonelist_cache *zlc; /* cached zonelist speedup info */ -+ int i; /* index of *z in zonelist zones */ -+ int n; /* node that zone *z is on */ -+ -+ zlc = zonelist->zlcache_ptr; -+ if (!zlc) -+ return 1; -+ -+ i = z - zonelist->_zonerefs; -+ n = zlc->z_to_n[i]; -+ -+ /* This zone is worth trying if it is allowed but not full */ -+ return node_isset(n, *allowednodes) && !test_bit(i, zlc->fullzones); -+} -+ -+/* -+ * Given 'z' scanning a zonelist, set the corresponding bit in -+ * zlc->fullzones, so that subsequent attempts to allocate a page -+ * from that zone don't waste time re-examining it. -+ */ -+static void zlc_mark_zone_full(struct zonelist *zonelist, struct zoneref *z) -+{ -+ struct zonelist_cache *zlc; /* cached zonelist speedup info */ -+ int i; /* index of *z in zonelist zones */ -+ -+ zlc = zonelist->zlcache_ptr; -+ if (!zlc) -+ return; -+ -+ i = z - zonelist->_zonerefs; -+ -+ set_bit(i, zlc->fullzones); -+} -+ -+/* -+ * clear all zones full, called after direct reclaim makes progress so that -+ * a zone that was recently full is not skipped over for up to a second -+ */ -+static void zlc_clear_zones_full(struct zonelist *zonelist) -+{ -+ struct zonelist_cache *zlc; /* cached zonelist speedup info */ -+ -+ zlc = zonelist->zlcache_ptr; -+ if (!zlc) -+ return; -+ -+ bitmap_zero(zlc->fullzones, MAX_ZONES_PER_ZONELIST); -+} -+ -+static bool zone_local(struct zone *local_zone, struct zone *zone) -+{ -+ return local_zone->node == zone->node; -+} -+ -+static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone) -+{ -+ return node_distance(zone_to_nid(local_zone), zone_to_nid(zone)) < -+ RECLAIM_DISTANCE; -+} -+ -+#else /* CONFIG_NUMA */ -+ -+static nodemask_t *zlc_setup(struct zonelist *zonelist, int alloc_flags) -+{ -+ return NULL; -+} -+ -+static int zlc_zone_worth_trying(struct zonelist *zonelist, struct zoneref *z, -+ nodemask_t *allowednodes) -+{ -+ return 1; -+} -+ -+static void zlc_mark_zone_full(struct zonelist *zonelist, struct zoneref *z) -+{ -+} -+ -+static void zlc_clear_zones_full(struct zonelist *zonelist) -+{ -+} -+ -+static bool zone_local(struct zone *local_zone, struct zone *zone) -+{ -+ return true; -+} -+ -+static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone) -+{ -+ return true; -+} -+ -+#endif /* CONFIG_NUMA */ -+ -+static void reset_alloc_batches(struct zone *preferred_zone) -+{ -+ struct zone *zone = preferred_zone->zone_pgdat->node_zones; -+ -+ do { -+ mod_zone_page_state(zone, NR_ALLOC_BATCH, -+ high_wmark_pages(zone) - low_wmark_pages(zone) - -+ atomic_long_read(&zone->vm_stat[NR_ALLOC_BATCH])); -+ clear_bit(ZONE_FAIR_DEPLETED, &zone->flags); -+ } while (zone++ != preferred_zone); -+} -+ -+/* -+ * get_page_from_freelist goes through the zonelist trying to allocate -+ * a page. -+ */ -+static struct page * -+get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags, -+ const struct alloc_context *ac) -+{ -+ struct zonelist *zonelist = ac->zonelist; -+ struct zoneref *z; -+ struct page *page = NULL; -+ struct zone *zone; -+ nodemask_t *allowednodes = NULL;/* zonelist_cache approximation */ -+ int zlc_active = 0; /* set if using zonelist_cache */ -+ int did_zlc_setup = 0; /* just call zlc_setup() one time */ -+ bool consider_zone_dirty = (alloc_flags & ALLOC_WMARK_LOW) && -+ (gfp_mask & __GFP_WRITE); -+ int nr_fair_skipped = 0; -+ bool zonelist_rescan; -+ -+zonelist_scan: -+ zonelist_rescan = false; -+ -+ /* -+ * Scan zonelist, looking for a zone with enough free. -+ * See also __cpuset_node_allowed() comment in kernel/cpuset.c. -+ */ -+ for_each_zone_zonelist_nodemask(zone, z, zonelist, ac->high_zoneidx, -+ ac->nodemask) { -+ unsigned long mark; -+ -+ if (IS_ENABLED(CONFIG_NUMA) && zlc_active && -+ !zlc_zone_worth_trying(zonelist, z, allowednodes)) -+ continue; -+ if (cpusets_enabled() && -+ (alloc_flags & ALLOC_CPUSET) && -+ !cpuset_zone_allowed(zone, gfp_mask)) -+ continue; -+ /* -+ * Distribute pages in proportion to the individual -+ * zone size to ensure fair page aging. The zone a -+ * page was allocated in should have no effect on the -+ * time the page has in memory before being reclaimed. -+ */ -+ if (alloc_flags & ALLOC_FAIR) { -+ if (!zone_local(ac->preferred_zone, zone)) -+ break; -+ if (test_bit(ZONE_FAIR_DEPLETED, &zone->flags)) { -+ nr_fair_skipped++; -+ continue; -+ } -+ } -+ /* -+ * When allocating a page cache page for writing, we -+ * want to get it from a zone that is within its dirty -+ * limit, such that no single zone holds more than its -+ * proportional share of globally allowed dirty pages. -+ * The dirty limits take into account the zone's -+ * lowmem reserves and high watermark so that kswapd -+ * should be able to balance it without having to -+ * write pages from its LRU list. -+ * -+ * This may look like it could increase pressure on -+ * lower zones by failing allocations in higher zones -+ * before they are full. But the pages that do spill -+ * over are limited as the lower zones are protected -+ * by this very same mechanism. It should not become -+ * a practical burden to them. -+ * -+ * XXX: For now, allow allocations to potentially -+ * exceed the per-zone dirty limit in the slowpath -+ * (ALLOC_WMARK_LOW unset) before going into reclaim, -+ * which is important when on a NUMA setup the allowed -+ * zones are together not big enough to reach the -+ * global limit. The proper fix for these situations -+ * will require awareness of zones in the -+ * dirty-throttling and the flusher threads. -+ */ -+ if (consider_zone_dirty && !zone_dirty_ok(zone)) -+ continue; -+ -+ mark = zone->watermark[alloc_flags & ALLOC_WMARK_MASK]; -+ if (!zone_watermark_ok(zone, order, mark, -+ ac->classzone_idx, alloc_flags)) { -+ int ret; -+ -+ /* Checked here to keep the fast path fast */ -+ BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK); -+ if (alloc_flags & ALLOC_NO_WATERMARKS) -+ goto try_this_zone; -+ -+ if (IS_ENABLED(CONFIG_NUMA) && -+ !did_zlc_setup && nr_online_nodes > 1) { -+ /* -+ * we do zlc_setup if there are multiple nodes -+ * and before considering the first zone allowed -+ * by the cpuset. -+ */ -+ allowednodes = zlc_setup(zonelist, alloc_flags); -+ zlc_active = 1; -+ did_zlc_setup = 1; -+ } -+ -+ if (zone_reclaim_mode == 0 || -+ !zone_allows_reclaim(ac->preferred_zone, zone)) -+ goto this_zone_full; -+ -+ /* -+ * As we may have just activated ZLC, check if the first -+ * eligible zone has failed zone_reclaim recently. -+ */ -+ if (IS_ENABLED(CONFIG_NUMA) && zlc_active && -+ !zlc_zone_worth_trying(zonelist, z, allowednodes)) -+ continue; -+ -+ ret = zone_reclaim(zone, gfp_mask, order); -+ switch (ret) { -+ case ZONE_RECLAIM_NOSCAN: -+ /* did not scan */ -+ continue; -+ case ZONE_RECLAIM_FULL: -+ /* scanned but unreclaimable */ -+ continue; -+ default: -+ /* did we reclaim enough */ -+ if (zone_watermark_ok(zone, order, mark, -+ ac->classzone_idx, alloc_flags)) -+ goto try_this_zone; -+ -+ /* -+ * Failed to reclaim enough to meet watermark. -+ * Only mark the zone full if checking the min -+ * watermark or if we failed to reclaim just -+ * 1<preferred_zone, zone, order, -+ gfp_mask, ac->migratetype); -+ if (page) { -+ if (prep_new_page(page, order, gfp_mask, alloc_flags)) -+ goto try_this_zone; -+ return page; -+ } -+this_zone_full: -+ if (IS_ENABLED(CONFIG_NUMA) && zlc_active) -+ zlc_mark_zone_full(zonelist, z); -+ } -+ -+ /* -+ * The first pass makes sure allocations are spread fairly within the -+ * local node. However, the local node might have free pages left -+ * after the fairness batches are exhausted, and remote zones haven't -+ * even been considered yet. Try once more without fairness, and -+ * include remote zones now, before entering the slowpath and waking -+ * kswapd: prefer spilling to a remote zone over swapping locally. -+ */ -+ if (alloc_flags & ALLOC_FAIR) { -+ alloc_flags &= ~ALLOC_FAIR; -+ if (nr_fair_skipped) { -+ zonelist_rescan = true; -+ reset_alloc_batches(ac->preferred_zone); -+ } -+ if (nr_online_nodes > 1) -+ zonelist_rescan = true; -+ } -+ -+ if (unlikely(IS_ENABLED(CONFIG_NUMA) && zlc_active)) { -+ /* Disable zlc cache for second zonelist scan */ -+ zlc_active = 0; -+ zonelist_rescan = true; -+ } -+ -+ if (zonelist_rescan) -+ goto zonelist_scan; -+ -+ return NULL; -+} -+ -+/* -+ * Large machines with many possible nodes should not always dump per-node -+ * meminfo in irq context. -+ */ -+static inline bool should_suppress_show_mem(void) -+{ -+ bool ret = false; -+ -+#if NODES_SHIFT > 8 -+ ret = in_interrupt(); -+#endif -+ return ret; -+} -+ -+static DEFINE_RATELIMIT_STATE(nopage_rs, -+ DEFAULT_RATELIMIT_INTERVAL, -+ DEFAULT_RATELIMIT_BURST); -+ -+void warn_alloc_failed(gfp_t gfp_mask, int order, const char *fmt, ...) -+{ -+ unsigned int filter = SHOW_MEM_FILTER_NODES; -+ -+ if ((gfp_mask & __GFP_NOWARN) || !__ratelimit(&nopage_rs) || -+ debug_guardpage_minorder() > 0) -+ return; -+ -+ /* -+ * This documents exceptions given to allocations in certain -+ * contexts that are allowed to allocate outside current's set -+ * of allowed nodes. -+ */ -+ if (!(gfp_mask & __GFP_NOMEMALLOC)) -+ if (test_thread_flag(TIF_MEMDIE) || -+ (current->flags & (PF_MEMALLOC | PF_EXITING))) -+ filter &= ~SHOW_MEM_FILTER_NODES; -+ if (in_interrupt() || !(gfp_mask & __GFP_WAIT)) -+ filter &= ~SHOW_MEM_FILTER_NODES; -+ -+ if (fmt) { -+ struct va_format vaf; -+ va_list args; -+ -+ va_start(args, fmt); -+ -+ vaf.fmt = fmt; -+ vaf.va = &args; -+ -+ pr_warn("%pV", &vaf); -+ -+ va_end(args); -+ } -+ -+ pr_warn("%s: page allocation failure: order:%d, mode:0x%x\n", -+ current->comm, order, gfp_mask); -+ -+ dump_stack(); -+ if (!should_suppress_show_mem()) -+ show_mem(filter); -+} -+ -+static inline int -+should_alloc_retry(gfp_t gfp_mask, unsigned int order, -+ unsigned long did_some_progress, -+ unsigned long pages_reclaimed) -+{ -+ /* Do not loop if specifically requested */ -+ if (gfp_mask & __GFP_NORETRY) -+ return 0; -+ -+ /* Always retry if specifically requested */ -+ if (gfp_mask & __GFP_NOFAIL) -+ return 1; -+ -+ /* -+ * Suspend converts GFP_KERNEL to __GFP_WAIT which can prevent reclaim -+ * making forward progress without invoking OOM. Suspend also disables -+ * storage devices so kswapd will not help. Bail if we are suspending. -+ */ -+ if (!did_some_progress && pm_suspended_storage()) -+ return 0; -+ -+ /* -+ * In this implementation, order <= PAGE_ALLOC_COSTLY_ORDER -+ * means __GFP_NOFAIL, but that may not be true in other -+ * implementations. -+ */ -+ if (order <= PAGE_ALLOC_COSTLY_ORDER) -+ return 1; -+ -+ /* -+ * For order > PAGE_ALLOC_COSTLY_ORDER, if __GFP_REPEAT is -+ * specified, then we retry until we no longer reclaim any pages -+ * (above), or we've reclaimed an order of pages at least as -+ * large as the allocation's order. In both cases, if the -+ * allocation still fails, we stop retrying. -+ */ -+ if (gfp_mask & __GFP_REPEAT && pages_reclaimed < (1 << order)) -+ return 1; -+ -+ return 0; -+} -+ -+static inline struct page * -+__alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order, -+ const struct alloc_context *ac, unsigned long *did_some_progress) -+{ -+ struct page *page; -+ -+ *did_some_progress = 0; -+ -+ /* -+ * Acquire the per-zone oom lock for each zone. If that -+ * fails, somebody else is making progress for us. -+ */ -+ if (!oom_zonelist_trylock(ac->zonelist, gfp_mask)) { -+ *did_some_progress = 1; -+ schedule_timeout_uninterruptible(1); -+ return NULL; -+ } -+ -+ /* -+ * Go through the zonelist yet one more time, keep very high watermark -+ * here, this is only to catch a parallel oom killing, we must fail if -+ * we're still under heavy pressure. -+ */ -+ page = get_page_from_freelist(gfp_mask | __GFP_HARDWALL, order, -+ ALLOC_WMARK_HIGH|ALLOC_CPUSET, ac); -+ if (page) -+ goto out; -+ -+ if (!(gfp_mask & __GFP_NOFAIL)) { -+ /* Coredumps can quickly deplete all memory reserves */ -+ if (current->flags & PF_DUMPCORE) -+ goto out; -+ /* The OOM killer will not help higher order allocs */ -+ if (order > PAGE_ALLOC_COSTLY_ORDER) -+ goto out; -+ /* The OOM killer does not needlessly kill tasks for lowmem */ -+ if (ac->high_zoneidx < ZONE_NORMAL) -+ goto out; -+ /* The OOM killer does not compensate for light reclaim */ -+ if (!(gfp_mask & __GFP_FS)) { -+ /* -+ * XXX: Page reclaim didn't yield anything, -+ * and the OOM killer can't be invoked, but -+ * keep looping as per should_alloc_retry(). -+ */ -+ *did_some_progress = 1; -+ goto out; -+ } -+ /* The OOM killer may not free memory on a specific node */ -+ if (gfp_mask & __GFP_THISNODE) -+ goto out; -+ } -+ /* Exhausted what can be done so it's blamo time */ -+ if (out_of_memory(ac->zonelist, gfp_mask, order, ac->nodemask, false) -+ || WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL)) -+ *did_some_progress = 1; -+out: -+ oom_zonelist_unlock(ac->zonelist, gfp_mask); -+ return page; -+} -+ -+#ifdef CONFIG_COMPACTION -+/* Try memory compaction for high-order allocations before reclaim */ -+static struct page * -+__alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, -+ int alloc_flags, const struct alloc_context *ac, -+ enum migrate_mode mode, int *contended_compaction, -+ bool *deferred_compaction) -+{ -+ unsigned long compact_result; -+ struct page *page; -+ -+ if (!order) -+ return NULL; -+ -+ current->flags |= PF_MEMALLOC; -+ compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac, -+ mode, contended_compaction); -+ current->flags &= ~PF_MEMALLOC; -+ -+ switch (compact_result) { -+ case COMPACT_DEFERRED: -+ *deferred_compaction = true; -+ /* fall-through */ -+ case COMPACT_SKIPPED: -+ return NULL; -+ default: -+ break; -+ } -+ -+ /* -+ * At least in one zone compaction wasn't deferred or skipped, so let's -+ * count a compaction stall -+ */ -+ count_vm_event(COMPACTSTALL); -+ -+ page = get_page_from_freelist(gfp_mask, order, -+ alloc_flags & ~ALLOC_NO_WATERMARKS, ac); -+ -+ if (page) { -+ struct zone *zone = page_zone(page); -+ -+ zone->compact_blockskip_flush = false; -+ compaction_defer_reset(zone, order, true); -+ count_vm_event(COMPACTSUCCESS); -+ return page; -+ } -+ -+ /* -+ * It's bad if compaction run occurs and fails. The most likely reason -+ * is that pages exist, but not enough to satisfy watermarks. -+ */ -+ count_vm_event(COMPACTFAIL); -+ -+ cond_resched(); -+ -+ return NULL; -+} -+#else -+static inline struct page * -+__alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, -+ int alloc_flags, const struct alloc_context *ac, -+ enum migrate_mode mode, int *contended_compaction, -+ bool *deferred_compaction) -+{ -+ return NULL; -+} -+#endif /* CONFIG_COMPACTION */ -+ -+/* Perform direct synchronous page reclaim */ -+static int -+__perform_reclaim(gfp_t gfp_mask, unsigned int order, -+ const struct alloc_context *ac) -+{ -+ struct reclaim_state reclaim_state; -+ int progress; -+ -+ cond_resched(); -+ -+ /* We now go into synchronous reclaim */ -+ cpuset_memory_pressure_bump(); -+ current->flags |= PF_MEMALLOC; -+ lockdep_set_current_reclaim_state(gfp_mask); -+ reclaim_state.reclaimed_slab = 0; -+ current->reclaim_state = &reclaim_state; -+ -+ progress = try_to_free_pages(ac->zonelist, order, gfp_mask, -+ ac->nodemask); -+ -+ current->reclaim_state = NULL; -+ lockdep_clear_current_reclaim_state(); -+ current->flags &= ~PF_MEMALLOC; -+ -+ cond_resched(); -+ -+ return progress; -+} -+ -+/* The really slow allocator path where we enter direct reclaim */ -+static inline struct page * -+__alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order, -+ int alloc_flags, const struct alloc_context *ac, -+ unsigned long *did_some_progress) -+{ -+ struct page *page = NULL; -+ bool drained = false; -+ -+ *did_some_progress = __perform_reclaim(gfp_mask, order, ac); -+ if (unlikely(!(*did_some_progress))) -+ return NULL; -+ -+ /* After successful reclaim, reconsider all zones for allocation */ -+ if (IS_ENABLED(CONFIG_NUMA)) -+ zlc_clear_zones_full(ac->zonelist); -+ -+retry: -+ page = get_page_from_freelist(gfp_mask, order, -+ alloc_flags & ~ALLOC_NO_WATERMARKS, ac); -+ -+ /* -+ * If an allocation failed after direct reclaim, it could be because -+ * pages are pinned on the per-cpu lists. Drain them and try again -+ */ -+ if (!page && !drained) { -+ drain_all_pages(NULL); -+ drained = true; -+ goto retry; -+ } -+ -+ return page; -+} -+ -+/* -+ * This is called in the allocator slow-path if the allocation request is of -+ * sufficient urgency to ignore watermarks and take other desperate measures -+ */ -+static inline struct page * -+__alloc_pages_high_priority(gfp_t gfp_mask, unsigned int order, -+ const struct alloc_context *ac) -+{ -+ struct page *page; -+ -+ do { -+ page = get_page_from_freelist(gfp_mask, order, -+ ALLOC_NO_WATERMARKS, ac); -+ -+ if (!page && gfp_mask & __GFP_NOFAIL) -+ wait_iff_congested(ac->preferred_zone, BLK_RW_ASYNC, -+ HZ/50); -+ } while (!page && (gfp_mask & __GFP_NOFAIL)); -+ -+ return page; -+} -+ -+static void wake_all_kswapds(unsigned int order, const struct alloc_context *ac) -+{ -+ struct zoneref *z; -+ struct zone *zone; -+ -+ for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, -+ ac->high_zoneidx, ac->nodemask) -+ wakeup_kswapd(zone, order, zone_idx(ac->preferred_zone)); -+} -+ -+static inline int -+gfp_to_alloc_flags(gfp_t gfp_mask) -+{ -+ int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET; -+ const bool atomic = !(gfp_mask & (__GFP_WAIT | __GFP_NO_KSWAPD)); -+ -+ /* __GFP_HIGH is assumed to be the same as ALLOC_HIGH to save a branch. */ -+ BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_HIGH); -+ -+ /* -+ * The caller may dip into page reserves a bit more if the caller -+ * cannot run direct reclaim, or if the caller has realtime scheduling -+ * policy or is asking for __GFP_HIGH memory. GFP_ATOMIC requests will -+ * set both ALLOC_HARDER (atomic == true) and ALLOC_HIGH (__GFP_HIGH). -+ */ -+ alloc_flags |= (__force int) (gfp_mask & __GFP_HIGH); -+ -+ if (atomic) { -+ /* -+ * Not worth trying to allocate harder for __GFP_NOMEMALLOC even -+ * if it can't schedule. -+ */ -+ if (!(gfp_mask & __GFP_NOMEMALLOC)) -+ alloc_flags |= ALLOC_HARDER; -+ /* -+ * Ignore cpuset mems for GFP_ATOMIC rather than fail, see the -+ * comment for __cpuset_node_allowed(). -+ */ -+ alloc_flags &= ~ALLOC_CPUSET; -+ } else if (unlikely(rt_task(current)) && !in_interrupt()) -+ alloc_flags |= ALLOC_HARDER; -+ -+ if (likely(!(gfp_mask & __GFP_NOMEMALLOC))) { -+ if (gfp_mask & __GFP_MEMALLOC) -+ alloc_flags |= ALLOC_NO_WATERMARKS; -+ else if (in_serving_softirq() && (current->flags & PF_MEMALLOC)) -+ alloc_flags |= ALLOC_NO_WATERMARKS; -+ else if (!in_interrupt() && -+ ((current->flags & PF_MEMALLOC) || -+ unlikely(test_thread_flag(TIF_MEMDIE)))) -+ alloc_flags |= ALLOC_NO_WATERMARKS; -+ } -+#ifdef CONFIG_CMA -+ if (gfpflags_to_migratetype(gfp_mask) == MIGRATE_MOVABLE) -+ alloc_flags |= ALLOC_CMA; -+#endif -+ return alloc_flags; -+} -+ -+bool gfp_pfmemalloc_allowed(gfp_t gfp_mask) -+{ -+ return !!(gfp_to_alloc_flags(gfp_mask) & ALLOC_NO_WATERMARKS); -+} -+ -+static inline struct page * -+__alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, -+ struct alloc_context *ac) -+{ -+ const gfp_t wait = gfp_mask & __GFP_WAIT; -+ struct page *page = NULL; -+ int alloc_flags; -+ unsigned long pages_reclaimed = 0; -+ unsigned long did_some_progress; -+ enum migrate_mode migration_mode = MIGRATE_ASYNC; -+ bool deferred_compaction = false; -+ int contended_compaction = COMPACT_CONTENDED_NONE; -+ -+ /* -+ * In the slowpath, we sanity check order to avoid ever trying to -+ * reclaim >= MAX_ORDER areas which will never succeed. Callers may -+ * be using allocators in order of preference for an area that is -+ * too large. -+ */ -+ if (order >= MAX_ORDER) { -+ WARN_ON_ONCE(!(gfp_mask & __GFP_NOWARN)); -+ return NULL; -+ } -+ -+ /* -+ * If this allocation cannot block and it is for a specific node, then -+ * fail early. There's no need to wakeup kswapd or retry for a -+ * speculative node-specific allocation. -+ */ -+ if (IS_ENABLED(CONFIG_NUMA) && (gfp_mask & __GFP_THISNODE) && !wait) -+ goto nopage; -+ -+retry: -+ if (!(gfp_mask & __GFP_NO_KSWAPD)) -+ wake_all_kswapds(order, ac); -+ -+ /* -+ * OK, we're below the kswapd watermark and have kicked background -+ * reclaim. Now things get more complex, so set up alloc_flags according -+ * to how we want to proceed. -+ */ -+ alloc_flags = gfp_to_alloc_flags(gfp_mask); -+ -+ /* -+ * Find the true preferred zone if the allocation is unconstrained by -+ * cpusets. -+ */ -+ if (!(alloc_flags & ALLOC_CPUSET) && !ac->nodemask) { -+ struct zoneref *preferred_zoneref; -+ preferred_zoneref = first_zones_zonelist(ac->zonelist, -+ ac->high_zoneidx, NULL, &ac->preferred_zone); -+ ac->classzone_idx = zonelist_zone_idx(preferred_zoneref); -+ } -+ -+ /* This is the last chance, in general, before the goto nopage. */ -+ page = get_page_from_freelist(gfp_mask, order, -+ alloc_flags & ~ALLOC_NO_WATERMARKS, ac); -+ if (page) -+ goto got_pg; -+ -+ /* Allocate without watermarks if the context allows */ -+ if (alloc_flags & ALLOC_NO_WATERMARKS) { -+ /* -+ * Ignore mempolicies if ALLOC_NO_WATERMARKS on the grounds -+ * the allocation is high priority and these type of -+ * allocations are system rather than user orientated -+ */ -+ ac->zonelist = node_zonelist(numa_node_id(), gfp_mask); -+ -+ page = __alloc_pages_high_priority(gfp_mask, order, ac); -+ -+ if (page) { -+ goto got_pg; -+ } -+ } -+ -+ /* Atomic allocations - we can't balance anything */ -+ if (!wait) { -+ /* -+ * All existing users of the deprecated __GFP_NOFAIL are -+ * blockable, so warn of any new users that actually allow this -+ * type of allocation to fail. -+ */ -+ WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL); -+ goto nopage; -+ } -+ -+ /* Avoid recursion of direct reclaim */ -+ if (current->flags & PF_MEMALLOC) -+ goto nopage; -+ -+ /* Avoid allocations with no watermarks from looping endlessly */ -+ if (test_thread_flag(TIF_MEMDIE) && !(gfp_mask & __GFP_NOFAIL)) -+ goto nopage; -+ -+ /* -+ * Try direct compaction. The first pass is asynchronous. Subsequent -+ * attempts after direct reclaim are synchronous -+ */ -+ page = __alloc_pages_direct_compact(gfp_mask, order, alloc_flags, ac, -+ migration_mode, -+ &contended_compaction, -+ &deferred_compaction); -+ if (page) -+ goto got_pg; -+ -+ /* Checks for THP-specific high-order allocations */ -+ if ((gfp_mask & GFP_TRANSHUGE) == GFP_TRANSHUGE) { -+ /* -+ * If compaction is deferred for high-order allocations, it is -+ * because sync compaction recently failed. If this is the case -+ * and the caller requested a THP allocation, we do not want -+ * to heavily disrupt the system, so we fail the allocation -+ * instead of entering direct reclaim. -+ */ -+ if (deferred_compaction) -+ goto nopage; -+ -+ /* -+ * In all zones where compaction was attempted (and not -+ * deferred or skipped), lock contention has been detected. -+ * For THP allocation we do not want to disrupt the others -+ * so we fallback to base pages instead. -+ */ -+ if (contended_compaction == COMPACT_CONTENDED_LOCK) -+ goto nopage; -+ -+ /* -+ * If compaction was aborted due to need_resched(), we do not -+ * want to further increase allocation latency, unless it is -+ * khugepaged trying to collapse. -+ */ -+ if (contended_compaction == COMPACT_CONTENDED_SCHED -+ && !(current->flags & PF_KTHREAD)) -+ goto nopage; -+ } -+ -+ /* -+ * It can become very expensive to allocate transparent hugepages at -+ * fault, so use asynchronous memory compaction for THP unless it is -+ * khugepaged trying to collapse. -+ */ -+ if ((gfp_mask & GFP_TRANSHUGE) != GFP_TRANSHUGE || -+ (current->flags & PF_KTHREAD)) -+ migration_mode = MIGRATE_SYNC_LIGHT; -+ -+ /* Try direct reclaim and then allocating */ -+ page = __alloc_pages_direct_reclaim(gfp_mask, order, alloc_flags, ac, -+ &did_some_progress); -+ if (page) -+ goto got_pg; -+ -+ /* Check if we should retry the allocation */ -+ pages_reclaimed += did_some_progress; -+ if (should_alloc_retry(gfp_mask, order, did_some_progress, -+ pages_reclaimed)) { -+ /* -+ * If we fail to make progress by freeing individual -+ * pages, but the allocation wants us to keep going, -+ * start OOM killing tasks. -+ */ -+ if (!did_some_progress) { -+ page = __alloc_pages_may_oom(gfp_mask, order, ac, -+ &did_some_progress); -+ if (page) -+ goto got_pg; -+ if (!did_some_progress) -+ goto nopage; -+ } -+ /* Wait for some write requests to complete then retry */ -+ wait_iff_congested(ac->preferred_zone, BLK_RW_ASYNC, HZ/50); -+ goto retry; -+ } else { -+ /* -+ * High-order allocations do not necessarily loop after -+ * direct reclaim and reclaim/compaction depends on compaction -+ * being called after reclaim so call directly if necessary -+ */ -+ page = __alloc_pages_direct_compact(gfp_mask, order, -+ alloc_flags, ac, migration_mode, -+ &contended_compaction, -+ &deferred_compaction); -+ if (page) -+ goto got_pg; -+ } -+ -+nopage: -+ warn_alloc_failed(gfp_mask, order, NULL); -+got_pg: -+ return page; -+} -+ -+/* -+ * This is the 'heart' of the zoned buddy allocator. -+ */ -+struct page * -+__alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order, -+ struct zonelist *zonelist, nodemask_t *nodemask) -+{ -+ struct zoneref *preferred_zoneref; -+ struct page *page = NULL; -+ unsigned int cpuset_mems_cookie; -+ int alloc_flags = ALLOC_WMARK_LOW|ALLOC_CPUSET|ALLOC_FAIR; -+ gfp_t alloc_mask; /* The gfp_t that was actually used for allocation */ -+ struct alloc_context ac = { -+ .high_zoneidx = gfp_zone(gfp_mask), -+ .nodemask = nodemask, -+ .migratetype = gfpflags_to_migratetype(gfp_mask), -+ }; -+ -+ gfp_mask &= gfp_allowed_mask; -+ -+ lockdep_trace_alloc(gfp_mask); -+ -+ might_sleep_if(gfp_mask & __GFP_WAIT); -+ -+ if (should_fail_alloc_page(gfp_mask, order)) -+ return NULL; -+ -+ /* -+ * Check the zones suitable for the gfp_mask contain at least one -+ * valid zone. It's possible to have an empty zonelist as a result -+ * of __GFP_THISNODE and a memoryless node -+ */ -+ if (unlikely(!zonelist->_zonerefs->zone)) -+ return NULL; -+ -+ if (IS_ENABLED(CONFIG_CMA) && ac.migratetype == MIGRATE_MOVABLE) -+ alloc_flags |= ALLOC_CMA; -+ -+retry_cpuset: -+ cpuset_mems_cookie = read_mems_allowed_begin(); -+ -+ /* We set it here, as __alloc_pages_slowpath might have changed it */ -+ ac.zonelist = zonelist; -+ /* The preferred zone is used for statistics later */ -+ preferred_zoneref = first_zones_zonelist(ac.zonelist, ac.high_zoneidx, -+ ac.nodemask ? : &cpuset_current_mems_allowed, -+ &ac.preferred_zone); -+ if (!ac.preferred_zone) -+ goto out; -+ ac.classzone_idx = zonelist_zone_idx(preferred_zoneref); -+ -+ /* First allocation attempt */ -+ alloc_mask = gfp_mask|__GFP_HARDWALL; -+ page = get_page_from_freelist(alloc_mask, order, alloc_flags, &ac); -+ if (unlikely(!page)) { -+ /* -+ * Runtime PM, block IO and its error handling path -+ * can deadlock because I/O on the device might not -+ * complete. -+ */ -+ alloc_mask = memalloc_noio_flags(gfp_mask); -+ -+ page = __alloc_pages_slowpath(alloc_mask, order, &ac); -+ } -+ -+ if (kmemcheck_enabled && page) -+ kmemcheck_pagealloc_alloc(page, order, gfp_mask); -+ -+ trace_mm_page_alloc(page, order, alloc_mask, ac.migratetype); -+ -+out: -+ /* -+ * When updating a task's mems_allowed, it is possible to race with -+ * parallel threads in such a way that an allocation can fail while -+ * the mask is being updated. If a page allocation is about to fail, -+ * check if the cpuset changed during allocation and if so, retry. -+ */ -+ if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie))) -+ goto retry_cpuset; -+ -+ return page; -+} -+EXPORT_SYMBOL(__alloc_pages_nodemask); -+ -+/* -+ * Common helper functions. -+ */ -+unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order) -+{ -+ struct page *page; -+ -+ /* -+ * __get_free_pages() returns a 32-bit address, which cannot represent -+ * a highmem page -+ */ -+ VM_BUG_ON((gfp_mask & __GFP_HIGHMEM) != 0); -+ -+ page = alloc_pages(gfp_mask, order); -+ if (!page) -+ return 0; -+ return (unsigned long) page_address(page); -+} -+EXPORT_SYMBOL(__get_free_pages); -+ -+unsigned long get_zeroed_page(gfp_t gfp_mask) -+{ -+ return __get_free_pages(gfp_mask | __GFP_ZERO, 0); -+} -+EXPORT_SYMBOL(get_zeroed_page); -+ -+void __free_pages(struct page *page, unsigned int order) -+{ -+ if (put_page_testzero(page)) { -+ if (order == 0) -+ free_hot_cold_page(page, false); -+ else -+ __free_pages_ok(page, order); -+ } -+} -+ -+EXPORT_SYMBOL(__free_pages); -+ -+void free_pages(unsigned long addr, unsigned int order) -+{ -+ if (addr != 0) { -+ VM_BUG_ON(!virt_addr_valid((void *)addr)); -+ __free_pages(virt_to_page((void *)addr), order); -+ } -+} -+ -+EXPORT_SYMBOL(free_pages); -+ -+/* -+ * alloc_kmem_pages charges newly allocated pages to the kmem resource counter -+ * of the current memory cgroup. -+ * -+ * It should be used when the caller would like to use kmalloc, but since the -+ * allocation is large, it has to fall back to the page allocator. -+ */ -+struct page *alloc_kmem_pages(gfp_t gfp_mask, unsigned int order) -+{ -+ struct page *page; -+ struct mem_cgroup *memcg = NULL; -+ -+ if (!memcg_kmem_newpage_charge(gfp_mask, &memcg, order)) -+ return NULL; -+ page = alloc_pages(gfp_mask, order); -+ memcg_kmem_commit_charge(page, memcg, order); -+ return page; -+} -+ -+struct page *alloc_kmem_pages_node(int nid, gfp_t gfp_mask, unsigned int order) -+{ -+ struct page *page; -+ struct mem_cgroup *memcg = NULL; -+ -+ if (!memcg_kmem_newpage_charge(gfp_mask, &memcg, order)) -+ return NULL; -+ page = alloc_pages_node(nid, gfp_mask, order); -+ memcg_kmem_commit_charge(page, memcg, order); -+ return page; -+} -+ -+/* -+ * __free_kmem_pages and free_kmem_pages will free pages allocated with -+ * alloc_kmem_pages. -+ */ -+void __free_kmem_pages(struct page *page, unsigned int order) -+{ -+ memcg_kmem_uncharge_pages(page, order); -+ __free_pages(page, order); -+} -+ -+void free_kmem_pages(unsigned long addr, unsigned int order) -+{ -+ if (addr != 0) { -+ VM_BUG_ON(!virt_addr_valid((void *)addr)); -+ __free_kmem_pages(virt_to_page((void *)addr), order); -+ } -+} -+ -+static void *make_alloc_exact(unsigned long addr, unsigned order, size_t size) -+{ -+ if (addr) { -+ unsigned long alloc_end = addr + (PAGE_SIZE << order); -+ unsigned long used = addr + PAGE_ALIGN(size); -+ -+ split_page(virt_to_page((void *)addr), order); -+ while (used < alloc_end) { -+ free_page(used); -+ used += PAGE_SIZE; -+ } -+ } -+ return (void *)addr; -+} -+ -+/** -+ * alloc_pages_exact - allocate an exact number physically-contiguous pages. -+ * @size: the number of bytes to allocate -+ * @gfp_mask: GFP flags for the allocation -+ * -+ * This function is similar to alloc_pages(), except that it allocates the -+ * minimum number of pages to satisfy the request. alloc_pages() can only -+ * allocate memory in power-of-two pages. -+ * -+ * This function is also limited by MAX_ORDER. -+ * -+ * Memory allocated by this function must be released by free_pages_exact(). -+ */ -+void *alloc_pages_exact(size_t size, gfp_t gfp_mask) -+{ -+ unsigned int order = get_order(size); -+ unsigned long addr; -+ -+ addr = __get_free_pages(gfp_mask, order); -+ return make_alloc_exact(addr, order, size); -+} -+EXPORT_SYMBOL(alloc_pages_exact); -+ -+/** -+ * alloc_pages_exact_nid - allocate an exact number of physically-contiguous -+ * pages on a node. -+ * @nid: the preferred node ID where memory should be allocated -+ * @size: the number of bytes to allocate -+ * @gfp_mask: GFP flags for the allocation -+ * -+ * Like alloc_pages_exact(), but try to allocate on node nid first before falling -+ * back. -+ * Note this is not alloc_pages_exact_node() which allocates on a specific node, -+ * but is not exact. -+ */ -+void * __meminit alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask) -+{ -+ unsigned order = get_order(size); -+ struct page *p = alloc_pages_node(nid, gfp_mask, order); -+ if (!p) -+ return NULL; -+ return make_alloc_exact((unsigned long)page_address(p), order, size); -+} -+ -+/** -+ * free_pages_exact - release memory allocated via alloc_pages_exact() -+ * @virt: the value returned by alloc_pages_exact. -+ * @size: size of allocation, same value as passed to alloc_pages_exact(). -+ * -+ * Release the memory allocated by a previous call to alloc_pages_exact. -+ */ -+void free_pages_exact(void *virt, size_t size) -+{ -+ unsigned long addr = (unsigned long)virt; -+ unsigned long end = addr + PAGE_ALIGN(size); -+ -+ while (addr < end) { -+ free_page(addr); -+ addr += PAGE_SIZE; -+ } -+} -+EXPORT_SYMBOL(free_pages_exact); -+ -+/** -+ * nr_free_zone_pages - count number of pages beyond high watermark -+ * @offset: The zone index of the highest zone -+ * -+ * nr_free_zone_pages() counts the number of counts pages which are beyond the -+ * high watermark within all zones at or below a given zone index. For each -+ * zone, the number of pages is calculated as: -+ * managed_pages - high_pages -+ */ -+static unsigned long nr_free_zone_pages(int offset) -+{ -+ struct zoneref *z; -+ struct zone *zone; -+ -+ /* Just pick one node, since fallback list is circular */ -+ unsigned long sum = 0; -+ -+ struct zonelist *zonelist = node_zonelist(numa_node_id(), GFP_KERNEL); -+ -+ for_each_zone_zonelist(zone, z, zonelist, offset) { -+ unsigned long size = zone->managed_pages; -+ unsigned long high = high_wmark_pages(zone); -+ if (size > high) -+ sum += size - high; -+ } -+ -+ return sum; -+} -+ -+/** -+ * nr_free_buffer_pages - count number of pages beyond high watermark -+ * -+ * nr_free_buffer_pages() counts the number of pages which are beyond the high -+ * watermark within ZONE_DMA and ZONE_NORMAL. -+ */ -+unsigned long nr_free_buffer_pages(void) -+{ -+ return nr_free_zone_pages(gfp_zone(GFP_USER)); -+} -+EXPORT_SYMBOL_GPL(nr_free_buffer_pages); -+ -+/** -+ * nr_free_pagecache_pages - count number of pages beyond high watermark -+ * -+ * nr_free_pagecache_pages() counts the number of pages which are beyond the -+ * high watermark within all zones. -+ */ -+unsigned long nr_free_pagecache_pages(void) -+{ -+ return nr_free_zone_pages(gfp_zone(GFP_HIGHUSER_MOVABLE)); -+} -+ -+static inline void show_node(struct zone *zone) -+{ -+ if (IS_ENABLED(CONFIG_NUMA)) -+ printk("Node %d ", zone_to_nid(zone)); -+} -+ -+void si_meminfo(struct sysinfo *val) -+{ -+ val->totalram = totalram_pages; -+ val->sharedram = global_page_state(NR_SHMEM); -+ val->freeram = global_page_state(NR_FREE_PAGES); -+ val->bufferram = nr_blockdev_pages(); -+ val->totalhigh = totalhigh_pages; -+ val->freehigh = nr_free_highpages(); -+ val->mem_unit = PAGE_SIZE; -+} -+ -+EXPORT_SYMBOL(si_meminfo); -+ -+#ifdef CONFIG_NUMA -+void si_meminfo_node(struct sysinfo *val, int nid) -+{ -+ int zone_type; /* needs to be signed */ -+ unsigned long managed_pages = 0; -+ pg_data_t *pgdat = NODE_DATA(nid); -+ -+ for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) -+ managed_pages += pgdat->node_zones[zone_type].managed_pages; -+ val->totalram = managed_pages; -+ val->sharedram = node_page_state(nid, NR_SHMEM); -+ val->freeram = node_page_state(nid, NR_FREE_PAGES); -+#ifdef CONFIG_HIGHMEM -+ val->totalhigh = pgdat->node_zones[ZONE_HIGHMEM].managed_pages; -+ val->freehigh = zone_page_state(&pgdat->node_zones[ZONE_HIGHMEM], -+ NR_FREE_PAGES); -+#else -+ val->totalhigh = 0; -+ val->freehigh = 0; -+#endif -+ val->mem_unit = PAGE_SIZE; -+} -+#endif -+ -+/* -+ * Determine whether the node should be displayed or not, depending on whether -+ * SHOW_MEM_FILTER_NODES was passed to show_free_areas(). -+ */ -+bool skip_free_areas_node(unsigned int flags, int nid) -+{ -+ bool ret = false; -+ unsigned int cpuset_mems_cookie; -+ -+ if (!(flags & SHOW_MEM_FILTER_NODES)) -+ goto out; -+ -+ do { -+ cpuset_mems_cookie = read_mems_allowed_begin(); -+ ret = !node_isset(nid, cpuset_current_mems_allowed); -+ } while (read_mems_allowed_retry(cpuset_mems_cookie)); -+out: -+ return ret; -+} -+ -+#define K(x) ((x) << (PAGE_SHIFT-10)) -+ -+static void show_migration_types(unsigned char type) -+{ -+ static const char types[MIGRATE_TYPES] = { -+ [MIGRATE_UNMOVABLE] = 'U', -+ [MIGRATE_RECLAIMABLE] = 'E', -+ [MIGRATE_MOVABLE] = 'M', -+ [MIGRATE_RESERVE] = 'R', -+#ifdef CONFIG_CMA -+ [MIGRATE_CMA] = 'C', -+#endif -+#ifdef CONFIG_MEMORY_ISOLATION -+ [MIGRATE_ISOLATE] = 'I', -+#endif -+ }; -+ char tmp[MIGRATE_TYPES + 1]; -+ char *p = tmp; -+ int i; -+ -+ for (i = 0; i < MIGRATE_TYPES; i++) { -+ if (type & (1 << i)) -+ *p++ = types[i]; -+ } -+ -+ *p = '\0'; -+ printk("(%s) ", tmp); -+} -+ -+/* -+ * Show free area list (used inside shift_scroll-lock stuff) -+ * We also calculate the percentage fragmentation. We do this by counting the -+ * memory on each free list with the exception of the first item on the list. -+ * -+ * Bits in @filter: -+ * SHOW_MEM_FILTER_NODES: suppress nodes that are not allowed by current's -+ * cpuset. -+ */ -+void show_free_areas(unsigned int filter) -+{ -+ unsigned long free_pcp = 0; -+ int cpu; -+ struct zone *zone; -+ -+ for_each_populated_zone(zone) { -+ if (skip_free_areas_node(filter, zone_to_nid(zone))) -+ continue; -+ -+ for_each_online_cpu(cpu) -+ free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count; -+ } -+ -+ printk("active_anon:%lu inactive_anon:%lu isolated_anon:%lu\n" -+ " active_file:%lu inactive_file:%lu isolated_file:%lu\n" -+ " unevictable:%lu dirty:%lu writeback:%lu unstable:%lu\n" -+ " slab_reclaimable:%lu slab_unreclaimable:%lu\n" -+ " mapped:%lu shmem:%lu pagetables:%lu bounce:%lu\n" -+ " free:%lu free_pcp:%lu free_cma:%lu\n", -+ global_page_state(NR_ACTIVE_ANON), -+ global_page_state(NR_INACTIVE_ANON), -+ global_page_state(NR_ISOLATED_ANON), -+ global_page_state(NR_ACTIVE_FILE), -+ global_page_state(NR_INACTIVE_FILE), -+ global_page_state(NR_ISOLATED_FILE), -+ global_page_state(NR_UNEVICTABLE), -+ global_page_state(NR_FILE_DIRTY), -+ global_page_state(NR_WRITEBACK), -+ global_page_state(NR_UNSTABLE_NFS), -+ global_page_state(NR_SLAB_RECLAIMABLE), -+ global_page_state(NR_SLAB_UNRECLAIMABLE), -+ global_page_state(NR_FILE_MAPPED), -+ global_page_state(NR_SHMEM), -+ global_page_state(NR_PAGETABLE), -+ global_page_state(NR_BOUNCE), -+ global_page_state(NR_FREE_PAGES), -+ free_pcp, -+ global_page_state(NR_FREE_CMA_PAGES)); -+ -+ for_each_populated_zone(zone) { -+ int i; -+ -+ if (skip_free_areas_node(filter, zone_to_nid(zone))) -+ continue; -+ -+ free_pcp = 0; -+ for_each_online_cpu(cpu) -+ free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count; -+ -+ show_node(zone); -+ printk("%s" -+ " free:%lukB" -+ " min:%lukB" -+ " low:%lukB" -+ " high:%lukB" -+ " active_anon:%lukB" -+ " inactive_anon:%lukB" -+ " active_file:%lukB" -+ " inactive_file:%lukB" -+ " unevictable:%lukB" -+ " isolated(anon):%lukB" -+ " isolated(file):%lukB" -+ " present:%lukB" -+ " managed:%lukB" -+ " mlocked:%lukB" -+ " dirty:%lukB" -+ " writeback:%lukB" -+ " mapped:%lukB" -+ " shmem:%lukB" -+ " slab_reclaimable:%lukB" -+ " slab_unreclaimable:%lukB" -+ " kernel_stack:%lukB" -+ " pagetables:%lukB" -+ " unstable:%lukB" -+ " bounce:%lukB" -+ " free_pcp:%lukB" -+ " local_pcp:%ukB" -+ " free_cma:%lukB" -+ " writeback_tmp:%lukB" -+ " pages_scanned:%lu" -+ " all_unreclaimable? %s" -+ "\n", -+ zone->name, -+ K(zone_page_state(zone, NR_FREE_PAGES)), -+ K(min_wmark_pages(zone)), -+ K(low_wmark_pages(zone)), -+ K(high_wmark_pages(zone)), -+ K(zone_page_state(zone, NR_ACTIVE_ANON)), -+ K(zone_page_state(zone, NR_INACTIVE_ANON)), -+ K(zone_page_state(zone, NR_ACTIVE_FILE)), -+ K(zone_page_state(zone, NR_INACTIVE_FILE)), -+ K(zone_page_state(zone, NR_UNEVICTABLE)), -+ K(zone_page_state(zone, NR_ISOLATED_ANON)), -+ K(zone_page_state(zone, NR_ISOLATED_FILE)), -+ K(zone->present_pages), -+ K(zone->managed_pages), -+ K(zone_page_state(zone, NR_MLOCK)), -+ K(zone_page_state(zone, NR_FILE_DIRTY)), -+ K(zone_page_state(zone, NR_WRITEBACK)), -+ K(zone_page_state(zone, NR_FILE_MAPPED)), -+ K(zone_page_state(zone, NR_SHMEM)), -+ K(zone_page_state(zone, NR_SLAB_RECLAIMABLE)), -+ K(zone_page_state(zone, NR_SLAB_UNRECLAIMABLE)), -+ zone_page_state(zone, NR_KERNEL_STACK) * -+ THREAD_SIZE / 1024, -+ K(zone_page_state(zone, NR_PAGETABLE)), -+ K(zone_page_state(zone, NR_UNSTABLE_NFS)), -+ K(zone_page_state(zone, NR_BOUNCE)), -+ K(free_pcp), -+ K(this_cpu_read(zone->pageset->pcp.count)), -+ K(zone_page_state(zone, NR_FREE_CMA_PAGES)), -+ K(zone_page_state(zone, NR_WRITEBACK_TEMP)), -+ K(zone_page_state(zone, NR_PAGES_SCANNED)), -+ (!zone_reclaimable(zone) ? "yes" : "no") -+ ); -+ printk("lowmem_reserve[]:"); -+ for (i = 0; i < MAX_NR_ZONES; i++) -+ printk(" %ld", zone->lowmem_reserve[i]); -+ printk("\n"); -+ } -+ -+ for_each_populated_zone(zone) { -+ unsigned long nr[MAX_ORDER], flags, order, total = 0; -+ unsigned char types[MAX_ORDER]; -+ -+ if (skip_free_areas_node(filter, zone_to_nid(zone))) -+ continue; -+ show_node(zone); -+ printk("%s: ", zone->name); -+ -+ spin_lock_irqsave(&zone->lock, flags); -+ for (order = 0; order < MAX_ORDER; order++) { -+ struct free_area *area = &zone->free_area[order]; -+ int type; -+ -+ nr[order] = area->nr_free; -+ total += nr[order] << order; -+ -+ types[order] = 0; -+ for (type = 0; type < MIGRATE_TYPES; type++) { -+ if (!list_empty(&area->free_list[type])) -+ types[order] |= 1 << type; -+ } -+ } -+ spin_unlock_irqrestore(&zone->lock, flags); -+ for (order = 0; order < MAX_ORDER; order++) { -+ printk("%lu*%lukB ", nr[order], K(1UL) << order); -+ if (nr[order]) -+ show_migration_types(types[order]); -+ } -+ printk("= %lukB\n", K(total)); -+ } -+ -+ hugetlb_show_meminfo(); -+ -+ printk("%ld total pagecache pages\n", global_page_state(NR_FILE_PAGES)); -+ -+ show_swap_cache_info(); -+} -+ -+static void zoneref_set_zone(struct zone *zone, struct zoneref *zoneref) -+{ -+ zoneref->zone = zone; -+ zoneref->zone_idx = zone_idx(zone); -+} -+ -+/* -+ * Builds allocation fallback zone lists. -+ * -+ * Add all populated zones of a node to the zonelist. -+ */ -+static int build_zonelists_node(pg_data_t *pgdat, struct zonelist *zonelist, -+ int nr_zones) -+{ -+ struct zone *zone; -+ enum zone_type zone_type = MAX_NR_ZONES; -+ -+ do { -+ zone_type--; -+ zone = pgdat->node_zones + zone_type; -+ if (populated_zone(zone)) { -+ zoneref_set_zone(zone, -+ &zonelist->_zonerefs[nr_zones++]); -+ check_highest_zone(zone_type); -+ } -+ } while (zone_type); -+ -+ return nr_zones; -+} -+ -+ -+/* -+ * zonelist_order: -+ * 0 = automatic detection of better ordering. -+ * 1 = order by ([node] distance, -zonetype) -+ * 2 = order by (-zonetype, [node] distance) -+ * -+ * If not NUMA, ZONELIST_ORDER_ZONE and ZONELIST_ORDER_NODE will create -+ * the same zonelist. So only NUMA can configure this param. -+ */ -+#define ZONELIST_ORDER_DEFAULT 0 -+#define ZONELIST_ORDER_NODE 1 -+#define ZONELIST_ORDER_ZONE 2 -+ -+/* zonelist order in the kernel. -+ * set_zonelist_order() will set this to NODE or ZONE. -+ */ -+static int current_zonelist_order = ZONELIST_ORDER_DEFAULT; -+static char zonelist_order_name[3][8] = {"Default", "Node", "Zone"}; -+ -+ -+#ifdef CONFIG_NUMA -+/* The value user specified ....changed by config */ -+static int user_zonelist_order = ZONELIST_ORDER_DEFAULT; -+/* string for sysctl */ -+#define NUMA_ZONELIST_ORDER_LEN 16 -+char numa_zonelist_order[16] = "default"; -+ -+/* -+ * interface for configure zonelist ordering. -+ * command line option "numa_zonelist_order" -+ * = "[dD]efault - default, automatic configuration. -+ * = "[nN]ode - order by node locality, then by zone within node -+ * = "[zZ]one - order by zone, then by locality within zone -+ */ -+ -+static int __parse_numa_zonelist_order(char *s) -+{ -+ if (*s == 'd' || *s == 'D') { -+ user_zonelist_order = ZONELIST_ORDER_DEFAULT; -+ } else if (*s == 'n' || *s == 'N') { -+ user_zonelist_order = ZONELIST_ORDER_NODE; -+ } else if (*s == 'z' || *s == 'Z') { -+ user_zonelist_order = ZONELIST_ORDER_ZONE; -+ } else { -+ printk(KERN_WARNING -+ "Ignoring invalid numa_zonelist_order value: " -+ "%s\n", s); -+ return -EINVAL; -+ } -+ return 0; -+} -+ -+static __init int setup_numa_zonelist_order(char *s) -+{ -+ int ret; -+ -+ if (!s) -+ return 0; -+ -+ ret = __parse_numa_zonelist_order(s); -+ if (ret == 0) -+ strlcpy(numa_zonelist_order, s, NUMA_ZONELIST_ORDER_LEN); -+ -+ return ret; -+} -+early_param("numa_zonelist_order", setup_numa_zonelist_order); -+ -+/* -+ * sysctl handler for numa_zonelist_order -+ */ -+int numa_zonelist_order_handler(struct ctl_table *table, int write, -+ void __user *buffer, size_t *length, -+ loff_t *ppos) -+{ -+ char saved_string[NUMA_ZONELIST_ORDER_LEN]; -+ int ret; -+ static DEFINE_MUTEX(zl_order_mutex); -+ -+ mutex_lock(&zl_order_mutex); -+ if (write) { -+ if (strlen((char *)table->data) >= NUMA_ZONELIST_ORDER_LEN) { -+ ret = -EINVAL; -+ goto out; -+ } -+ strcpy(saved_string, (char *)table->data); -+ } -+ ret = proc_dostring(table, write, buffer, length, ppos); -+ if (ret) -+ goto out; -+ if (write) { -+ int oldval = user_zonelist_order; -+ -+ ret = __parse_numa_zonelist_order((char *)table->data); -+ if (ret) { -+ /* -+ * bogus value. restore saved string -+ */ -+ strncpy((char *)table->data, saved_string, -+ NUMA_ZONELIST_ORDER_LEN); -+ user_zonelist_order = oldval; -+ } else if (oldval != user_zonelist_order) { -+ mutex_lock(&zonelists_mutex); -+ build_all_zonelists(NULL, NULL); -+ mutex_unlock(&zonelists_mutex); -+ } -+ } -+out: -+ mutex_unlock(&zl_order_mutex); -+ return ret; -+} -+ -+ -+#define MAX_NODE_LOAD (nr_online_nodes) -+static int node_load[MAX_NUMNODES]; -+ -+/** -+ * find_next_best_node - find the next node that should appear in a given node's fallback list -+ * @node: node whose fallback list we're appending -+ * @used_node_mask: nodemask_t of already used nodes -+ * -+ * We use a number of factors to determine which is the next node that should -+ * appear on a given node's fallback list. The node should not have appeared -+ * already in @node's fallback list, and it should be the next closest node -+ * according to the distance array (which contains arbitrary distance values -+ * from each node to each node in the system), and should also prefer nodes -+ * with no CPUs, since presumably they'll have very little allocation pressure -+ * on them otherwise. -+ * It returns -1 if no node is found. -+ */ -+static int find_next_best_node(int node, nodemask_t *used_node_mask) -+{ -+ int n, val; -+ int min_val = INT_MAX; -+ int best_node = NUMA_NO_NODE; -+ const struct cpumask *tmp = cpumask_of_node(0); -+ -+ /* Use the local node if we haven't already */ -+ if (!node_isset(node, *used_node_mask)) { -+ node_set(node, *used_node_mask); -+ return node; -+ } -+ -+ for_each_node_state(n, N_MEMORY) { -+ -+ /* Don't want a node to appear more than once */ -+ if (node_isset(n, *used_node_mask)) -+ continue; -+ -+ /* Use the distance array to find the distance */ -+ val = node_distance(node, n); -+ -+ /* Penalize nodes under us ("prefer the next node") */ -+ val += (n < node); -+ -+ /* Give preference to headless and unused nodes */ -+ tmp = cpumask_of_node(n); -+ if (!cpumask_empty(tmp)) -+ val += PENALTY_FOR_NODE_WITH_CPUS; -+ -+ /* Slight preference for less loaded node */ -+ val *= (MAX_NODE_LOAD*MAX_NUMNODES); -+ val += node_load[n]; -+ -+ if (val < min_val) { -+ min_val = val; -+ best_node = n; -+ } -+ } -+ -+ if (best_node >= 0) -+ node_set(best_node, *used_node_mask); -+ -+ return best_node; -+} -+ -+ -+/* -+ * Build zonelists ordered by node and zones within node. -+ * This results in maximum locality--normal zone overflows into local -+ * DMA zone, if any--but risks exhausting DMA zone. -+ */ -+static void build_zonelists_in_node_order(pg_data_t *pgdat, int node) -+{ -+ int j; -+ struct zonelist *zonelist; -+ -+ zonelist = &pgdat->node_zonelists[0]; -+ for (j = 0; zonelist->_zonerefs[j].zone != NULL; j++) -+ ; -+ j = build_zonelists_node(NODE_DATA(node), zonelist, j); -+ zonelist->_zonerefs[j].zone = NULL; -+ zonelist->_zonerefs[j].zone_idx = 0; -+} -+ -+/* -+ * Build gfp_thisnode zonelists -+ */ -+static void build_thisnode_zonelists(pg_data_t *pgdat) -+{ -+ int j; -+ struct zonelist *zonelist; -+ -+ zonelist = &pgdat->node_zonelists[1]; -+ j = build_zonelists_node(pgdat, zonelist, 0); -+ zonelist->_zonerefs[j].zone = NULL; -+ zonelist->_zonerefs[j].zone_idx = 0; -+} -+ -+/* -+ * Build zonelists ordered by zone and nodes within zones. -+ * This results in conserving DMA zone[s] until all Normal memory is -+ * exhausted, but results in overflowing to remote node while memory -+ * may still exist in local DMA zone. -+ */ -+static int node_order[MAX_NUMNODES]; -+ -+static void build_zonelists_in_zone_order(pg_data_t *pgdat, int nr_nodes) -+{ -+ int pos, j, node; -+ int zone_type; /* needs to be signed */ -+ struct zone *z; -+ struct zonelist *zonelist; -+ -+ zonelist = &pgdat->node_zonelists[0]; -+ pos = 0; -+ for (zone_type = MAX_NR_ZONES - 1; zone_type >= 0; zone_type--) { -+ for (j = 0; j < nr_nodes; j++) { -+ node = node_order[j]; -+ z = &NODE_DATA(node)->node_zones[zone_type]; -+ if (populated_zone(z)) { -+ zoneref_set_zone(z, -+ &zonelist->_zonerefs[pos++]); -+ check_highest_zone(zone_type); -+ } -+ } -+ } -+ zonelist->_zonerefs[pos].zone = NULL; -+ zonelist->_zonerefs[pos].zone_idx = 0; -+} -+ -+#if defined(CONFIG_64BIT) -+/* -+ * Devices that require DMA32/DMA are relatively rare and do not justify a -+ * penalty to every machine in case the specialised case applies. Default -+ * to Node-ordering on 64-bit NUMA machines -+ */ -+static int default_zonelist_order(void) -+{ -+ return ZONELIST_ORDER_NODE; -+} -+#else -+/* -+ * On 32-bit, the Normal zone needs to be preserved for allocations accessible -+ * by the kernel. If processes running on node 0 deplete the low memory zone -+ * then reclaim will occur more frequency increasing stalls and potentially -+ * be easier to OOM if a large percentage of the zone is under writeback or -+ * dirty. The problem is significantly worse if CONFIG_HIGHPTE is not set. -+ * Hence, default to zone ordering on 32-bit. -+ */ -+static int default_zonelist_order(void) -+{ -+ return ZONELIST_ORDER_ZONE; -+} -+#endif /* CONFIG_64BIT */ -+ -+static void set_zonelist_order(void) -+{ -+ if (user_zonelist_order == ZONELIST_ORDER_DEFAULT) -+ current_zonelist_order = default_zonelist_order(); -+ else -+ current_zonelist_order = user_zonelist_order; -+} -+ -+static void build_zonelists(pg_data_t *pgdat) -+{ -+ int j, node, load; -+ enum zone_type i; -+ nodemask_t used_mask; -+ int local_node, prev_node; -+ struct zonelist *zonelist; -+ int order = current_zonelist_order; -+ -+ /* initialize zonelists */ -+ for (i = 0; i < MAX_ZONELISTS; i++) { -+ zonelist = pgdat->node_zonelists + i; -+ zonelist->_zonerefs[0].zone = NULL; -+ zonelist->_zonerefs[0].zone_idx = 0; -+ } -+ -+ /* NUMA-aware ordering of nodes */ -+ local_node = pgdat->node_id; -+ load = nr_online_nodes; -+ prev_node = local_node; -+ nodes_clear(used_mask); -+ -+ memset(node_order, 0, sizeof(node_order)); -+ j = 0; -+ -+ while ((node = find_next_best_node(local_node, &used_mask)) >= 0) { -+ /* -+ * We don't want to pressure a particular node. -+ * So adding penalty to the first node in same -+ * distance group to make it round-robin. -+ */ -+ if (node_distance(local_node, node) != -+ node_distance(local_node, prev_node)) -+ node_load[node] = load; -+ -+ prev_node = node; -+ load--; -+ if (order == ZONELIST_ORDER_NODE) -+ build_zonelists_in_node_order(pgdat, node); -+ else -+ node_order[j++] = node; /* remember order */ -+ } -+ -+ if (order == ZONELIST_ORDER_ZONE) { -+ /* calculate node order -- i.e., DMA last! */ -+ build_zonelists_in_zone_order(pgdat, j); -+ } -+ -+ build_thisnode_zonelists(pgdat); -+} -+ -+/* Construct the zonelist performance cache - see further mmzone.h */ -+static void build_zonelist_cache(pg_data_t *pgdat) -+{ -+ struct zonelist *zonelist; -+ struct zonelist_cache *zlc; -+ struct zoneref *z; -+ -+ zonelist = &pgdat->node_zonelists[0]; -+ zonelist->zlcache_ptr = zlc = &zonelist->zlcache; -+ bitmap_zero(zlc->fullzones, MAX_ZONES_PER_ZONELIST); -+ for (z = zonelist->_zonerefs; z->zone; z++) -+ zlc->z_to_n[z - zonelist->_zonerefs] = zonelist_node_idx(z); -+} -+ -+#ifdef CONFIG_HAVE_MEMORYLESS_NODES -+/* -+ * Return node id of node used for "local" allocations. -+ * I.e., first node id of first zone in arg node's generic zonelist. -+ * Used for initializing percpu 'numa_mem', which is used primarily -+ * for kernel allocations, so use GFP_KERNEL flags to locate zonelist. -+ */ -+int local_memory_node(int node) -+{ -+ struct zone *zone; -+ -+ (void)first_zones_zonelist(node_zonelist(node, GFP_KERNEL), -+ gfp_zone(GFP_KERNEL), -+ NULL, -+ &zone); -+ return zone->node; -+} -+#endif -+ -+#else /* CONFIG_NUMA */ -+ -+static void set_zonelist_order(void) -+{ -+ current_zonelist_order = ZONELIST_ORDER_ZONE; -+} -+ -+static void build_zonelists(pg_data_t *pgdat) -+{ -+ int node, local_node; -+ enum zone_type j; -+ struct zonelist *zonelist; -+ -+ local_node = pgdat->node_id; -+ -+ zonelist = &pgdat->node_zonelists[0]; -+ j = build_zonelists_node(pgdat, zonelist, 0); -+ -+ /* -+ * Now we build the zonelist so that it contains the zones -+ * of all the other nodes. -+ * We don't want to pressure a particular node, so when -+ * building the zones for node N, we make sure that the -+ * zones coming right after the local ones are those from -+ * node N+1 (modulo N) -+ */ -+ for (node = local_node + 1; node < MAX_NUMNODES; node++) { -+ if (!node_online(node)) -+ continue; -+ j = build_zonelists_node(NODE_DATA(node), zonelist, j); -+ } -+ for (node = 0; node < local_node; node++) { -+ if (!node_online(node)) -+ continue; -+ j = build_zonelists_node(NODE_DATA(node), zonelist, j); -+ } -+ -+ zonelist->_zonerefs[j].zone = NULL; -+ zonelist->_zonerefs[j].zone_idx = 0; -+} -+ -+/* non-NUMA variant of zonelist performance cache - just NULL zlcache_ptr */ -+static void build_zonelist_cache(pg_data_t *pgdat) -+{ -+ pgdat->node_zonelists[0].zlcache_ptr = NULL; -+} -+ -+#endif /* CONFIG_NUMA */ -+ -+/* -+ * Boot pageset table. One per cpu which is going to be used for all -+ * zones and all nodes. The parameters will be set in such a way -+ * that an item put on a list will immediately be handed over to -+ * the buddy list. This is safe since pageset manipulation is done -+ * with interrupts disabled. -+ * -+ * The boot_pagesets must be kept even after bootup is complete for -+ * unused processors and/or zones. They do play a role for bootstrapping -+ * hotplugged processors. -+ * -+ * zoneinfo_show() and maybe other functions do -+ * not check if the processor is online before following the pageset pointer. -+ * Other parts of the kernel may not check if the zone is available. -+ */ -+static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch); -+static DEFINE_PER_CPU(struct per_cpu_pageset, boot_pageset); -+static void setup_zone_pageset(struct zone *zone); -+ -+/* -+ * Global mutex to protect against size modification of zonelists -+ * as well as to serialize pageset setup for the new populated zone. -+ */ -+DEFINE_MUTEX(zonelists_mutex); -+ -+/* return values int ....just for stop_machine() */ -+static int __build_all_zonelists(void *data) -+{ -+ int nid; -+ int cpu; -+ pg_data_t *self = data; -+ -+#ifdef CONFIG_NUMA -+ memset(node_load, 0, sizeof(node_load)); -+#endif -+ -+ if (self && !node_online(self->node_id)) { -+ build_zonelists(self); -+ build_zonelist_cache(self); -+ } -+ -+ for_each_online_node(nid) { -+ pg_data_t *pgdat = NODE_DATA(nid); -+ -+ build_zonelists(pgdat); -+ build_zonelist_cache(pgdat); -+ } -+ -+ /* -+ * Initialize the boot_pagesets that are going to be used -+ * for bootstrapping processors. The real pagesets for -+ * each zone will be allocated later when the per cpu -+ * allocator is available. -+ * -+ * boot_pagesets are used also for bootstrapping offline -+ * cpus if the system is already booted because the pagesets -+ * are needed to initialize allocators on a specific cpu too. -+ * F.e. the percpu allocator needs the page allocator which -+ * needs the percpu allocator in order to allocate its pagesets -+ * (a chicken-egg dilemma). -+ */ -+ for_each_possible_cpu(cpu) { -+ setup_pageset(&per_cpu(boot_pageset, cpu), 0); -+ -+#ifdef CONFIG_HAVE_MEMORYLESS_NODES -+ /* -+ * We now know the "local memory node" for each node-- -+ * i.e., the node of the first zone in the generic zonelist. -+ * Set up numa_mem percpu variable for on-line cpus. During -+ * boot, only the boot cpu should be on-line; we'll init the -+ * secondary cpus' numa_mem as they come on-line. During -+ * node/memory hotplug, we'll fixup all on-line cpus. -+ */ -+ if (cpu_online(cpu)) -+ set_cpu_numa_mem(cpu, local_memory_node(cpu_to_node(cpu))); -+#endif -+ } -+ -+ return 0; -+} -+ -+static noinline void __init -+build_all_zonelists_init(void) -+{ -+ __build_all_zonelists(NULL); -+ mminit_verify_zonelist(); -+ cpuset_init_current_mems_allowed(); -+} -+ -+/* -+ * Called with zonelists_mutex held always -+ * unless system_state == SYSTEM_BOOTING. -+ * -+ * __ref due to (1) call of __meminit annotated setup_zone_pageset -+ * [we're only called with non-NULL zone through __meminit paths] and -+ * (2) call of __init annotated helper build_all_zonelists_init -+ * [protected by SYSTEM_BOOTING]. -+ */ -+void __ref build_all_zonelists(pg_data_t *pgdat, struct zone *zone) -+{ -+ set_zonelist_order(); -+ -+ if (system_state == SYSTEM_BOOTING) { -+ build_all_zonelists_init(); -+ } else { -+#ifdef CONFIG_MEMORY_HOTPLUG -+ if (zone) -+ setup_zone_pageset(zone); -+#endif -+ /* we have to stop all cpus to guarantee there is no user -+ of zonelist */ -+ stop_machine(__build_all_zonelists, pgdat, NULL); -+ /* cpuset refresh routine should be here */ -+ } -+ vm_total_pages = nr_free_pagecache_pages(); -+ /* -+ * Disable grouping by mobility if the number of pages in the -+ * system is too low to allow the mechanism to work. It would be -+ * more accurate, but expensive to check per-zone. This check is -+ * made on memory-hotadd so a system can start with mobility -+ * disabled and enable it later -+ */ -+ if (vm_total_pages < (pageblock_nr_pages * MIGRATE_TYPES)) -+ page_group_by_mobility_disabled = 1; -+ else -+ page_group_by_mobility_disabled = 0; -+ -+ pr_info("Built %i zonelists in %s order, mobility grouping %s. " -+ "Total pages: %ld\n", -+ nr_online_nodes, -+ zonelist_order_name[current_zonelist_order], -+ page_group_by_mobility_disabled ? "off" : "on", -+ vm_total_pages); -+#ifdef CONFIG_NUMA -+ pr_info("Policy zone: %s\n", zone_names[policy_zone]); -+#endif -+} -+ -+/* -+ * Helper functions to size the waitqueue hash table. -+ * Essentially these want to choose hash table sizes sufficiently -+ * large so that collisions trying to wait on pages are rare. -+ * But in fact, the number of active page waitqueues on typical -+ * systems is ridiculously low, less than 200. So this is even -+ * conservative, even though it seems large. -+ * -+ * The constant PAGES_PER_WAITQUEUE specifies the ratio of pages to -+ * waitqueues, i.e. the size of the waitq table given the number of pages. -+ */ -+#define PAGES_PER_WAITQUEUE 256 -+ -+#ifndef CONFIG_MEMORY_HOTPLUG -+static inline unsigned long wait_table_hash_nr_entries(unsigned long pages) -+{ -+ unsigned long size = 1; -+ -+ pages /= PAGES_PER_WAITQUEUE; -+ -+ while (size < pages) -+ size <<= 1; -+ -+ /* -+ * Once we have dozens or even hundreds of threads sleeping -+ * on IO we've got bigger problems than wait queue collision. -+ * Limit the size of the wait table to a reasonable size. -+ */ -+ size = min(size, 4096UL); -+ -+ return max(size, 4UL); -+} -+#else -+/* -+ * A zone's size might be changed by hot-add, so it is not possible to determine -+ * a suitable size for its wait_table. So we use the maximum size now. -+ * -+ * The max wait table size = 4096 x sizeof(wait_queue_head_t). ie: -+ * -+ * i386 (preemption config) : 4096 x 16 = 64Kbyte. -+ * ia64, x86-64 (no preemption): 4096 x 20 = 80Kbyte. -+ * ia64, x86-64 (preemption) : 4096 x 24 = 96Kbyte. -+ * -+ * The maximum entries are prepared when a zone's memory is (512K + 256) pages -+ * or more by the traditional way. (See above). It equals: -+ * -+ * i386, x86-64, powerpc(4K page size) : = ( 2G + 1M)byte. -+ * ia64(16K page size) : = ( 8G + 4M)byte. -+ * powerpc (64K page size) : = (32G +16M)byte. -+ */ -+static inline unsigned long wait_table_hash_nr_entries(unsigned long pages) -+{ -+ return 4096UL; -+} -+#endif -+ -+/* -+ * This is an integer logarithm so that shifts can be used later -+ * to extract the more random high bits from the multiplicative -+ * hash function before the remainder is taken. -+ */ -+static inline unsigned long wait_table_bits(unsigned long size) -+{ -+ return ffz(~size); -+} -+ -+/* -+ * Check if a pageblock contains reserved pages -+ */ -+static int pageblock_is_reserved(unsigned long start_pfn, unsigned long end_pfn) -+{ -+ unsigned long pfn; -+ -+ for (pfn = start_pfn; pfn < end_pfn; pfn++) { -+ if (!pfn_valid_within(pfn) || PageReserved(pfn_to_page(pfn))) -+ return 1; -+ } -+ return 0; -+} -+ -+/* -+ * Mark a number of pageblocks as MIGRATE_RESERVE. The number -+ * of blocks reserved is based on min_wmark_pages(zone). The memory within -+ * the reserve will tend to store contiguous free pages. Setting min_free_kbytes -+ * higher will lead to a bigger reserve which will get freed as contiguous -+ * blocks as reclaim kicks in -+ */ -+static void setup_zone_migrate_reserve(struct zone *zone) -+{ -+ unsigned long start_pfn, pfn, end_pfn, block_end_pfn; -+ struct page *page; -+ unsigned long block_migratetype; -+ int reserve; -+ int old_reserve; -+ -+ /* -+ * Get the start pfn, end pfn and the number of blocks to reserve -+ * We have to be careful to be aligned to pageblock_nr_pages to -+ * make sure that we always check pfn_valid for the first page in -+ * the block. -+ */ -+ start_pfn = zone->zone_start_pfn; -+ end_pfn = zone_end_pfn(zone); -+ start_pfn = roundup(start_pfn, pageblock_nr_pages); -+ reserve = roundup(min_wmark_pages(zone), pageblock_nr_pages) >> -+ pageblock_order; -+ -+ /* -+ * Reserve blocks are generally in place to help high-order atomic -+ * allocations that are short-lived. A min_free_kbytes value that -+ * would result in more than 2 reserve blocks for atomic allocations -+ * is assumed to be in place to help anti-fragmentation for the -+ * future allocation of hugepages at runtime. -+ */ -+ reserve = min(2, reserve); -+ old_reserve = zone->nr_migrate_reserve_block; -+ -+ /* When memory hot-add, we almost always need to do nothing */ -+ if (reserve == old_reserve) -+ return; -+ zone->nr_migrate_reserve_block = reserve; -+ -+ for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) { -+ if (!pfn_valid(pfn)) -+ continue; -+ page = pfn_to_page(pfn); -+ -+ /* Watch out for overlapping nodes */ -+ if (page_to_nid(page) != zone_to_nid(zone)) -+ continue; -+ -+ block_migratetype = get_pageblock_migratetype(page); -+ -+ /* Only test what is necessary when the reserves are not met */ -+ if (reserve > 0) { -+ /* -+ * Blocks with reserved pages will never free, skip -+ * them. -+ */ -+ block_end_pfn = min(pfn + pageblock_nr_pages, end_pfn); -+ if (pageblock_is_reserved(pfn, block_end_pfn)) -+ continue; -+ -+ /* If this block is reserved, account for it */ -+ if (block_migratetype == MIGRATE_RESERVE) { -+ reserve--; -+ continue; -+ } -+ -+ /* Suitable for reserving if this block is movable */ -+ if (block_migratetype == MIGRATE_MOVABLE) { -+ set_pageblock_migratetype(page, -+ MIGRATE_RESERVE); -+ move_freepages_block(zone, page, -+ MIGRATE_RESERVE); -+ reserve--; -+ continue; -+ } -+ } else if (!old_reserve) { -+ /* -+ * At boot time we don't need to scan the whole zone -+ * for turning off MIGRATE_RESERVE. -+ */ -+ break; -+ } -+ -+ /* -+ * If the reserve is met and this is a previous reserved block, -+ * take it back -+ */ -+ if (block_migratetype == MIGRATE_RESERVE) { -+ set_pageblock_migratetype(page, MIGRATE_MOVABLE); -+ move_freepages_block(zone, page, MIGRATE_MOVABLE); -+ } -+ } -+} -+ -+/* -+ * Initially all pages are reserved - free ones are freed -+ * up by free_all_bootmem() once the early boot process is -+ * done. Non-atomic initialization, single-pass. -+ */ -+void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone, -+ unsigned long start_pfn, enum memmap_context context) -+{ -+ struct page *page; -+ unsigned long end_pfn = start_pfn + size; -+ unsigned long pfn; -+ struct zone *z; -+ -+ if (highest_memmap_pfn < end_pfn - 1) -+ highest_memmap_pfn = end_pfn - 1; -+ -+ z = &NODE_DATA(nid)->node_zones[zone]; -+ for (pfn = start_pfn; pfn < end_pfn; pfn++) { -+ /* -+ * There can be holes in boot-time mem_map[]s -+ * handed to this function. They do not -+ * exist on hotplugged memory. -+ */ -+ if (context == MEMMAP_EARLY) { -+ if (!early_pfn_valid(pfn)) -+ continue; -+ if (!early_pfn_in_nid(pfn, nid)) -+ continue; -+ } -+ page = pfn_to_page(pfn); -+ set_page_links(page, zone, nid, pfn); -+ mminit_verify_page_links(page, zone, nid, pfn); -+ init_page_count(page); -+ page_mapcount_reset(page); -+ page_cpupid_reset_last(page); -+ SetPageReserved(page); -+ /* -+ * Mark the block movable so that blocks are reserved for -+ * movable at startup. This will force kernel allocations -+ * to reserve their blocks rather than leaking throughout -+ * the address space during boot when many long-lived -+ * kernel allocations are made. Later some blocks near -+ * the start are marked MIGRATE_RESERVE by -+ * setup_zone_migrate_reserve() -+ * -+ * bitmap is created for zone's valid pfn range. but memmap -+ * can be created for invalid pages (for alignment) -+ * check here not to call set_pageblock_migratetype() against -+ * pfn out of zone. -+ */ -+ if ((z->zone_start_pfn <= pfn) -+ && (pfn < zone_end_pfn(z)) -+ && !(pfn & (pageblock_nr_pages - 1))) -+ set_pageblock_migratetype(page, MIGRATE_MOVABLE); -+ -+ INIT_LIST_HEAD(&page->lru); -+#ifdef WANT_PAGE_VIRTUAL -+ /* The shift won't overflow because ZONE_NORMAL is below 4G. */ -+ if (!is_highmem_idx(zone)) -+ set_page_address(page, __va(pfn << PAGE_SHIFT)); -+#endif -+ } -+} -+ -+static void __meminit zone_init_free_lists(struct zone *zone) -+{ -+ unsigned int order, t; -+ for_each_migratetype_order(order, t) { -+ INIT_LIST_HEAD(&zone->free_area[order].free_list[t]); -+ zone->free_area[order].nr_free = 0; -+ } -+} -+ -+#ifndef __HAVE_ARCH_MEMMAP_INIT -+#define memmap_init(size, nid, zone, start_pfn) \ -+ memmap_init_zone((size), (nid), (zone), (start_pfn), MEMMAP_EARLY) -+#endif -+ -+static int zone_batchsize(struct zone *zone) -+{ -+#ifdef CONFIG_MMU -+ int batch; -+ -+ /* -+ * The per-cpu-pages pools are set to around 1000th of the -+ * size of the zone. But no more than 1/2 of a meg. -+ * -+ * OK, so we don't know how big the cache is. So guess. -+ */ -+ batch = zone->managed_pages / 1024; -+ if (batch * PAGE_SIZE > 512 * 1024) -+ batch = (512 * 1024) / PAGE_SIZE; -+ batch /= 4; /* We effectively *= 4 below */ -+ if (batch < 1) -+ batch = 1; -+ -+ /* -+ * Clamp the batch to a 2^n - 1 value. Having a power -+ * of 2 value was found to be more likely to have -+ * suboptimal cache aliasing properties in some cases. -+ * -+ * For example if 2 tasks are alternately allocating -+ * batches of pages, one task can end up with a lot -+ * of pages of one half of the possible page colors -+ * and the other with pages of the other colors. -+ */ -+ batch = rounddown_pow_of_two(batch + batch/2) - 1; -+ -+ return batch; -+ -+#else -+ /* The deferral and batching of frees should be suppressed under NOMMU -+ * conditions. -+ * -+ * The problem is that NOMMU needs to be able to allocate large chunks -+ * of contiguous memory as there's no hardware page translation to -+ * assemble apparent contiguous memory from discontiguous pages. -+ * -+ * Queueing large contiguous runs of pages for batching, however, -+ * causes the pages to actually be freed in smaller chunks. As there -+ * can be a significant delay between the individual batches being -+ * recycled, this leads to the once large chunks of space being -+ * fragmented and becoming unavailable for high-order allocations. -+ */ -+ return 0; -+#endif -+} -+ -+/* -+ * pcp->high and pcp->batch values are related and dependent on one another: -+ * ->batch must never be higher then ->high. -+ * The following function updates them in a safe manner without read side -+ * locking. -+ * -+ * Any new users of pcp->batch and pcp->high should ensure they can cope with -+ * those fields changing asynchronously (acording the the above rule). -+ * -+ * mutex_is_locked(&pcp_batch_high_lock) required when calling this function -+ * outside of boot time (or some other assurance that no concurrent updaters -+ * exist). -+ */ -+static void pageset_update(struct per_cpu_pages *pcp, unsigned long high, -+ unsigned long batch) -+{ -+ /* start with a fail safe value for batch */ -+ pcp->batch = 1; -+ smp_wmb(); -+ -+ /* Update high, then batch, in order */ -+ pcp->high = high; -+ smp_wmb(); -+ -+ pcp->batch = batch; -+} -+ -+/* a companion to pageset_set_high() */ -+static void pageset_set_batch(struct per_cpu_pageset *p, unsigned long batch) -+{ -+ pageset_update(&p->pcp, 6 * batch, max(1UL, 1 * batch)); -+} -+ -+static void pageset_init(struct per_cpu_pageset *p) -+{ -+ struct per_cpu_pages *pcp; -+ int migratetype; -+ -+ memset(p, 0, sizeof(*p)); -+ -+ pcp = &p->pcp; -+ pcp->count = 0; -+ for (migratetype = 0; migratetype < MIGRATE_PCPTYPES; migratetype++) -+ INIT_LIST_HEAD(&pcp->lists[migratetype]); -+} -+ -+static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch) -+{ -+ pageset_init(p); -+ pageset_set_batch(p, batch); -+} -+ -+/* -+ * pageset_set_high() sets the high water mark for hot per_cpu_pagelist -+ * to the value high for the pageset p. -+ */ -+static void pageset_set_high(struct per_cpu_pageset *p, -+ unsigned long high) -+{ -+ unsigned long batch = max(1UL, high / 4); -+ if ((high / 4) > (PAGE_SHIFT * 8)) -+ batch = PAGE_SHIFT * 8; -+ -+ pageset_update(&p->pcp, high, batch); -+} -+ -+static void pageset_set_high_and_batch(struct zone *zone, -+ struct per_cpu_pageset *pcp) -+{ -+ if (percpu_pagelist_fraction) -+ pageset_set_high(pcp, -+ (zone->managed_pages / -+ percpu_pagelist_fraction)); -+ else -+ pageset_set_batch(pcp, zone_batchsize(zone)); -+} -+ -+static void __meminit zone_pageset_init(struct zone *zone, int cpu) -+{ -+ struct per_cpu_pageset *pcp = per_cpu_ptr(zone->pageset, cpu); -+ -+ pageset_init(pcp); -+ pageset_set_high_and_batch(zone, pcp); -+} -+ -+static void __meminit setup_zone_pageset(struct zone *zone) -+{ -+ int cpu; -+ zone->pageset = alloc_percpu(struct per_cpu_pageset); -+ for_each_possible_cpu(cpu) -+ zone_pageset_init(zone, cpu); -+} -+ -+/* -+ * Allocate per cpu pagesets and initialize them. -+ * Before this call only boot pagesets were available. -+ */ -+void __init setup_per_cpu_pageset(void) -+{ -+ struct zone *zone; -+ -+ for_each_populated_zone(zone) -+ setup_zone_pageset(zone); -+} -+ -+static noinline __init_refok -+int zone_wait_table_init(struct zone *zone, unsigned long zone_size_pages) -+{ -+ int i; -+ size_t alloc_size; -+ -+ /* -+ * The per-page waitqueue mechanism uses hashed waitqueues -+ * per zone. -+ */ -+ zone->wait_table_hash_nr_entries = -+ wait_table_hash_nr_entries(zone_size_pages); -+ zone->wait_table_bits = -+ wait_table_bits(zone->wait_table_hash_nr_entries); -+ alloc_size = zone->wait_table_hash_nr_entries -+ * sizeof(wait_queue_head_t); -+ -+ if (!slab_is_available()) { -+ zone->wait_table = (wait_queue_head_t *) -+ memblock_virt_alloc_node_nopanic( -+ alloc_size, zone->zone_pgdat->node_id); -+ } else { -+ /* -+ * This case means that a zone whose size was 0 gets new memory -+ * via memory hot-add. -+ * But it may be the case that a new node was hot-added. In -+ * this case vmalloc() will not be able to use this new node's -+ * memory - this wait_table must be initialized to use this new -+ * node itself as well. -+ * To use this new node's memory, further consideration will be -+ * necessary. -+ */ -+ zone->wait_table = vmalloc(alloc_size); -+ } -+ if (!zone->wait_table) -+ return -ENOMEM; -+ -+ for (i = 0; i < zone->wait_table_hash_nr_entries; ++i) -+ init_waitqueue_head(zone->wait_table + i); -+ -+ return 0; -+} -+ -+static __meminit void zone_pcp_init(struct zone *zone) -+{ -+ /* -+ * per cpu subsystem is not up at this point. The following code -+ * relies on the ability of the linker to provide the -+ * offset of a (static) per cpu variable into the per cpu area. -+ */ -+ zone->pageset = &boot_pageset; -+ -+ if (populated_zone(zone)) -+ printk(KERN_DEBUG " %s zone: %lu pages, LIFO batch:%u\n", -+ zone->name, zone->present_pages, -+ zone_batchsize(zone)); -+} -+ -+int __meminit init_currently_empty_zone(struct zone *zone, -+ unsigned long zone_start_pfn, -+ unsigned long size, -+ enum memmap_context context) -+{ -+ struct pglist_data *pgdat = zone->zone_pgdat; -+ int ret; -+ ret = zone_wait_table_init(zone, size); -+ if (ret) -+ return ret; -+ pgdat->nr_zones = zone_idx(zone) + 1; -+ -+ zone->zone_start_pfn = zone_start_pfn; -+ -+ mminit_dprintk(MMINIT_TRACE, "memmap_init", -+ "Initialising map node %d zone %lu pfns %lu -> %lu\n", -+ pgdat->node_id, -+ (unsigned long)zone_idx(zone), -+ zone_start_pfn, (zone_start_pfn + size)); -+ -+ zone_init_free_lists(zone); -+ -+ return 0; -+} -+ -+#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP -+#ifndef CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID -+/* -+ * Required by SPARSEMEM. Given a PFN, return what node the PFN is on. -+ */ -+int __meminit __early_pfn_to_nid(unsigned long pfn) -+{ -+ unsigned long start_pfn, end_pfn; -+ int nid; -+ /* -+ * NOTE: The following SMP-unsafe globals are only used early in boot -+ * when the kernel is running single-threaded. -+ */ -+ static unsigned long __meminitdata last_start_pfn, last_end_pfn; -+ static int __meminitdata last_nid; -+ -+ if (last_start_pfn <= pfn && pfn < last_end_pfn) -+ return last_nid; -+ -+ nid = memblock_search_pfn_nid(pfn, &start_pfn, &end_pfn); -+ if (nid != -1) { -+ last_start_pfn = start_pfn; -+ last_end_pfn = end_pfn; -+ last_nid = nid; -+ } -+ -+ return nid; -+} -+#endif /* CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID */ -+ -+int __meminit early_pfn_to_nid(unsigned long pfn) -+{ -+ int nid; -+ -+ nid = __early_pfn_to_nid(pfn); -+ if (nid >= 0) -+ return nid; -+ /* just returns 0 */ -+ return 0; -+} -+ -+#ifdef CONFIG_NODES_SPAN_OTHER_NODES -+bool __meminit early_pfn_in_nid(unsigned long pfn, int node) -+{ -+ int nid; -+ -+ nid = __early_pfn_to_nid(pfn); -+ if (nid >= 0 && nid != node) -+ return false; -+ return true; -+} -+#endif -+ -+/** -+ * free_bootmem_with_active_regions - Call memblock_free_early_nid for each active range -+ * @nid: The node to free memory on. If MAX_NUMNODES, all nodes are freed. -+ * @max_low_pfn: The highest PFN that will be passed to memblock_free_early_nid -+ * -+ * If an architecture guarantees that all ranges registered contain no holes -+ * and may be freed, this this function may be used instead of calling -+ * memblock_free_early_nid() manually. -+ */ -+void __init free_bootmem_with_active_regions(int nid, unsigned long max_low_pfn) -+{ -+ unsigned long start_pfn, end_pfn; -+ int i, this_nid; -+ -+ for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, &this_nid) { -+ start_pfn = min(start_pfn, max_low_pfn); -+ end_pfn = min(end_pfn, max_low_pfn); -+ -+ if (start_pfn < end_pfn) -+ memblock_free_early_nid(PFN_PHYS(start_pfn), -+ (end_pfn - start_pfn) << PAGE_SHIFT, -+ this_nid); -+ } -+} -+ -+/** -+ * sparse_memory_present_with_active_regions - Call memory_present for each active range -+ * @nid: The node to call memory_present for. If MAX_NUMNODES, all nodes will be used. -+ * -+ * If an architecture guarantees that all ranges registered contain no holes and may -+ * be freed, this function may be used instead of calling memory_present() manually. -+ */ -+void __init sparse_memory_present_with_active_regions(int nid) -+{ -+ unsigned long start_pfn, end_pfn; -+ int i, this_nid; -+ -+ for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, &this_nid) -+ memory_present(this_nid, start_pfn, end_pfn); -+} -+ -+/** -+ * get_pfn_range_for_nid - Return the start and end page frames for a node -+ * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned. -+ * @start_pfn: Passed by reference. On return, it will have the node start_pfn. -+ * @end_pfn: Passed by reference. On return, it will have the node end_pfn. -+ * -+ * It returns the start and end page frame of a node based on information -+ * provided by memblock_set_node(). If called for a node -+ * with no available memory, a warning is printed and the start and end -+ * PFNs will be 0. -+ */ -+void __meminit get_pfn_range_for_nid(unsigned int nid, -+ unsigned long *start_pfn, unsigned long *end_pfn) -+{ -+ unsigned long this_start_pfn, this_end_pfn; -+ int i; -+ -+ *start_pfn = -1UL; -+ *end_pfn = 0; -+ -+ for_each_mem_pfn_range(i, nid, &this_start_pfn, &this_end_pfn, NULL) { -+ *start_pfn = min(*start_pfn, this_start_pfn); -+ *end_pfn = max(*end_pfn, this_end_pfn); -+ } -+ -+ if (*start_pfn == -1UL) -+ *start_pfn = 0; -+} -+ -+/* -+ * This finds a zone that can be used for ZONE_MOVABLE pages. The -+ * assumption is made that zones within a node are ordered in monotonic -+ * increasing memory addresses so that the "highest" populated zone is used -+ */ -+static void __init find_usable_zone_for_movable(void) -+{ -+ int zone_index; -+ for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) { -+ if (zone_index == ZONE_MOVABLE) -+ continue; -+ -+ if (arch_zone_highest_possible_pfn[zone_index] > -+ arch_zone_lowest_possible_pfn[zone_index]) -+ break; -+ } -+ -+ VM_BUG_ON(zone_index == -1); -+ movable_zone = zone_index; -+} -+ -+/* -+ * The zone ranges provided by the architecture do not include ZONE_MOVABLE -+ * because it is sized independent of architecture. Unlike the other zones, -+ * the starting point for ZONE_MOVABLE is not fixed. It may be different -+ * in each node depending on the size of each node and how evenly kernelcore -+ * is distributed. This helper function adjusts the zone ranges -+ * provided by the architecture for a given node by using the end of the -+ * highest usable zone for ZONE_MOVABLE. This preserves the assumption that -+ * zones within a node are in order of monotonic increases memory addresses -+ */ -+static void __meminit adjust_zone_range_for_zone_movable(int nid, -+ unsigned long zone_type, -+ unsigned long node_start_pfn, -+ unsigned long node_end_pfn, -+ unsigned long *zone_start_pfn, -+ unsigned long *zone_end_pfn) -+{ -+ /* Only adjust if ZONE_MOVABLE is on this node */ -+ if (zone_movable_pfn[nid]) { -+ /* Size ZONE_MOVABLE */ -+ if (zone_type == ZONE_MOVABLE) { -+ *zone_start_pfn = zone_movable_pfn[nid]; -+ *zone_end_pfn = min(node_end_pfn, -+ arch_zone_highest_possible_pfn[movable_zone]); -+ -+ /* Adjust for ZONE_MOVABLE starting within this range */ -+ } else if (*zone_start_pfn < zone_movable_pfn[nid] && -+ *zone_end_pfn > zone_movable_pfn[nid]) { -+ *zone_end_pfn = zone_movable_pfn[nid]; -+ -+ /* Check if this whole range is within ZONE_MOVABLE */ -+ } else if (*zone_start_pfn >= zone_movable_pfn[nid]) -+ *zone_start_pfn = *zone_end_pfn; -+ } -+} -+ -+/* -+ * Return the number of pages a zone spans in a node, including holes -+ * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node() -+ */ -+static unsigned long __meminit zone_spanned_pages_in_node(int nid, -+ unsigned long zone_type, -+ unsigned long node_start_pfn, -+ unsigned long node_end_pfn, -+ unsigned long *ignored) -+{ -+ unsigned long zone_start_pfn, zone_end_pfn; -+ -+ /* Get the start and end of the zone */ -+ zone_start_pfn = arch_zone_lowest_possible_pfn[zone_type]; -+ zone_end_pfn = arch_zone_highest_possible_pfn[zone_type]; -+ adjust_zone_range_for_zone_movable(nid, zone_type, -+ node_start_pfn, node_end_pfn, -+ &zone_start_pfn, &zone_end_pfn); -+ -+ /* Check that this node has pages within the zone's required range */ -+ if (zone_end_pfn < node_start_pfn || zone_start_pfn > node_end_pfn) -+ return 0; -+ -+ /* Move the zone boundaries inside the node if necessary */ -+ zone_end_pfn = min(zone_end_pfn, node_end_pfn); -+ zone_start_pfn = max(zone_start_pfn, node_start_pfn); -+ -+ /* Return the spanned pages */ -+ return zone_end_pfn - zone_start_pfn; -+} -+ -+/* -+ * Return the number of holes in a range on a node. If nid is MAX_NUMNODES, -+ * then all holes in the requested range will be accounted for. -+ */ -+unsigned long __meminit __absent_pages_in_range(int nid, -+ unsigned long range_start_pfn, -+ unsigned long range_end_pfn) -+{ -+ unsigned long nr_absent = range_end_pfn - range_start_pfn; -+ unsigned long start_pfn, end_pfn; -+ int i; -+ -+ for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { -+ start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn); -+ end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn); -+ nr_absent -= end_pfn - start_pfn; -+ } -+ return nr_absent; -+} -+ -+/** -+ * absent_pages_in_range - Return number of page frames in holes within a range -+ * @start_pfn: The start PFN to start searching for holes -+ * @end_pfn: The end PFN to stop searching for holes -+ * -+ * It returns the number of pages frames in memory holes within a range. -+ */ -+unsigned long __init absent_pages_in_range(unsigned long start_pfn, -+ unsigned long end_pfn) -+{ -+ return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn); -+} -+ -+/* Return the number of page frames in holes in a zone on a node */ -+static unsigned long __meminit zone_absent_pages_in_node(int nid, -+ unsigned long zone_type, -+ unsigned long node_start_pfn, -+ unsigned long node_end_pfn, -+ unsigned long *ignored) -+{ -+ unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type]; -+ unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type]; -+ unsigned long zone_start_pfn, zone_end_pfn; -+ -+ zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high); -+ zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high); -+ -+ adjust_zone_range_for_zone_movable(nid, zone_type, -+ node_start_pfn, node_end_pfn, -+ &zone_start_pfn, &zone_end_pfn); -+ return __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn); -+} -+ -+#else /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ -+static inline unsigned long __meminit zone_spanned_pages_in_node(int nid, -+ unsigned long zone_type, -+ unsigned long node_start_pfn, -+ unsigned long node_end_pfn, -+ unsigned long *zones_size) -+{ -+ return zones_size[zone_type]; -+} -+ -+static inline unsigned long __meminit zone_absent_pages_in_node(int nid, -+ unsigned long zone_type, -+ unsigned long node_start_pfn, -+ unsigned long node_end_pfn, -+ unsigned long *zholes_size) -+{ -+ if (!zholes_size) -+ return 0; -+ -+ return zholes_size[zone_type]; -+} -+ -+#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ -+ -+static void __meminit calculate_node_totalpages(struct pglist_data *pgdat, -+ unsigned long node_start_pfn, -+ unsigned long node_end_pfn, -+ unsigned long *zones_size, -+ unsigned long *zholes_size) -+{ -+ unsigned long realtotalpages, totalpages = 0; -+ enum zone_type i; -+ -+ for (i = 0; i < MAX_NR_ZONES; i++) -+ totalpages += zone_spanned_pages_in_node(pgdat->node_id, i, -+ node_start_pfn, -+ node_end_pfn, -+ zones_size); -+ pgdat->node_spanned_pages = totalpages; -+ -+ realtotalpages = totalpages; -+ for (i = 0; i < MAX_NR_ZONES; i++) -+ realtotalpages -= -+ zone_absent_pages_in_node(pgdat->node_id, i, -+ node_start_pfn, node_end_pfn, -+ zholes_size); -+ pgdat->node_present_pages = realtotalpages; -+ printk(KERN_DEBUG "On node %d totalpages: %lu\n", pgdat->node_id, -+ realtotalpages); -+} -+ -+#ifndef CONFIG_SPARSEMEM -+/* -+ * Calculate the size of the zone->blockflags rounded to an unsigned long -+ * Start by making sure zonesize is a multiple of pageblock_order by rounding -+ * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally -+ * round what is now in bits to nearest long in bits, then return it in -+ * bytes. -+ */ -+static unsigned long __init usemap_size(unsigned long zone_start_pfn, unsigned long zonesize) -+{ -+ unsigned long usemapsize; -+ -+ zonesize += zone_start_pfn & (pageblock_nr_pages-1); -+ usemapsize = roundup(zonesize, pageblock_nr_pages); -+ usemapsize = usemapsize >> pageblock_order; -+ usemapsize *= NR_PAGEBLOCK_BITS; -+ usemapsize = roundup(usemapsize, 8 * sizeof(unsigned long)); -+ -+ return usemapsize / 8; -+} -+ -+static void __init setup_usemap(struct pglist_data *pgdat, -+ struct zone *zone, -+ unsigned long zone_start_pfn, -+ unsigned long zonesize) -+{ -+ unsigned long usemapsize = usemap_size(zone_start_pfn, zonesize); -+ zone->pageblock_flags = NULL; -+ if (usemapsize) -+ zone->pageblock_flags = -+ memblock_virt_alloc_node_nopanic(usemapsize, -+ pgdat->node_id); -+} -+#else -+static inline void setup_usemap(struct pglist_data *pgdat, struct zone *zone, -+ unsigned long zone_start_pfn, unsigned long zonesize) {} -+#endif /* CONFIG_SPARSEMEM */ -+ -+#ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE -+ -+/* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */ -+void __paginginit set_pageblock_order(void) -+{ -+ unsigned int order; -+ -+ /* Check that pageblock_nr_pages has not already been setup */ -+ if (pageblock_order) -+ return; -+ -+ if (HPAGE_SHIFT > PAGE_SHIFT) -+ order = HUGETLB_PAGE_ORDER; -+ else -+ order = MAX_ORDER - 1; -+ -+ /* -+ * Assume the largest contiguous order of interest is a huge page. -+ * This value may be variable depending on boot parameters on IA64 and -+ * powerpc. -+ */ -+ pageblock_order = order; -+} -+#else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ -+ -+/* -+ * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order() -+ * is unused as pageblock_order is set at compile-time. See -+ * include/linux/pageblock-flags.h for the values of pageblock_order based on -+ * the kernel config -+ */ -+void __paginginit set_pageblock_order(void) -+{ -+} -+ -+#endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ -+ -+static unsigned long __paginginit calc_memmap_size(unsigned long spanned_pages, -+ unsigned long present_pages) -+{ -+ unsigned long pages = spanned_pages; -+ -+ /* -+ * Provide a more accurate estimation if there are holes within -+ * the zone and SPARSEMEM is in use. If there are holes within the -+ * zone, each populated memory region may cost us one or two extra -+ * memmap pages due to alignment because memmap pages for each -+ * populated regions may not naturally algined on page boundary. -+ * So the (present_pages >> 4) heuristic is a tradeoff for that. -+ */ -+ if (spanned_pages > present_pages + (present_pages >> 4) && -+ IS_ENABLED(CONFIG_SPARSEMEM)) -+ pages = present_pages; -+ -+ return PAGE_ALIGN(pages * sizeof(struct page)) >> PAGE_SHIFT; -+} -+ -+/* -+ * Set up the zone data structures: -+ * - mark all pages reserved -+ * - mark all memory queues empty -+ * - clear the memory bitmaps -+ * -+ * NOTE: pgdat should get zeroed by caller. -+ */ -+static void __paginginit free_area_init_core(struct pglist_data *pgdat, -+ unsigned long node_start_pfn, unsigned long node_end_pfn, -+ unsigned long *zones_size, unsigned long *zholes_size) -+{ -+ enum zone_type j; -+ int nid = pgdat->node_id; -+ unsigned long zone_start_pfn = pgdat->node_start_pfn; -+ int ret; -+ -+ pgdat_resize_init(pgdat); -+#ifdef CONFIG_NUMA_BALANCING -+ spin_lock_init(&pgdat->numabalancing_migrate_lock); -+ pgdat->numabalancing_migrate_nr_pages = 0; -+ pgdat->numabalancing_migrate_next_window = jiffies; -+#endif -+ init_waitqueue_head(&pgdat->kswapd_wait); -+ init_waitqueue_head(&pgdat->pfmemalloc_wait); -+ pgdat_page_ext_init(pgdat); -+ -+ for (j = 0; j < MAX_NR_ZONES; j++) { -+ struct zone *zone = pgdat->node_zones + j; -+ unsigned long size, realsize, freesize, memmap_pages; -+ -+ size = zone_spanned_pages_in_node(nid, j, node_start_pfn, -+ node_end_pfn, zones_size); -+ realsize = freesize = size - zone_absent_pages_in_node(nid, j, -+ node_start_pfn, -+ node_end_pfn, -+ zholes_size); -+ -+ /* -+ * Adjust freesize so that it accounts for how much memory -+ * is used by this zone for memmap. This affects the watermark -+ * and per-cpu initialisations -+ */ -+ memmap_pages = calc_memmap_size(size, realsize); -+ if (!is_highmem_idx(j)) { -+ if (freesize >= memmap_pages) { -+ freesize -= memmap_pages; -+ if (memmap_pages) -+ printk(KERN_DEBUG -+ " %s zone: %lu pages used for memmap\n", -+ zone_names[j], memmap_pages); -+ } else -+ printk(KERN_WARNING -+ " %s zone: %lu pages exceeds freesize %lu\n", -+ zone_names[j], memmap_pages, freesize); -+ } -+ -+ /* Account for reserved pages */ -+ if (j == 0 && freesize > dma_reserve) { -+ freesize -= dma_reserve; -+ printk(KERN_DEBUG " %s zone: %lu pages reserved\n", -+ zone_names[0], dma_reserve); -+ } -+ -+ if (!is_highmem_idx(j)) -+ nr_kernel_pages += freesize; -+ /* Charge for highmem memmap if there are enough kernel pages */ -+ else if (nr_kernel_pages > memmap_pages * 2) -+ nr_kernel_pages -= memmap_pages; -+ nr_all_pages += freesize; -+ -+ zone->spanned_pages = size; -+ zone->present_pages = realsize; -+ /* -+ * Set an approximate value for lowmem here, it will be adjusted -+ * when the bootmem allocator frees pages into the buddy system. -+ * And all highmem pages will be managed by the buddy system. -+ */ -+ zone->managed_pages = is_highmem_idx(j) ? realsize : freesize; -+#ifdef CONFIG_NUMA -+ zone->node = nid; -+ zone->min_unmapped_pages = (freesize*sysctl_min_unmapped_ratio) -+ / 100; -+ zone->min_slab_pages = (freesize * sysctl_min_slab_ratio) / 100; -+#endif -+ zone->name = zone_names[j]; -+ spin_lock_init(&zone->lock); -+ spin_lock_init(&zone->lru_lock); -+ zone_seqlock_init(zone); -+ zone->zone_pgdat = pgdat; -+ zone_pcp_init(zone); -+ -+ /* For bootup, initialized properly in watermark setup */ -+ mod_zone_page_state(zone, NR_ALLOC_BATCH, zone->managed_pages); -+ -+ lruvec_init(&zone->lruvec); -+ if (!size) -+ continue; -+ -+ set_pageblock_order(); -+ setup_usemap(pgdat, zone, zone_start_pfn, size); -+ ret = init_currently_empty_zone(zone, zone_start_pfn, -+ size, MEMMAP_EARLY); -+ BUG_ON(ret); -+ memmap_init(size, nid, j, zone_start_pfn); -+ zone_start_pfn += size; -+ } -+} -+ -+static void __init_refok alloc_node_mem_map(struct pglist_data *pgdat) -+{ -+ /* Skip empty nodes */ -+ if (!pgdat->node_spanned_pages) -+ return; -+ -+#ifdef CONFIG_FLAT_NODE_MEM_MAP -+ /* ia64 gets its own node_mem_map, before this, without bootmem */ -+ if (!pgdat->node_mem_map) { -+ unsigned long size, start, end; -+ struct page *map; -+ -+ /* -+ * The zone's endpoints aren't required to be MAX_ORDER -+ * aligned but the node_mem_map endpoints must be in order -+ * for the buddy allocator to function correctly. -+ */ -+ start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1); -+ end = pgdat_end_pfn(pgdat); -+ end = ALIGN(end, MAX_ORDER_NR_PAGES); -+ size = (end - start) * sizeof(struct page); -+ map = alloc_remap(pgdat->node_id, size); -+ if (!map) -+ map = memblock_virt_alloc_node_nopanic(size, -+ pgdat->node_id); -+ pgdat->node_mem_map = map + (pgdat->node_start_pfn - start); -+ } -+#ifndef CONFIG_NEED_MULTIPLE_NODES -+ /* -+ * With no DISCONTIG, the global mem_map is just set as node 0's -+ */ -+ if (pgdat == NODE_DATA(0)) { -+ mem_map = NODE_DATA(0)->node_mem_map; -+#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP -+ if (page_to_pfn(mem_map) != pgdat->node_start_pfn) -+ mem_map -= (pgdat->node_start_pfn - ARCH_PFN_OFFSET); -+#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ -+ } -+#endif -+#endif /* CONFIG_FLAT_NODE_MEM_MAP */ -+} -+ -+void __paginginit free_area_init_node(int nid, unsigned long *zones_size, -+ unsigned long node_start_pfn, unsigned long *zholes_size) -+{ -+ pg_data_t *pgdat = NODE_DATA(nid); -+ unsigned long start_pfn = 0; -+ unsigned long end_pfn = 0; -+ -+ /* pg_data_t should be reset to zero when it's allocated */ -+ WARN_ON(pgdat->nr_zones || pgdat->classzone_idx); -+ -+ pgdat->node_id = nid; -+ pgdat->node_start_pfn = node_start_pfn; -+#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP -+ get_pfn_range_for_nid(nid, &start_pfn, &end_pfn); -+ pr_info("Initmem setup node %d [mem %#018Lx-%#018Lx]\n", nid, -+ (u64)start_pfn << PAGE_SHIFT, ((u64)end_pfn << PAGE_SHIFT) - 1); -+#endif -+ calculate_node_totalpages(pgdat, start_pfn, end_pfn, -+ zones_size, zholes_size); -+ -+ alloc_node_mem_map(pgdat); -+#ifdef CONFIG_FLAT_NODE_MEM_MAP -+ printk(KERN_DEBUG "free_area_init_node: node %d, pgdat %08lx, node_mem_map %08lx\n", -+ nid, (unsigned long)pgdat, -+ (unsigned long)pgdat->node_mem_map); -+#endif -+ -+ free_area_init_core(pgdat, start_pfn, end_pfn, -+ zones_size, zholes_size); -+} -+ -+#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP -+ -+#if MAX_NUMNODES > 1 -+/* -+ * Figure out the number of possible node ids. -+ */ -+void __init setup_nr_node_ids(void) -+{ -+ unsigned int node; -+ unsigned int highest = 0; -+ -+ for_each_node_mask(node, node_possible_map) -+ highest = node; -+ nr_node_ids = highest + 1; -+} -+#endif -+ -+/** -+ * node_map_pfn_alignment - determine the maximum internode alignment -+ * -+ * This function should be called after node map is populated and sorted. -+ * It calculates the maximum power of two alignment which can distinguish -+ * all the nodes. -+ * -+ * For example, if all nodes are 1GiB and aligned to 1GiB, the return value -+ * would indicate 1GiB alignment with (1 << (30 - PAGE_SHIFT)). If the -+ * nodes are shifted by 256MiB, 256MiB. Note that if only the last node is -+ * shifted, 1GiB is enough and this function will indicate so. -+ * -+ * This is used to test whether pfn -> nid mapping of the chosen memory -+ * model has fine enough granularity to avoid incorrect mapping for the -+ * populated node map. -+ * -+ * Returns the determined alignment in pfn's. 0 if there is no alignment -+ * requirement (single node). -+ */ -+unsigned long __init node_map_pfn_alignment(void) -+{ -+ unsigned long accl_mask = 0, last_end = 0; -+ unsigned long start, end, mask; -+ int last_nid = -1; -+ int i, nid; -+ -+ for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) { -+ if (!start || last_nid < 0 || last_nid == nid) { -+ last_nid = nid; -+ last_end = end; -+ continue; -+ } -+ -+ /* -+ * Start with a mask granular enough to pin-point to the -+ * start pfn and tick off bits one-by-one until it becomes -+ * too coarse to separate the current node from the last. -+ */ -+ mask = ~((1 << __ffs(start)) - 1); -+ while (mask && last_end <= (start & (mask << 1))) -+ mask <<= 1; -+ -+ /* accumulate all internode masks */ -+ accl_mask |= mask; -+ } -+ -+ /* convert mask to number of pages */ -+ return ~accl_mask + 1; -+} -+ -+/* Find the lowest pfn for a node */ -+static unsigned long __init find_min_pfn_for_node(int nid) -+{ -+ unsigned long min_pfn = ULONG_MAX; -+ unsigned long start_pfn; -+ int i; -+ -+ for_each_mem_pfn_range(i, nid, &start_pfn, NULL, NULL) -+ min_pfn = min(min_pfn, start_pfn); -+ -+ if (min_pfn == ULONG_MAX) { -+ printk(KERN_WARNING -+ "Could not find start_pfn for node %d\n", nid); -+ return 0; -+ } -+ -+ return min_pfn; -+} -+ -+/** -+ * find_min_pfn_with_active_regions - Find the minimum PFN registered -+ * -+ * It returns the minimum PFN based on information provided via -+ * memblock_set_node(). -+ */ -+unsigned long __init find_min_pfn_with_active_regions(void) -+{ -+ return find_min_pfn_for_node(MAX_NUMNODES); -+} -+ -+/* -+ * early_calculate_totalpages() -+ * Sum pages in active regions for movable zone. -+ * Populate N_MEMORY for calculating usable_nodes. -+ */ -+static unsigned long __init early_calculate_totalpages(void) -+{ -+ unsigned long totalpages = 0; -+ unsigned long start_pfn, end_pfn; -+ int i, nid; -+ -+ for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { -+ unsigned long pages = end_pfn - start_pfn; -+ -+ totalpages += pages; -+ if (pages) -+ node_set_state(nid, N_MEMORY); -+ } -+ return totalpages; -+} -+ -+/* -+ * Find the PFN the Movable zone begins in each node. Kernel memory -+ * is spread evenly between nodes as long as the nodes have enough -+ * memory. When they don't, some nodes will have more kernelcore than -+ * others -+ */ -+static void __init find_zone_movable_pfns_for_nodes(void) -+{ -+ int i, nid; -+ unsigned long usable_startpfn; -+ unsigned long kernelcore_node, kernelcore_remaining; -+ /* save the state before borrow the nodemask */ -+ nodemask_t saved_node_state = node_states[N_MEMORY]; -+ unsigned long totalpages = early_calculate_totalpages(); -+ int usable_nodes = nodes_weight(node_states[N_MEMORY]); -+ struct memblock_region *r; -+ -+ /* Need to find movable_zone earlier when movable_node is specified. */ -+ find_usable_zone_for_movable(); -+ -+ /* -+ * If movable_node is specified, ignore kernelcore and movablecore -+ * options. -+ */ -+ if (movable_node_is_enabled()) { -+ for_each_memblock(memory, r) { -+ if (!memblock_is_hotpluggable(r)) -+ continue; -+ -+ nid = r->nid; -+ -+ usable_startpfn = PFN_DOWN(r->base); -+ zone_movable_pfn[nid] = zone_movable_pfn[nid] ? -+ min(usable_startpfn, zone_movable_pfn[nid]) : -+ usable_startpfn; -+ } -+ -+ goto out2; -+ } -+ -+ /* -+ * If movablecore=nn[KMG] was specified, calculate what size of -+ * kernelcore that corresponds so that memory usable for -+ * any allocation type is evenly spread. If both kernelcore -+ * and movablecore are specified, then the value of kernelcore -+ * will be used for required_kernelcore if it's greater than -+ * what movablecore would have allowed. -+ */ -+ if (required_movablecore) { -+ unsigned long corepages; -+ -+ /* -+ * Round-up so that ZONE_MOVABLE is at least as large as what -+ * was requested by the user -+ */ -+ required_movablecore = -+ roundup(required_movablecore, MAX_ORDER_NR_PAGES); -+ corepages = totalpages - required_movablecore; -+ -+ required_kernelcore = max(required_kernelcore, corepages); -+ } -+ -+ /* If kernelcore was not specified, there is no ZONE_MOVABLE */ -+ if (!required_kernelcore) -+ goto out; -+ -+ /* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */ -+ usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone]; -+ -+restart: -+ /* Spread kernelcore memory as evenly as possible throughout nodes */ -+ kernelcore_node = required_kernelcore / usable_nodes; -+ for_each_node_state(nid, N_MEMORY) { -+ unsigned long start_pfn, end_pfn; -+ -+ /* -+ * Recalculate kernelcore_node if the division per node -+ * now exceeds what is necessary to satisfy the requested -+ * amount of memory for the kernel -+ */ -+ if (required_kernelcore < kernelcore_node) -+ kernelcore_node = required_kernelcore / usable_nodes; -+ -+ /* -+ * As the map is walked, we track how much memory is usable -+ * by the kernel using kernelcore_remaining. When it is -+ * 0, the rest of the node is usable by ZONE_MOVABLE -+ */ -+ kernelcore_remaining = kernelcore_node; -+ -+ /* Go through each range of PFNs within this node */ -+ for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { -+ unsigned long size_pages; -+ -+ start_pfn = max(start_pfn, zone_movable_pfn[nid]); -+ if (start_pfn >= end_pfn) -+ continue; -+ -+ /* Account for what is only usable for kernelcore */ -+ if (start_pfn < usable_startpfn) { -+ unsigned long kernel_pages; -+ kernel_pages = min(end_pfn, usable_startpfn) -+ - start_pfn; -+ -+ kernelcore_remaining -= min(kernel_pages, -+ kernelcore_remaining); -+ required_kernelcore -= min(kernel_pages, -+ required_kernelcore); -+ -+ /* Continue if range is now fully accounted */ -+ if (end_pfn <= usable_startpfn) { -+ -+ /* -+ * Push zone_movable_pfn to the end so -+ * that if we have to rebalance -+ * kernelcore across nodes, we will -+ * not double account here -+ */ -+ zone_movable_pfn[nid] = end_pfn; -+ continue; -+ } -+ start_pfn = usable_startpfn; -+ } -+ -+ /* -+ * The usable PFN range for ZONE_MOVABLE is from -+ * start_pfn->end_pfn. Calculate size_pages as the -+ * number of pages used as kernelcore -+ */ -+ size_pages = end_pfn - start_pfn; -+ if (size_pages > kernelcore_remaining) -+ size_pages = kernelcore_remaining; -+ zone_movable_pfn[nid] = start_pfn + size_pages; -+ -+ /* -+ * Some kernelcore has been met, update counts and -+ * break if the kernelcore for this node has been -+ * satisfied -+ */ -+ required_kernelcore -= min(required_kernelcore, -+ size_pages); -+ kernelcore_remaining -= size_pages; -+ if (!kernelcore_remaining) -+ break; -+ } -+ } -+ -+ /* -+ * If there is still required_kernelcore, we do another pass with one -+ * less node in the count. This will push zone_movable_pfn[nid] further -+ * along on the nodes that still have memory until kernelcore is -+ * satisfied -+ */ -+ usable_nodes--; -+ if (usable_nodes && required_kernelcore > usable_nodes) -+ goto restart; -+ -+out2: -+ /* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */ -+ for (nid = 0; nid < MAX_NUMNODES; nid++) -+ zone_movable_pfn[nid] = -+ roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES); -+ -+out: -+ /* restore the node_state */ -+ node_states[N_MEMORY] = saved_node_state; -+} -+ -+/* Any regular or high memory on that node ? */ -+static void check_for_memory(pg_data_t *pgdat, int nid) -+{ -+ enum zone_type zone_type; -+ -+ if (N_MEMORY == N_NORMAL_MEMORY) -+ return; -+ -+ for (zone_type = 0; zone_type <= ZONE_MOVABLE - 1; zone_type++) { -+ struct zone *zone = &pgdat->node_zones[zone_type]; -+ if (populated_zone(zone)) { -+ node_set_state(nid, N_HIGH_MEMORY); -+ if (N_NORMAL_MEMORY != N_HIGH_MEMORY && -+ zone_type <= ZONE_NORMAL) -+ node_set_state(nid, N_NORMAL_MEMORY); -+ break; -+ } -+ } -+} -+ -+/** -+ * free_area_init_nodes - Initialise all pg_data_t and zone data -+ * @max_zone_pfn: an array of max PFNs for each zone -+ * -+ * This will call free_area_init_node() for each active node in the system. -+ * Using the page ranges provided by memblock_set_node(), the size of each -+ * zone in each node and their holes is calculated. If the maximum PFN -+ * between two adjacent zones match, it is assumed that the zone is empty. -+ * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed -+ * that arch_max_dma32_pfn has no pages. It is also assumed that a zone -+ * starts where the previous one ended. For example, ZONE_DMA32 starts -+ * at arch_max_dma_pfn. -+ */ -+void __init free_area_init_nodes(unsigned long *max_zone_pfn) -+{ -+ unsigned long start_pfn, end_pfn; -+ int i, nid; -+ -+ /* Record where the zone boundaries are */ -+ memset(arch_zone_lowest_possible_pfn, 0, -+ sizeof(arch_zone_lowest_possible_pfn)); -+ memset(arch_zone_highest_possible_pfn, 0, -+ sizeof(arch_zone_highest_possible_pfn)); -+ arch_zone_lowest_possible_pfn[0] = find_min_pfn_with_active_regions(); -+ arch_zone_highest_possible_pfn[0] = max_zone_pfn[0]; -+ for (i = 1; i < MAX_NR_ZONES; i++) { -+ if (i == ZONE_MOVABLE) -+ continue; -+ arch_zone_lowest_possible_pfn[i] = -+ arch_zone_highest_possible_pfn[i-1]; -+ arch_zone_highest_possible_pfn[i] = -+ max(max_zone_pfn[i], arch_zone_lowest_possible_pfn[i]); -+ } -+ arch_zone_lowest_possible_pfn[ZONE_MOVABLE] = 0; -+ arch_zone_highest_possible_pfn[ZONE_MOVABLE] = 0; -+ -+ /* Find the PFNs that ZONE_MOVABLE begins at in each node */ -+ memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn)); -+ find_zone_movable_pfns_for_nodes(); -+ -+ /* Print out the zone ranges */ -+ pr_info("Zone ranges:\n"); -+ for (i = 0; i < MAX_NR_ZONES; i++) { -+ if (i == ZONE_MOVABLE) -+ continue; -+ pr_info(" %-8s ", zone_names[i]); -+ if (arch_zone_lowest_possible_pfn[i] == -+ arch_zone_highest_possible_pfn[i]) -+ pr_cont("empty\n"); -+ else -+ pr_cont("[mem %#018Lx-%#018Lx]\n", -+ (u64)arch_zone_lowest_possible_pfn[i] -+ << PAGE_SHIFT, -+ ((u64)arch_zone_highest_possible_pfn[i] -+ << PAGE_SHIFT) - 1); -+ } -+ -+ /* Print out the PFNs ZONE_MOVABLE begins at in each node */ -+ pr_info("Movable zone start for each node\n"); -+ for (i = 0; i < MAX_NUMNODES; i++) { -+ if (zone_movable_pfn[i]) -+ pr_info(" Node %d: %#018Lx\n", i, -+ (u64)zone_movable_pfn[i] << PAGE_SHIFT); -+ } -+ -+ /* Print out the early node map */ -+ pr_info("Early memory node ranges\n"); -+ for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) -+ pr_info(" node %3d: [mem %#018Lx-%#018Lx]\n", nid, -+ (u64)start_pfn << PAGE_SHIFT, -+ ((u64)end_pfn << PAGE_SHIFT) - 1); -+ -+ /* Initialise every node */ -+ mminit_verify_pageflags_layout(); -+ setup_nr_node_ids(); -+ for_each_online_node(nid) { -+ pg_data_t *pgdat = NODE_DATA(nid); -+ free_area_init_node(nid, NULL, -+ find_min_pfn_for_node(nid), NULL); -+ -+ /* Any memory on that node */ -+ if (pgdat->node_present_pages) -+ node_set_state(nid, N_MEMORY); -+ check_for_memory(pgdat, nid); -+ } -+} -+ -+static int __init cmdline_parse_core(char *p, unsigned long *core) -+{ -+ unsigned long long coremem; -+ if (!p) -+ return -EINVAL; -+ -+ coremem = memparse(p, &p); -+ *core = coremem >> PAGE_SHIFT; -+ -+ /* Paranoid check that UL is enough for the coremem value */ -+ WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX); -+ -+ return 0; -+} -+ -+/* -+ * kernelcore=size sets the amount of memory for use for allocations that -+ * cannot be reclaimed or migrated. -+ */ -+static int __init cmdline_parse_kernelcore(char *p) -+{ -+ return cmdline_parse_core(p, &required_kernelcore); -+} -+ -+/* -+ * movablecore=size sets the amount of memory for use for allocations that -+ * can be reclaimed or migrated. -+ */ -+static int __init cmdline_parse_movablecore(char *p) -+{ -+ return cmdline_parse_core(p, &required_movablecore); -+} -+ -+early_param("kernelcore", cmdline_parse_kernelcore); -+early_param("movablecore", cmdline_parse_movablecore); -+ -+#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ -+ -+void adjust_managed_page_count(struct page *page, long count) -+{ -+ spin_lock(&managed_page_count_lock); -+ page_zone(page)->managed_pages += count; -+ totalram_pages += count; -+#ifdef CONFIG_HIGHMEM -+ if (PageHighMem(page)) -+ totalhigh_pages += count; -+#endif -+ spin_unlock(&managed_page_count_lock); -+} -+EXPORT_SYMBOL(adjust_managed_page_count); -+ -+unsigned long free_reserved_area(void *start, void *end, int poison, char *s) -+{ -+ void *pos; -+ unsigned long pages = 0; -+ -+ start = (void *)PAGE_ALIGN((unsigned long)start); -+ end = (void *)((unsigned long)end & PAGE_MASK); -+ for (pos = start; pos < end; pos += PAGE_SIZE, pages++) { -+ if ((unsigned int)poison <= 0xFF) -+ memset(pos, poison, PAGE_SIZE); -+ free_reserved_page(virt_to_page(pos)); -+ } -+ -+ if (pages && s) -+ pr_info("Freeing %s memory: %ldK (%p - %p)\n", -+ s, pages << (PAGE_SHIFT - 10), start, end); -+ -+ return pages; -+} -+EXPORT_SYMBOL(free_reserved_area); -+ -+#ifdef CONFIG_HIGHMEM -+void free_highmem_page(struct page *page) -+{ -+ __free_reserved_page(page); -+ totalram_pages++; -+ page_zone(page)->managed_pages++; -+ totalhigh_pages++; -+} -+#endif -+ -+ -+void __init mem_init_print_info(const char *str) -+{ -+ unsigned long physpages, codesize, datasize, rosize, bss_size; -+ unsigned long init_code_size, init_data_size; -+ -+ physpages = get_num_physpages(); -+ codesize = _etext - _stext; -+ datasize = _edata - _sdata; -+ rosize = __end_rodata - __start_rodata; -+ bss_size = __bss_stop - __bss_start; -+ init_data_size = __init_end - __init_begin; -+ init_code_size = _einittext - _sinittext; -+ -+ /* -+ * Detect special cases and adjust section sizes accordingly: -+ * 1) .init.* may be embedded into .data sections -+ * 2) .init.text.* may be out of [__init_begin, __init_end], -+ * please refer to arch/tile/kernel/vmlinux.lds.S. -+ * 3) .rodata.* may be embedded into .text or .data sections. -+ */ -+#define adj_init_size(start, end, size, pos, adj) \ -+ do { \ -+ if (start <= pos && pos < end && size > adj) \ -+ size -= adj; \ -+ } while (0) -+ -+ adj_init_size(__init_begin, __init_end, init_data_size, -+ _sinittext, init_code_size); -+ adj_init_size(_stext, _etext, codesize, _sinittext, init_code_size); -+ adj_init_size(_sdata, _edata, datasize, __init_begin, init_data_size); -+ adj_init_size(_stext, _etext, codesize, __start_rodata, rosize); -+ adj_init_size(_sdata, _edata, datasize, __start_rodata, rosize); -+ -+#undef adj_init_size -+ -+ pr_info("Memory: %luK/%luK available " -+ "(%luK kernel code, %luK rwdata, %luK rodata, " -+ "%luK init, %luK bss, %luK reserved, %luK cma-reserved" -+#ifdef CONFIG_HIGHMEM -+ ", %luK highmem" -+#endif -+ "%s%s)\n", -+ nr_free_pages() << (PAGE_SHIFT-10), physpages << (PAGE_SHIFT-10), -+ codesize >> 10, datasize >> 10, rosize >> 10, -+ (init_data_size + init_code_size) >> 10, bss_size >> 10, -+ (physpages - totalram_pages - totalcma_pages) << (PAGE_SHIFT-10), -+ totalcma_pages << (PAGE_SHIFT-10), -+#ifdef CONFIG_HIGHMEM -+ totalhigh_pages << (PAGE_SHIFT-10), -+#endif -+ str ? ", " : "", str ? str : ""); -+} -+ -+/** -+ * set_dma_reserve - set the specified number of pages reserved in the first zone -+ * @new_dma_reserve: The number of pages to mark reserved -+ * -+ * The per-cpu batchsize and zone watermarks are determined by present_pages. -+ * In the DMA zone, a significant percentage may be consumed by kernel image -+ * and other unfreeable allocations which can skew the watermarks badly. This -+ * function may optionally be used to account for unfreeable pages in the -+ * first zone (e.g., ZONE_DMA). The effect will be lower watermarks and -+ * smaller per-cpu batchsize. -+ */ -+void __init set_dma_reserve(unsigned long new_dma_reserve) -+{ -+ dma_reserve = new_dma_reserve; -+} -+ -+void __init free_area_init(unsigned long *zones_size) -+{ -+ free_area_init_node(0, zones_size, -+ __pa(PAGE_OFFSET) >> PAGE_SHIFT, NULL); -+} -+ -+static int page_alloc_cpu_notify(struct notifier_block *self, -+ unsigned long action, void *hcpu) -+{ -+ int cpu = (unsigned long)hcpu; -+ -+ if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) { -+ lru_add_drain_cpu(cpu); -+ drain_pages(cpu); -+ -+ /* -+ * Spill the event counters of the dead processor -+ * into the current processors event counters. -+ * This artificially elevates the count of the current -+ * processor. -+ */ -+ vm_events_fold_cpu(cpu); -+ -+ /* -+ * Zero the differential counters of the dead processor -+ * so that the vm statistics are consistent. -+ * -+ * This is only okay since the processor is dead and cannot -+ * race with what we are doing. -+ */ -+ cpu_vm_stats_fold(cpu); -+ } -+ return NOTIFY_OK; -+} -+ -+void __init page_alloc_init(void) -+{ -+ hotcpu_notifier(page_alloc_cpu_notify, 0); -+} -+ -+/* -+ * calculate_totalreserve_pages - called when sysctl_lower_zone_reserve_ratio -+ * or min_free_kbytes changes. -+ */ -+static void calculate_totalreserve_pages(void) -+{ -+ struct pglist_data *pgdat; -+ unsigned long reserve_pages = 0; -+ enum zone_type i, j; -+ -+ for_each_online_pgdat(pgdat) { -+ for (i = 0; i < MAX_NR_ZONES; i++) { -+ struct zone *zone = pgdat->node_zones + i; -+ long max = 0; -+ -+ /* Find valid and maximum lowmem_reserve in the zone */ -+ for (j = i; j < MAX_NR_ZONES; j++) { -+ if (zone->lowmem_reserve[j] > max) -+ max = zone->lowmem_reserve[j]; -+ } -+ -+ /* we treat the high watermark as reserved pages. */ -+ max += high_wmark_pages(zone); -+ -+ if (max > zone->managed_pages) -+ max = zone->managed_pages; -+ reserve_pages += max; -+ /* -+ * Lowmem reserves are not available to -+ * GFP_HIGHUSER page cache allocations and -+ * kswapd tries to balance zones to their high -+ * watermark. As a result, neither should be -+ * regarded as dirtyable memory, to prevent a -+ * situation where reclaim has to clean pages -+ * in order to balance the zones. -+ */ -+ zone->dirty_balance_reserve = max; -+ } -+ } -+ dirty_balance_reserve = reserve_pages; -+ totalreserve_pages = reserve_pages; -+} -+ -+/* -+ * setup_per_zone_lowmem_reserve - called whenever -+ * sysctl_lower_zone_reserve_ratio changes. Ensures that each zone -+ * has a correct pages reserved value, so an adequate number of -+ * pages are left in the zone after a successful __alloc_pages(). -+ */ -+static void setup_per_zone_lowmem_reserve(void) -+{ -+ struct pglist_data *pgdat; -+ enum zone_type j, idx; -+ -+ for_each_online_pgdat(pgdat) { -+ for (j = 0; j < MAX_NR_ZONES; j++) { -+ struct zone *zone = pgdat->node_zones + j; -+ unsigned long managed_pages = zone->managed_pages; -+ -+ zone->lowmem_reserve[j] = 0; -+ -+ idx = j; -+ while (idx) { -+ struct zone *lower_zone; -+ -+ idx--; -+ -+ if (sysctl_lowmem_reserve_ratio[idx] < 1) -+ sysctl_lowmem_reserve_ratio[idx] = 1; -+ -+ lower_zone = pgdat->node_zones + idx; -+ lower_zone->lowmem_reserve[j] = managed_pages / -+ sysctl_lowmem_reserve_ratio[idx]; -+ managed_pages += lower_zone->managed_pages; -+ } -+ } -+ } -+ -+ /* update totalreserve_pages */ -+ calculate_totalreserve_pages(); -+} -+ -+static void __setup_per_zone_wmarks(void) -+{ -+ unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10); -+ unsigned long lowmem_pages = 0; -+ struct zone *zone; -+ unsigned long flags; -+ -+ /* Calculate total number of !ZONE_HIGHMEM pages */ -+ for_each_zone(zone) { -+ if (!is_highmem(zone)) -+ lowmem_pages += zone->managed_pages; -+ } -+ -+ for_each_zone(zone) { -+ u64 tmp; -+ -+ spin_lock_irqsave(&zone->lock, flags); -+ tmp = (u64)pages_min * zone->managed_pages; -+ do_div(tmp, lowmem_pages); -+ if (is_highmem(zone)) { -+ /* -+ * __GFP_HIGH and PF_MEMALLOC allocations usually don't -+ * need highmem pages, so cap pages_min to a small -+ * value here. -+ * -+ * The WMARK_HIGH-WMARK_LOW and (WMARK_LOW-WMARK_MIN) -+ * deltas control asynch page reclaim, and so should -+ * not be capped for highmem. -+ */ -+ unsigned long min_pages; -+ -+ min_pages = zone->managed_pages / 1024; -+ min_pages = clamp(min_pages, SWAP_CLUSTER_MAX, 128UL); -+ zone->watermark[WMARK_MIN] = min_pages; -+ } else { -+ /* -+ * If it's a lowmem zone, reserve a number of pages -+ * proportionate to the zone's size. -+ */ -+ zone->watermark[WMARK_MIN] = tmp; -+ } -+ -+ zone->watermark[WMARK_LOW] = min_wmark_pages(zone) + (tmp >> 2); -+ zone->watermark[WMARK_HIGH] = min_wmark_pages(zone) + (tmp >> 1); -+ -+ __mod_zone_page_state(zone, NR_ALLOC_BATCH, -+ high_wmark_pages(zone) - low_wmark_pages(zone) - -+ atomic_long_read(&zone->vm_stat[NR_ALLOC_BATCH])); -+ -+ setup_zone_migrate_reserve(zone); -+ spin_unlock_irqrestore(&zone->lock, flags); -+ } -+ -+ /* update totalreserve_pages */ -+ calculate_totalreserve_pages(); -+} -+ -+/** -+ * setup_per_zone_wmarks - called when min_free_kbytes changes -+ * or when memory is hot-{added|removed} -+ * -+ * Ensures that the watermark[min,low,high] values for each zone are set -+ * correctly with respect to min_free_kbytes. -+ */ -+void setup_per_zone_wmarks(void) -+{ -+ mutex_lock(&zonelists_mutex); -+ __setup_per_zone_wmarks(); -+ mutex_unlock(&zonelists_mutex); -+} -+ -+/* -+ * The inactive anon list should be small enough that the VM never has to -+ * do too much work, but large enough that each inactive page has a chance -+ * to be referenced again before it is swapped out. -+ * -+ * The inactive_anon ratio is the target ratio of ACTIVE_ANON to -+ * INACTIVE_ANON pages on this zone's LRU, maintained by the -+ * pageout code. A zone->inactive_ratio of 3 means 3:1 or 25% of -+ * the anonymous pages are kept on the inactive list. -+ * -+ * total target max -+ * memory ratio inactive anon -+ * ------------------------------------- -+ * 10MB 1 5MB -+ * 100MB 1 50MB -+ * 1GB 3 250MB -+ * 10GB 10 0.9GB -+ * 100GB 31 3GB -+ * 1TB 101 10GB -+ * 10TB 320 32GB -+ */ -+static void __meminit calculate_zone_inactive_ratio(struct zone *zone) -+{ -+ unsigned int gb, ratio; -+ -+ /* Zone size in gigabytes */ -+ gb = zone->managed_pages >> (30 - PAGE_SHIFT); -+ if (gb) -+ ratio = int_sqrt(10 * gb); -+ else -+ ratio = 1; -+ -+ zone->inactive_ratio = ratio; -+} -+ -+static void __meminit setup_per_zone_inactive_ratio(void) -+{ -+ struct zone *zone; -+ -+ for_each_zone(zone) -+ calculate_zone_inactive_ratio(zone); -+} -+ -+/* -+ * Initialise min_free_kbytes. -+ * -+ * For small machines we want it small (128k min). For large machines -+ * we want it large (64MB max). But it is not linear, because network -+ * bandwidth does not increase linearly with machine size. We use -+ * -+ * min_free_kbytes = 4 * sqrt(lowmem_kbytes), for better accuracy: -+ * min_free_kbytes = sqrt(lowmem_kbytes * 16) -+ * -+ * which yields -+ * -+ * 16MB: 512k -+ * 32MB: 724k -+ * 64MB: 1024k -+ * 128MB: 1448k -+ * 256MB: 2048k -+ * 512MB: 2896k -+ * 1024MB: 4096k -+ * 2048MB: 5792k -+ * 4096MB: 8192k -+ * 8192MB: 11584k -+ * 16384MB: 16384k -+ */ -+int __meminit init_per_zone_wmark_min(void) -+{ -+ unsigned long lowmem_kbytes; -+ int new_min_free_kbytes; -+ -+ lowmem_kbytes = nr_free_buffer_pages() * (PAGE_SIZE >> 10); -+ new_min_free_kbytes = int_sqrt(lowmem_kbytes * 16); -+ -+ if (new_min_free_kbytes > user_min_free_kbytes) { -+ min_free_kbytes = new_min_free_kbytes; -+ if (min_free_kbytes < 128) -+ min_free_kbytes = 128; -+ if (min_free_kbytes > 65536) -+ min_free_kbytes = 65536; -+ } else { -+ pr_warn("min_free_kbytes is not updated to %d because user defined value %d is preferred\n", -+ new_min_free_kbytes, user_min_free_kbytes); -+ } -+ setup_per_zone_wmarks(); -+ refresh_zone_stat_thresholds(); -+ setup_per_zone_lowmem_reserve(); -+ setup_per_zone_inactive_ratio(); -+ return 0; -+} -+module_init(init_per_zone_wmark_min) -+ -+/* -+ * min_free_kbytes_sysctl_handler - just a wrapper around proc_dointvec() so -+ * that we can call two helper functions whenever min_free_kbytes -+ * changes. -+ */ -+int min_free_kbytes_sysctl_handler(struct ctl_table *table, int write, -+ void __user *buffer, size_t *length, loff_t *ppos) -+{ -+ int rc; -+ -+ rc = proc_dointvec_minmax(table, write, buffer, length, ppos); -+ if (rc) -+ return rc; -+ -+ if (write) { -+ user_min_free_kbytes = min_free_kbytes; -+ setup_per_zone_wmarks(); -+ } -+ return 0; -+} -+ -+#ifdef CONFIG_NUMA -+int sysctl_min_unmapped_ratio_sysctl_handler(struct ctl_table *table, int write, -+ void __user *buffer, size_t *length, loff_t *ppos) -+{ -+ struct zone *zone; -+ int rc; -+ -+ rc = proc_dointvec_minmax(table, write, buffer, length, ppos); -+ if (rc) -+ return rc; -+ -+ for_each_zone(zone) -+ zone->min_unmapped_pages = (zone->managed_pages * -+ sysctl_min_unmapped_ratio) / 100; -+ return 0; -+} -+ -+int sysctl_min_slab_ratio_sysctl_handler(struct ctl_table *table, int write, -+ void __user *buffer, size_t *length, loff_t *ppos) -+{ -+ struct zone *zone; -+ int rc; -+ -+ rc = proc_dointvec_minmax(table, write, buffer, length, ppos); -+ if (rc) -+ return rc; -+ -+ for_each_zone(zone) -+ zone->min_slab_pages = (zone->managed_pages * -+ sysctl_min_slab_ratio) / 100; -+ return 0; -+} -+#endif -+ -+/* -+ * lowmem_reserve_ratio_sysctl_handler - just a wrapper around -+ * proc_dointvec() so that we can call setup_per_zone_lowmem_reserve() -+ * whenever sysctl_lowmem_reserve_ratio changes. -+ * -+ * The reserve ratio obviously has absolutely no relation with the -+ * minimum watermarks. The lowmem reserve ratio can only make sense -+ * if in function of the boot time zone sizes. -+ */ -+int lowmem_reserve_ratio_sysctl_handler(struct ctl_table *table, int write, -+ void __user *buffer, size_t *length, loff_t *ppos) -+{ -+ proc_dointvec_minmax(table, write, buffer, length, ppos); -+ setup_per_zone_lowmem_reserve(); -+ return 0; -+} -+ -+/* -+ * percpu_pagelist_fraction - changes the pcp->high for each zone on each -+ * cpu. It is the fraction of total pages in each zone that a hot per cpu -+ * pagelist can have before it gets flushed back to buddy allocator. -+ */ -+int percpu_pagelist_fraction_sysctl_handler(struct ctl_table *table, int write, -+ void __user *buffer, size_t *length, loff_t *ppos) -+{ -+ struct zone *zone; -+ int old_percpu_pagelist_fraction; -+ int ret; -+ -+ mutex_lock(&pcp_batch_high_lock); -+ old_percpu_pagelist_fraction = percpu_pagelist_fraction; -+ -+ ret = proc_dointvec_minmax(table, write, buffer, length, ppos); -+ if (!write || ret < 0) -+ goto out; -+ -+ /* Sanity checking to avoid pcp imbalance */ -+ if (percpu_pagelist_fraction && -+ percpu_pagelist_fraction < MIN_PERCPU_PAGELIST_FRACTION) { -+ percpu_pagelist_fraction = old_percpu_pagelist_fraction; -+ ret = -EINVAL; -+ goto out; -+ } -+ -+ /* No change? */ -+ if (percpu_pagelist_fraction == old_percpu_pagelist_fraction) -+ goto out; -+ -+ for_each_populated_zone(zone) { -+ unsigned int cpu; -+ -+ for_each_possible_cpu(cpu) -+ pageset_set_high_and_batch(zone, -+ per_cpu_ptr(zone->pageset, cpu)); -+ } -+out: -+ mutex_unlock(&pcp_batch_high_lock); -+ return ret; -+} -+ -+int hashdist = HASHDIST_DEFAULT; -+ -+#ifdef CONFIG_NUMA -+static int __init set_hashdist(char *str) -+{ -+ if (!str) -+ return 0; -+ hashdist = simple_strtoul(str, &str, 0); -+ return 1; -+} -+__setup("hashdist=", set_hashdist); -+#endif -+ -+/* -+ * allocate a large system hash table from bootmem -+ * - it is assumed that the hash table must contain an exact power-of-2 -+ * quantity of entries -+ * - limit is the number of hash buckets, not the total allocation size -+ */ -+void *__init alloc_large_system_hash(const char *tablename, -+ unsigned long bucketsize, -+ unsigned long numentries, -+ int scale, -+ int flags, -+ unsigned int *_hash_shift, -+ unsigned int *_hash_mask, -+ unsigned long low_limit, -+ unsigned long high_limit) -+{ -+ unsigned long long max = high_limit; -+ unsigned long log2qty, size; -+ void *table = NULL; -+ -+ /* allow the kernel cmdline to have a say */ -+ if (!numentries) { -+ /* round applicable memory size up to nearest megabyte */ -+ numentries = nr_kernel_pages; -+ -+ /* It isn't necessary when PAGE_SIZE >= 1MB */ -+ if (PAGE_SHIFT < 20) -+ numentries = round_up(numentries, (1<<20)/PAGE_SIZE); -+ -+ /* limit to 1 bucket per 2^scale bytes of low memory */ -+ if (scale > PAGE_SHIFT) -+ numentries >>= (scale - PAGE_SHIFT); -+ else -+ numentries <<= (PAGE_SHIFT - scale); -+ -+ /* Make sure we've got at least a 0-order allocation.. */ -+ if (unlikely(flags & HASH_SMALL)) { -+ /* Makes no sense without HASH_EARLY */ -+ WARN_ON(!(flags & HASH_EARLY)); -+ if (!(numentries >> *_hash_shift)) { -+ numentries = 1UL << *_hash_shift; -+ BUG_ON(!numentries); -+ } -+ } else if (unlikely((numentries * bucketsize) < PAGE_SIZE)) -+ numentries = PAGE_SIZE / bucketsize; -+ } -+ numentries = roundup_pow_of_two(numentries); -+ -+ /* limit allocation size to 1/16 total memory by default */ -+ if (max == 0) { -+ max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4; -+ do_div(max, bucketsize); -+ } -+ max = min(max, 0x80000000ULL); -+ -+ if (numentries < low_limit) -+ numentries = low_limit; -+ if (numentries > max) -+ numentries = max; -+ -+ log2qty = ilog2(numentries); -+ -+ do { -+ size = bucketsize << log2qty; -+ if (flags & HASH_EARLY) -+ table = memblock_virt_alloc_nopanic(size, 0); -+ else if (hashdist) -+ table = __vmalloc(size, GFP_ATOMIC, PAGE_KERNEL); -+ else { -+ /* -+ * If bucketsize is not a power-of-two, we may free -+ * some pages at the end of hash table which -+ * alloc_pages_exact() automatically does -+ */ -+ if (get_order(size) < MAX_ORDER) { -+ table = alloc_pages_exact(size, GFP_ATOMIC); -+ kmemleak_alloc(table, size, 1, GFP_ATOMIC); -+ } -+ } -+ } while (!table && size > PAGE_SIZE && --log2qty); -+ -+ if (!table) -+ panic("Failed to allocate %s hash table\n", tablename); -+ -+ printk(KERN_INFO "%s hash table entries: %ld (order: %d, %lu bytes)\n", -+ tablename, -+ (1UL << log2qty), -+ ilog2(size) - PAGE_SHIFT, -+ size); -+ -+ if (_hash_shift) -+ *_hash_shift = log2qty; -+ if (_hash_mask) -+ *_hash_mask = (1 << log2qty) - 1; -+ -+ return table; -+} -+ -+/* Return a pointer to the bitmap storing bits affecting a block of pages */ -+static inline unsigned long *get_pageblock_bitmap(struct zone *zone, -+ unsigned long pfn) -+{ -+#ifdef CONFIG_SPARSEMEM -+ return __pfn_to_section(pfn)->pageblock_flags; -+#else -+ return zone->pageblock_flags; -+#endif /* CONFIG_SPARSEMEM */ -+} -+ -+static inline int pfn_to_bitidx(struct zone *zone, unsigned long pfn) -+{ -+#ifdef CONFIG_SPARSEMEM -+ pfn &= (PAGES_PER_SECTION-1); -+ return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS; -+#else -+ pfn = pfn - round_down(zone->zone_start_pfn, pageblock_nr_pages); -+ return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS; -+#endif /* CONFIG_SPARSEMEM */ -+} -+ -+/** -+ * get_pfnblock_flags_mask - Return the requested group of flags for the pageblock_nr_pages block of pages -+ * @page: The page within the block of interest -+ * @pfn: The target page frame number -+ * @end_bitidx: The last bit of interest to retrieve -+ * @mask: mask of bits that the caller is interested in -+ * -+ * Return: pageblock_bits flags -+ */ -+unsigned long get_pfnblock_flags_mask(struct page *page, unsigned long pfn, -+ unsigned long end_bitidx, -+ unsigned long mask) -+{ -+ struct zone *zone; -+ unsigned long *bitmap; -+ unsigned long bitidx, word_bitidx; -+ unsigned long word; -+ -+ zone = page_zone(page); -+ bitmap = get_pageblock_bitmap(zone, pfn); -+ bitidx = pfn_to_bitidx(zone, pfn); -+ word_bitidx = bitidx / BITS_PER_LONG; -+ bitidx &= (BITS_PER_LONG-1); -+ -+ word = bitmap[word_bitidx]; -+ bitidx += end_bitidx; -+ return (word >> (BITS_PER_LONG - bitidx - 1)) & mask; -+} -+ -+/** -+ * set_pfnblock_flags_mask - Set the requested group of flags for a pageblock_nr_pages block of pages -+ * @page: The page within the block of interest -+ * @flags: The flags to set -+ * @pfn: The target page frame number -+ * @end_bitidx: The last bit of interest -+ * @mask: mask of bits that the caller is interested in -+ */ -+void set_pfnblock_flags_mask(struct page *page, unsigned long flags, -+ unsigned long pfn, -+ unsigned long end_bitidx, -+ unsigned long mask) -+{ -+ struct zone *zone; -+ unsigned long *bitmap; -+ unsigned long bitidx, word_bitidx; -+ unsigned long old_word, word; -+ -+ BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4); -+ -+ zone = page_zone(page); -+ bitmap = get_pageblock_bitmap(zone, pfn); -+ bitidx = pfn_to_bitidx(zone, pfn); -+ word_bitidx = bitidx / BITS_PER_LONG; -+ bitidx &= (BITS_PER_LONG-1); -+ -+ VM_BUG_ON_PAGE(!zone_spans_pfn(zone, pfn), page); -+ -+ bitidx += end_bitidx; -+ mask <<= (BITS_PER_LONG - bitidx - 1); -+ flags <<= (BITS_PER_LONG - bitidx - 1); -+ -+ word = READ_ONCE(bitmap[word_bitidx]); -+ for (;;) { -+ old_word = cmpxchg(&bitmap[word_bitidx], word, (word & ~mask) | flags); -+ if (word == old_word) -+ break; -+ word = old_word; -+ } -+} -+ -+/* -+ * This function checks whether pageblock includes unmovable pages or not. -+ * If @count is not zero, it is okay to include less @count unmovable pages -+ * -+ * PageLRU check without isolation or lru_lock could race so that -+ * MIGRATE_MOVABLE block might include unmovable pages. It means you can't -+ * expect this function should be exact. -+ */ -+bool has_unmovable_pages(struct zone *zone, struct page *page, int count, -+ bool skip_hwpoisoned_pages) -+{ -+ unsigned long pfn, iter, found; -+ int mt; -+ -+ /* -+ * For avoiding noise data, lru_add_drain_all() should be called -+ * If ZONE_MOVABLE, the zone never contains unmovable pages -+ */ -+ if (zone_idx(zone) == ZONE_MOVABLE) -+ return false; -+ mt = get_pageblock_migratetype(page); -+ if (mt == MIGRATE_MOVABLE || is_migrate_cma(mt)) -+ return false; -+ -+ pfn = page_to_pfn(page); -+ for (found = 0, iter = 0; iter < pageblock_nr_pages; iter++) { -+ unsigned long check = pfn + iter; -+ -+ if (!pfn_valid_within(check)) -+ continue; -+ -+ page = pfn_to_page(check); -+ -+ /* -+ * Hugepages are not in LRU lists, but they're movable. -+ * We need not scan over tail pages bacause we don't -+ * handle each tail page individually in migration. -+ */ -+ if (PageHuge(page)) { -+ iter = round_up(iter + 1, 1<_count is zero at all time. -+ */ -+ if (!atomic_read(&page->_count)) { -+ if (PageBuddy(page)) -+ iter += (1 << page_order(page)) - 1; -+ continue; -+ } -+ -+ /* -+ * The HWPoisoned page may be not in buddy system, and -+ * page_count() is not 0. -+ */ -+ if (skip_hwpoisoned_pages && PageHWPoison(page)) -+ continue; -+ -+ if (!PageLRU(page)) -+ found++; -+ /* -+ * If there are RECLAIMABLE pages, we need to check -+ * it. But now, memory offline itself doesn't call -+ * shrink_node_slabs() and it still to be fixed. -+ */ -+ /* -+ * If the page is not RAM, page_count()should be 0. -+ * we don't need more check. This is an _used_ not-movable page. -+ * -+ * The problematic thing here is PG_reserved pages. PG_reserved -+ * is set to both of a memory hole page and a _used_ kernel -+ * page at boot. -+ */ -+ if (found > count) -+ return true; -+ } -+ return false; -+} -+ -+bool is_pageblock_removable_nolock(struct page *page) -+{ -+ struct zone *zone; -+ unsigned long pfn; -+ -+ /* -+ * We have to be careful here because we are iterating over memory -+ * sections which are not zone aware so we might end up outside of -+ * the zone but still within the section. -+ * We have to take care about the node as well. If the node is offline -+ * its NODE_DATA will be NULL - see page_zone. -+ */ -+ if (!node_online(page_to_nid(page))) -+ return false; -+ -+ zone = page_zone(page); -+ pfn = page_to_pfn(page); -+ if (!zone_spans_pfn(zone, pfn)) -+ return false; -+ -+ return !has_unmovable_pages(zone, page, 0, true); -+} -+ -+#ifdef CONFIG_CMA -+ -+static unsigned long pfn_max_align_down(unsigned long pfn) -+{ -+ return pfn & ~(max_t(unsigned long, MAX_ORDER_NR_PAGES, -+ pageblock_nr_pages) - 1); -+} -+ -+static unsigned long pfn_max_align_up(unsigned long pfn) -+{ -+ return ALIGN(pfn, max_t(unsigned long, MAX_ORDER_NR_PAGES, -+ pageblock_nr_pages)); -+} -+ -+/* [start, end) must belong to a single zone. */ -+static int __alloc_contig_migrate_range(struct compact_control *cc, -+ unsigned long start, unsigned long end) -+{ -+ /* This function is based on compact_zone() from compaction.c. */ -+ unsigned long nr_reclaimed; -+ unsigned long pfn = start; -+ unsigned int tries = 0; -+ int ret = 0; -+ -+ migrate_prep(); -+ -+ while (pfn < end || !list_empty(&cc->migratepages)) { -+ if (fatal_signal_pending(current)) { -+ ret = -EINTR; -+ break; -+ } -+ -+ if (list_empty(&cc->migratepages)) { -+ cc->nr_migratepages = 0; -+ pfn = isolate_migratepages_range(cc, pfn, end); -+ if (!pfn) { -+ ret = -EINTR; -+ break; -+ } -+ tries = 0; -+ } else if (++tries == 5) { -+ ret = ret < 0 ? ret : -EBUSY; -+ break; -+ } -+ -+ nr_reclaimed = reclaim_clean_pages_from_list(cc->zone, -+ &cc->migratepages); -+ cc->nr_migratepages -= nr_reclaimed; -+ -+ ret = migrate_pages(&cc->migratepages, alloc_migrate_target, -+ NULL, 0, cc->mode, MR_CMA); -+ } -+ if (ret < 0) { -+ putback_movable_pages(&cc->migratepages); -+ return ret; -+ } -+ return 0; -+} -+ -+/** -+ * alloc_contig_range() -- tries to allocate given range of pages -+ * @start: start PFN to allocate -+ * @end: one-past-the-last PFN to allocate -+ * @migratetype: migratetype of the underlaying pageblocks (either -+ * #MIGRATE_MOVABLE or #MIGRATE_CMA). All pageblocks -+ * in range must have the same migratetype and it must -+ * be either of the two. -+ * -+ * The PFN range does not have to be pageblock or MAX_ORDER_NR_PAGES -+ * aligned, however it's the caller's responsibility to guarantee that -+ * we are the only thread that changes migrate type of pageblocks the -+ * pages fall in. -+ * -+ * The PFN range must belong to a single zone. -+ * -+ * Returns zero on success or negative error code. On success all -+ * pages which PFN is in [start, end) are allocated for the caller and -+ * need to be freed with free_contig_range(). -+ */ -+int alloc_contig_range(unsigned long start, unsigned long end, -+ unsigned migratetype) -+{ -+ unsigned long outer_start, outer_end; -+ int ret = 0, order; -+ -+ struct compact_control cc = { -+ .nr_migratepages = 0, -+ .order = -1, -+ .zone = page_zone(pfn_to_page(start)), -+ .mode = MIGRATE_SYNC, -+ .ignore_skip_hint = true, -+ }; -+ INIT_LIST_HEAD(&cc.migratepages); -+ -+ /* -+ * What we do here is we mark all pageblocks in range as -+ * MIGRATE_ISOLATE. Because pageblock and max order pages may -+ * have different sizes, and due to the way page allocator -+ * work, we align the range to biggest of the two pages so -+ * that page allocator won't try to merge buddies from -+ * different pageblocks and change MIGRATE_ISOLATE to some -+ * other migration type. -+ * -+ * Once the pageblocks are marked as MIGRATE_ISOLATE, we -+ * migrate the pages from an unaligned range (ie. pages that -+ * we are interested in). This will put all the pages in -+ * range back to page allocator as MIGRATE_ISOLATE. -+ * -+ * When this is done, we take the pages in range from page -+ * allocator removing them from the buddy system. This way -+ * page allocator will never consider using them. -+ * -+ * This lets us mark the pageblocks back as -+ * MIGRATE_CMA/MIGRATE_MOVABLE so that free pages in the -+ * aligned range but not in the unaligned, original range are -+ * put back to page allocator so that buddy can use them. -+ */ -+ -+ ret = start_isolate_page_range(pfn_max_align_down(start), -+ pfn_max_align_up(end), migratetype, -+ false); -+ if (ret) -+ return ret; -+ -+ ret = __alloc_contig_migrate_range(&cc, start, end); -+ if (ret) -+ goto done; -+ -+ /* -+ * Pages from [start, end) are within a MAX_ORDER_NR_PAGES -+ * aligned blocks that are marked as MIGRATE_ISOLATE. What's -+ * more, all pages in [start, end) are free in page allocator. -+ * What we are going to do is to allocate all pages from -+ * [start, end) (that is remove them from page allocator). -+ * -+ * The only problem is that pages at the beginning and at the -+ * end of interesting range may be not aligned with pages that -+ * page allocator holds, ie. they can be part of higher order -+ * pages. Because of this, we reserve the bigger range and -+ * once this is done free the pages we are not interested in. -+ * -+ * We don't have to hold zone->lock here because the pages are -+ * isolated thus they won't get removed from buddy. -+ */ -+ -+ lru_add_drain_all(); -+ drain_all_pages(cc.zone); -+ -+ order = 0; -+ outer_start = start; -+ while (!PageBuddy(pfn_to_page(outer_start))) { -+ if (++order >= MAX_ORDER) { -+ ret = -EBUSY; -+ goto done; -+ } -+ outer_start &= ~0UL << order; -+ } -+ -+ /* Make sure the range is really isolated. */ -+ if (test_pages_isolated(outer_start, end, false)) { -+ pr_info("%s: [%lx, %lx) PFNs busy\n", -+ __func__, outer_start, end); -+ ret = -EBUSY; -+ goto done; -+ } -+ -+ /* Grab isolated pages from freelists. */ -+ outer_end = isolate_freepages_range(&cc, outer_start, end); -+ if (!outer_end) { -+ ret = -EBUSY; -+ goto done; -+ } -+ -+ /* Free head and tail (if any) */ -+ if (start != outer_start) -+ free_contig_range(outer_start, start - outer_start); -+ if (end != outer_end) -+ free_contig_range(end, outer_end - end); -+ -+done: -+ undo_isolate_page_range(pfn_max_align_down(start), -+ pfn_max_align_up(end), migratetype); -+ return ret; -+} -+ -+void free_contig_range(unsigned long pfn, unsigned nr_pages) -+{ -+ unsigned int count = 0; -+ -+ for (; nr_pages--; pfn++) { -+ struct page *page = pfn_to_page(pfn); -+ -+ count += page_count(page) != 1; -+ __free_page(page); -+ } -+ WARN(count != 0, "%d pages are still in use!\n", count); -+} -+#endif -+ -+#ifdef CONFIG_MEMORY_HOTPLUG -+/* -+ * The zone indicated has a new number of managed_pages; batch sizes and percpu -+ * page high values need to be recalulated. -+ */ -+void __meminit zone_pcp_update(struct zone *zone) -+{ -+ unsigned cpu; -+ mutex_lock(&pcp_batch_high_lock); -+ for_each_possible_cpu(cpu) -+ pageset_set_high_and_batch(zone, -+ per_cpu_ptr(zone->pageset, cpu)); -+ mutex_unlock(&pcp_batch_high_lock); -+} -+#endif -+ -+void zone_pcp_reset(struct zone *zone) -+{ -+ unsigned long flags; -+ int cpu; -+ struct per_cpu_pageset *pset; -+ -+ /* avoid races with drain_pages() */ -+ local_irq_save(flags); -+ if (zone->pageset != &boot_pageset) { -+ for_each_online_cpu(cpu) { -+ pset = per_cpu_ptr(zone->pageset, cpu); -+ drain_zonestat(zone, pset); -+ } -+ free_percpu(zone->pageset); -+ zone->pageset = &boot_pageset; -+ } -+ local_irq_restore(flags); -+} -+ -+#ifdef CONFIG_MEMORY_HOTREMOVE -+/* -+ * All pages in the range must be isolated before calling this. -+ */ -+void -+__offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn) -+{ -+ struct page *page; -+ struct zone *zone; -+ unsigned int order, i; -+ unsigned long pfn; -+ unsigned long flags; -+ /* find the first valid pfn */ -+ for (pfn = start_pfn; pfn < end_pfn; pfn++) -+ if (pfn_valid(pfn)) -+ break; -+ if (pfn == end_pfn) -+ return; -+ zone = page_zone(pfn_to_page(pfn)); -+ spin_lock_irqsave(&zone->lock, flags); -+ pfn = start_pfn; -+ while (pfn < end_pfn) { -+ if (!pfn_valid(pfn)) { -+ pfn++; -+ continue; -+ } -+ page = pfn_to_page(pfn); -+ /* -+ * The HWPoisoned page may be not in buddy system, and -+ * page_count() is not 0. -+ */ -+ if (unlikely(!PageBuddy(page) && PageHWPoison(page))) { -+ pfn++; -+ SetPageReserved(page); -+ continue; -+ } -+ -+ BUG_ON(page_count(page)); -+ BUG_ON(!PageBuddy(page)); -+ order = page_order(page); -+#ifdef CONFIG_DEBUG_VM -+ printk(KERN_INFO "remove from free list %lx %d %lx\n", -+ pfn, 1 << order, end_pfn); -+#endif -+ list_del(&page->lru); -+ rmv_page_order(page); -+ zone->free_area[order].nr_free--; -+ for (i = 0; i < (1 << order); i++) -+ SetPageReserved((page+i)); -+ pfn += (1 << order); -+ } -+ spin_unlock_irqrestore(&zone->lock, flags); -+} -+#endif -+ -+#ifdef CONFIG_MEMORY_FAILURE -+bool is_free_buddy_page(struct page *page) -+{ -+ struct zone *zone = page_zone(page); -+ unsigned long pfn = page_to_pfn(page); -+ unsigned long flags; -+ unsigned int order; -+ -+ spin_lock_irqsave(&zone->lock, flags); -+ for (order = 0; order < MAX_ORDER; order++) { -+ struct page *page_head = page - (pfn & ((1 << order) - 1)); -+ -+ if (PageBuddy(page_head) && page_order(page_head) >= order) -+ break; -+ } -+ spin_unlock_irqrestore(&zone->lock, flags); -+ -+ return order < MAX_ORDER; -+} -+#endif -diff -Nur linux-4.1.10.orig/mm/slab.h linux-4.1.10/mm/slab.h ---- linux-4.1.10.orig/mm/slab.h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/mm/slab.h 2015-10-07 18:00:08.000000000 +0200 -@@ -330,7 +330,11 @@ - * The slab lists for all objects. - */ - struct kmem_cache_node { -+#ifdef CONFIG_SLUB -+ raw_spinlock_t list_lock; -+#else - spinlock_t list_lock; -+#endif - - #ifdef CONFIG_SLAB - struct list_head slabs_partial; /* partial list first, better asm code */ -diff -Nur linux-4.1.10.orig/mm/slub.c linux-4.1.10/mm/slub.c ---- linux-4.1.10.orig/mm/slub.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/mm/slub.c 2015-10-07 18:00:08.000000000 +0200 -@@ -1069,7 +1069,7 @@ - { - struct kmem_cache_node *n = get_node(s, page_to_nid(page)); - -- spin_lock_irqsave(&n->list_lock, *flags); -+ raw_spin_lock_irqsave(&n->list_lock, *flags); - slab_lock(page); - - if (!check_slab(s, page)) -@@ -1116,7 +1116,7 @@ - - fail: - slab_unlock(page); -- spin_unlock_irqrestore(&n->list_lock, *flags); -+ raw_spin_unlock_irqrestore(&n->list_lock, *flags); - slab_fix(s, "Object at 0x%p not freed", object); - return NULL; - } -@@ -1242,6 +1242,12 @@ - - #endif /* CONFIG_SLUB_DEBUG */ - -+struct slub_free_list { -+ raw_spinlock_t lock; -+ struct list_head list; -+}; -+static DEFINE_PER_CPU(struct slub_free_list, slub_free_list); -+ - /* - * Hooks for other subsystems that check memory allocations. In a typical - * production configuration these hooks all should produce no code at all. -@@ -1306,6 +1312,17 @@ - kasan_slab_free(s, x); - } - -+static void setup_object(struct kmem_cache *s, struct page *page, -+ void *object) -+{ -+ setup_object_debug(s, page, object); -+ if (unlikely(s->ctor)) { -+ kasan_unpoison_object_data(s, object); -+ s->ctor(object); -+ kasan_poison_object_data(s, object); -+ } -+} -+ - /* - * Slab allocation and freeing - */ -@@ -1336,10 +1353,17 @@ - struct page *page; - struct kmem_cache_order_objects oo = s->oo; - gfp_t alloc_gfp; -+ void *start, *p; -+ int idx, order; -+ bool enableirqs; - - flags &= gfp_allowed_mask; - -- if (flags & __GFP_WAIT) -+ enableirqs = (flags & __GFP_WAIT) != 0; -+#ifdef CONFIG_PREEMPT_RT_FULL -+ enableirqs |= system_state == SYSTEM_RUNNING; -+#endif -+ if (enableirqs) - local_irq_enable(); - - flags |= s->allocflags; -@@ -1359,13 +1383,13 @@ - * Try a lower order alloc if possible - */ - page = alloc_slab_page(s, alloc_gfp, node, oo); -- -- if (page) -- stat(s, ORDER_FALLBACK); -+ if (unlikely(!page)) -+ goto out; -+ stat(s, ORDER_FALLBACK); - } - -- if (kmemcheck_enabled && page -- && !(s->flags & (SLAB_NOTRACK | DEBUG_DEFAULT_FLAGS))) { -+ if (kmemcheck_enabled && -+ !(s->flags & (SLAB_NOTRACK | DEBUG_DEFAULT_FLAGS))) { - int pages = 1 << oo_order(oo); - - kmemcheck_alloc_shadow(page, oo_order(oo), alloc_gfp, node); -@@ -1380,51 +1404,9 @@ - kmemcheck_mark_unallocated_pages(page, pages); - } - -- if (flags & __GFP_WAIT) -- local_irq_disable(); -- if (!page) -- return NULL; -- - page->objects = oo_objects(oo); -- mod_zone_page_state(page_zone(page), -- (s->flags & SLAB_RECLAIM_ACCOUNT) ? -- NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE, -- 1 << oo_order(oo)); -- -- return page; --} -- --static void setup_object(struct kmem_cache *s, struct page *page, -- void *object) --{ -- setup_object_debug(s, page, object); -- if (unlikely(s->ctor)) { -- kasan_unpoison_object_data(s, object); -- s->ctor(object); -- kasan_poison_object_data(s, object); -- } --} -- --static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node) --{ -- struct page *page; -- void *start; -- void *p; -- int order; -- int idx; -- -- if (unlikely(flags & GFP_SLAB_BUG_MASK)) { -- pr_emerg("gfp: %u\n", flags & GFP_SLAB_BUG_MASK); -- BUG(); -- } -- -- page = allocate_slab(s, -- flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node); -- if (!page) -- goto out; - - order = compound_order(page); -- inc_slabs_node(s, page_to_nid(page), page->objects); - page->slab_cache = s; - __SetPageSlab(page); - if (page_is_pfmemalloc(page)) -@@ -1448,10 +1430,34 @@ - page->freelist = start; - page->inuse = page->objects; - page->frozen = 1; -+ - out: -+ if (enableirqs) -+ local_irq_disable(); -+ if (!page) -+ return NULL; -+ -+ mod_zone_page_state(page_zone(page), -+ (s->flags & SLAB_RECLAIM_ACCOUNT) ? -+ NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE, -+ 1 << oo_order(oo)); -+ -+ inc_slabs_node(s, page_to_nid(page), page->objects); -+ - return page; - } - -+static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node) -+{ -+ if (unlikely(flags & GFP_SLAB_BUG_MASK)) { -+ pr_emerg("gfp: %u\n", flags & GFP_SLAB_BUG_MASK); -+ BUG(); -+ } -+ -+ return allocate_slab(s, -+ flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node); -+} -+ - static void __free_slab(struct kmem_cache *s, struct page *page) - { - int order = compound_order(page); -@@ -1483,6 +1489,16 @@ - memcg_uncharge_slab(s, order); - } - -+static void free_delayed(struct list_head *h) -+{ -+ while(!list_empty(h)) { -+ struct page *page = list_first_entry(h, struct page, lru); -+ -+ list_del(&page->lru); -+ __free_slab(page->slab_cache, page); -+ } -+} -+ - #define need_reserve_slab_rcu \ - (sizeof(((struct page *)NULL)->lru) < sizeof(struct rcu_head)) - -@@ -1517,6 +1533,12 @@ - } - - call_rcu(head, rcu_free_slab); -+ } else if (irqs_disabled()) { -+ struct slub_free_list *f = this_cpu_ptr(&slub_free_list); -+ -+ raw_spin_lock(&f->lock); -+ list_add(&page->lru, &f->list); -+ raw_spin_unlock(&f->lock); - } else - __free_slab(s, page); - } -@@ -1630,7 +1652,7 @@ - if (!n || !n->nr_partial) - return NULL; - -- spin_lock(&n->list_lock); -+ raw_spin_lock(&n->list_lock); - list_for_each_entry_safe(page, page2, &n->partial, lru) { - void *t; - -@@ -1655,7 +1677,7 @@ - break; - - } -- spin_unlock(&n->list_lock); -+ raw_spin_unlock(&n->list_lock); - return object; - } - -@@ -1901,7 +1923,7 @@ - * that acquire_slab() will see a slab page that - * is frozen - */ -- spin_lock(&n->list_lock); -+ raw_spin_lock(&n->list_lock); - } - } else { - m = M_FULL; -@@ -1912,7 +1934,7 @@ - * slabs from diagnostic functions will not see - * any frozen slabs. - */ -- spin_lock(&n->list_lock); -+ raw_spin_lock(&n->list_lock); - } - } - -@@ -1947,7 +1969,7 @@ - goto redo; - - if (lock) -- spin_unlock(&n->list_lock); -+ raw_spin_unlock(&n->list_lock); - - if (m == M_FREE) { - stat(s, DEACTIVATE_EMPTY); -@@ -1979,10 +2001,10 @@ - n2 = get_node(s, page_to_nid(page)); - if (n != n2) { - if (n) -- spin_unlock(&n->list_lock); -+ raw_spin_unlock(&n->list_lock); - - n = n2; -- spin_lock(&n->list_lock); -+ raw_spin_lock(&n->list_lock); - } - - do { -@@ -2011,7 +2033,7 @@ - } - - if (n) -- spin_unlock(&n->list_lock); -+ raw_spin_unlock(&n->list_lock); - - while (discard_page) { - page = discard_page; -@@ -2050,14 +2072,21 @@ - pobjects = oldpage->pobjects; - pages = oldpage->pages; - if (drain && pobjects > s->cpu_partial) { -+ struct slub_free_list *f; - unsigned long flags; -+ LIST_HEAD(tofree); - /* - * partial array is full. Move the existing - * set to the per node partial list. - */ - local_irq_save(flags); - unfreeze_partials(s, this_cpu_ptr(s->cpu_slab)); -+ f = this_cpu_ptr(&slub_free_list); -+ raw_spin_lock(&f->lock); -+ list_splice_init(&f->list, &tofree); -+ raw_spin_unlock(&f->lock); - local_irq_restore(flags); -+ free_delayed(&tofree); - oldpage = NULL; - pobjects = 0; - pages = 0; -@@ -2129,7 +2158,22 @@ - - static void flush_all(struct kmem_cache *s) - { -+ LIST_HEAD(tofree); -+ int cpu; -+ - on_each_cpu_cond(has_cpu_slab, flush_cpu_slab, s, 1, GFP_ATOMIC); -+ for_each_online_cpu(cpu) { -+ struct slub_free_list *f; -+ -+ if (!has_cpu_slab(cpu, s)) -+ continue; -+ -+ f = &per_cpu(slub_free_list, cpu); -+ raw_spin_lock_irq(&f->lock); -+ list_splice_init(&f->list, &tofree); -+ raw_spin_unlock_irq(&f->lock); -+ free_delayed(&tofree); -+ } - } - - /* -@@ -2165,10 +2209,10 @@ - unsigned long x = 0; - struct page *page; - -- spin_lock_irqsave(&n->list_lock, flags); -+ raw_spin_lock_irqsave(&n->list_lock, flags); - list_for_each_entry(page, &n->partial, lru) - x += get_count(page); -- spin_unlock_irqrestore(&n->list_lock, flags); -+ raw_spin_unlock_irqrestore(&n->list_lock, flags); - return x; - } - #endif /* CONFIG_SLUB_DEBUG || CONFIG_SYSFS */ -@@ -2305,9 +2349,11 @@ - static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, - unsigned long addr, struct kmem_cache_cpu *c) - { -+ struct slub_free_list *f; - void *freelist; - struct page *page; - unsigned long flags; -+ LIST_HEAD(tofree); - - local_irq_save(flags); - #ifdef CONFIG_PREEMPT -@@ -2375,7 +2421,13 @@ - VM_BUG_ON(!c->page->frozen); - c->freelist = get_freepointer(s, freelist); - c->tid = next_tid(c->tid); -+out: -+ f = this_cpu_ptr(&slub_free_list); -+ raw_spin_lock(&f->lock); -+ list_splice_init(&f->list, &tofree); -+ raw_spin_unlock(&f->lock); - local_irq_restore(flags); -+ free_delayed(&tofree); - return freelist; - - new_slab: -@@ -2392,8 +2444,7 @@ - - if (unlikely(!freelist)) { - slab_out_of_memory(s, gfpflags, node); -- local_irq_restore(flags); -- return NULL; -+ goto out; - } - - page = c->page; -@@ -2408,8 +2459,7 @@ - deactivate_slab(s, page, get_freepointer(s, freelist)); - c->page = NULL; - c->freelist = NULL; -- local_irq_restore(flags); -- return freelist; -+ goto out; - } - - /* -@@ -2593,7 +2643,7 @@ - - do { - if (unlikely(n)) { -- spin_unlock_irqrestore(&n->list_lock, flags); -+ raw_spin_unlock_irqrestore(&n->list_lock, flags); - n = NULL; - } - prior = page->freelist; -@@ -2625,7 +2675,7 @@ - * Otherwise the list_lock will synchronize with - * other processors updating the list of slabs. - */ -- spin_lock_irqsave(&n->list_lock, flags); -+ raw_spin_lock_irqsave(&n->list_lock, flags); - - } - } -@@ -2667,7 +2717,7 @@ - add_partial(n, page, DEACTIVATE_TO_TAIL); - stat(s, FREE_ADD_PARTIAL); - } -- spin_unlock_irqrestore(&n->list_lock, flags); -+ raw_spin_unlock_irqrestore(&n->list_lock, flags); - return; - - slab_empty: -@@ -2682,7 +2732,7 @@ - remove_full(s, n, page); - } - -- spin_unlock_irqrestore(&n->list_lock, flags); -+ raw_spin_unlock_irqrestore(&n->list_lock, flags); - stat(s, FREE_SLAB); - discard_slab(s, page); - } -@@ -2881,7 +2931,7 @@ - init_kmem_cache_node(struct kmem_cache_node *n) - { - n->nr_partial = 0; -- spin_lock_init(&n->list_lock); -+ raw_spin_lock_init(&n->list_lock); - INIT_LIST_HEAD(&n->partial); - #ifdef CONFIG_SLUB_DEBUG - atomic_long_set(&n->nr_slabs, 0); -@@ -3463,7 +3513,7 @@ - for (i = 0; i < SHRINK_PROMOTE_MAX; i++) - INIT_LIST_HEAD(promote + i); - -- spin_lock_irqsave(&n->list_lock, flags); -+ raw_spin_lock_irqsave(&n->list_lock, flags); - - /* - * Build lists of slabs to discard or promote. -@@ -3494,7 +3544,7 @@ - for (i = SHRINK_PROMOTE_MAX - 1; i >= 0; i--) - list_splice(promote + i, &n->partial); - -- spin_unlock_irqrestore(&n->list_lock, flags); -+ raw_spin_unlock_irqrestore(&n->list_lock, flags); - - /* Release empty slabs */ - list_for_each_entry_safe(page, t, &discard, lru) -@@ -3670,6 +3720,12 @@ - { - static __initdata struct kmem_cache boot_kmem_cache, - boot_kmem_cache_node; -+ int cpu; -+ -+ for_each_possible_cpu(cpu) { -+ raw_spin_lock_init(&per_cpu(slub_free_list, cpu).lock); -+ INIT_LIST_HEAD(&per_cpu(slub_free_list, cpu).list); -+ } - - if (debug_guardpage_minorder()) - slub_max_order = 0; -@@ -3912,7 +3968,7 @@ - struct page *page; - unsigned long flags; - -- spin_lock_irqsave(&n->list_lock, flags); -+ raw_spin_lock_irqsave(&n->list_lock, flags); - - list_for_each_entry(page, &n->partial, lru) { - validate_slab_slab(s, page, map); -@@ -3934,7 +3990,7 @@ - s->name, count, atomic_long_read(&n->nr_slabs)); - - out: -- spin_unlock_irqrestore(&n->list_lock, flags); -+ raw_spin_unlock_irqrestore(&n->list_lock, flags); - return count; - } - -@@ -4122,12 +4178,12 @@ - if (!atomic_long_read(&n->nr_slabs)) - continue; - -- spin_lock_irqsave(&n->list_lock, flags); -+ raw_spin_lock_irqsave(&n->list_lock, flags); - list_for_each_entry(page, &n->partial, lru) - process_slab(&t, s, page, alloc, map); - list_for_each_entry(page, &n->full, lru) - process_slab(&t, s, page, alloc, map); -- spin_unlock_irqrestore(&n->list_lock, flags); -+ raw_spin_unlock_irqrestore(&n->list_lock, flags); - } - - for (i = 0; i < t.count; i++) { -diff -Nur linux-4.1.10.orig/mm/slub.c.orig linux-4.1.10/mm/slub.c.orig ---- linux-4.1.10.orig/mm/slub.c.orig 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.10/mm/slub.c.orig 2015-10-03 13:49:38.000000000 +0200 -@@ -0,0 +1,5344 @@ -+/* -+ * SLUB: A slab allocator that limits cache line use instead of queuing -+ * objects in per cpu and per node lists. -+ * -+ * The allocator synchronizes using per slab locks or atomic operatios -+ * and only uses a centralized lock to manage a pool of partial slabs. -+ * -+ * (C) 2007 SGI, Christoph Lameter -+ * (C) 2011 Linux Foundation, Christoph Lameter -+ */ -+ -+#include -+#include /* struct reclaim_state */ -+#include -+#include -+#include -+#include -+#include -+#include "slab.h" -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include -+ -+#include "internal.h" -+ -+/* -+ * Lock order: -+ * 1. slab_mutex (Global Mutex) -+ * 2. node->list_lock -+ * 3. slab_lock(page) (Only on some arches and for debugging) -+ * -+ * slab_mutex -+ * -+ * The role of the slab_mutex is to protect the list of all the slabs -+ * and to synchronize major metadata changes to slab cache structures. -+ * -+ * The slab_lock is only used for debugging and on arches that do not -+ * have the ability to do a cmpxchg_double. It only protects the second -+ * double word in the page struct. Meaning -+ * A. page->freelist -> List of object free in a page -+ * B. page->counters -> Counters of objects -+ * C. page->frozen -> frozen state -+ * -+ * If a slab is frozen then it is exempt from list management. It is not -+ * on any list. The processor that froze the slab is the one who can -+ * perform list operations on the page. Other processors may put objects -+ * onto the freelist but the processor that froze the slab is the only -+ * one that can retrieve the objects from the page's freelist. -+ * -+ * The list_lock protects the partial and full list on each node and -+ * the partial slab counter. If taken then no new slabs may be added or -+ * removed from the lists nor make the number of partial slabs be modified. -+ * (Note that the total number of slabs is an atomic value that may be -+ * modified without taking the list lock). -+ * -+ * The list_lock is a centralized lock and thus we avoid taking it as -+ * much as possible. As long as SLUB does not have to handle partial -+ * slabs, operations can continue without any centralized lock. F.e. -+ * allocating a long series of objects that fill up slabs does not require -+ * the list lock. -+ * Interrupts are disabled during allocation and deallocation in order to -+ * make the slab allocator safe to use in the context of an irq. In addition -+ * interrupts are disabled to ensure that the processor does not change -+ * while handling per_cpu slabs, due to kernel preemption. -+ * -+ * SLUB assigns one slab for allocation to each processor. -+ * Allocations only occur from these slabs called cpu slabs. -+ * -+ * Slabs with free elements are kept on a partial list and during regular -+ * operations no list for full slabs is used. If an object in a full slab is -+ * freed then the slab will show up again on the partial lists. -+ * We track full slabs for debugging purposes though because otherwise we -+ * cannot scan all objects. -+ * -+ * Slabs are freed when they become empty. Teardown and setup is -+ * minimal so we rely on the page allocators per cpu caches for -+ * fast frees and allocs. -+ * -+ * Overloading of page flags that are otherwise used for LRU management. -+ * -+ * PageActive The slab is frozen and exempt from list processing. -+ * This means that the slab is dedicated to a purpose -+ * such as satisfying allocations for a specific -+ * processor. Objects may be freed in the slab while -+ * it is frozen but slab_free will then skip the usual -+ * list operations. It is up to the processor holding -+ * the slab to integrate the slab into the slab lists -+ * when the slab is no longer needed. -+ * -+ * One use of this flag is to mark slabs that are -+ * used for allocations. Then such a slab becomes a cpu -+ * slab. The cpu slab may be equipped with an additional -+ * freelist that allows lockless access to -+ * free objects in addition to the regular freelist -+ * that requires the slab lock. -+ * -+ * PageError Slab requires special handling due to debug -+ * options set. This moves slab handling out of -+ * the fast path and disables lockless freelists. -+ */ -+ -+static inline int kmem_cache_debug(struct kmem_cache *s) -+{ -+#ifdef CONFIG_SLUB_DEBUG -+ return unlikely(s->flags & SLAB_DEBUG_FLAGS); -+#else -+ return 0; -+#endif -+} -+ -+static inline bool kmem_cache_has_cpu_partial(struct kmem_cache *s) -+{ -+#ifdef CONFIG_SLUB_CPU_PARTIAL -+ return !kmem_cache_debug(s); -+#else -+ return false; -+#endif -+} -+ -+/* -+ * Issues still to be resolved: -+ * -+ * - Support PAGE_ALLOC_DEBUG. Should be easy to do. -+ * -+ * - Variable sizing of the per node arrays -+ */ -+ -+/* Enable to test recovery from slab corruption on boot */ -+#undef SLUB_RESILIENCY_TEST -+ -+/* Enable to log cmpxchg failures */ -+#undef SLUB_DEBUG_CMPXCHG -+ -+/* -+ * Mininum number of partial slabs. These will be left on the partial -+ * lists even if they are empty. kmem_cache_shrink may reclaim them. -+ */ -+#define MIN_PARTIAL 5 -+ -+/* -+ * Maximum number of desirable partial slabs. -+ * The existence of more partial slabs makes kmem_cache_shrink -+ * sort the partial list by the number of objects in use. -+ */ -+#define MAX_PARTIAL 10 -+ -+#define DEBUG_DEFAULT_FLAGS (SLAB_DEBUG_FREE | SLAB_RED_ZONE | \ -+ SLAB_POISON | SLAB_STORE_USER) -+ -+/* -+ * Debugging flags that require metadata to be stored in the slab. These get -+ * disabled when slub_debug=O is used and a cache's min order increases with -+ * metadata. -+ */ -+#define DEBUG_METADATA_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER) -+ -+#define OO_SHIFT 16 -+#define OO_MASK ((1 << OO_SHIFT) - 1) -+#define MAX_OBJS_PER_PAGE 32767 /* since page.objects is u15 */ -+ -+/* Internal SLUB flags */ -+#define __OBJECT_POISON 0x80000000UL /* Poison object */ -+#define __CMPXCHG_DOUBLE 0x40000000UL /* Use cmpxchg_double */ -+ -+#ifdef CONFIG_SMP -+static struct notifier_block slab_notifier; -+#endif -+ -+/* -+ * Tracking user of a slab. -+ */ -+#define TRACK_ADDRS_COUNT 16 -+struct track { -+ unsigned long addr; /* Called from address */ -+#ifdef CONFIG_STACKTRACE -+ unsigned long addrs[TRACK_ADDRS_COUNT]; /* Called from address */ -+#endif -+ int cpu; /* Was running on cpu */ -+ int pid; /* Pid context */ -+ unsigned long when; /* When did the operation occur */ -+}; -+ -+enum track_item { TRACK_ALLOC, TRACK_FREE }; -+ -+#ifdef CONFIG_SYSFS -+static int sysfs_slab_add(struct kmem_cache *); -+static int sysfs_slab_alias(struct kmem_cache *, const char *); -+static void memcg_propagate_slab_attrs(struct kmem_cache *s); -+#else -+static inline int sysfs_slab_add(struct kmem_cache *s) { return 0; } -+static inline int sysfs_slab_alias(struct kmem_cache *s, const char *p) -+ { return 0; } -+static inline void memcg_propagate_slab_attrs(struct kmem_cache *s) { } -+#endif -+ -+static inline void stat(const struct kmem_cache *s, enum stat_item si) -+{ -+#ifdef CONFIG_SLUB_STATS -+ /* -+ * The rmw is racy on a preemptible kernel but this is acceptable, so -+ * avoid this_cpu_add()'s irq-disable overhead. -+ */ -+ raw_cpu_inc(s->cpu_slab->stat[si]); -+#endif -+} -+ -+/******************************************************************** -+ * Core slab cache functions -+ *******************************************************************/ -+ -+/* Verify that a pointer has an address that is valid within a slab page */ -+static inline int check_valid_pointer(struct kmem_cache *s, -+ struct page *page, const void *object) -+{ -+ void *base; -+ -+ if (!object) -+ return 1; -+ -+ base = page_address(page); -+ if (object < base || object >= base + page->objects * s->size || -+ (object - base) % s->size) { -+ return 0; -+ } -+ -+ return 1; -+} -+ -+static inline void *get_freepointer(struct kmem_cache *s, void *object) -+{ -+ return *(void **)(object + s->offset); -+} -+ -+static void prefetch_freepointer(const struct kmem_cache *s, void *object) -+{ -+ prefetch(object + s->offset); -+} -+ -+static inline void *get_freepointer_safe(struct kmem_cache *s, void *object) -+{ -+ void *p; -+ -+#ifdef CONFIG_DEBUG_PAGEALLOC -+ probe_kernel_read(&p, (void **)(object + s->offset), sizeof(p)); -+#else -+ p = get_freepointer(s, object); -+#endif -+ return p; -+} -+ -+static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp) -+{ -+ *(void **)(object + s->offset) = fp; -+} -+ -+/* Loop over all objects in a slab */ -+#define for_each_object(__p, __s, __addr, __objects) \ -+ for (__p = (__addr); __p < (__addr) + (__objects) * (__s)->size;\ -+ __p += (__s)->size) -+ -+#define for_each_object_idx(__p, __idx, __s, __addr, __objects) \ -+ for (__p = (__addr), __idx = 1; __idx <= __objects;\ -+ __p += (__s)->size, __idx++) -+ -+/* Determine object index from a given position */ -+static inline int slab_index(void *p, struct kmem_cache *s, void *addr) -+{ -+ return (p - addr) / s->size; -+} -+ -+static inline size_t slab_ksize(const struct kmem_cache *s) -+{ -+#ifdef CONFIG_SLUB_DEBUG -+ /* -+ * Debugging requires use of the padding between object -+ * and whatever may come after it. -+ */ -+ if (s->flags & (SLAB_RED_ZONE | SLAB_POISON)) -+ return s->object_size; -+ -+#endif -+ /* -+ * If we have the need to store the freelist pointer -+ * back there or track user information then we can -+ * only use the space before that information. -+ */ -+ if (s->flags & (SLAB_DESTROY_BY_RCU | SLAB_STORE_USER)) -+ return s->inuse; -+ /* -+ * Else we can use all the padding etc for the allocation -+ */ -+ return s->size; -+} -+ -+static inline int order_objects(int order, unsigned long size, int reserved) -+{ -+ return ((PAGE_SIZE << order) - reserved) / size; -+} -+ -+static inline struct kmem_cache_order_objects oo_make(int order, -+ unsigned long size, int reserved) -+{ -+ struct kmem_cache_order_objects x = { -+ (order << OO_SHIFT) + order_objects(order, size, reserved) -+ }; -+ -+ return x; -+} -+ -+static inline int oo_order(struct kmem_cache_order_objects x) -+{ -+ return x.x >> OO_SHIFT; -+} -+ -+static inline int oo_objects(struct kmem_cache_order_objects x) -+{ -+ return x.x & OO_MASK; -+} -+ -+/* -+ * Per slab locking using the pagelock -+ */ -+static __always_inline void slab_lock(struct page *page) -+{ -+ bit_spin_lock(PG_locked, &page->flags); -+} -+ -+static __always_inline void slab_unlock(struct page *page) -+{ -+ __bit_spin_unlock(PG_locked, &page->flags); -+} -+ -+static inline void set_page_slub_counters(struct page *page, unsigned long counters_new) -+{ -+ struct page tmp; -+ tmp.counters = counters_new; -+ /* -+ * page->counters can cover frozen/inuse/objects as well -+ * as page->_count. If we assign to ->counters directly -+ * we run the risk of losing updates to page->_count, so -+ * be careful and only assign to the fields we need. -+ */ -+ page->frozen = tmp.frozen; -+ page->inuse = tmp.inuse; -+ page->objects = tmp.objects; -+} -+ -+/* Interrupts must be disabled (for the fallback code to work right) */ -+static inline bool __cmpxchg_double_slab(struct kmem_cache *s, struct page *page, -+ void *freelist_old, unsigned long counters_old, -+ void *freelist_new, unsigned long counters_new, -+ const char *n) -+{ -+ VM_BUG_ON(!irqs_disabled()); -+#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \ -+ defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE) -+ if (s->flags & __CMPXCHG_DOUBLE) { -+ if (cmpxchg_double(&page->freelist, &page->counters, -+ freelist_old, counters_old, -+ freelist_new, counters_new)) -+ return true; -+ } else -+#endif -+ { -+ slab_lock(page); -+ if (page->freelist == freelist_old && -+ page->counters == counters_old) { -+ page->freelist = freelist_new; -+ set_page_slub_counters(page, counters_new); -+ slab_unlock(page); -+ return true; -+ } -+ slab_unlock(page); -+ } -+ -+ cpu_relax(); -+ stat(s, CMPXCHG_DOUBLE_FAIL); -+ -+#ifdef SLUB_DEBUG_CMPXCHG -+ pr_info("%s %s: cmpxchg double redo ", n, s->name); -+#endif -+ -+ return false; -+} -+ -+static inline bool cmpxchg_double_slab(struct kmem_cache *s, struct page *page, -+ void *freelist_old, unsigned long counters_old, -+ void *freelist_new, unsigned long counters_new, -+ const char *n) -+{ -+#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \ -+ defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE) -+ if (s->flags & __CMPXCHG_DOUBLE) { -+ if (cmpxchg_double(&page->freelist, &page->counters, -+ freelist_old, counters_old, -+ freelist_new, counters_new)) -+ return true; -+ } else -+#endif -+ { -+ unsigned long flags; -+ -+ local_irq_save(flags); -+ slab_lock(page); -+ if (page->freelist == freelist_old && -+ page->counters == counters_old) { -+ page->freelist = freelist_new; -+ set_page_slub_counters(page, counters_new); -+ slab_unlock(page); -+ local_irq_restore(flags); -+ return true; -+ } -+ slab_unlock(page); -+ local_irq_restore(flags); -+ } -+ -+ cpu_relax(); -+ stat(s, CMPXCHG_DOUBLE_FAIL); -+ -+#ifdef SLUB_DEBUG_CMPXCHG -+ pr_info("%s %s: cmpxchg double redo ", n, s->name); -+#endif -+ -+ return false; -+} -+ -+#ifdef CONFIG_SLUB_DEBUG -+/* -+ * Determine a map of object in use on a page. -+ * -+ * Node listlock must be held to guarantee that the page does -+ * not vanish from under us. -+ */ -+static void get_map(struct kmem_cache *s, struct page *page, unsigned long *map) -+{ -+ void *p; -+ void *addr = page_address(page); -+ -+ for (p = page->freelist; p; p = get_freepointer(s, p)) -+ set_bit(slab_index(p, s, addr), map); -+} -+ -+/* -+ * Debug settings: -+ */ -+#ifdef CONFIG_SLUB_DEBUG_ON -+static int slub_debug = DEBUG_DEFAULT_FLAGS; -+#else -+static int slub_debug; -+#endif -+ -+static char *slub_debug_slabs; -+static int disable_higher_order_debug; -+ -+/* -+ * slub is about to manipulate internal object metadata. This memory lies -+ * outside the range of the allocated object, so accessing it would normally -+ * be reported by kasan as a bounds error. metadata_access_enable() is used -+ * to tell kasan that these accesses are OK. -+ */ -+static inline void metadata_access_enable(void) -+{ -+ kasan_disable_current(); -+} -+ -+static inline void metadata_access_disable(void) -+{ -+ kasan_enable_current(); -+} -+ -+/* -+ * Object debugging -+ */ -+static void print_section(char *text, u8 *addr, unsigned int length) -+{ -+ metadata_access_enable(); -+ print_hex_dump(KERN_ERR, text, DUMP_PREFIX_ADDRESS, 16, 1, addr, -+ length, 1); -+ metadata_access_disable(); -+} -+ -+static struct track *get_track(struct kmem_cache *s, void *object, -+ enum track_item alloc) -+{ -+ struct track *p; -+ -+ if (s->offset) -+ p = object + s->offset + sizeof(void *); -+ else -+ p = object + s->inuse; -+ -+ return p + alloc; -+} -+ -+static void set_track(struct kmem_cache *s, void *object, -+ enum track_item alloc, unsigned long addr) -+{ -+ struct track *p = get_track(s, object, alloc); -+ -+ if (addr) { -+#ifdef CONFIG_STACKTRACE -+ struct stack_trace trace; -+ int i; -+ -+ trace.nr_entries = 0; -+ trace.max_entries = TRACK_ADDRS_COUNT; -+ trace.entries = p->addrs; -+ trace.skip = 3; -+ metadata_access_enable(); -+ save_stack_trace(&trace); -+ metadata_access_disable(); -+ -+ /* See rant in lockdep.c */ -+ if (trace.nr_entries != 0 && -+ trace.entries[trace.nr_entries - 1] == ULONG_MAX) -+ trace.nr_entries--; -+ -+ for (i = trace.nr_entries; i < TRACK_ADDRS_COUNT; i++) -+ p->addrs[i] = 0; -+#endif -+ p->addr = addr; -+ p->cpu = smp_processor_id(); -+ p->pid = current->pid; -+ p->when = jiffies; -+ } else -+ memset(p, 0, sizeof(struct track)); -+} -+ -+static void init_tracking(struct kmem_cache *s, void *object) -+{ -+ if (!(s->flags & SLAB_STORE_USER)) -+ return; -+ -+ set_track(s, object, TRACK_FREE, 0UL); -+ set_track(s, object, TRACK_ALLOC, 0UL); -+} -+ -+static void print_track(const char *s, struct track *t) -+{ -+ if (!t->addr) -+ return; -+ -+ pr_err("INFO: %s in %pS age=%lu cpu=%u pid=%d\n", -+ s, (void *)t->addr, jiffies - t->when, t->cpu, t->pid); -+#ifdef CONFIG_STACKTRACE -+ { -+ int i; -+ for (i = 0; i < TRACK_ADDRS_COUNT; i++) -+ if (t->addrs[i]) -+ pr_err("\t%pS\n", (void *)t->addrs[i]); -+ else -+ break; -+ } -+#endif -+} -+ -+static void print_tracking(struct kmem_cache *s, void *object) -+{ -+ if (!(s->flags & SLAB_STORE_USER)) -+ return; -+ -+ print_track("Allocated", get_track(s, object, TRACK_ALLOC)); -+ print_track("Freed", get_track(s, object, TRACK_FREE)); -+} -+ -+static void print_page_info(struct page *page) -+{ -+ pr_err("INFO: Slab 0x%p objects=%u used=%u fp=0x%p flags=0x%04lx\n", -+ page, page->objects, page->inuse, page->freelist, page->flags); -+ -+} -+ -+static void slab_bug(struct kmem_cache *s, char *fmt, ...) -+{ -+ struct va_format vaf; -+ va_list args; -+ -+ va_start(args, fmt); -+ vaf.fmt = fmt; -+ vaf.va = &args; -+ pr_err("=============================================================================\n"); -+ pr_err("BUG %s (%s): %pV\n", s->name, print_tainted(), &vaf); -+ pr_err("-----------------------------------------------------------------------------\n\n"); -+ -+ add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); -+ va_end(args); -+} -+ -+static void slab_fix(struct kmem_cache *s, char *fmt, ...) -+{ -+ struct va_format vaf; -+ va_list args; -+ -+ va_start(args, fmt); -+ vaf.fmt = fmt; -+ vaf.va = &args; -+ pr_err("FIX %s: %pV\n", s->name, &vaf); -+ va_end(args); -+} -+ -+static void print_trailer(struct kmem_cache *s, struct page *page, u8 *p) -+{ -+ unsigned int off; /* Offset of last byte */ -+ u8 *addr = page_address(page); -+ -+ print_tracking(s, p); -+ -+ print_page_info(page); -+ -+ pr_err("INFO: Object 0x%p @offset=%tu fp=0x%p\n\n", -+ p, p - addr, get_freepointer(s, p)); -+ -+ if (p > addr + 16) -+ print_section("Bytes b4 ", p - 16, 16); -+ -+ print_section("Object ", p, min_t(unsigned long, s->object_size, -+ PAGE_SIZE)); -+ if (s->flags & SLAB_RED_ZONE) -+ print_section("Redzone ", p + s->object_size, -+ s->inuse - s->object_size); -+ -+ if (s->offset) -+ off = s->offset + sizeof(void *); -+ else -+ off = s->inuse; -+ -+ if (s->flags & SLAB_STORE_USER) -+ off += 2 * sizeof(struct track); -+ -+ if (off != s->size) -+ /* Beginning of the filler is the free pointer */ -+ print_section("Padding ", p + off, s->size - off); -+ -+ dump_stack(); -+} -+ -+void object_err(struct kmem_cache *s, struct page *page, -+ u8 *object, char *reason) -+{ -+ slab_bug(s, "%s", reason); -+ print_trailer(s, page, object); -+} -+ -+static void slab_err(struct kmem_cache *s, struct page *page, -+ const char *fmt, ...) -+{ -+ va_list args; -+ char buf[100]; -+ -+ va_start(args, fmt); -+ vsnprintf(buf, sizeof(buf), fmt, args); -+ va_end(args); -+ slab_bug(s, "%s", buf); -+ print_page_info(page); -+ dump_stack(); -+} -+ -+static void init_object(struct kmem_cache *s, void *object, u8 val) -+{ -+ u8 *p = object; -+ -+ if (s->flags & __OBJECT_POISON) { -+ memset(p, POISON_FREE, s->object_size - 1); -+ p[s->object_size - 1] = POISON_END; -+ } -+ -+ if (s->flags & SLAB_RED_ZONE) -+ memset(p + s->object_size, val, s->inuse - s->object_size); -+} -+ -+static void restore_bytes(struct kmem_cache *s, char *message, u8 data, -+ void *from, void *to) -+{ -+ slab_fix(s, "Restoring 0x%p-0x%p=0x%x\n", from, to - 1, data); -+ memset(from, data, to - from); -+} -+ -+static int check_bytes_and_report(struct kmem_cache *s, struct page *page, -+ u8 *object, char *what, -+ u8 *start, unsigned int value, unsigned int bytes) -+{ -+ u8 *fault; -+ u8 *end; -+ -+ metadata_access_enable(); -+ fault = memchr_inv(start, value, bytes); -+ metadata_access_disable(); -+ if (!fault) -+ return 1; -+ -+ end = start + bytes; -+ while (end > fault && end[-1] == value) -+ end--; -+ -+ slab_bug(s, "%s overwritten", what); -+ pr_err("INFO: 0x%p-0x%p. First byte 0x%x instead of 0x%x\n", -+ fault, end - 1, fault[0], value); -+ print_trailer(s, page, object); -+ -+ restore_bytes(s, what, value, fault, end); -+ return 0; -+} -+ -+/* -+ * Object layout: -+ * -+ * object address -+ * Bytes of the object to be managed. -+ * If the freepointer may overlay the object then the free -+ * pointer is the first word of the object. -+ * -+ * Poisoning uses 0x6b (POISON_FREE) and the last byte is -+ * 0xa5 (POISON_END) -+ * -+ * object + s->object_size -+ * Padding to reach word boundary. This is also used for Redzoning. -+ * Padding is extended by another word if Redzoning is enabled and -+ * object_size == inuse. -+ * -+ * We fill with 0xbb (RED_INACTIVE) for inactive objects and with -+ * 0xcc (RED_ACTIVE) for objects in use. -+ * -+ * object + s->inuse -+ * Meta data starts here. -+ * -+ * A. Free pointer (if we cannot overwrite object on free) -+ * B. Tracking data for SLAB_STORE_USER -+ * C. Padding to reach required alignment boundary or at mininum -+ * one word if debugging is on to be able to detect writes -+ * before the word boundary. -+ * -+ * Padding is done using 0x5a (POISON_INUSE) -+ * -+ * object + s->size -+ * Nothing is used beyond s->size. -+ * -+ * If slabcaches are merged then the object_size and inuse boundaries are mostly -+ * ignored. And therefore no slab options that rely on these boundaries -+ * may be used with merged slabcaches. -+ */ -+ -+static int check_pad_bytes(struct kmem_cache *s, struct page *page, u8 *p) -+{ -+ unsigned long off = s->inuse; /* The end of info */ -+ -+ if (s->offset) -+ /* Freepointer is placed after the object. */ -+ off += sizeof(void *); -+ -+ if (s->flags & SLAB_STORE_USER) -+ /* We also have user information there */ -+ off += 2 * sizeof(struct track); -+ -+ if (s->size == off) -+ return 1; -+ -+ return check_bytes_and_report(s, page, p, "Object padding", -+ p + off, POISON_INUSE, s->size - off); -+} -+ -+/* Check the pad bytes at the end of a slab page */ -+static int slab_pad_check(struct kmem_cache *s, struct page *page) -+{ -+ u8 *start; -+ u8 *fault; -+ u8 *end; -+ int length; -+ int remainder; -+ -+ if (!(s->flags & SLAB_POISON)) -+ return 1; -+ -+ start = page_address(page); -+ length = (PAGE_SIZE << compound_order(page)) - s->reserved; -+ end = start + length; -+ remainder = length % s->size; -+ if (!remainder) -+ return 1; -+ -+ metadata_access_enable(); -+ fault = memchr_inv(end - remainder, POISON_INUSE, remainder); -+ metadata_access_disable(); -+ if (!fault) -+ return 1; -+ while (end > fault && end[-1] == POISON_INUSE) -+ end--; -+ -+ slab_err(s, page, "Padding overwritten. 0x%p-0x%p", fault, end - 1); -+ print_section("Padding ", end - remainder, remainder); -+ -+ restore_bytes(s, "slab padding", POISON_INUSE, end - remainder, end); -+ return 0; -+} -+ -+static int check_object(struct kmem_cache *s, struct page *page, -+ void *object, u8 val) -+{ -+ u8 *p = object; -+ u8 *endobject = object + s->object_size; -+ -+ if (s->flags & SLAB_RED_ZONE) { -+ if (!check_bytes_and_report(s, page, object, "Redzone", -+ endobject, val, s->inuse - s->object_size)) -+ return 0; -+ } else { -+ if ((s->flags & SLAB_POISON) && s->object_size < s->inuse) { -+ check_bytes_and_report(s, page, p, "Alignment padding", -+ endobject, POISON_INUSE, -+ s->inuse - s->object_size); -+ } -+ } -+ -+ if (s->flags & SLAB_POISON) { -+ if (val != SLUB_RED_ACTIVE && (s->flags & __OBJECT_POISON) && -+ (!check_bytes_and_report(s, page, p, "Poison", p, -+ POISON_FREE, s->object_size - 1) || -+ !check_bytes_and_report(s, page, p, "Poison", -+ p + s->object_size - 1, POISON_END, 1))) -+ return 0; -+ /* -+ * check_pad_bytes cleans up on its own. -+ */ -+ check_pad_bytes(s, page, p); -+ } -+ -+ if (!s->offset && val == SLUB_RED_ACTIVE) -+ /* -+ * Object and freepointer overlap. Cannot check -+ * freepointer while object is allocated. -+ */ -+ return 1; -+ -+ /* Check free pointer validity */ -+ if (!check_valid_pointer(s, page, get_freepointer(s, p))) { -+ object_err(s, page, p, "Freepointer corrupt"); -+ /* -+ * No choice but to zap it and thus lose the remainder -+ * of the free objects in this slab. May cause -+ * another error because the object count is now wrong. -+ */ -+ set_freepointer(s, p, NULL); -+ return 0; -+ } -+ return 1; -+} -+ -+static int check_slab(struct kmem_cache *s, struct page *page) -+{ -+ int maxobj; -+ -+ VM_BUG_ON(!irqs_disabled()); -+ -+ if (!PageSlab(page)) { -+ slab_err(s, page, "Not a valid slab page"); -+ return 0; -+ } -+ -+ maxobj = order_objects(compound_order(page), s->size, s->reserved); -+ if (page->objects > maxobj) { -+ slab_err(s, page, "objects %u > max %u", -+ page->objects, maxobj); -+ return 0; -+ } -+ if (page->inuse > page->objects) { -+ slab_err(s, page, "inuse %u > max %u", -+ page->inuse, page->objects); -+ return 0; -+ } -+ /* Slab_pad_check fixes things up after itself */ -+ slab_pad_check(s, page); -+ return 1; -+} -+ -+/* -+ * Determine if a certain object on a page is on the freelist. Must hold the -+ * slab lock to guarantee that the chains are in a consistent state. -+ */ -+static int on_freelist(struct kmem_cache *s, struct page *page, void *search) -+{ -+ int nr = 0; -+ void *fp; -+ void *object = NULL; -+ int max_objects; -+ -+ fp = page->freelist; -+ while (fp && nr <= page->objects) { -+ if (fp == search) -+ return 1; -+ if (!check_valid_pointer(s, page, fp)) { -+ if (object) { -+ object_err(s, page, object, -+ "Freechain corrupt"); -+ set_freepointer(s, object, NULL); -+ } else { -+ slab_err(s, page, "Freepointer corrupt"); -+ page->freelist = NULL; -+ page->inuse = page->objects; -+ slab_fix(s, "Freelist cleared"); -+ return 0; -+ } -+ break; -+ } -+ object = fp; -+ fp = get_freepointer(s, object); -+ nr++; -+ } -+ -+ max_objects = order_objects(compound_order(page), s->size, s->reserved); -+ if (max_objects > MAX_OBJS_PER_PAGE) -+ max_objects = MAX_OBJS_PER_PAGE; -+ -+ if (page->objects != max_objects) { -+ slab_err(s, page, "Wrong number of objects. Found %d but " -+ "should be %d", page->objects, max_objects); -+ page->objects = max_objects; -+ slab_fix(s, "Number of objects adjusted."); -+ } -+ if (page->inuse != page->objects - nr) { -+ slab_err(s, page, "Wrong object count. Counter is %d but " -+ "counted were %d", page->inuse, page->objects - nr); -+ page->inuse = page->objects - nr; -+ slab_fix(s, "Object count adjusted."); -+ } -+ return search == NULL; -+} -+ -+static void trace(struct kmem_cache *s, struct page *page, void *object, -+ int alloc) -+{ -+ if (s->flags & SLAB_TRACE) { -+ pr_info("TRACE %s %s 0x%p inuse=%d fp=0x%p\n", -+ s->name, -+ alloc ? "alloc" : "free", -+ object, page->inuse, -+ page->freelist); -+ -+ if (!alloc) -+ print_section("Object ", (void *)object, -+ s->object_size); -+ -+ dump_stack(); -+ } -+} -+ -+/* -+ * Tracking of fully allocated slabs for debugging purposes. -+ */ -+static void add_full(struct kmem_cache *s, -+ struct kmem_cache_node *n, struct page *page) -+{ -+ if (!(s->flags & SLAB_STORE_USER)) -+ return; -+ -+ lockdep_assert_held(&n->list_lock); -+ list_add(&page->lru, &n->full); -+} -+ -+static void remove_full(struct kmem_cache *s, struct kmem_cache_node *n, struct page *page) -+{ -+ if (!(s->flags & SLAB_STORE_USER)) -+ return; -+ -+ lockdep_assert_held(&n->list_lock); -+ list_del(&page->lru); -+} -+ -+/* Tracking of the number of slabs for debugging purposes */ -+static inline unsigned long slabs_node(struct kmem_cache *s, int node) -+{ -+ struct kmem_cache_node *n = get_node(s, node); -+ -+ return atomic_long_read(&n->nr_slabs); -+} -+ -+static inline unsigned long node_nr_slabs(struct kmem_cache_node *n) -+{ -+ return atomic_long_read(&n->nr_slabs); -+} -+ -+static inline void inc_slabs_node(struct kmem_cache *s, int node, int objects) -+{ -+ struct kmem_cache_node *n = get_node(s, node); -+ -+ /* -+ * May be called early in order to allocate a slab for the -+ * kmem_cache_node structure. Solve the chicken-egg -+ * dilemma by deferring the increment of the count during -+ * bootstrap (see early_kmem_cache_node_alloc). -+ */ -+ if (likely(n)) { -+ atomic_long_inc(&n->nr_slabs); -+ atomic_long_add(objects, &n->total_objects); -+ } -+} -+static inline void dec_slabs_node(struct kmem_cache *s, int node, int objects) -+{ -+ struct kmem_cache_node *n = get_node(s, node); -+ -+ atomic_long_dec(&n->nr_slabs); -+ atomic_long_sub(objects, &n->total_objects); -+} -+ -+/* Object debug checks for alloc/free paths */ -+static void setup_object_debug(struct kmem_cache *s, struct page *page, -+ void *object) -+{ -+ if (!(s->flags & (SLAB_STORE_USER|SLAB_RED_ZONE|__OBJECT_POISON))) -+ return; -+ -+ init_object(s, object, SLUB_RED_INACTIVE); -+ init_tracking(s, object); -+} -+ -+static noinline int alloc_debug_processing(struct kmem_cache *s, -+ struct page *page, -+ void *object, unsigned long addr) -+{ -+ if (!check_slab(s, page)) -+ goto bad; -+ -+ if (!check_valid_pointer(s, page, object)) { -+ object_err(s, page, object, "Freelist Pointer check fails"); -+ goto bad; -+ } -+ -+ if (!check_object(s, page, object, SLUB_RED_INACTIVE)) -+ goto bad; -+ -+ /* Success perform special debug activities for allocs */ -+ if (s->flags & SLAB_STORE_USER) -+ set_track(s, object, TRACK_ALLOC, addr); -+ trace(s, page, object, 1); -+ init_object(s, object, SLUB_RED_ACTIVE); -+ return 1; -+ -+bad: -+ if (PageSlab(page)) { -+ /* -+ * If this is a slab page then lets do the best we can -+ * to avoid issues in the future. Marking all objects -+ * as used avoids touching the remaining objects. -+ */ -+ slab_fix(s, "Marking all objects used"); -+ page->inuse = page->objects; -+ page->freelist = NULL; -+ } -+ return 0; -+} -+ -+static noinline struct kmem_cache_node *free_debug_processing( -+ struct kmem_cache *s, struct page *page, void *object, -+ unsigned long addr, unsigned long *flags) -+{ -+ struct kmem_cache_node *n = get_node(s, page_to_nid(page)); -+ -+ spin_lock_irqsave(&n->list_lock, *flags); -+ slab_lock(page); -+ -+ if (!check_slab(s, page)) -+ goto fail; -+ -+ if (!check_valid_pointer(s, page, object)) { -+ slab_err(s, page, "Invalid object pointer 0x%p", object); -+ goto fail; -+ } -+ -+ if (on_freelist(s, page, object)) { -+ object_err(s, page, object, "Object already free"); -+ goto fail; -+ } -+ -+ if (!check_object(s, page, object, SLUB_RED_ACTIVE)) -+ goto out; -+ -+ if (unlikely(s != page->slab_cache)) { -+ if (!PageSlab(page)) { -+ slab_err(s, page, "Attempt to free object(0x%p) " -+ "outside of slab", object); -+ } else if (!page->slab_cache) { -+ pr_err("SLUB : no slab for object 0x%p.\n", -+ object); -+ dump_stack(); -+ } else -+ object_err(s, page, object, -+ "page slab pointer corrupt."); -+ goto fail; -+ } -+ -+ if (s->flags & SLAB_STORE_USER) -+ set_track(s, object, TRACK_FREE, addr); -+ trace(s, page, object, 0); -+ init_object(s, object, SLUB_RED_INACTIVE); -+out: -+ slab_unlock(page); -+ /* -+ * Keep node_lock to preserve integrity -+ * until the object is actually freed -+ */ -+ return n; -+ -+fail: -+ slab_unlock(page); -+ spin_unlock_irqrestore(&n->list_lock, *flags); -+ slab_fix(s, "Object at 0x%p not freed", object); -+ return NULL; -+} -+ -+static int __init setup_slub_debug(char *str) -+{ -+ slub_debug = DEBUG_DEFAULT_FLAGS; -+ if (*str++ != '=' || !*str) -+ /* -+ * No options specified. Switch on full debugging. -+ */ -+ goto out; -+ -+ if (*str == ',') -+ /* -+ * No options but restriction on slabs. This means full -+ * debugging for slabs matching a pattern. -+ */ -+ goto check_slabs; -+ -+ slub_debug = 0; -+ if (*str == '-') -+ /* -+ * Switch off all debugging measures. -+ */ -+ goto out; -+ -+ /* -+ * Determine which debug features should be switched on -+ */ -+ for (; *str && *str != ','; str++) { -+ switch (tolower(*str)) { -+ case 'f': -+ slub_debug |= SLAB_DEBUG_FREE; -+ break; -+ case 'z': -+ slub_debug |= SLAB_RED_ZONE; -+ break; -+ case 'p': -+ slub_debug |= SLAB_POISON; -+ break; -+ case 'u': -+ slub_debug |= SLAB_STORE_USER; -+ break; -+ case 't': -+ slub_debug |= SLAB_TRACE; -+ break; -+ case 'a': -+ slub_debug |= SLAB_FAILSLAB; -+ break; -+ case 'o': -+ /* -+ * Avoid enabling debugging on caches if its minimum -+ * order would increase as a result. -+ */ -+ disable_higher_order_debug = 1; -+ break; -+ default: -+ pr_err("slub_debug option '%c' unknown. skipped\n", -+ *str); -+ } -+ } -+ -+check_slabs: -+ if (*str == ',') -+ slub_debug_slabs = str + 1; -+out: -+ return 1; -+} -+ -+__setup("slub_debug", setup_slub_debug); -+ -+unsigned long kmem_cache_flags(unsigned long object_size, -+ unsigned long flags, const char *name, -+ void (*ctor)(void *)) -+{ -+ /* -+ * Enable debugging if selected on the kernel commandline. -+ */ -+ if (slub_debug && (!slub_debug_slabs || (name && -+ !strncmp(slub_debug_slabs, name, strlen(slub_debug_slabs))))) -+ flags |= slub_debug; -+ -+ return flags; -+} -+#else -+static inline void setup_object_debug(struct kmem_cache *s, -+ struct page *page, void *object) {} -+ -+static inline int alloc_debug_processing(struct kmem_cache *s, -+ struct page *page, void *object, unsigned long addr) { return 0; } -+ -+static inline struct kmem_cache_node *free_debug_processing( -+ struct kmem_cache *s, struct page *page, void *object, -+ unsigned long addr, unsigned long *flags) { return NULL; } -+ -+static inline int slab_pad_check(struct kmem_cache *s, struct page *page) -+ { return 1; } -+static inline int check_object(struct kmem_cache *s, struct page *page, -+ void *object, u8 val) { return 1; } -+static inline void add_full(struct kmem_cache *s, struct kmem_cache_node *n, -+ struct page *page) {} -+static inline void remove_full(struct kmem_cache *s, struct kmem_cache_node *n, -+ struct page *page) {} -+unsigned long kmem_cache_flags(unsigned long object_size, -+ unsigned long flags, const char *name, -+ void (*ctor)(void *)) -+{ -+ return flags; -+} -+#define slub_debug 0 -+ -+#define disable_higher_order_debug 0 -+ -+static inline unsigned long slabs_node(struct kmem_cache *s, int node) -+ { return 0; } -+static inline unsigned long node_nr_slabs(struct kmem_cache_node *n) -+ { return 0; } -+static inline void inc_slabs_node(struct kmem_cache *s, int node, -+ int objects) {} -+static inline void dec_slabs_node(struct kmem_cache *s, int node, -+ int objects) {} -+ -+#endif /* CONFIG_SLUB_DEBUG */ -+ -+/* -+ * Hooks for other subsystems that check memory allocations. In a typical -+ * production configuration these hooks all should produce no code at all. -+ */ -+static inline void kmalloc_large_node_hook(void *ptr, size_t size, gfp_t flags) -+{ -+ kmemleak_alloc(ptr, size, 1, flags); -+ kasan_kmalloc_large(ptr, size); -+} -+ -+static inline void kfree_hook(const void *x) -+{ -+ kmemleak_free(x); -+ kasan_kfree_large(x); -+} -+ -+static inline struct kmem_cache *slab_pre_alloc_hook(struct kmem_cache *s, -+ gfp_t flags) -+{ -+ flags &= gfp_allowed_mask; -+ lockdep_trace_alloc(flags); -+ might_sleep_if(flags & __GFP_WAIT); -+ -+ if (should_failslab(s->object_size, flags, s->flags)) -+ return NULL; -+ -+ return memcg_kmem_get_cache(s, flags); -+} -+ -+static inline void slab_post_alloc_hook(struct kmem_cache *s, -+ gfp_t flags, void *object) -+{ -+ flags &= gfp_allowed_mask; -+ kmemcheck_slab_alloc(s, flags, object, slab_ksize(s)); -+ kmemleak_alloc_recursive(object, s->object_size, 1, s->flags, flags); -+ memcg_kmem_put_cache(s); -+ kasan_slab_alloc(s, object); -+} -+ -+static inline void slab_free_hook(struct kmem_cache *s, void *x) -+{ -+ kmemleak_free_recursive(x, s->flags); -+ -+ /* -+ * Trouble is that we may no longer disable interrupts in the fast path -+ * So in order to make the debug calls that expect irqs to be -+ * disabled we need to disable interrupts temporarily. -+ */ -+#if defined(CONFIG_KMEMCHECK) || defined(CONFIG_LOCKDEP) -+ { -+ unsigned long flags; -+ -+ local_irq_save(flags); -+ kmemcheck_slab_free(s, x, s->object_size); -+ debug_check_no_locks_freed(x, s->object_size); -+ local_irq_restore(flags); -+ } -+#endif -+ if (!(s->flags & SLAB_DEBUG_OBJECTS)) -+ debug_check_no_obj_freed(x, s->object_size); -+ -+ kasan_slab_free(s, x); -+} -+ -+/* -+ * Slab allocation and freeing -+ */ -+static inline struct page *alloc_slab_page(struct kmem_cache *s, -+ gfp_t flags, int node, struct kmem_cache_order_objects oo) -+{ -+ struct page *page; -+ int order = oo_order(oo); -+ -+ flags |= __GFP_NOTRACK; -+ -+ if (memcg_charge_slab(s, flags, order)) -+ return NULL; -+ -+ if (node == NUMA_NO_NODE) -+ page = alloc_pages(flags, order); -+ else -+ page = alloc_pages_exact_node(node, flags, order); -+ -+ if (!page) -+ memcg_uncharge_slab(s, order); -+ -+ return page; -+} -+ -+static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) -+{ -+ struct page *page; -+ struct kmem_cache_order_objects oo = s->oo; -+ gfp_t alloc_gfp; -+ -+ flags &= gfp_allowed_mask; -+ -+ if (flags & __GFP_WAIT) -+ local_irq_enable(); -+ -+ flags |= s->allocflags; -+ -+ /* -+ * Let the initial higher-order allocation fail under memory pressure -+ * so we fall-back to the minimum order allocation. -+ */ -+ alloc_gfp = (flags | __GFP_NOWARN | __GFP_NORETRY) & ~__GFP_NOFAIL; -+ -+ page = alloc_slab_page(s, alloc_gfp, node, oo); -+ if (unlikely(!page)) { -+ oo = s->min; -+ alloc_gfp = flags; -+ /* -+ * Allocation may have failed due to fragmentation. -+ * Try a lower order alloc if possible -+ */ -+ page = alloc_slab_page(s, alloc_gfp, node, oo); -+ -+ if (page) -+ stat(s, ORDER_FALLBACK); -+ } -+ -+ if (kmemcheck_enabled && page -+ && !(s->flags & (SLAB_NOTRACK | DEBUG_DEFAULT_FLAGS))) { -+ int pages = 1 << oo_order(oo); -+ -+ kmemcheck_alloc_shadow(page, oo_order(oo), alloc_gfp, node); -+ -+ /* -+ * Objects from caches that have a constructor don't get -+ * cleared when they're allocated, so we need to do it here. -+ */ -+ if (s->ctor) -+ kmemcheck_mark_uninitialized_pages(page, pages); -+ else -+ kmemcheck_mark_unallocated_pages(page, pages); -+ } -+ -+ if (flags & __GFP_WAIT) -+ local_irq_disable(); -+ if (!page) -+ return NULL; -+ -+ page->objects = oo_objects(oo); -+ mod_zone_page_state(page_zone(page), -+ (s->flags & SLAB_RECLAIM_ACCOUNT) ? -+ NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE, -+ 1 << oo_order(oo)); -+ -+ return page; -+} -+ -+static void setup_object(struct kmem_cache *s, struct page *page, -+ void *object) -+{ -+ setup_object_debug(s, page, object); -+ if (unlikely(s->ctor)) { -+ kasan_unpoison_object_data(s, object); -+ s->ctor(object); -+ kasan_poison_object_data(s, object); -+ } -+} -+ -+static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node) -+{ -+ struct page *page; -+ void *start; -+ void *p; -+ int order; -+ int idx; -+ -+ if (unlikely(flags & GFP_SLAB_BUG_MASK)) { -+ pr_emerg("gfp: %u\n", flags & GFP_SLAB_BUG_MASK); -+ BUG(); -+ } -+ -+ page = allocate_slab(s, -+ flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node); -+ if (!page) -+ goto out; -+ -+ order = compound_order(page); -+ inc_slabs_node(s, page_to_nid(page), page->objects); -+ page->slab_cache = s; -+ __SetPageSlab(page); -+ if (page_is_pfmemalloc(page)) -+ SetPageSlabPfmemalloc(page); -+ -+ start = page_address(page); -+ -+ if (unlikely(s->flags & SLAB_POISON)) -+ memset(start, POISON_INUSE, PAGE_SIZE << order); -+ -+ kasan_poison_slab(page); -+ -+ for_each_object_idx(p, idx, s, start, page->objects) { -+ setup_object(s, page, p); -+ if (likely(idx < page->objects)) -+ set_freepointer(s, p, p + s->size); -+ else -+ set_freepointer(s, p, NULL); -+ } -+ -+ page->freelist = start; -+ page->inuse = page->objects; -+ page->frozen = 1; -+out: -+ return page; -+} -+ -+static void __free_slab(struct kmem_cache *s, struct page *page) -+{ -+ int order = compound_order(page); -+ int pages = 1 << order; -+ -+ if (kmem_cache_debug(s)) { -+ void *p; -+ -+ slab_pad_check(s, page); -+ for_each_object(p, s, page_address(page), -+ page->objects) -+ check_object(s, page, p, SLUB_RED_INACTIVE); -+ } -+ -+ kmemcheck_free_shadow(page, compound_order(page)); -+ -+ mod_zone_page_state(page_zone(page), -+ (s->flags & SLAB_RECLAIM_ACCOUNT) ? -+ NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE, -+ -pages); -+ -+ __ClearPageSlabPfmemalloc(page); -+ __ClearPageSlab(page); -+ -+ page_mapcount_reset(page); -+ if (current->reclaim_state) -+ current->reclaim_state->reclaimed_slab += pages; -+ __free_pages(page, order); -+ memcg_uncharge_slab(s, order); -+} -+ -+#define need_reserve_slab_rcu \ -+ (sizeof(((struct page *)NULL)->lru) < sizeof(struct rcu_head)) -+ -+static void rcu_free_slab(struct rcu_head *h) -+{ -+ struct page *page; -+ -+ if (need_reserve_slab_rcu) -+ page = virt_to_head_page(h); -+ else -+ page = container_of((struct list_head *)h, struct page, lru); -+ -+ __free_slab(page->slab_cache, page); -+} -+ -+static void free_slab(struct kmem_cache *s, struct page *page) -+{ -+ if (unlikely(s->flags & SLAB_DESTROY_BY_RCU)) { -+ struct rcu_head *head; -+ -+ if (need_reserve_slab_rcu) { -+ int order = compound_order(page); -+ int offset = (PAGE_SIZE << order) - s->reserved; -+ -+ VM_BUG_ON(s->reserved != sizeof(*head)); -+ head = page_address(page) + offset; -+ } else { -+ /* -+ * RCU free overloads the RCU head over the LRU -+ */ -+ head = (void *)&page->lru; -+ } -+ -+ call_rcu(head, rcu_free_slab); -+ } else -+ __free_slab(s, page); -+} -+ -+static void discard_slab(struct kmem_cache *s, struct page *page) -+{ -+ dec_slabs_node(s, page_to_nid(page), page->objects); -+ free_slab(s, page); -+} -+ -+/* -+ * Management of partially allocated slabs. -+ */ -+static inline void -+__add_partial(struct kmem_cache_node *n, struct page *page, int tail) -+{ -+ n->nr_partial++; -+ if (tail == DEACTIVATE_TO_TAIL) -+ list_add_tail(&page->lru, &n->partial); -+ else -+ list_add(&page->lru, &n->partial); -+} -+ -+static inline void add_partial(struct kmem_cache_node *n, -+ struct page *page, int tail) -+{ -+ lockdep_assert_held(&n->list_lock); -+ __add_partial(n, page, tail); -+} -+ -+static inline void -+__remove_partial(struct kmem_cache_node *n, struct page *page) -+{ -+ list_del(&page->lru); -+ n->nr_partial--; -+} -+ -+static inline void remove_partial(struct kmem_cache_node *n, -+ struct page *page) -+{ -+ lockdep_assert_held(&n->list_lock); -+ __remove_partial(n, page); -+} -+ -+/* -+ * Remove slab from the partial list, freeze it and -+ * return the pointer to the freelist. -+ * -+ * Returns a list of objects or NULL if it fails. -+ */ -+static inline void *acquire_slab(struct kmem_cache *s, -+ struct kmem_cache_node *n, struct page *page, -+ int mode, int *objects) -+{ -+ void *freelist; -+ unsigned long counters; -+ struct page new; -+ -+ lockdep_assert_held(&n->list_lock); -+ -+ /* -+ * Zap the freelist and set the frozen bit. -+ * The old freelist is the list of objects for the -+ * per cpu allocation list. -+ */ -+ freelist = page->freelist; -+ counters = page->counters; -+ new.counters = counters; -+ *objects = new.objects - new.inuse; -+ if (mode) { -+ new.inuse = page->objects; -+ new.freelist = NULL; -+ } else { -+ new.freelist = freelist; -+ } -+ -+ VM_BUG_ON(new.frozen); -+ new.frozen = 1; -+ -+ if (!__cmpxchg_double_slab(s, page, -+ freelist, counters, -+ new.freelist, new.counters, -+ "acquire_slab")) -+ return NULL; -+ -+ remove_partial(n, page); -+ WARN_ON(!freelist); -+ return freelist; -+} -+ -+static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain); -+static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags); -+ -+/* -+ * Try to allocate a partial slab from a specific node. -+ */ -+static void *get_partial_node(struct kmem_cache *s, struct kmem_cache_node *n, -+ struct kmem_cache_cpu *c, gfp_t flags) -+{ -+ struct page *page, *page2; -+ void *object = NULL; -+ int available = 0; -+ int objects; -+ -+ /* -+ * Racy check. If we mistakenly see no partial slabs then we -+ * just allocate an empty slab. If we mistakenly try to get a -+ * partial slab and there is none available then get_partials() -+ * will return NULL. -+ */ -+ if (!n || !n->nr_partial) -+ return NULL; -+ -+ spin_lock(&n->list_lock); -+ list_for_each_entry_safe(page, page2, &n->partial, lru) { -+ void *t; -+ -+ if (!pfmemalloc_match(page, flags)) -+ continue; -+ -+ t = acquire_slab(s, n, page, object == NULL, &objects); -+ if (!t) -+ break; -+ -+ available += objects; -+ if (!object) { -+ c->page = page; -+ stat(s, ALLOC_FROM_PARTIAL); -+ object = t; -+ } else { -+ put_cpu_partial(s, page, 0); -+ stat(s, CPU_PARTIAL_NODE); -+ } -+ if (!kmem_cache_has_cpu_partial(s) -+ || available > s->cpu_partial / 2) -+ break; -+ -+ } -+ spin_unlock(&n->list_lock); -+ return object; -+} -+ -+/* -+ * Get a page from somewhere. Search in increasing NUMA distances. -+ */ -+static void *get_any_partial(struct kmem_cache *s, gfp_t flags, -+ struct kmem_cache_cpu *c) -+{ -+#ifdef CONFIG_NUMA -+ struct zonelist *zonelist; -+ struct zoneref *z; -+ struct zone *zone; -+ enum zone_type high_zoneidx = gfp_zone(flags); -+ void *object; -+ unsigned int cpuset_mems_cookie; -+ -+ /* -+ * The defrag ratio allows a configuration of the tradeoffs between -+ * inter node defragmentation and node local allocations. A lower -+ * defrag_ratio increases the tendency to do local allocations -+ * instead of attempting to obtain partial slabs from other nodes. -+ * -+ * If the defrag_ratio is set to 0 then kmalloc() always -+ * returns node local objects. If the ratio is higher then kmalloc() -+ * may return off node objects because partial slabs are obtained -+ * from other nodes and filled up. -+ * -+ * If /sys/kernel/slab/xx/defrag_ratio is set to 100 (which makes -+ * defrag_ratio = 1000) then every (well almost) allocation will -+ * first attempt to defrag slab caches on other nodes. This means -+ * scanning over all nodes to look for partial slabs which may be -+ * expensive if we do it every time we are trying to find a slab -+ * with available objects. -+ */ -+ if (!s->remote_node_defrag_ratio || -+ get_cycles() % 1024 > s->remote_node_defrag_ratio) -+ return NULL; -+ -+ do { -+ cpuset_mems_cookie = read_mems_allowed_begin(); -+ zonelist = node_zonelist(mempolicy_slab_node(), flags); -+ for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) { -+ struct kmem_cache_node *n; -+ -+ n = get_node(s, zone_to_nid(zone)); -+ -+ if (n && cpuset_zone_allowed(zone, flags) && -+ n->nr_partial > s->min_partial) { -+ object = get_partial_node(s, n, c, flags); -+ if (object) { -+ /* -+ * Don't check read_mems_allowed_retry() -+ * here - if mems_allowed was updated in -+ * parallel, that was a harmless race -+ * between allocation and the cpuset -+ * update -+ */ -+ return object; -+ } -+ } -+ } -+ } while (read_mems_allowed_retry(cpuset_mems_cookie)); -+#endif -+ return NULL; -+} -+ -+/* -+ * Get a partial page, lock it and return it. -+ */ -+static void *get_partial(struct kmem_cache *s, gfp_t flags, int node, -+ struct kmem_cache_cpu *c) -+{ -+ void *object; -+ int searchnode = node; -+ -+ if (node == NUMA_NO_NODE) -+ searchnode = numa_mem_id(); -+ else if (!node_present_pages(node)) -+ searchnode = node_to_mem_node(node); -+ -+ object = get_partial_node(s, get_node(s, searchnode), c, flags); -+ if (object || node != NUMA_NO_NODE) -+ return object; -+ -+ return get_any_partial(s, flags, c); -+} -+ -+#ifdef CONFIG_PREEMPT -+/* -+ * Calculate the next globally unique transaction for disambiguiation -+ * during cmpxchg. The transactions start with the cpu number and are then -+ * incremented by CONFIG_NR_CPUS. -+ */ -+#define TID_STEP roundup_pow_of_two(CONFIG_NR_CPUS) -+#else -+/* -+ * No preemption supported therefore also no need to check for -+ * different cpus. -+ */ -+#define TID_STEP 1 -+#endif -+ -+static inline unsigned long next_tid(unsigned long tid) -+{ -+ return tid + TID_STEP; -+} -+ -+static inline unsigned int tid_to_cpu(unsigned long tid) -+{ -+ return tid % TID_STEP; -+} -+ -+static inline unsigned long tid_to_event(unsigned long tid) -+{ -+ return tid / TID_STEP; -+} -+ -+static inline unsigned int init_tid(int cpu) -+{ -+ return cpu; -+} -+ -+static inline void note_cmpxchg_failure(const char *n, -+ const struct kmem_cache *s, unsigned long tid) -+{ -+#ifdef SLUB_DEBUG_CMPXCHG -+ unsigned long actual_tid = __this_cpu_read(s->cpu_slab->tid); -+ -+ pr_info("%s %s: cmpxchg redo ", n, s->name); -+ -+#ifdef CONFIG_PREEMPT -+ if (tid_to_cpu(tid) != tid_to_cpu(actual_tid)) -+ pr_warn("due to cpu change %d -> %d\n", -+ tid_to_cpu(tid), tid_to_cpu(actual_tid)); -+ else -+#endif -+ if (tid_to_event(tid) != tid_to_event(actual_tid)) -+ pr_warn("due to cpu running other code. Event %ld->%ld\n", -+ tid_to_event(tid), tid_to_event(actual_tid)); -+ else -+ pr_warn("for unknown reason: actual=%lx was=%lx target=%lx\n", -+ actual_tid, tid, next_tid(tid)); -+#endif -+ stat(s, CMPXCHG_DOUBLE_CPU_FAIL); -+} -+ -+static void init_kmem_cache_cpus(struct kmem_cache *s) -+{ -+ int cpu; -+ -+ for_each_possible_cpu(cpu) -+ per_cpu_ptr(s->cpu_slab, cpu)->tid = init_tid(cpu); -+} -+ -+/* -+ * Remove the cpu slab -+ */ -+static void deactivate_slab(struct kmem_cache *s, struct page *page, -+ void *freelist) -+{ -+ enum slab_modes { M_NONE, M_PARTIAL, M_FULL, M_FREE }; -+ struct kmem_cache_node *n = get_node(s, page_to_nid(page)); -+ int lock = 0; -+ enum slab_modes l = M_NONE, m = M_NONE; -+ void *nextfree; -+ int tail = DEACTIVATE_TO_HEAD; -+ struct page new; -+ struct page old; -+ -+ if (page->freelist) { -+ stat(s, DEACTIVATE_REMOTE_FREES); -+ tail = DEACTIVATE_TO_TAIL; -+ } -+ -+ /* -+ * Stage one: Free all available per cpu objects back -+ * to the page freelist while it is still frozen. Leave the -+ * last one. -+ * -+ * There is no need to take the list->lock because the page -+ * is still frozen. -+ */ -+ while (freelist && (nextfree = get_freepointer(s, freelist))) { -+ void *prior; -+ unsigned long counters; -+ -+ do { -+ prior = page->freelist; -+ counters = page->counters; -+ set_freepointer(s, freelist, prior); -+ new.counters = counters; -+ new.inuse--; -+ VM_BUG_ON(!new.frozen); -+ -+ } while (!__cmpxchg_double_slab(s, page, -+ prior, counters, -+ freelist, new.counters, -+ "drain percpu freelist")); -+ -+ freelist = nextfree; -+ } -+ -+ /* -+ * Stage two: Ensure that the page is unfrozen while the -+ * list presence reflects the actual number of objects -+ * during unfreeze. -+ * -+ * We setup the list membership and then perform a cmpxchg -+ * with the count. If there is a mismatch then the page -+ * is not unfrozen but the page is on the wrong list. -+ * -+ * Then we restart the process which may have to remove -+ * the page from the list that we just put it on again -+ * because the number of objects in the slab may have -+ * changed. -+ */ -+redo: -+ -+ old.freelist = page->freelist; -+ old.counters = page->counters; -+ VM_BUG_ON(!old.frozen); -+ -+ /* Determine target state of the slab */ -+ new.counters = old.counters; -+ if (freelist) { -+ new.inuse--; -+ set_freepointer(s, freelist, old.freelist); -+ new.freelist = freelist; -+ } else -+ new.freelist = old.freelist; -+ -+ new.frozen = 0; -+ -+ if (!new.inuse && n->nr_partial >= s->min_partial) -+ m = M_FREE; -+ else if (new.freelist) { -+ m = M_PARTIAL; -+ if (!lock) { -+ lock = 1; -+ /* -+ * Taking the spinlock removes the possiblity -+ * that acquire_slab() will see a slab page that -+ * is frozen -+ */ -+ spin_lock(&n->list_lock); -+ } -+ } else { -+ m = M_FULL; -+ if (kmem_cache_debug(s) && !lock) { -+ lock = 1; -+ /* -+ * This also ensures that the scanning of full -+ * slabs from diagnostic functions will not see -+ * any frozen slabs. -+ */ -+ spin_lock(&n->list_lock); -+ } -+ } -+ -+ if (l != m) { -+ -+ if (l == M_PARTIAL) -+ -+ remove_partial(n, page); -+ -+ else if (l == M_FULL) -+ -+ remove_full(s, n, page); -+ -+ if (m == M_PARTIAL) { -+ -+ add_partial(n, page, tail); -+ stat(s, tail); -+ -+ } else if (m == M_FULL) { -+ -+ stat(s, DEACTIVATE_FULL); -+ add_full(s, n, page); -+ -+ } -+ } -+ -+ l = m; -+ if (!__cmpxchg_double_slab(s, page, -+ old.freelist, old.counters, -+ new.freelist, new.counters, -+ "unfreezing slab")) -+ goto redo; -+ -+ if (lock) -+ spin_unlock(&n->list_lock); -+ -+ if (m == M_FREE) { -+ stat(s, DEACTIVATE_EMPTY); -+ discard_slab(s, page); -+ stat(s, FREE_SLAB); -+ } -+} -+ -+/* -+ * Unfreeze all the cpu partial slabs. -+ * -+ * This function must be called with interrupts disabled -+ * for the cpu using c (or some other guarantee must be there -+ * to guarantee no concurrent accesses). -+ */ -+static void unfreeze_partials(struct kmem_cache *s, -+ struct kmem_cache_cpu *c) -+{ -+#ifdef CONFIG_SLUB_CPU_PARTIAL -+ struct kmem_cache_node *n = NULL, *n2 = NULL; -+ struct page *page, *discard_page = NULL; -+ -+ while ((page = c->partial)) { -+ struct page new; -+ struct page old; -+ -+ c->partial = page->next; -+ -+ n2 = get_node(s, page_to_nid(page)); -+ if (n != n2) { -+ if (n) -+ spin_unlock(&n->list_lock); -+ -+ n = n2; -+ spin_lock(&n->list_lock); -+ } -+ -+ do { -+ -+ old.freelist = page->freelist; -+ old.counters = page->counters; -+ VM_BUG_ON(!old.frozen); -+ -+ new.counters = old.counters; -+ new.freelist = old.freelist; -+ -+ new.frozen = 0; -+ -+ } while (!__cmpxchg_double_slab(s, page, -+ old.freelist, old.counters, -+ new.freelist, new.counters, -+ "unfreezing slab")); -+ -+ if (unlikely(!new.inuse && n->nr_partial >= s->min_partial)) { -+ page->next = discard_page; -+ discard_page = page; -+ } else { -+ add_partial(n, page, DEACTIVATE_TO_TAIL); -+ stat(s, FREE_ADD_PARTIAL); -+ } -+ } -+ -+ if (n) -+ spin_unlock(&n->list_lock); -+ -+ while (discard_page) { -+ page = discard_page; -+ discard_page = discard_page->next; -+ -+ stat(s, DEACTIVATE_EMPTY); -+ discard_slab(s, page); -+ stat(s, FREE_SLAB); -+ } -+#endif -+} -+ -+/* -+ * Put a page that was just frozen (in __slab_free) into a partial page -+ * slot if available. This is done without interrupts disabled and without -+ * preemption disabled. The cmpxchg is racy and may put the partial page -+ * onto a random cpus partial slot. -+ * -+ * If we did not find a slot then simply move all the partials to the -+ * per node partial list. -+ */ -+static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain) -+{ -+#ifdef CONFIG_SLUB_CPU_PARTIAL -+ struct page *oldpage; -+ int pages; -+ int pobjects; -+ -+ preempt_disable(); -+ do { -+ pages = 0; -+ pobjects = 0; -+ oldpage = this_cpu_read(s->cpu_slab->partial); -+ -+ if (oldpage) { -+ pobjects = oldpage->pobjects; -+ pages = oldpage->pages; -+ if (drain && pobjects > s->cpu_partial) { -+ unsigned long flags; -+ /* -+ * partial array is full. Move the existing -+ * set to the per node partial list. -+ */ -+ local_irq_save(flags); -+ unfreeze_partials(s, this_cpu_ptr(s->cpu_slab)); -+ local_irq_restore(flags); -+ oldpage = NULL; -+ pobjects = 0; -+ pages = 0; -+ stat(s, CPU_PARTIAL_DRAIN); -+ } -+ } -+ -+ pages++; -+ pobjects += page->objects - page->inuse; -+ -+ page->pages = pages; -+ page->pobjects = pobjects; -+ page->next = oldpage; -+ -+ } while (this_cpu_cmpxchg(s->cpu_slab->partial, oldpage, page) -+ != oldpage); -+ if (unlikely(!s->cpu_partial)) { -+ unsigned long flags; -+ -+ local_irq_save(flags); -+ unfreeze_partials(s, this_cpu_ptr(s->cpu_slab)); -+ local_irq_restore(flags); -+ } -+ preempt_enable(); -+#endif -+} -+ -+static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c) -+{ -+ stat(s, CPUSLAB_FLUSH); -+ deactivate_slab(s, c->page, c->freelist); -+ -+ c->tid = next_tid(c->tid); -+ c->page = NULL; -+ c->freelist = NULL; -+} -+ -+/* -+ * Flush cpu slab. -+ * -+ * Called from IPI handler with interrupts disabled. -+ */ -+static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu) -+{ -+ struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu); -+ -+ if (likely(c)) { -+ if (c->page) -+ flush_slab(s, c); -+ -+ unfreeze_partials(s, c); -+ } -+} -+ -+static void flush_cpu_slab(void *d) -+{ -+ struct kmem_cache *s = d; -+ -+ __flush_cpu_slab(s, smp_processor_id()); -+} -+ -+static bool has_cpu_slab(int cpu, void *info) -+{ -+ struct kmem_cache *s = info; -+ struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu); -+ -+ return c->page || c->partial; -+} -+ -+static void flush_all(struct kmem_cache *s) -+{ -+ on_each_cpu_cond(has_cpu_slab, flush_cpu_slab, s, 1, GFP_ATOMIC); -+} -+ -+/* -+ * Check if the objects in a per cpu structure fit numa -+ * locality expectations. -+ */ -+static inline int node_match(struct page *page, int node) -+{ -+#ifdef CONFIG_NUMA -+ if (!page || (node != NUMA_NO_NODE && page_to_nid(page) != node)) -+ return 0; -+#endif -+ return 1; -+} -+ -+#ifdef CONFIG_SLUB_DEBUG -+static int count_free(struct page *page) -+{ -+ return page->objects - page->inuse; -+} -+ -+static inline unsigned long node_nr_objs(struct kmem_cache_node *n) -+{ -+ return atomic_long_read(&n->total_objects); -+} -+#endif /* CONFIG_SLUB_DEBUG */ -+ -+#if defined(CONFIG_SLUB_DEBUG) || defined(CONFIG_SYSFS) -+static unsigned long count_partial(struct kmem_cache_node *n, -+ int (*get_count)(struct page *)) -+{ -+ unsigned long flags; -+ unsigned long x = 0; -+ struct page *page; -+ -+ spin_lock_irqsave(&n->list_lock, flags); -+ list_for_each_entry(page, &n->partial, lru) -+ x += get_count(page); -+ spin_unlock_irqrestore(&n->list_lock, flags); -+ return x; -+} -+#endif /* CONFIG_SLUB_DEBUG || CONFIG_SYSFS */ -+ -+static noinline void -+slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid) -+{ -+#ifdef CONFIG_SLUB_DEBUG -+ static DEFINE_RATELIMIT_STATE(slub_oom_rs, DEFAULT_RATELIMIT_INTERVAL, -+ DEFAULT_RATELIMIT_BURST); -+ int node; -+ struct kmem_cache_node *n; -+ -+ if ((gfpflags & __GFP_NOWARN) || !__ratelimit(&slub_oom_rs)) -+ return; -+ -+ pr_warn("SLUB: Unable to allocate memory on node %d (gfp=0x%x)\n", -+ nid, gfpflags); -+ pr_warn(" cache: %s, object size: %d, buffer size: %d, default order: %d, min order: %d\n", -+ s->name, s->object_size, s->size, oo_order(s->oo), -+ oo_order(s->min)); -+ -+ if (oo_order(s->min) > get_order(s->object_size)) -+ pr_warn(" %s debugging increased min order, use slub_debug=O to disable.\n", -+ s->name); -+ -+ for_each_kmem_cache_node(s, node, n) { -+ unsigned long nr_slabs; -+ unsigned long nr_objs; -+ unsigned long nr_free; -+ -+ nr_free = count_partial(n, count_free); -+ nr_slabs = node_nr_slabs(n); -+ nr_objs = node_nr_objs(n); -+ -+ pr_warn(" node %d: slabs: %ld, objs: %ld, free: %ld\n", -+ node, nr_slabs, nr_objs, nr_free); -+ } -+#endif -+} -+ -+static inline void *new_slab_objects(struct kmem_cache *s, gfp_t flags, -+ int node, struct kmem_cache_cpu **pc) -+{ -+ void *freelist; -+ struct kmem_cache_cpu *c = *pc; -+ struct page *page; -+ -+ freelist = get_partial(s, flags, node, c); -+ -+ if (freelist) -+ return freelist; -+ -+ page = new_slab(s, flags, node); -+ if (page) { -+ c = raw_cpu_ptr(s->cpu_slab); -+ if (c->page) -+ flush_slab(s, c); -+ -+ /* -+ * No other reference to the page yet so we can -+ * muck around with it freely without cmpxchg -+ */ -+ freelist = page->freelist; -+ page->freelist = NULL; -+ -+ stat(s, ALLOC_SLAB); -+ c->page = page; -+ *pc = c; -+ } else -+ freelist = NULL; -+ -+ return freelist; -+} -+ -+static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags) -+{ -+ if (unlikely(PageSlabPfmemalloc(page))) -+ return gfp_pfmemalloc_allowed(gfpflags); -+ -+ return true; -+} -+ -+/* -+ * Check the page->freelist of a page and either transfer the freelist to the -+ * per cpu freelist or deactivate the page. -+ * -+ * The page is still frozen if the return value is not NULL. -+ * -+ * If this function returns NULL then the page has been unfrozen. -+ * -+ * This function must be called with interrupt disabled. -+ */ -+static inline void *get_freelist(struct kmem_cache *s, struct page *page) -+{ -+ struct page new; -+ unsigned long counters; -+ void *freelist; -+ -+ do { -+ freelist = page->freelist; -+ counters = page->counters; -+ -+ new.counters = counters; -+ VM_BUG_ON(!new.frozen); -+ -+ new.inuse = page->objects; -+ new.frozen = freelist != NULL; -+ -+ } while (!__cmpxchg_double_slab(s, page, -+ freelist, counters, -+ NULL, new.counters, -+ "get_freelist")); -+ -+ return freelist; -+} -+ -+/* -+ * Slow path. The lockless freelist is empty or we need to perform -+ * debugging duties. -+ * -+ * Processing is still very fast if new objects have been freed to the -+ * regular freelist. In that case we simply take over the regular freelist -+ * as the lockless freelist and zap the regular freelist. -+ * -+ * If that is not working then we fall back to the partial lists. We take the -+ * first element of the freelist as the object to allocate now and move the -+ * rest of the freelist to the lockless freelist. -+ * -+ * And if we were unable to get a new slab from the partial slab lists then -+ * we need to allocate a new slab. This is the slowest path since it involves -+ * a call to the page allocator and the setup of a new slab. -+ */ -+static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, -+ unsigned long addr, struct kmem_cache_cpu *c) -+{ -+ void *freelist; -+ struct page *page; -+ unsigned long flags; -+ -+ local_irq_save(flags); -+#ifdef CONFIG_PREEMPT -+ /* -+ * We may have been preempted and rescheduled on a different -+ * cpu before disabling interrupts. Need to reload cpu area -+ * pointer. -+ */ -+ c = this_cpu_ptr(s->cpu_slab); -+#endif -+ -+ page = c->page; -+ if (!page) -+ goto new_slab; -+redo: -+ -+ if (unlikely(!node_match(page, node))) { -+ int searchnode = node; -+ -+ if (node != NUMA_NO_NODE && !node_present_pages(node)) -+ searchnode = node_to_mem_node(node); -+ -+ if (unlikely(!node_match(page, searchnode))) { -+ stat(s, ALLOC_NODE_MISMATCH); -+ deactivate_slab(s, page, c->freelist); -+ c->page = NULL; -+ c->freelist = NULL; -+ goto new_slab; -+ } -+ } -+ -+ /* -+ * By rights, we should be searching for a slab page that was -+ * PFMEMALLOC but right now, we are losing the pfmemalloc -+ * information when the page leaves the per-cpu allocator -+ */ -+ if (unlikely(!pfmemalloc_match(page, gfpflags))) { -+ deactivate_slab(s, page, c->freelist); -+ c->page = NULL; -+ c->freelist = NULL; -+ goto new_slab; -+ } -+ -+ /* must check again c->freelist in case of cpu migration or IRQ */ -+ freelist = c->freelist; -+ if (freelist) -+ goto load_freelist; -+ -+ freelist = get_freelist(s, page); -+ -+ if (!freelist) { -+ c->page = NULL; -+ stat(s, DEACTIVATE_BYPASS); -+ goto new_slab; -+ } -+ -+ stat(s, ALLOC_REFILL); -+ -+load_freelist: -+ /* -+ * freelist is pointing to the list of objects to be used. -+ * page is pointing to the page from which the objects are obtained. -+ * That page must be frozen for per cpu allocations to work. -+ */ -+ VM_BUG_ON(!c->page->frozen); -+ c->freelist = get_freepointer(s, freelist); -+ c->tid = next_tid(c->tid); -+ local_irq_restore(flags); -+ return freelist; -+ -+new_slab: -+ -+ if (c->partial) { -+ page = c->page = c->partial; -+ c->partial = page->next; -+ stat(s, CPU_PARTIAL_ALLOC); -+ c->freelist = NULL; -+ goto redo; -+ } -+ -+ freelist = new_slab_objects(s, gfpflags, node, &c); -+ -+ if (unlikely(!freelist)) { -+ slab_out_of_memory(s, gfpflags, node); -+ local_irq_restore(flags); -+ return NULL; -+ } -+ -+ page = c->page; -+ if (likely(!kmem_cache_debug(s) && pfmemalloc_match(page, gfpflags))) -+ goto load_freelist; -+ -+ /* Only entered in the debug case */ -+ if (kmem_cache_debug(s) && -+ !alloc_debug_processing(s, page, freelist, addr)) -+ goto new_slab; /* Slab failed checks. Next slab needed */ -+ -+ deactivate_slab(s, page, get_freepointer(s, freelist)); -+ c->page = NULL; -+ c->freelist = NULL; -+ local_irq_restore(flags); -+ return freelist; -+} -+ -+/* -+ * Inlined fastpath so that allocation functions (kmalloc, kmem_cache_alloc) -+ * have the fastpath folded into their functions. So no function call -+ * overhead for requests that can be satisfied on the fastpath. -+ * -+ * The fastpath works by first checking if the lockless freelist can be used. -+ * If not then __slab_alloc is called for slow processing. -+ * -+ * Otherwise we can simply pick the next object from the lockless free list. -+ */ -+static __always_inline void *slab_alloc_node(struct kmem_cache *s, -+ gfp_t gfpflags, int node, unsigned long addr) -+{ -+ void **object; -+ struct kmem_cache_cpu *c; -+ struct page *page; -+ unsigned long tid; -+ -+ s = slab_pre_alloc_hook(s, gfpflags); -+ if (!s) -+ return NULL; -+redo: -+ /* -+ * Must read kmem_cache cpu data via this cpu ptr. Preemption is -+ * enabled. We may switch back and forth between cpus while -+ * reading from one cpu area. That does not matter as long -+ * as we end up on the original cpu again when doing the cmpxchg. -+ * -+ * We should guarantee that tid and kmem_cache are retrieved on -+ * the same cpu. It could be different if CONFIG_PREEMPT so we need -+ * to check if it is matched or not. -+ */ -+ do { -+ tid = this_cpu_read(s->cpu_slab->tid); -+ c = raw_cpu_ptr(s->cpu_slab); -+ } while (IS_ENABLED(CONFIG_PREEMPT) && -+ unlikely(tid != READ_ONCE(c->tid))); -+ -+ /* -+ * Irqless object alloc/free algorithm used here depends on sequence -+ * of fetching cpu_slab's data. tid should be fetched before anything -+ * on c to guarantee that object and page associated with previous tid -+ * won't be used with current tid. If we fetch tid first, object and -+ * page could be one associated with next tid and our alloc/free -+ * request will be failed. In this case, we will retry. So, no problem. -+ */ -+ barrier(); -+ -+ /* -+ * The transaction ids are globally unique per cpu and per operation on -+ * a per cpu queue. Thus they can be guarantee that the cmpxchg_double -+ * occurs on the right processor and that there was no operation on the -+ * linked list in between. -+ */ -+ -+ object = c->freelist; -+ page = c->page; -+ if (unlikely(!object || !node_match(page, node))) { -+ object = __slab_alloc(s, gfpflags, node, addr, c); -+ stat(s, ALLOC_SLOWPATH); -+ } else { -+ void *next_object = get_freepointer_safe(s, object); -+ -+ /* -+ * The cmpxchg will only match if there was no additional -+ * operation and if we are on the right processor. -+ * -+ * The cmpxchg does the following atomically (without lock -+ * semantics!) -+ * 1. Relocate first pointer to the current per cpu area. -+ * 2. Verify that tid and freelist have not been changed -+ * 3. If they were not changed replace tid and freelist -+ * -+ * Since this is without lock semantics the protection is only -+ * against code executing on this cpu *not* from access by -+ * other cpus. -+ */ -+ if (unlikely(!this_cpu_cmpxchg_double( -+ s->cpu_slab->freelist, s->cpu_slab->tid, -+ object, tid, -+ next_object, next_tid(tid)))) { -+ -+ note_cmpxchg_failure("slab_alloc", s, tid); -+ goto redo; -+ } -+ prefetch_freepointer(s, next_object); -+ stat(s, ALLOC_FASTPATH); -+ } -+ -+ if (unlikely(gfpflags & __GFP_ZERO) && object) -+ memset(object, 0, s->object_size); -+ -+ slab_post_alloc_hook(s, gfpflags, object); -+ -+ return object; -+} -+ -+static __always_inline void *slab_alloc(struct kmem_cache *s, -+ gfp_t gfpflags, unsigned long addr) -+{ -+ return slab_alloc_node(s, gfpflags, NUMA_NO_NODE, addr); -+} -+ -+void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags) -+{ -+ void *ret = slab_alloc(s, gfpflags, _RET_IP_); -+ -+ trace_kmem_cache_alloc(_RET_IP_, ret, s->object_size, -+ s->size, gfpflags); -+ -+ return ret; -+} -+EXPORT_SYMBOL(kmem_cache_alloc); -+ -+#ifdef CONFIG_TRACING -+void *kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size) -+{ -+ void *ret = slab_alloc(s, gfpflags, _RET_IP_); -+ trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags); -+ kasan_kmalloc(s, ret, size); -+ return ret; -+} -+EXPORT_SYMBOL(kmem_cache_alloc_trace); -+#endif -+ -+#ifdef CONFIG_NUMA -+void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node) -+{ -+ void *ret = slab_alloc_node(s, gfpflags, node, _RET_IP_); -+ -+ trace_kmem_cache_alloc_node(_RET_IP_, ret, -+ s->object_size, s->size, gfpflags, node); -+ -+ return ret; -+} -+EXPORT_SYMBOL(kmem_cache_alloc_node); -+ -+#ifdef CONFIG_TRACING -+void *kmem_cache_alloc_node_trace(struct kmem_cache *s, -+ gfp_t gfpflags, -+ int node, size_t size) -+{ -+ void *ret = slab_alloc_node(s, gfpflags, node, _RET_IP_); -+ -+ trace_kmalloc_node(_RET_IP_, ret, -+ size, s->size, gfpflags, node); -+ -+ kasan_kmalloc(s, ret, size); -+ return ret; -+} -+EXPORT_SYMBOL(kmem_cache_alloc_node_trace); -+#endif -+#endif -+ -+/* -+ * Slow path handling. This may still be called frequently since objects -+ * have a longer lifetime than the cpu slabs in most processing loads. -+ * -+ * So we still attempt to reduce cache line usage. Just take the slab -+ * lock and free the item. If there is no additional partial page -+ * handling required then we can return immediately. -+ */ -+static void __slab_free(struct kmem_cache *s, struct page *page, -+ void *x, unsigned long addr) -+{ -+ void *prior; -+ void **object = (void *)x; -+ int was_frozen; -+ struct page new; -+ unsigned long counters; -+ struct kmem_cache_node *n = NULL; -+ unsigned long uninitialized_var(flags); -+ -+ stat(s, FREE_SLOWPATH); -+ -+ if (kmem_cache_debug(s) && -+ !(n = free_debug_processing(s, page, x, addr, &flags))) -+ return; -+ -+ do { -+ if (unlikely(n)) { -+ spin_unlock_irqrestore(&n->list_lock, flags); -+ n = NULL; -+ } -+ prior = page->freelist; -+ counters = page->counters; -+ set_freepointer(s, object, prior); -+ new.counters = counters; -+ was_frozen = new.frozen; -+ new.inuse--; -+ if ((!new.inuse || !prior) && !was_frozen) { -+ -+ if (kmem_cache_has_cpu_partial(s) && !prior) { -+ -+ /* -+ * Slab was on no list before and will be -+ * partially empty -+ * We can defer the list move and instead -+ * freeze it. -+ */ -+ new.frozen = 1; -+ -+ } else { /* Needs to be taken off a list */ -+ -+ n = get_node(s, page_to_nid(page)); -+ /* -+ * Speculatively acquire the list_lock. -+ * If the cmpxchg does not succeed then we may -+ * drop the list_lock without any processing. -+ * -+ * Otherwise the list_lock will synchronize with -+ * other processors updating the list of slabs. -+ */ -+ spin_lock_irqsave(&n->list_lock, flags); -+ -+ } -+ } -+ -+ } while (!cmpxchg_double_slab(s, page, -+ prior, counters, -+ object, new.counters, -+ "__slab_free")); -+ -+ if (likely(!n)) { -+ -+ /* -+ * If we just froze the page then put it onto the -+ * per cpu partial list. -+ */ -+ if (new.frozen && !was_frozen) { -+ put_cpu_partial(s, page, 1); -+ stat(s, CPU_PARTIAL_FREE); -+ } -+ /* -+ * The list lock was not taken therefore no list -+ * activity can be necessary. -+ */ -+ if (was_frozen) -+ stat(s, FREE_FROZEN); -+ return; -+ } -+ -+ if (unlikely(!new.inuse && n->nr_partial >= s->min_partial)) -+ goto slab_empty; -+ -+ /* -+ * Objects left in the slab. If it was not on the partial list before -+ * then add it. -+ */ -+ if (!kmem_cache_has_cpu_partial(s) && unlikely(!prior)) { -+ if (kmem_cache_debug(s)) -+ remove_full(s, n, page); -+ add_partial(n, page, DEACTIVATE_TO_TAIL); -+ stat(s, FREE_ADD_PARTIAL); -+ } -+ spin_unlock_irqrestore(&n->list_lock, flags); -+ return; -+ -+slab_empty: -+ if (prior) { -+ /* -+ * Slab on the partial list. -+ */ -+ remove_partial(n, page); -+ stat(s, FREE_REMOVE_PARTIAL); -+ } else { -+ /* Slab must be on the full list */ -+ remove_full(s, n, page); -+ } -+ -+ spin_unlock_irqrestore(&n->list_lock, flags); -+ stat(s, FREE_SLAB); -+ discard_slab(s, page); -+} -+ -+/* -+ * Fastpath with forced inlining to produce a kfree and kmem_cache_free that -+ * can perform fastpath freeing without additional function calls. -+ * -+ * The fastpath is only possible if we are freeing to the current cpu slab -+ * of this processor. This typically the case if we have just allocated -+ * the item before. -+ * -+ * If fastpath is not possible then fall back to __slab_free where we deal -+ * with all sorts of special processing. -+ */ -+static __always_inline void slab_free(struct kmem_cache *s, -+ struct page *page, void *x, unsigned long addr) -+{ -+ void **object = (void *)x; -+ struct kmem_cache_cpu *c; -+ unsigned long tid; -+ -+ slab_free_hook(s, x); -+ -+redo: -+ /* -+ * Determine the currently cpus per cpu slab. -+ * The cpu may change afterward. However that does not matter since -+ * data is retrieved via this pointer. If we are on the same cpu -+ * during the cmpxchg then the free will succedd. -+ */ -+ do { -+ tid = this_cpu_read(s->cpu_slab->tid); -+ c = raw_cpu_ptr(s->cpu_slab); -+ } while (IS_ENABLED(CONFIG_PREEMPT) && -+ unlikely(tid != READ_ONCE(c->tid))); -+ -+ /* Same with comment on barrier() in slab_alloc_node() */ -+ barrier(); -+ -+ if (likely(page == c->page)) { -+ set_freepointer(s, object, c->freelist); -+ -+ if (unlikely(!this_cpu_cmpxchg_double( -+ s->cpu_slab->freelist, s->cpu_slab->tid, -+ c->freelist, tid, -+ object, next_tid(tid)))) { -+ -+ note_cmpxchg_failure("slab_free", s, tid); -+ goto redo; -+ } -+ stat(s, FREE_FASTPATH); -+ } else -+ __slab_free(s, page, x, addr); -+ -+} -+ -+void kmem_cache_free(struct kmem_cache *s, void *x) -+{ -+ s = cache_from_obj(s, x); -+ if (!s) -+ return; -+ slab_free(s, virt_to_head_page(x), x, _RET_IP_); -+ trace_kmem_cache_free(_RET_IP_, x); -+} -+EXPORT_SYMBOL(kmem_cache_free); -+ -+/* -+ * Object placement in a slab is made very easy because we always start at -+ * offset 0. If we tune the size of the object to the alignment then we can -+ * get the required alignment by putting one properly sized object after -+ * another. -+ * -+ * Notice that the allocation order determines the sizes of the per cpu -+ * caches. Each processor has always one slab available for allocations. -+ * Increasing the allocation order reduces the number of times that slabs -+ * must be moved on and off the partial lists and is therefore a factor in -+ * locking overhead. -+ */ -+ -+/* -+ * Mininum / Maximum order of slab pages. This influences locking overhead -+ * and slab fragmentation. A higher order reduces the number of partial slabs -+ * and increases the number of allocations possible without having to -+ * take the list_lock. -+ */ -+static int slub_min_order; -+static int slub_max_order = PAGE_ALLOC_COSTLY_ORDER; -+static int slub_min_objects; -+ -+/* -+ * Calculate the order of allocation given an slab object size. -+ * -+ * The order of allocation has significant impact on performance and other -+ * system components. Generally order 0 allocations should be preferred since -+ * order 0 does not cause fragmentation in the page allocator. Larger objects -+ * be problematic to put into order 0 slabs because there may be too much -+ * unused space left. We go to a higher order if more than 1/16th of the slab -+ * would be wasted. -+ * -+ * In order to reach satisfactory performance we must ensure that a minimum -+ * number of objects is in one slab. Otherwise we may generate too much -+ * activity on the partial lists which requires taking the list_lock. This is -+ * less a concern for large slabs though which are rarely used. -+ * -+ * slub_max_order specifies the order where we begin to stop considering the -+ * number of objects in a slab as critical. If we reach slub_max_order then -+ * we try to keep the page order as low as possible. So we accept more waste -+ * of space in favor of a small page order. -+ * -+ * Higher order allocations also allow the placement of more objects in a -+ * slab and thereby reduce object handling overhead. If the user has -+ * requested a higher mininum order then we start with that one instead of -+ * the smallest order which will fit the object. -+ */ -+static inline int slab_order(int size, int min_objects, -+ int max_order, int fract_leftover, int reserved) -+{ -+ int order; -+ int rem; -+ int min_order = slub_min_order; -+ -+ if (order_objects(min_order, size, reserved) > MAX_OBJS_PER_PAGE) -+ return get_order(size * MAX_OBJS_PER_PAGE) - 1; -+ -+ for (order = max(min_order, -+ fls(min_objects * size - 1) - PAGE_SHIFT); -+ order <= max_order; order++) { -+ -+ unsigned long slab_size = PAGE_SIZE << order; -+ -+ if (slab_size < min_objects * size + reserved) -+ continue; -+ -+ rem = (slab_size - reserved) % size; -+ -+ if (rem <= slab_size / fract_leftover) -+ break; -+ -+ } -+ -+ return order; -+} -+ -+static inline int calculate_order(int size, int reserved) -+{ -+ int order; -+ int min_objects; -+ int fraction; -+ int max_objects; -+ -+ /* -+ * Attempt to find best configuration for a slab. This -+ * works by first attempting to generate a layout with -+ * the best configuration and backing off gradually. -+ * -+ * First we reduce the acceptable waste in a slab. Then -+ * we reduce the minimum objects required in a slab. -+ */ -+ min_objects = slub_min_objects; -+ if (!min_objects) -+ min_objects = 4 * (fls(nr_cpu_ids) + 1); -+ max_objects = order_objects(slub_max_order, size, reserved); -+ min_objects = min(min_objects, max_objects); -+ -+ while (min_objects > 1) { -+ fraction = 16; -+ while (fraction >= 4) { -+ order = slab_order(size, min_objects, -+ slub_max_order, fraction, reserved); -+ if (order <= slub_max_order) -+ return order; -+ fraction /= 2; -+ } -+ min_objects--; -+ } -+ -+ /* -+ * We were unable to place multiple objects in a slab. Now -+ * lets see if we can place a single object there. -+ */ -+ order = slab_order(size, 1, slub_max_order, 1, reserved); -+ if (order <= slub_max_order) -+ return order; -+ -+ /* -+ * Doh this slab cannot be placed using slub_max_order. -+ */ -+ order = slab_order(size, 1, MAX_ORDER, 1, reserved); -+ if (order < MAX_ORDER) -+ return order; -+ return -ENOSYS; -+} -+ -+static void -+init_kmem_cache_node(struct kmem_cache_node *n) -+{ -+ n->nr_partial = 0; -+ spin_lock_init(&n->list_lock); -+ INIT_LIST_HEAD(&n->partial); -+#ifdef CONFIG_SLUB_DEBUG -+ atomic_long_set(&n->nr_slabs, 0); -+ atomic_long_set(&n->total_objects, 0); -+ INIT_LIST_HEAD(&n->full); -+#endif -+} -+ -+static inline int alloc_kmem_cache_cpus(struct kmem_cache *s) -+{ -+ BUILD_BUG_ON(PERCPU_DYNAMIC_EARLY_SIZE < -+ KMALLOC_SHIFT_HIGH * sizeof(struct kmem_cache_cpu)); -+ -+ /* -+ * Must align to double word boundary for the double cmpxchg -+ * instructions to work; see __pcpu_double_call_return_bool(). -+ */ -+ s->cpu_slab = __alloc_percpu(sizeof(struct kmem_cache_cpu), -+ 2 * sizeof(void *)); -+ -+ if (!s->cpu_slab) -+ return 0; -+ -+ init_kmem_cache_cpus(s); -+ -+ return 1; -+} -+ -+static struct kmem_cache *kmem_cache_node; -+ -+/* -+ * No kmalloc_node yet so do it by hand. We know that this is the first -+ * slab on the node for this slabcache. There are no concurrent accesses -+ * possible. -+ * -+ * Note that this function only works on the kmem_cache_node -+ * when allocating for the kmem_cache_node. This is used for bootstrapping -+ * memory on a fresh node that has no slab structures yet. -+ */ -+static void early_kmem_cache_node_alloc(int node) -+{ -+ struct page *page; -+ struct kmem_cache_node *n; -+ -+ BUG_ON(kmem_cache_node->size < sizeof(struct kmem_cache_node)); -+ -+ page = new_slab(kmem_cache_node, GFP_NOWAIT, node); -+ -+ BUG_ON(!page); -+ if (page_to_nid(page) != node) { -+ pr_err("SLUB: Unable to allocate memory from node %d\n", node); -+ pr_err("SLUB: Allocating a useless per node structure in order to be able to continue\n"); -+ } -+ -+ n = page->freelist; -+ BUG_ON(!n); -+ page->freelist = get_freepointer(kmem_cache_node, n); -+ page->inuse = 1; -+ page->frozen = 0; -+ kmem_cache_node->node[node] = n; -+#ifdef CONFIG_SLUB_DEBUG -+ init_object(kmem_cache_node, n, SLUB_RED_ACTIVE); -+ init_tracking(kmem_cache_node, n); -+#endif -+ kasan_kmalloc(kmem_cache_node, n, sizeof(struct kmem_cache_node)); -+ init_kmem_cache_node(n); -+ inc_slabs_node(kmem_cache_node, node, page->objects); -+ -+ /* -+ * No locks need to be taken here as it has just been -+ * initialized and there is no concurrent access. -+ */ -+ __add_partial(n, page, DEACTIVATE_TO_HEAD); -+} -+ -+static void free_kmem_cache_nodes(struct kmem_cache *s) -+{ -+ int node; -+ struct kmem_cache_node *n; -+ -+ for_each_kmem_cache_node(s, node, n) { -+ kmem_cache_free(kmem_cache_node, n); -+ s->node[node] = NULL; -+ } -+} -+ -+static int init_kmem_cache_nodes(struct kmem_cache *s) -+{ -+ int node; -+ -+ for_each_node_state(node, N_NORMAL_MEMORY) { -+ struct kmem_cache_node *n; -+ -+ if (slab_state == DOWN) { -+ early_kmem_cache_node_alloc(node); -+ continue; -+ } -+ n = kmem_cache_alloc_node(kmem_cache_node, -+ GFP_KERNEL, node); -+ -+ if (!n) { -+ free_kmem_cache_nodes(s); -+ return 0; -+ } -+ -+ s->node[node] = n; -+ init_kmem_cache_node(n); -+ } -+ return 1; -+} -+ -+static void set_min_partial(struct kmem_cache *s, unsigned long min) -+{ -+ if (min < MIN_PARTIAL) -+ min = MIN_PARTIAL; -+ else if (min > MAX_PARTIAL) -+ min = MAX_PARTIAL; -+ s->min_partial = min; -+} -+ -+/* -+ * calculate_sizes() determines the order and the distribution of data within -+ * a slab object. -+ */ -+static int calculate_sizes(struct kmem_cache *s, int forced_order) -+{ -+ unsigned long flags = s->flags; -+ unsigned long size = s->object_size; -+ int order; -+ -+ /* -+ * Round up object size to the next word boundary. We can only -+ * place the free pointer at word boundaries and this determines -+ * the possible location of the free pointer. -+ */ -+ size = ALIGN(size, sizeof(void *)); -+ -+#ifdef CONFIG_SLUB_DEBUG -+ /* -+ * Determine if we can poison the object itself. If the user of -+ * the slab may touch the object after free or before allocation -+ * then we should never poison the object itself. -+ */ -+ if ((flags & SLAB_POISON) && !(flags & SLAB_DESTROY_BY_RCU) && -+ !s->ctor) -+ s->flags |= __OBJECT_POISON; -+ else -+ s->flags &= ~__OBJECT_POISON; -+ -+ -+ /* -+ * If we are Redzoning then check if there is some space between the -+ * end of the object and the free pointer. If not then add an -+ * additional word to have some bytes to store Redzone information. -+ */ -+ if ((flags & SLAB_RED_ZONE) && size == s->object_size) -+ size += sizeof(void *); -+#endif -+ -+ /* -+ * With that we have determined the number of bytes in actual use -+ * by the object. This is the potential offset to the free pointer. -+ */ -+ s->inuse = size; -+ -+ if (((flags & (SLAB_DESTROY_BY_RCU | SLAB_POISON)) || -+ s->ctor)) { -+ /* -+ * Relocate free pointer after the object if it is not -+ * permitted to overwrite the first word of the object on -+ * kmem_cache_free. -+ * -+ * This is the case if we do RCU, have a constructor or -+ * destructor or are poisoning the objects. -+ */ -+ s->offset = size; -+ size += sizeof(void *); -+ } -+ -+#ifdef CONFIG_SLUB_DEBUG -+ if (flags & SLAB_STORE_USER) -+ /* -+ * Need to store information about allocs and frees after -+ * the object. -+ */ -+ size += 2 * sizeof(struct track); -+ -+ if (flags & SLAB_RED_ZONE) -+ /* -+ * Add some empty padding so that we can catch -+ * overwrites from earlier objects rather than let -+ * tracking information or the free pointer be -+ * corrupted if a user writes before the start -+ * of the object. -+ */ -+ size += sizeof(void *); -+#endif -+ -+ /* -+ * SLUB stores one object immediately after another beginning from -+ * offset 0. In order to align the objects we have to simply size -+ * each object to conform to the alignment. -+ */ -+ size = ALIGN(size, s->align); -+ s->size = size; -+ if (forced_order >= 0) -+ order = forced_order; -+ else -+ order = calculate_order(size, s->reserved); -+ -+ if (order < 0) -+ return 0; -+ -+ s->allocflags = 0; -+ if (order) -+ s->allocflags |= __GFP_COMP; -+ -+ if (s->flags & SLAB_CACHE_DMA) -+ s->allocflags |= GFP_DMA; -+ -+ if (s->flags & SLAB_RECLAIM_ACCOUNT) -+ s->allocflags |= __GFP_RECLAIMABLE; -+ -+ /* -+ * Determine the number of objects per slab -+ */ -+ s->oo = oo_make(order, size, s->reserved); -+ s->min = oo_make(get_order(size), size, s->reserved); -+ if (oo_objects(s->oo) > oo_objects(s->max)) -+ s->max = s->oo; -+ -+ return !!oo_objects(s->oo); -+} -+ -+static int kmem_cache_open(struct kmem_cache *s, unsigned long flags) -+{ -+ s->flags = kmem_cache_flags(s->size, flags, s->name, s->ctor); -+ s->reserved = 0; -+ -+ if (need_reserve_slab_rcu && (s->flags & SLAB_DESTROY_BY_RCU)) -+ s->reserved = sizeof(struct rcu_head); -+ -+ if (!calculate_sizes(s, -1)) -+ goto error; -+ if (disable_higher_order_debug) { -+ /* -+ * Disable debugging flags that store metadata if the min slab -+ * order increased. -+ */ -+ if (get_order(s->size) > get_order(s->object_size)) { -+ s->flags &= ~DEBUG_METADATA_FLAGS; -+ s->offset = 0; -+ if (!calculate_sizes(s, -1)) -+ goto error; -+ } -+ } -+ -+#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \ -+ defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE) -+ if (system_has_cmpxchg_double() && (s->flags & SLAB_DEBUG_FLAGS) == 0) -+ /* Enable fast mode */ -+ s->flags |= __CMPXCHG_DOUBLE; -+#endif -+ -+ /* -+ * The larger the object size is, the more pages we want on the partial -+ * list to avoid pounding the page allocator excessively. -+ */ -+ set_min_partial(s, ilog2(s->size) / 2); -+ -+ /* -+ * cpu_partial determined the maximum number of objects kept in the -+ * per cpu partial lists of a processor. -+ * -+ * Per cpu partial lists mainly contain slabs that just have one -+ * object freed. If they are used for allocation then they can be -+ * filled up again with minimal effort. The slab will never hit the -+ * per node partial lists and therefore no locking will be required. -+ * -+ * This setting also determines -+ * -+ * A) The number of objects from per cpu partial slabs dumped to the -+ * per node list when we reach the limit. -+ * B) The number of objects in cpu partial slabs to extract from the -+ * per node list when we run out of per cpu objects. We only fetch -+ * 50% to keep some capacity around for frees. -+ */ -+ if (!kmem_cache_has_cpu_partial(s)) -+ s->cpu_partial = 0; -+ else if (s->size >= PAGE_SIZE) -+ s->cpu_partial = 2; -+ else if (s->size >= 1024) -+ s->cpu_partial = 6; -+ else if (s->size >= 256) -+ s->cpu_partial = 13; -+ else -+ s->cpu_partial = 30; -+ -+#ifdef CONFIG_NUMA -+ s->remote_node_defrag_ratio = 1000; -+#endif -+ if (!init_kmem_cache_nodes(s)) -+ goto error; -+ -+ if (alloc_kmem_cache_cpus(s)) -+ return 0; -+ -+ free_kmem_cache_nodes(s); -+error: -+ if (flags & SLAB_PANIC) -+ panic("Cannot create slab %s size=%lu realsize=%u " -+ "order=%u offset=%u flags=%lx\n", -+ s->name, (unsigned long)s->size, s->size, -+ oo_order(s->oo), s->offset, flags); -+ return -EINVAL; -+} -+ -+static void list_slab_objects(struct kmem_cache *s, struct page *page, -+ const char *text) -+{ -+#ifdef CONFIG_SLUB_DEBUG -+ void *addr = page_address(page); -+ void *p; -+ unsigned long *map = kzalloc(BITS_TO_LONGS(page->objects) * -+ sizeof(long), GFP_ATOMIC); -+ if (!map) -+ return; -+ slab_err(s, page, text, s->name); -+ slab_lock(page); -+ -+ get_map(s, page, map); -+ for_each_object(p, s, addr, page->objects) { -+ -+ if (!test_bit(slab_index(p, s, addr), map)) { -+ pr_err("INFO: Object 0x%p @offset=%tu\n", p, p - addr); -+ print_tracking(s, p); -+ } -+ } -+ slab_unlock(page); -+ kfree(map); -+#endif -+} -+ -+/* -+ * Attempt to free all partial slabs on a node. -+ * This is called from kmem_cache_close(). We must be the last thread -+ * using the cache and therefore we do not need to lock anymore. -+ */ -+static void free_partial(struct kmem_cache *s, struct kmem_cache_node *n) -+{ -+ struct page *page, *h; -+ -+ list_for_each_entry_safe(page, h, &n->partial, lru) { -+ if (!page->inuse) { -+ __remove_partial(n, page); -+ discard_slab(s, page); -+ } else { -+ list_slab_objects(s, page, -+ "Objects remaining in %s on kmem_cache_close()"); -+ } -+ } -+} -+ -+/* -+ * Release all resources used by a slab cache. -+ */ -+static inline int kmem_cache_close(struct kmem_cache *s) -+{ -+ int node; -+ struct kmem_cache_node *n; -+ -+ flush_all(s); -+ /* Attempt to free all objects */ -+ for_each_kmem_cache_node(s, node, n) { -+ free_partial(s, n); -+ if (n->nr_partial || slabs_node(s, node)) -+ return 1; -+ } -+ free_percpu(s->cpu_slab); -+ free_kmem_cache_nodes(s); -+ return 0; -+} -+ -+int __kmem_cache_shutdown(struct kmem_cache *s) -+{ -+ return kmem_cache_close(s); -+} -+ -+/******************************************************************** -+ * Kmalloc subsystem -+ *******************************************************************/ -+ -+static int __init setup_slub_min_order(char *str) -+{ -+ get_option(&str, &slub_min_order); -+ -+ return 1; -+} -+ -+__setup("slub_min_order=", setup_slub_min_order); -+ -+static int __init setup_slub_max_order(char *str) -+{ -+ get_option(&str, &slub_max_order); -+ slub_max_order = min(slub_max_order, MAX_ORDER - 1); -+ -+ return 1; -+} -+ -+__setup("slub_max_order=", setup_slub_max_order); -+ -+static int __init setup_slub_min_objects(char *str) -+{ -+ get_option(&str, &slub_min_objects); -+ -+ return 1; -+} -+ -+__setup("slub_min_objects=", setup_slub_min_objects); -+ -+void *__kmalloc(size_t size, gfp_t flags) -+{ -+ struct kmem_cache *s; -+ void *ret; -+ -+ if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) -+ return kmalloc_large(size, flags); -+ -+ s = kmalloc_slab(size, flags); -+ -+ if (unlikely(ZERO_OR_NULL_PTR(s))) -+ return s; -+ -+ ret = slab_alloc(s, flags, _RET_IP_); -+ -+ trace_kmalloc(_RET_IP_, ret, size, s->size, flags); -+ -+ kasan_kmalloc(s, ret, size); -+ -+ return ret; -+} -+EXPORT_SYMBOL(__kmalloc); -+ -+#ifdef CONFIG_NUMA -+static void *kmalloc_large_node(size_t size, gfp_t flags, int node) -+{ -+ struct page *page; -+ void *ptr = NULL; -+ -+ flags |= __GFP_COMP | __GFP_NOTRACK; -+ page = alloc_kmem_pages_node(node, flags, get_order(size)); -+ if (page) -+ ptr = page_address(page); -+ -+ kmalloc_large_node_hook(ptr, size, flags); -+ return ptr; -+} -+ -+void *__kmalloc_node(size_t size, gfp_t flags, int node) -+{ -+ struct kmem_cache *s; -+ void *ret; -+ -+ if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) { -+ ret = kmalloc_large_node(size, flags, node); -+ -+ trace_kmalloc_node(_RET_IP_, ret, -+ size, PAGE_SIZE << get_order(size), -+ flags, node); -+ -+ return ret; -+ } -+ -+ s = kmalloc_slab(size, flags); -+ -+ if (unlikely(ZERO_OR_NULL_PTR(s))) -+ return s; -+ -+ ret = slab_alloc_node(s, flags, node, _RET_IP_); -+ -+ trace_kmalloc_node(_RET_IP_, ret, size, s->size, flags, node); -+ -+ kasan_kmalloc(s, ret, size); -+ -+ return ret; -+} -+EXPORT_SYMBOL(__kmalloc_node); -+#endif -+ -+static size_t __ksize(const void *object) -+{ -+ struct page *page; -+ -+ if (unlikely(object == ZERO_SIZE_PTR)) -+ return 0; -+ -+ page = virt_to_head_page(object); -+ -+ if (unlikely(!PageSlab(page))) { -+ WARN_ON(!PageCompound(page)); -+ return PAGE_SIZE << compound_order(page); -+ } -+ -+ return slab_ksize(page->slab_cache); -+} -+ -+size_t ksize(const void *object) -+{ -+ size_t size = __ksize(object); -+ /* We assume that ksize callers could use whole allocated area, -+ so we need unpoison this area. */ -+ kasan_krealloc(object, size); -+ return size; -+} -+EXPORT_SYMBOL(ksize); -+ -+void kfree(const void *x) -+{ -+ struct page *page; -+ void *object = (void *)x; -+ -+ trace_kfree(_RET_IP_, x); -+ -+ if (unlikely(ZERO_OR_NULL_PTR(x))) -+ return; -+ -+ page = virt_to_head_page(x); -+ if (unlikely(!PageSlab(page))) { -+ BUG_ON(!PageCompound(page)); -+ kfree_hook(x); -+ __free_kmem_pages(page, compound_order(page)); -+ return; -+ } -+ slab_free(page->slab_cache, page, object, _RET_IP_); -+} -+EXPORT_SYMBOL(kfree); -+ -+#define SHRINK_PROMOTE_MAX 32 -+ -+/* -+ * kmem_cache_shrink discards empty slabs and promotes the slabs filled -+ * up most to the head of the partial lists. New allocations will then -+ * fill those up and thus they can be removed from the partial lists. -+ * -+ * The slabs with the least items are placed last. This results in them -+ * being allocated from last increasing the chance that the last objects -+ * are freed in them. -+ */ -+int __kmem_cache_shrink(struct kmem_cache *s, bool deactivate) -+{ -+ int node; -+ int i; -+ struct kmem_cache_node *n; -+ struct page *page; -+ struct page *t; -+ struct list_head discard; -+ struct list_head promote[SHRINK_PROMOTE_MAX]; -+ unsigned long flags; -+ int ret = 0; -+ -+ if (deactivate) { -+ /* -+ * Disable empty slabs caching. Used to avoid pinning offline -+ * memory cgroups by kmem pages that can be freed. -+ */ -+ s->cpu_partial = 0; -+ s->min_partial = 0; -+ -+ /* -+ * s->cpu_partial is checked locklessly (see put_cpu_partial), -+ * so we have to make sure the change is visible. -+ */ -+ kick_all_cpus_sync(); -+ } -+ -+ flush_all(s); -+ for_each_kmem_cache_node(s, node, n) { -+ INIT_LIST_HEAD(&discard); -+ for (i = 0; i < SHRINK_PROMOTE_MAX; i++) -+ INIT_LIST_HEAD(promote + i); -+ -+ spin_lock_irqsave(&n->list_lock, flags); -+ -+ /* -+ * Build lists of slabs to discard or promote. -+ * -+ * Note that concurrent frees may occur while we hold the -+ * list_lock. page->inuse here is the upper limit. -+ */ -+ list_for_each_entry_safe(page, t, &n->partial, lru) { -+ int free = page->objects - page->inuse; -+ -+ /* Do not reread page->inuse */ -+ barrier(); -+ -+ /* We do not keep full slabs on the list */ -+ BUG_ON(free <= 0); -+ -+ if (free == page->objects) { -+ list_move(&page->lru, &discard); -+ n->nr_partial--; -+ } else if (free <= SHRINK_PROMOTE_MAX) -+ list_move(&page->lru, promote + free - 1); -+ } -+ -+ /* -+ * Promote the slabs filled up most to the head of the -+ * partial list. -+ */ -+ for (i = SHRINK_PROMOTE_MAX - 1; i >= 0; i--) -+ list_splice(promote + i, &n->partial); -+ -+ spin_unlock_irqrestore(&n->list_lock, flags); -+ -+ /* Release empty slabs */ -+ list_for_each_entry_safe(page, t, &discard, lru) -+ discard_slab(s, page); -+ -+ if (slabs_node(s, node)) -+ ret = 1; -+ } -+ -+ return ret; -+} -+ -+static int slab_mem_going_offline_callback(void *arg) -+{ -+ struct kmem_cache *s; -+ -+ mutex_lock(&slab_mutex); -+ list_for_each_entry(s, &slab_caches, list) -+ __kmem_cache_shrink(s, false); -+ mutex_unlock(&slab_mutex); -+ -+ return 0; -+} -+ -+static void slab_mem_offline_callback(void *arg) -+{ -+ struct kmem_cache_node *n; -+ struct kmem_cache *s; -+ struct memory_notify *marg = arg; -+ int offline_node; -+ -+ offline_node = marg->status_change_nid_normal; -+ -+ /* -+ * If the node still has available memory. we need kmem_cache_node -+ * for it yet. -+ */ -+ if (offline_node < 0) -+ return; -+ -+ mutex_lock(&slab_mutex); -+ list_for_each_entry(s, &slab_caches, list) { -+ n = get_node(s, offline_node); -+ if (n) { -+ /* -+ * if n->nr_slabs > 0, slabs still exist on the node -+ * that is going down. We were unable to free them, -+ * and offline_pages() function shouldn't call this -+ * callback. So, we must fail. -+ */ -+ BUG_ON(slabs_node(s, offline_node)); -+ -+ s->node[offline_node] = NULL; -+ kmem_cache_free(kmem_cache_node, n); -+ } -+ } -+ mutex_unlock(&slab_mutex); -+} -+ -+static int slab_mem_going_online_callback(void *arg) -+{ -+ struct kmem_cache_node *n; -+ struct kmem_cache *s; -+ struct memory_notify *marg = arg; -+ int nid = marg->status_change_nid_normal; -+ int ret = 0; -+ -+ /* -+ * If the node's memory is already available, then kmem_cache_node is -+ * already created. Nothing to do. -+ */ -+ if (nid < 0) -+ return 0; -+ -+ /* -+ * We are bringing a node online. No memory is available yet. We must -+ * allocate a kmem_cache_node structure in order to bring the node -+ * online. -+ */ -+ mutex_lock(&slab_mutex); -+ list_for_each_entry(s, &slab_caches, list) { -+ /* -+ * XXX: kmem_cache_alloc_node will fallback to other nodes -+ * since memory is not yet available from the node that -+ * is brought up. -+ */ -+ n = kmem_cache_alloc(kmem_cache_node, GFP_KERNEL); -+ if (!n) { -+ ret = -ENOMEM; -+ goto out; -+ } -+ init_kmem_cache_node(n); -+ s->node[nid] = n; -+ } -+out: -+ mutex_unlock(&slab_mutex); -+ return ret; -+} -+ -+static int slab_memory_callback(struct notifier_block *self, -+ unsigned long action, void *arg) -+{ -+ int ret = 0; -+ -+ switch (action) { -+ case MEM_GOING_ONLINE: -+ ret = slab_mem_going_online_callback(arg); -+ break; -+ case MEM_GOING_OFFLINE: -+ ret = slab_mem_going_offline_callback(arg); -+ break; -+ case MEM_OFFLINE: -+ case MEM_CANCEL_ONLINE: -+ slab_mem_offline_callback(arg); -+ break; -+ case MEM_ONLINE: -+ case MEM_CANCEL_OFFLINE: -+ break; -+ } -+ if (ret) -+ ret = notifier_from_errno(ret); -+ else -+ ret = NOTIFY_OK; -+ return ret; -+} -+ -+static struct notifier_block slab_memory_callback_nb = { -+ .notifier_call = slab_memory_callback, -+ .priority = SLAB_CALLBACK_PRI, -+}; -+ -+/******************************************************************** -+ * Basic setup of slabs -+ *******************************************************************/ -+ -+/* -+ * Used for early kmem_cache structures that were allocated using -+ * the page allocator. Allocate them properly then fix up the pointers -+ * that may be pointing to the wrong kmem_cache structure. -+ */ -+ -+static struct kmem_cache * __init bootstrap(struct kmem_cache *static_cache) -+{ -+ int node; -+ struct kmem_cache *s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT); -+ struct kmem_cache_node *n; -+ -+ memcpy(s, static_cache, kmem_cache->object_size); -+ -+ /* -+ * This runs very early, and only the boot processor is supposed to be -+ * up. Even if it weren't true, IRQs are not up so we couldn't fire -+ * IPIs around. -+ */ -+ __flush_cpu_slab(s, smp_processor_id()); -+ for_each_kmem_cache_node(s, node, n) { -+ struct page *p; -+ -+ list_for_each_entry(p, &n->partial, lru) -+ p->slab_cache = s; -+ -+#ifdef CONFIG_SLUB_DEBUG -+ list_for_each_entry(p, &n->full, lru) -+ p->slab_cache = s; -+#endif -+ } -+ slab_init_memcg_params(s); -+ list_add(&s->list, &slab_caches); -+ return s; -+} -+ -+void __init kmem_cache_init(void) -+{ -+ static __initdata struct kmem_cache boot_kmem_cache, -+ boot_kmem_cache_node; -+ -+ if (debug_guardpage_minorder()) -+ slub_max_order = 0; -+ -+ kmem_cache_node = &boot_kmem_cache_node; -+ kmem_cache = &boot_kmem_cache; -+ -+ create_boot_cache(kmem_cache_node, "kmem_cache_node", -+ sizeof(struct kmem_cache_node), SLAB_HWCACHE_ALIGN); -+ -+ register_hotmemory_notifier(&slab_memory_callback_nb); -+ -+ /* Able to allocate the per node structures */ -+ slab_state = PARTIAL; -+ -+ create_boot_cache(kmem_cache, "kmem_cache", -+ offsetof(struct kmem_cache, node) + -+ nr_node_ids * sizeof(struct kmem_cache_node *), -+ SLAB_HWCACHE_ALIGN); -+ -+ kmem_cache = bootstrap(&boot_kmem_cache); -+ -+ /* -+ * Allocate kmem_cache_node properly from the kmem_cache slab. -+ * kmem_cache_node is separately allocated so no need to -+ * update any list pointers. -+ */ -+ kmem_cache_node = bootstrap(&boot_kmem_cache_node); -+ -+ /* Now we can use the kmem_cache to allocate kmalloc slabs */ -+ create_kmalloc_caches(0); -+ -+#ifdef CONFIG_SMP -+ register_cpu_notifier(&slab_notifier); -+#endif -+ -+ pr_info("SLUB: HWalign=%d, Order=%d-%d, MinObjects=%d, CPUs=%d, Nodes=%d\n", -+ cache_line_size(), -+ slub_min_order, slub_max_order, slub_min_objects, -+ nr_cpu_ids, nr_node_ids); -+} -+ -+void __init kmem_cache_init_late(void) -+{ -+} -+ -+struct kmem_cache * -+__kmem_cache_alias(const char *name, size_t size, size_t align, -+ unsigned long flags, void (*ctor)(void *)) -+{ -+ struct kmem_cache *s, *c; -+ -+ s = find_mergeable(size, align, flags, name, ctor); -+ if (s) { -+ s->refcount++; -+ -+ /* -+ * Adjust the object sizes so that we clear -+ * the complete object on kzalloc. -+ */ -+ s->object_size = max(s->object_size, (int)size); -+ s->inuse = max_t(int, s->inuse, ALIGN(size, sizeof(void *))); -+ -+ for_each_memcg_cache(c, s) { -+ c->object_size = s->object_size; -+ c->inuse = max_t(int, c->inuse, -+ ALIGN(size, sizeof(void *))); -+ } -+ -+ if (sysfs_slab_alias(s, name)) { -+ s->refcount--; -+ s = NULL; -+ } -+ } -+ -+ return s; -+} -+ -+int __kmem_cache_create(struct kmem_cache *s, unsigned long flags) -+{ -+ int err; -+ -+ err = kmem_cache_open(s, flags); -+ if (err) -+ return err; -+ -+ /* Mutex is not taken during early boot */ -+ if (slab_state <= UP) -+ return 0; -+ -+ memcg_propagate_slab_attrs(s); -+ err = sysfs_slab_add(s); -+ if (err) -+ kmem_cache_close(s); -+ -+ return err; -+} -+ -+#ifdef CONFIG_SMP -+/* -+ * Use the cpu notifier to insure that the cpu slabs are flushed when -+ * necessary. -+ */ -+static int slab_cpuup_callback(struct notifier_block *nfb, -+ unsigned long action, void *hcpu) -+{ -+ long cpu = (long)hcpu; -+ struct kmem_cache *s; -+ unsigned long flags; -+ -+ switch (action) { -+ case CPU_UP_CANCELED: -+ case CPU_UP_CANCELED_FROZEN: -+ case CPU_DEAD: -+ case CPU_DEAD_FROZEN: -+ mutex_lock(&slab_mutex); -+ list_for_each_entry(s, &slab_caches, list) { -+ local_irq_save(flags); -+ __flush_cpu_slab(s, cpu); -+ local_irq_restore(flags); -+ } -+ mutex_unlock(&slab_mutex); -+ break; -+ default: -+ break; -+ } -+ return NOTIFY_OK; -+} -+ -+static struct notifier_block slab_notifier = { -+ .notifier_call = slab_cpuup_callback -+}; -+ -+#endif -+ -+void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, unsigned long caller) -+{ -+ struct kmem_cache *s; -+ void *ret; -+ -+ if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) -+ return kmalloc_large(size, gfpflags); -+ -+ s = kmalloc_slab(size, gfpflags); -+ -+ if (unlikely(ZERO_OR_NULL_PTR(s))) -+ return s; -+ -+ ret = slab_alloc(s, gfpflags, caller); -+ -+ /* Honor the call site pointer we received. */ -+ trace_kmalloc(caller, ret, size, s->size, gfpflags); -+ -+ return ret; -+} -+ -+#ifdef CONFIG_NUMA -+void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags, -+ int node, unsigned long caller) -+{ -+ struct kmem_cache *s; -+ void *ret; -+ -+ if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) { -+ ret = kmalloc_large_node(size, gfpflags, node); -+ -+ trace_kmalloc_node(caller, ret, -+ size, PAGE_SIZE << get_order(size), -+ gfpflags, node); -+ -+ return ret; -+ } -+ -+ s = kmalloc_slab(size, gfpflags); -+ -+ if (unlikely(ZERO_OR_NULL_PTR(s))) -+ return s; -+ -+ ret = slab_alloc_node(s, gfpflags, node, caller); -+ -+ /* Honor the call site pointer we received. */ -+ trace_kmalloc_node(caller, ret, size, s->size, gfpflags, node); -+ -+ return ret; -+} -+#endif -+ -+#ifdef CONFIG_SYSFS -+static int count_inuse(struct page *page) -+{ -+ return page->inuse; -+} -+ -+static int count_total(struct page *page) -+{ -+ return page->objects; -+} -+#endif -+ -+#ifdef CONFIG_SLUB_DEBUG -+static int validate_slab(struct kmem_cache *s, struct page *page, -+ unsigned long *map) -+{ -+ void *p; -+ void *addr = page_address(page); -+ -+ if (!check_slab(s, page) || -+ !on_freelist(s, page, NULL)) -+ return 0; -+ -+ /* Now we know that a valid freelist exists */ -+ bitmap_zero(map, page->objects); -+ -+ get_map(s, page, map); -+ for_each_object(p, s, addr, page->objects) { -+ if (test_bit(slab_index(p, s, addr), map)) -+ if (!check_object(s, page, p, SLUB_RED_INACTIVE)) -+ return 0; -+ } -+ -+ for_each_object(p, s, addr, page->objects) -+ if (!test_bit(slab_index(p, s, addr), map)) -+ if (!check_object(s, page, p, SLUB_RED_ACTIVE)) -+ return 0; -+ return 1; -+} -+ -+static void validate_slab_slab(struct kmem_cache *s, struct page *page, -+ unsigned long *map) -+{ -+ slab_lock(page); -+ validate_slab(s, page, map); -+ slab_unlock(page); -+} -+ -+static int validate_slab_node(struct kmem_cache *s, -+ struct kmem_cache_node *n, unsigned long *map) -+{ -+ unsigned long count = 0; -+ struct page *page; -+ unsigned long flags; -+ -+ spin_lock_irqsave(&n->list_lock, flags); -+ -+ list_for_each_entry(page, &n->partial, lru) { -+ validate_slab_slab(s, page, map); -+ count++; -+ } -+ if (count != n->nr_partial) -+ pr_err("SLUB %s: %ld partial slabs counted but counter=%ld\n", -+ s->name, count, n->nr_partial); -+ -+ if (!(s->flags & SLAB_STORE_USER)) -+ goto out; -+ -+ list_for_each_entry(page, &n->full, lru) { -+ validate_slab_slab(s, page, map); -+ count++; -+ } -+ if (count != atomic_long_read(&n->nr_slabs)) -+ pr_err("SLUB: %s %ld slabs counted but counter=%ld\n", -+ s->name, count, atomic_long_read(&n->nr_slabs)); -+ -+out: -+ spin_unlock_irqrestore(&n->list_lock, flags); -+ return count; -+} -+ -+static long validate_slab_cache(struct kmem_cache *s) -+{ -+ int node; -+ unsigned long count = 0; -+ unsigned long *map = kmalloc(BITS_TO_LONGS(oo_objects(s->max)) * -+ sizeof(unsigned long), GFP_KERNEL); -+ struct kmem_cache_node *n; -+ -+ if (!map) -+ return -ENOMEM; -+ -+ flush_all(s); -+ for_each_kmem_cache_node(s, node, n) -+ count += validate_slab_node(s, n, map); -+ kfree(map); -+ return count; -+} -+/* -+ * Generate lists of code addresses where slabcache objects are allocated -+ * and freed. -+ */ -+ -+struct location { -+ unsigned long count; -+ unsigned long addr; -+ long long sum_time; -+ long min_time; -+ long max_time; -+ long min_pid; -+ long max_pid; -+ DECLARE_BITMAP(cpus, NR_CPUS); -+ nodemask_t nodes; -+}; -+ -+struct loc_track { -+ unsigned long max; -+ unsigned long count; -+ struct location *loc; -+}; -+ -+static void free_loc_track(struct loc_track *t) -+{ -+ if (t->max) -+ free_pages((unsigned long)t->loc, -+ get_order(sizeof(struct location) * t->max)); -+} -+ -+static int alloc_loc_track(struct loc_track *t, unsigned long max, gfp_t flags) -+{ -+ struct location *l; -+ int order; -+ -+ order = get_order(sizeof(struct location) * max); -+ -+ l = (void *)__get_free_pages(flags, order); -+ if (!l) -+ return 0; -+ -+ if (t->count) { -+ memcpy(l, t->loc, sizeof(struct location) * t->count); -+ free_loc_track(t); -+ } -+ t->max = max; -+ t->loc = l; -+ return 1; -+} -+ -+static int add_location(struct loc_track *t, struct kmem_cache *s, -+ const struct track *track) -+{ -+ long start, end, pos; -+ struct location *l; -+ unsigned long caddr; -+ unsigned long age = jiffies - track->when; -+ -+ start = -1; -+ end = t->count; -+ -+ for ( ; ; ) { -+ pos = start + (end - start + 1) / 2; -+ -+ /* -+ * There is nothing at "end". If we end up there -+ * we need to add something to before end. -+ */ -+ if (pos == end) -+ break; -+ -+ caddr = t->loc[pos].addr; -+ if (track->addr == caddr) { -+ -+ l = &t->loc[pos]; -+ l->count++; -+ if (track->when) { -+ l->sum_time += age; -+ if (age < l->min_time) -+ l->min_time = age; -+ if (age > l->max_time) -+ l->max_time = age; -+ -+ if (track->pid < l->min_pid) -+ l->min_pid = track->pid; -+ if (track->pid > l->max_pid) -+ l->max_pid = track->pid; -+ -+ cpumask_set_cpu(track->cpu, -+ to_cpumask(l->cpus)); -+ } -+ node_set(page_to_nid(virt_to_page(track)), l->nodes); -+ return 1; -+ } -+ -+ if (track->addr < caddr) -+ end = pos; -+ else -+ start = pos; -+ } -+ -+ /* -+ * Not found. Insert new tracking element. -+ */ -+ if (t->count >= t->max && !alloc_loc_track(t, 2 * t->max, GFP_ATOMIC)) -+ return 0; -+ -+ l = t->loc + pos; -+ if (pos < t->count) -+ memmove(l + 1, l, -+ (t->count - pos) * sizeof(struct location)); -+ t->count++; -+ l->count = 1; -+ l->addr = track->addr; -+ l->sum_time = age; -+ l->min_time = age; -+ l->max_time = age; -+ l->min_pid = track->pid; -+ l->max_pid = track->pid; -+ cpumask_clear(to_cpumask(l->cpus)); -+ cpumask_set_cpu(track->cpu, to_cpumask(l->cpus)); -+ nodes_clear(l->nodes); -+ node_set(page_to_nid(virt_to_page(track)), l->nodes); -+ return 1; -+} -+ -+static void process_slab(struct loc_track *t, struct kmem_cache *s, -+ struct page *page, enum track_item alloc, -+ unsigned long *map) -+{ -+ void *addr = page_address(page); -+ void *p; -+ -+ bitmap_zero(map, page->objects); -+ get_map(s, page, map); -+ -+ for_each_object(p, s, addr, page->objects) -+ if (!test_bit(slab_index(p, s, addr), map)) -+ add_location(t, s, get_track(s, p, alloc)); -+} -+ -+static int list_locations(struct kmem_cache *s, char *buf, -+ enum track_item alloc) -+{ -+ int len = 0; -+ unsigned long i; -+ struct loc_track t = { 0, 0, NULL }; -+ int node; -+ unsigned long *map = kmalloc(BITS_TO_LONGS(oo_objects(s->max)) * -+ sizeof(unsigned long), GFP_KERNEL); -+ struct kmem_cache_node *n; -+ -+ if (!map || !alloc_loc_track(&t, PAGE_SIZE / sizeof(struct location), -+ GFP_TEMPORARY)) { -+ kfree(map); -+ return sprintf(buf, "Out of memory\n"); -+ } -+ /* Push back cpu slabs */ -+ flush_all(s); -+ -+ for_each_kmem_cache_node(s, node, n) { -+ unsigned long flags; -+ struct page *page; -+ -+ if (!atomic_long_read(&n->nr_slabs)) -+ continue; -+ -+ spin_lock_irqsave(&n->list_lock, flags); -+ list_for_each_entry(page, &n->partial, lru) -+ process_slab(&t, s, page, alloc, map); -+ list_for_each_entry(page, &n->full, lru) -+ process_slab(&t, s, page, alloc, map); -+ spin_unlock_irqrestore(&n->list_lock, flags); -+ } -+ -+ for (i = 0; i < t.count; i++) { -+ struct location *l = &t.loc[i]; -+ -+ if (len > PAGE_SIZE - KSYM_SYMBOL_LEN - 100) -+ break; -+ len += sprintf(buf + len, "%7ld ", l->count); -+ -+ if (l->addr) -+ len += sprintf(buf + len, "%pS", (void *)l->addr); -+ else -+ len += sprintf(buf + len, ""); -+ -+ if (l->sum_time != l->min_time) { -+ len += sprintf(buf + len, " age=%ld/%ld/%ld", -+ l->min_time, -+ (long)div_u64(l->sum_time, l->count), -+ l->max_time); -+ } else -+ len += sprintf(buf + len, " age=%ld", -+ l->min_time); -+ -+ if (l->min_pid != l->max_pid) -+ len += sprintf(buf + len, " pid=%ld-%ld", -+ l->min_pid, l->max_pid); -+ else -+ len += sprintf(buf + len, " pid=%ld", -+ l->min_pid); -+ -+ if (num_online_cpus() > 1 && -+ !cpumask_empty(to_cpumask(l->cpus)) && -+ len < PAGE_SIZE - 60) -+ len += scnprintf(buf + len, PAGE_SIZE - len - 50, -+ " cpus=%*pbl", -+ cpumask_pr_args(to_cpumask(l->cpus))); -+ -+ if (nr_online_nodes > 1 && !nodes_empty(l->nodes) && -+ len < PAGE_SIZE - 60) -+ len += scnprintf(buf + len, PAGE_SIZE - len - 50, -+ " nodes=%*pbl", -+ nodemask_pr_args(&l->nodes)); -+ -+ len += sprintf(buf + len, "\n"); -+ } -+ -+ free_loc_track(&t); -+ kfree(map); -+ if (!t.count) -+ len += sprintf(buf, "No data\n"); -+ return len; -+} -+#endif -+ -+#ifdef SLUB_RESILIENCY_TEST -+static void __init resiliency_test(void) -+{ -+ u8 *p; -+ -+ BUILD_BUG_ON(KMALLOC_MIN_SIZE > 16 || KMALLOC_SHIFT_HIGH < 10); -+ -+ pr_err("SLUB resiliency testing\n"); -+ pr_err("-----------------------\n"); -+ pr_err("A. Corruption after allocation\n"); -+ -+ p = kzalloc(16, GFP_KERNEL); -+ p[16] = 0x12; -+ pr_err("\n1. kmalloc-16: Clobber Redzone/next pointer 0x12->0x%p\n\n", -+ p + 16); -+ -+ validate_slab_cache(kmalloc_caches[4]); -+ -+ /* Hmmm... The next two are dangerous */ -+ p = kzalloc(32, GFP_KERNEL); -+ p[32 + sizeof(void *)] = 0x34; -+ pr_err("\n2. kmalloc-32: Clobber next pointer/next slab 0x34 -> -0x%p\n", -+ p); -+ pr_err("If allocated object is overwritten then not detectable\n\n"); -+ -+ validate_slab_cache(kmalloc_caches[5]); -+ p = kzalloc(64, GFP_KERNEL); -+ p += 64 + (get_cycles() & 0xff) * sizeof(void *); -+ *p = 0x56; -+ pr_err("\n3. kmalloc-64: corrupting random byte 0x56->0x%p\n", -+ p); -+ pr_err("If allocated object is overwritten then not detectable\n\n"); -+ validate_slab_cache(kmalloc_caches[6]); -+ -+ pr_err("\nB. Corruption after free\n"); -+ p = kzalloc(128, GFP_KERNEL); -+ kfree(p); -+ *p = 0x78; -+ pr_err("1. kmalloc-128: Clobber first word 0x78->0x%p\n\n", p); -+ validate_slab_cache(kmalloc_caches[7]); -+ -+ p = kzalloc(256, GFP_KERNEL); -+ kfree(p); -+ p[50] = 0x9a; -+ pr_err("\n2. kmalloc-256: Clobber 50th byte 0x9a->0x%p\n\n", p); -+ validate_slab_cache(kmalloc_caches[8]); -+ -+ p = kzalloc(512, GFP_KERNEL); -+ kfree(p); -+ p[512] = 0xab; -+ pr_err("\n3. kmalloc-512: Clobber redzone 0xab->0x%p\n\n", p); -+ validate_slab_cache(kmalloc_caches[9]); -+} -+#else -+#ifdef CONFIG_SYSFS -+static void resiliency_test(void) {}; -+#endif -+#endif -+ -+#ifdef CONFIG_SYSFS -+enum slab_stat_type { -+ SL_ALL, /* All slabs */ -+ SL_PARTIAL, /* Only partially allocated slabs */ -+ SL_CPU, /* Only slabs used for cpu caches */ -+ SL_OBJECTS, /* Determine allocated objects not slabs */ -+ SL_TOTAL /* Determine object capacity not slabs */ -+}; -+ -+#define SO_ALL (1 << SL_ALL) -+#define SO_PARTIAL (1 << SL_PARTIAL) -+#define SO_CPU (1 << SL_CPU) -+#define SO_OBJECTS (1 << SL_OBJECTS) -+#define SO_TOTAL (1 << SL_TOTAL) -+ -+static ssize_t show_slab_objects(struct kmem_cache *s, -+ char *buf, unsigned long flags) -+{ -+ unsigned long total = 0; -+ int node; -+ int x; -+ unsigned long *nodes; -+ -+ nodes = kzalloc(sizeof(unsigned long) * nr_node_ids, GFP_KERNEL); -+ if (!nodes) -+ return -ENOMEM; -+ -+ if (flags & SO_CPU) { -+ int cpu; -+ -+ for_each_possible_cpu(cpu) { -+ struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, -+ cpu); -+ int node; -+ struct page *page; -+ -+ page = READ_ONCE(c->page); -+ if (!page) -+ continue; -+ -+ node = page_to_nid(page); -+ if (flags & SO_TOTAL) -+ x = page->objects; -+ else if (flags & SO_OBJECTS) -+ x = page->inuse; -+ else -+ x = 1; -+ -+ total += x; -+ nodes[node] += x; -+ -+ page = READ_ONCE(c->partial); -+ if (page) { -+ node = page_to_nid(page); -+ if (flags & SO_TOTAL) -+ WARN_ON_ONCE(1); -+ else if (flags & SO_OBJECTS) -+ WARN_ON_ONCE(1); -+ else -+ x = page->pages; -+ total += x; -+ nodes[node] += x; -+ } -+ } -+ } -+ -+ get_online_mems(); -+#ifdef CONFIG_SLUB_DEBUG -+ if (flags & SO_ALL) { -+ struct kmem_cache_node *n; -+ -+ for_each_kmem_cache_node(s, node, n) { -+ -+ if (flags & SO_TOTAL) -+ x = atomic_long_read(&n->total_objects); -+ else if (flags & SO_OBJECTS) -+ x = atomic_long_read(&n->total_objects) - -+ count_partial(n, count_free); -+ else -+ x = atomic_long_read(&n->nr_slabs); -+ total += x; -+ nodes[node] += x; -+ } -+ -+ } else -+#endif -+ if (flags & SO_PARTIAL) { -+ struct kmem_cache_node *n; -+ -+ for_each_kmem_cache_node(s, node, n) { -+ if (flags & SO_TOTAL) -+ x = count_partial(n, count_total); -+ else if (flags & SO_OBJECTS) -+ x = count_partial(n, count_inuse); -+ else -+ x = n->nr_partial; -+ total += x; -+ nodes[node] += x; -+ } -+ } -+ x = sprintf(buf, "%lu", total); -+#ifdef CONFIG_NUMA -+ for (node = 0; node < nr_node_ids; node++) -+ if (nodes[node]) -+ x += sprintf(buf + x, " N%d=%lu", -+ node, nodes[node]); -+#endif -+ put_online_mems(); -+ kfree(nodes); -+ return x + sprintf(buf + x, "\n"); -+} -+ -+#ifdef CONFIG_SLUB_DEBUG -+static int any_slab_objects(struct kmem_cache *s) -+{ -+ int node; -+ struct kmem_cache_node *n; -+ -+ for_each_kmem_cache_node(s, node, n) -+ if (atomic_long_read(&n->total_objects)) -+ return 1; -+ -+ return 0; -+} -+#endif -+ -+#define to_slab_attr(n) container_of(n, struct slab_attribute, attr) -+#define to_slab(n) container_of(n, struct kmem_cache, kobj) -+ -+struct slab_attribute { -+ struct attribute attr; -+ ssize_t (*show)(struct kmem_cache *s, char *buf); -+ ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count); -+}; -+ -+#define SLAB_ATTR_RO(_name) \ -+ static struct slab_attribute _name##_attr = \ -+ __ATTR(_name, 0400, _name##_show, NULL) -+ -+#define SLAB_ATTR(_name) \ -+ static struct slab_attribute _name##_attr = \ -+ __ATTR(_name, 0600, _name##_show, _name##_store) -+ -+static ssize_t slab_size_show(struct kmem_cache *s, char *buf) -+{ -+ return sprintf(buf, "%d\n", s->size); -+} -+SLAB_ATTR_RO(slab_size); -+ -+static ssize_t align_show(struct kmem_cache *s, char *buf) -+{ -+ return sprintf(buf, "%d\n", s->align); -+} -+SLAB_ATTR_RO(align); -+ -+static ssize_t object_size_show(struct kmem_cache *s, char *buf) -+{ -+ return sprintf(buf, "%d\n", s->object_size); -+} -+SLAB_ATTR_RO(object_size); -+ -+static ssize_t objs_per_slab_show(struct kmem_cache *s, char *buf) -+{ -+ return sprintf(buf, "%d\n", oo_objects(s->oo)); -+} -+SLAB_ATTR_RO(objs_per_slab); -+ -+static ssize_t order_store(struct kmem_cache *s, -+ const char *buf, size_t length) -+{ -+ unsigned long order; -+ int err; -+ -+ err = kstrtoul(buf, 10, &order); -+ if (err) -+ return err; -+ -+ if (order > slub_max_order || order < slub_min_order) -+ return -EINVAL; -+ -+ calculate_sizes(s, order); -+ return length; -+} -+ -+static ssize_t order_show(struct kmem_cache *s, char *buf) -+{ -+ return sprintf(buf, "%d\n", oo_order(s->oo)); -+} -+SLAB_ATTR(order); -+ -+static ssize_t min_partial_show(struct kmem_cache *s, char *buf) -+{ -+ return sprintf(buf, "%lu\n", s->min_partial); -+} -+ -+static ssize_t min_partial_store(struct kmem_cache *s, const char *buf, -+ size_t length) -+{ -+ unsigned long min; -+ int err; -+ -+ err = kstrtoul(buf, 10, &min); -+ if (err) -+ return err; -+ -+ set_min_partial(s, min); -+ return length; -+} -+SLAB_ATTR(min_partial); -+ -+static ssize_t cpu_partial_show(struct kmem_cache *s, char *buf) -+{ -+ return sprintf(buf, "%u\n", s->cpu_partial); -+} -+ -+static ssize_t cpu_partial_store(struct kmem_cache *s, const char *buf, -+ size_t length) -+{ -+ unsigned long objects; -+ int err; -+ -+ err = kstrtoul(buf, 10, &objects); -+ if (err) -+ return err; -+ if (objects && !kmem_cache_has_cpu_partial(s)) -+ return -EINVAL; -+ -+ s->cpu_partial = objects; -+ flush_all(s); -+ return length; -+} -+SLAB_ATTR(cpu_partial); -+ -+static ssize_t ctor_show(struct kmem_cache *s, char *buf) -+{ -+ if (!s->ctor) -+ return 0; -+ return sprintf(buf, "%pS\n", s->ctor); -+} -+SLAB_ATTR_RO(ctor); -+ -+static ssize_t aliases_show(struct kmem_cache *s, char *buf) -+{ -+ return sprintf(buf, "%d\n", s->refcount < 0 ? 0 : s->refcount - 1); -+} -+SLAB_ATTR_RO(aliases); -+ -+static ssize_t partial_show(struct kmem_cache *s, char *buf) -+{ -+ return show_slab_objects(s, buf, SO_PARTIAL); -+} -+SLAB_ATTR_RO(partial); -+ -+static ssize_t cpu_slabs_show(struct kmem_cache *s, char *buf) -+{ -+ return show_slab_objects(s, buf, SO_CPU); -+} -+SLAB_ATTR_RO(cpu_slabs); -+ -+static ssize_t objects_show(struct kmem_cache *s, char *buf) -+{ -+ return show_slab_objects(s, buf, SO_ALL|SO_OBJECTS); -+} -+SLAB_ATTR_RO(objects); -+ -+static ssize_t objects_partial_show(struct kmem_cache *s, char *buf) -+{ -+ return show_slab_objects(s, buf, SO_PARTIAL|SO_OBJECTS); -+} -+SLAB_ATTR_RO(objects_partial); -+ -+static ssize_t slabs_cpu_partial_show(struct kmem_cache *s, char *buf) -+{ -+ int objects = 0; -+ int pages = 0; -+ int cpu; -+ int len; -+ -+ for_each_online_cpu(cpu) { -+ struct page *page = per_cpu_ptr(s->cpu_slab, cpu)->partial; -+ -+ if (page) { -+ pages += page->pages; -+ objects += page->pobjects; -+ } -+ } -+ -+ len = sprintf(buf, "%d(%d)", objects, pages); -+ -+#ifdef CONFIG_SMP -+ for_each_online_cpu(cpu) { -+ struct page *page = per_cpu_ptr(s->cpu_slab, cpu) ->partial; -+ -+ if (page && len < PAGE_SIZE - 20) -+ len += sprintf(buf + len, " C%d=%d(%d)", cpu, -+ page->pobjects, page->pages); -+ } -+#endif -+ return len + sprintf(buf + len, "\n"); -+} -+SLAB_ATTR_RO(slabs_cpu_partial); -+ -+static ssize_t reclaim_account_show(struct kmem_cache *s, char *buf) -+{ -+ return sprintf(buf, "%d\n", !!(s->flags & SLAB_RECLAIM_ACCOUNT)); -+} -+ -+static ssize_t reclaim_account_store(struct kmem_cache *s, -+ const char *buf, size_t length) -+{ -+ s->flags &= ~SLAB_RECLAIM_ACCOUNT; -+ if (buf[0] == '1') -+ s->flags |= SLAB_RECLAIM_ACCOUNT; -+ return length; -+} -+SLAB_ATTR(reclaim_account); -+ -+static ssize_t hwcache_align_show(struct kmem_cache *s, char *buf) -+{ -+ return sprintf(buf, "%d\n", !!(s->flags & SLAB_HWCACHE_ALIGN)); -+} -+SLAB_ATTR_RO(hwcache_align); -+ -+#ifdef CONFIG_ZONE_DMA -+static ssize_t cache_dma_show(struct kmem_cache *s, char *buf) -+{ -+ return sprintf(buf, "%d\n", !!(s->flags & SLAB_CACHE_DMA)); -+} -+SLAB_ATTR_RO(cache_dma); -+#endif -+ -+static ssize_t destroy_by_rcu_show(struct kmem_cache *s, char *buf) -+{ -+ return sprintf(buf, "%d\n", !!(s->flags & SLAB_DESTROY_BY_RCU)); -+} -+SLAB_ATTR_RO(destroy_by_rcu); -+ -+static ssize_t reserved_show(struct kmem_cache *s, char *buf) -+{ -+ return sprintf(buf, "%d\n", s->reserved); -+} -+SLAB_ATTR_RO(reserved); -+ -+#ifdef CONFIG_SLUB_DEBUG -+static ssize_t slabs_show(struct kmem_cache *s, char *buf) -+{ -+ return show_slab_objects(s, buf, SO_ALL); -+} -+SLAB_ATTR_RO(slabs); -+ -+static ssize_t total_objects_show(struct kmem_cache *s, char *buf) -+{ -+ return show_slab_objects(s, buf, SO_ALL|SO_TOTAL); -+} -+SLAB_ATTR_RO(total_objects); -+ -+static ssize_t sanity_checks_show(struct kmem_cache *s, char *buf) -+{ -+ return sprintf(buf, "%d\n", !!(s->flags & SLAB_DEBUG_FREE)); -+} -+ -+static ssize_t sanity_checks_store(struct kmem_cache *s, -+ const char *buf, size_t length) -+{ -+ s->flags &= ~SLAB_DEBUG_FREE; -+ if (buf[0] == '1') { -+ s->flags &= ~__CMPXCHG_DOUBLE; -+ s->flags |= SLAB_DEBUG_FREE; -+ } -+ return length; -+} -+SLAB_ATTR(sanity_checks); -+ -+static ssize_t trace_show(struct kmem_cache *s, char *buf) -+{ -+ return sprintf(buf, "%d\n", !!(s->flags & SLAB_TRACE)); -+} -+ -+static ssize_t trace_store(struct kmem_cache *s, const char *buf, -+ size_t length) -+{ -+ /* -+ * Tracing a merged cache is going to give confusing results -+ * as well as cause other issues like converting a mergeable -+ * cache into an umergeable one. -+ */ -+ if (s->refcount > 1) -+ return -EINVAL; -+ -+ s->flags &= ~SLAB_TRACE; -+ if (buf[0] == '1') { -+ s->flags &= ~__CMPXCHG_DOUBLE; -+ s->flags |= SLAB_TRACE; -+ } -+ return length; -+} -+SLAB_ATTR(trace); -+ -+static ssize_t red_zone_show(struct kmem_cache *s, char *buf) -+{ -+ return sprintf(buf, "%d\n", !!(s->flags & SLAB_RED_ZONE)); -+} -+ -+static ssize_t red_zone_store(struct kmem_cache *s, -+ const char *buf, size_t length) -+{ -+ if (any_slab_objects(s)) -+ return -EBUSY; -+ -+ s->flags &= ~SLAB_RED_ZONE; -+ if (buf[0] == '1') { -+ s->flags &= ~__CMPXCHG_DOUBLE; -+ s->flags |= SLAB_RED_ZONE; -+ } -+ calculate_sizes(s, -1); -+ return length; -+} -+SLAB_ATTR(red_zone); -+ -+static ssize_t poison_show(struct kmem_cache *s, char *buf) -+{ -+ return sprintf(buf, "%d\n", !!(s->flags & SLAB_POISON)); -+} -+ -+static ssize_t poison_store(struct kmem_cache *s, -+ const char *buf, size_t length) -+{ -+ if (any_slab_objects(s)) -+ return -EBUSY; -+ -+ s->flags &= ~SLAB_POISON; -+ if (buf[0] == '1') { -+ s->flags &= ~__CMPXCHG_DOUBLE; -+ s->flags |= SLAB_POISON; -+ } -+ calculate_sizes(s, -1); -+ return length; -+} -+SLAB_ATTR(poison); -+ -+static ssize_t store_user_show(struct kmem_cache *s, char *buf) -+{ -+ return sprintf(buf, "%d\n", !!(s->flags & SLAB_STORE_USER)); -+} -+ -+static ssize_t store_user_store(struct kmem_cache *s, -+ const char *buf, size_t length) -+{ -+ if (any_slab_objects(s)) -+ return -EBUSY; -+ -+ s->flags &= ~SLAB_STORE_USER; -+ if (buf[0] == '1') { -+ s->flags &= ~__CMPXCHG_DOUBLE; -+ s->flags |= SLAB_STORE_USER; -+ } -+ calculate_sizes(s, -1); -+ return length; -+} -+SLAB_ATTR(store_user); -+ -+static ssize_t validate_show(struct kmem_cache *s, char *buf) -+{ -+ return 0; -+} -+ -+static ssize_t validate_store(struct kmem_cache *s, -+ const char *buf, size_t length) -+{ -+ int ret = -EINVAL; -+ -+ if (buf[0] == '1') { -+ ret = validate_slab_cache(s); -+ if (ret >= 0) -+ ret = length; -+ } -+ return ret; -+} -+SLAB_ATTR(validate); -+ -+static ssize_t alloc_calls_show(struct kmem_cache *s, char *buf) -+{ -+ if (!(s->flags & SLAB_STORE_USER)) -+ return -ENOSYS; -+ return list_locations(s, buf, TRACK_ALLOC); -+} -+SLAB_ATTR_RO(alloc_calls); -+ -+static ssize_t free_calls_show(struct kmem_cache *s, char *buf) -+{ -+ if (!(s->flags & SLAB_STORE_USER)) -+ return -ENOSYS; -+ return list_locations(s, buf, TRACK_FREE); -+} -+SLAB_ATTR_RO(free_calls); -+#endif /* CONFIG_SLUB_DEBUG */ -+ -+#ifdef CONFIG_FAILSLAB -+static ssize_t failslab_show(struct kmem_cache *s, char *buf) -+{ -+ return sprintf(buf, "%d\n", !!(s->flags & SLAB_FAILSLAB)); -+} -+ -+static ssize_t failslab_store(struct kmem_cache *s, const char *buf, -+ size_t length) -+{ -+ if (s->refcount > 1) -+ return -EINVAL; -+ -+ s->flags &= ~SLAB_FAILSLAB; -+ if (buf[0] == '1') -+ s->flags |= SLAB_FAILSLAB; -+ return length; -+} -+SLAB_ATTR(failslab); -+#endif -+ -+static ssize_t shrink_show(struct kmem_cache *s, char *buf) -+{ -+ return 0; -+} -+ -+static ssize_t shrink_store(struct kmem_cache *s, -+ const char *buf, size_t length) -+{ -+ if (buf[0] == '1') -+ kmem_cache_shrink(s); -+ else -+ return -EINVAL; -+ return length; -+} -+SLAB_ATTR(shrink); -+ -+#ifdef CONFIG_NUMA -+static ssize_t remote_node_defrag_ratio_show(struct kmem_cache *s, char *buf) -+{ -+ return sprintf(buf, "%d\n", s->remote_node_defrag_ratio / 10); -+} -+ -+static ssize_t remote_node_defrag_ratio_store(struct kmem_cache *s, -+ const char *buf, size_t length) -+{ -+ unsigned long ratio; -+ int err; -+ -+ err = kstrtoul(buf, 10, &ratio); -+ if (err) -+ return err; -+ -+ if (ratio <= 100) -+ s->remote_node_defrag_ratio = ratio * 10; -+ -+ return length; -+} -+SLAB_ATTR(remote_node_defrag_ratio); -+#endif -+ -+#ifdef CONFIG_SLUB_STATS -+static int show_stat(struct kmem_cache *s, char *buf, enum stat_item si) -+{ -+ unsigned long sum = 0; -+ int cpu; -+ int len; -+ int *data = kmalloc(nr_cpu_ids * sizeof(int), GFP_KERNEL); -+ -+ if (!data) -+ return -ENOMEM; -+ -+ for_each_online_cpu(cpu) { -+ unsigned x = per_cpu_ptr(s->cpu_slab, cpu)->stat[si]; -+ -+ data[cpu] = x; -+ sum += x; -+ } -+ -+ len = sprintf(buf, "%lu", sum); -+ -+#ifdef CONFIG_SMP -+ for_each_online_cpu(cpu) { -+ if (data[cpu] && len < PAGE_SIZE - 20) -+ len += sprintf(buf + len, " C%d=%u", cpu, data[cpu]); -+ } -+#endif -+ kfree(data); -+ return len + sprintf(buf + len, "\n"); -+} -+ -+static void clear_stat(struct kmem_cache *s, enum stat_item si) -+{ -+ int cpu; -+ -+ for_each_online_cpu(cpu) -+ per_cpu_ptr(s->cpu_slab, cpu)->stat[si] = 0; -+} -+ -+#define STAT_ATTR(si, text) \ -+static ssize_t text##_show(struct kmem_cache *s, char *buf) \ -+{ \ -+ return show_stat(s, buf, si); \ -+} \ -+static ssize_t text##_store(struct kmem_cache *s, \ -+ const char *buf, size_t length) \ -+{ \ -+ if (buf[0] != '0') \ -+ return -EINVAL; \ -+ clear_stat(s, si); \ -+ return length; \ -+} \ -+SLAB_ATTR(text); \ -+ -+STAT_ATTR(ALLOC_FASTPATH, alloc_fastpath); -+STAT_ATTR(ALLOC_SLOWPATH, alloc_slowpath); -+STAT_ATTR(FREE_FASTPATH, free_fastpath); -+STAT_ATTR(FREE_SLOWPATH, free_slowpath); -+STAT_ATTR(FREE_FROZEN, free_frozen); -+STAT_ATTR(FREE_ADD_PARTIAL, free_add_partial); -+STAT_ATTR(FREE_REMOVE_PARTIAL, free_remove_partial); -+STAT_ATTR(ALLOC_FROM_PARTIAL, alloc_from_partial); -+STAT_ATTR(ALLOC_SLAB, alloc_slab); -+STAT_ATTR(ALLOC_REFILL, alloc_refill); -+STAT_ATTR(ALLOC_NODE_MISMATCH, alloc_node_mismatch); -+STAT_ATTR(FREE_SLAB, free_slab); -+STAT_ATTR(CPUSLAB_FLUSH, cpuslab_flush); -+STAT_ATTR(DEACTIVATE_FULL, deactivate_full); -+STAT_ATTR(DEACTIVATE_EMPTY, deactivate_empty); -+STAT_ATTR(DEACTIVATE_TO_HEAD, deactivate_to_head); -+STAT_ATTR(DEACTIVATE_TO_TAIL, deactivate_to_tail); -+STAT_ATTR(DEACTIVATE_REMOTE_FREES, deactivate_remote_frees); -+STAT_ATTR(DEACTIVATE_BYPASS, deactivate_bypass); -+STAT_ATTR(ORDER_FALLBACK, order_fallback); -+STAT_ATTR(CMPXCHG_DOUBLE_CPU_FAIL, cmpxchg_double_cpu_fail); -+STAT_ATTR(CMPXCHG_DOUBLE_FAIL, cmpxchg_double_fail); -+STAT_ATTR(CPU_PARTIAL_ALLOC, cpu_partial_alloc); -+STAT_ATTR(CPU_PARTIAL_FREE, cpu_partial_free); -+STAT_ATTR(CPU_PARTIAL_NODE, cpu_partial_node); -+STAT_ATTR(CPU_PARTIAL_DRAIN, cpu_partial_drain); -+#endif -+ -+static struct attribute *slab_attrs[] = { -+ &slab_size_attr.attr, -+ &object_size_attr.attr, -+ &objs_per_slab_attr.attr, -+ &order_attr.attr, -+ &min_partial_attr.attr, -+ &cpu_partial_attr.attr, -+ &objects_attr.attr, -+ &objects_partial_attr.attr, -+ &partial_attr.attr, -+ &cpu_slabs_attr.attr, -+ &ctor_attr.attr, -+ &aliases_attr.attr, -+ &align_attr.attr, -+ &hwcache_align_attr.attr, -+ &reclaim_account_attr.attr, -+ &destroy_by_rcu_attr.attr, -+ &shrink_attr.attr, -+ &reserved_attr.attr, -+ &slabs_cpu_partial_attr.attr, -+#ifdef CONFIG_SLUB_DEBUG -+ &total_objects_attr.attr, -+ &slabs_attr.attr, -+ &sanity_checks_attr.attr, -+ &trace_attr.attr, -+ &red_zone_attr.attr, -+ &poison_attr.attr, -+ &store_user_attr.attr, -+ &validate_attr.attr, -+ &alloc_calls_attr.attr, -+ &free_calls_attr.attr, -+#endif -+#ifdef CONFIG_ZONE_DMA -+ &cache_dma_attr.attr, -+#endif -+#ifdef CONFIG_NUMA -+ &remote_node_defrag_ratio_attr.attr, -+#endif -+#ifdef CONFIG_SLUB_STATS -+ &alloc_fastpath_attr.attr, -+ &alloc_slowpath_attr.attr, -+ &free_fastpath_attr.attr, -+ &free_slowpath_attr.attr, -+ &free_frozen_attr.attr, -+ &free_add_partial_attr.attr, -+ &free_remove_partial_attr.attr, -+ &alloc_from_partial_attr.attr, -+ &alloc_slab_attr.attr, -+ &alloc_refill_attr.attr, -+ &alloc_node_mismatch_attr.attr, -+ &free_slab_attr.attr, -+ &cpuslab_flush_attr.attr, -+ &deactivate_full_attr.attr, -+ &deactivate_empty_attr.attr, -+ &deactivate_to_head_attr.attr, -+ &deactivate_to_tail_attr.attr, -+ &deactivate_remote_frees_attr.attr, -+ &deactivate_bypass_attr.attr, -+ &order_fallback_attr.attr, -+ &cmpxchg_double_fail_attr.attr, -+ &cmpxchg_double_cpu_fail_attr.attr, -+ &cpu_partial_alloc_attr.attr, -+ &cpu_partial_free_attr.attr, -+ &cpu_partial_node_attr.attr, -+ &cpu_partial_drain_attr.attr, -+#endif -+#ifdef CONFIG_FAILSLAB -+ &failslab_attr.attr, -+#endif -+ -+ NULL -+}; -+ -+static struct attribute_group slab_attr_group = { -+ .attrs = slab_attrs, -+}; -+ -+static ssize_t slab_attr_show(struct kobject *kobj, -+ struct attribute *attr, -+ char *buf) -+{ -+ struct slab_attribute *attribute; -+ struct kmem_cache *s; -+ int err; -+ -+ attribute = to_slab_attr(attr); -+ s = to_slab(kobj); -+ -+ if (!attribute->show) -+ return -EIO; -+ -+ err = attribute->show(s, buf); -+ -+ return err; -+} -+ -+static ssize_t slab_attr_store(struct kobject *kobj, -+ struct attribute *attr, -+ const char *buf, size_t len) -+{ -+ struct slab_attribute *attribute; -+ struct kmem_cache *s; -+ int err; -+ -+ attribute = to_slab_attr(attr); -+ s = to_slab(kobj); -+ -+ if (!attribute->store) -+ return -EIO; -+ -+ err = attribute->store(s, buf, len); -+#ifdef CONFIG_MEMCG_KMEM -+ if (slab_state >= FULL && err >= 0 && is_root_cache(s)) { -+ struct kmem_cache *c; -+ -+ mutex_lock(&slab_mutex); -+ if (s->max_attr_size < len) -+ s->max_attr_size = len; -+ -+ /* -+ * This is a best effort propagation, so this function's return -+ * value will be determined by the parent cache only. This is -+ * basically because not all attributes will have a well -+ * defined semantics for rollbacks - most of the actions will -+ * have permanent effects. -+ * -+ * Returning the error value of any of the children that fail -+ * is not 100 % defined, in the sense that users seeing the -+ * error code won't be able to know anything about the state of -+ * the cache. -+ * -+ * Only returning the error code for the parent cache at least -+ * has well defined semantics. The cache being written to -+ * directly either failed or succeeded, in which case we loop -+ * through the descendants with best-effort propagation. -+ */ -+ for_each_memcg_cache(c, s) -+ attribute->store(c, buf, len); -+ mutex_unlock(&slab_mutex); -+ } -+#endif -+ return err; -+} -+ -+static void memcg_propagate_slab_attrs(struct kmem_cache *s) -+{ -+#ifdef CONFIG_MEMCG_KMEM -+ int i; -+ char *buffer = NULL; -+ struct kmem_cache *root_cache; -+ -+ if (is_root_cache(s)) -+ return; -+ -+ root_cache = s->memcg_params.root_cache; -+ -+ /* -+ * This mean this cache had no attribute written. Therefore, no point -+ * in copying default values around -+ */ -+ if (!root_cache->max_attr_size) -+ return; -+ -+ for (i = 0; i < ARRAY_SIZE(slab_attrs); i++) { -+ char mbuf[64]; -+ char *buf; -+ struct slab_attribute *attr = to_slab_attr(slab_attrs[i]); -+ -+ if (!attr || !attr->store || !attr->show) -+ continue; -+ -+ /* -+ * It is really bad that we have to allocate here, so we will -+ * do it only as a fallback. If we actually allocate, though, -+ * we can just use the allocated buffer until the end. -+ * -+ * Most of the slub attributes will tend to be very small in -+ * size, but sysfs allows buffers up to a page, so they can -+ * theoretically happen. -+ */ -+ if (buffer) -+ buf = buffer; -+ else if (root_cache->max_attr_size < ARRAY_SIZE(mbuf)) -+ buf = mbuf; -+ else { -+ buffer = (char *) get_zeroed_page(GFP_KERNEL); -+ if (WARN_ON(!buffer)) -+ continue; -+ buf = buffer; -+ } -+ -+ attr->show(root_cache, buf); -+ attr->store(s, buf, strlen(buf)); -+ } -+ -+ if (buffer) -+ free_page((unsigned long)buffer); -+#endif -+} -+ -+static void kmem_cache_release(struct kobject *k) -+{ -+ slab_kmem_cache_release(to_slab(k)); -+} -+ -+static const struct sysfs_ops slab_sysfs_ops = { -+ .show = slab_attr_show, -+ .store = slab_attr_store, -+}; -+ -+static struct kobj_type slab_ktype = { -+ .sysfs_ops = &slab_sysfs_ops, -+ .release = kmem_cache_release, -+}; -+ -+static int uevent_filter(struct kset *kset, struct kobject *kobj) -+{ -+ struct kobj_type *ktype = get_ktype(kobj); -+ -+ if (ktype == &slab_ktype) -+ return 1; -+ return 0; -+} -+ -+static const struct kset_uevent_ops slab_uevent_ops = { -+ .filter = uevent_filter, -+}; -+ -+static struct kset *slab_kset; -+ -+static inline struct kset *cache_kset(struct kmem_cache *s) -+{ -+#ifdef CONFIG_MEMCG_KMEM -+ if (!is_root_cache(s)) -+ return s->memcg_params.root_cache->memcg_kset; -+#endif -+ return slab_kset; -+} -+ -+#define ID_STR_LENGTH 64 -+ -+/* Create a unique string id for a slab cache: -+ * -+ * Format :[flags-]size -+ */ -+static char *create_unique_id(struct kmem_cache *s) -+{ -+ char *name = kmalloc(ID_STR_LENGTH, GFP_KERNEL); -+ char *p = name; -+ -+ BUG_ON(!name); -+ -+ *p++ = ':'; -+ /* -+ * First flags affecting slabcache operations. We will only -+ * get here for aliasable slabs so we do not need to support -+ * too many flags. The flags here must cover all flags that -+ * are matched during merging to guarantee that the id is -+ * unique. -+ */ -+ if (s->flags & SLAB_CACHE_DMA) -+ *p++ = 'd'; -+ if (s->flags & SLAB_RECLAIM_ACCOUNT) -+ *p++ = 'a'; -+ if (s->flags & SLAB_DEBUG_FREE) -+ *p++ = 'F'; -+ if (!(s->flags & SLAB_NOTRACK)) -+ *p++ = 't'; -+ if (p != name + 1) -+ *p++ = '-'; -+ p += sprintf(p, "%07d", s->size); -+ -+ BUG_ON(p > name + ID_STR_LENGTH - 1); -+ return name; -+} -+ -+static int sysfs_slab_add(struct kmem_cache *s) -+{ -+ int err; -+ const char *name; -+ int unmergeable = slab_unmergeable(s); -+ -+ if (unmergeable) { -+ /* -+ * Slabcache can never be merged so we can use the name proper. -+ * This is typically the case for debug situations. In that -+ * case we can catch duplicate names easily. -+ */ -+ sysfs_remove_link(&slab_kset->kobj, s->name); -+ name = s->name; -+ } else { -+ /* -+ * Create a unique name for the slab as a target -+ * for the symlinks. -+ */ -+ name = create_unique_id(s); -+ } -+ -+ s->kobj.kset = cache_kset(s); -+ err = kobject_init_and_add(&s->kobj, &slab_ktype, NULL, "%s", name); -+ if (err) -+ goto out_put_kobj; -+ -+ err = sysfs_create_group(&s->kobj, &slab_attr_group); -+ if (err) -+ goto out_del_kobj; -+ -+#ifdef CONFIG_MEMCG_KMEM -+ if (is_root_cache(s)) { -+ s->memcg_kset = kset_create_and_add("cgroup", NULL, &s->kobj); -+ if (!s->memcg_kset) { -+ err = -ENOMEM; -+ goto out_del_kobj; -+ } -+ } -+#endif -+ -+ kobject_uevent(&s->kobj, KOBJ_ADD); -+ if (!unmergeable) { -+ /* Setup first alias */ -+ sysfs_slab_alias(s, s->name); -+ } -+out: -+ if (!unmergeable) -+ kfree(name); -+ return err; -+out_del_kobj: -+ kobject_del(&s->kobj); -+out_put_kobj: -+ kobject_put(&s->kobj); -+ goto out; -+} -+ -+void sysfs_slab_remove(struct kmem_cache *s) -+{ -+ if (slab_state < FULL) -+ /* -+ * Sysfs has not been setup yet so no need to remove the -+ * cache from sysfs. -+ */ -+ return; -+ -+#ifdef CONFIG_MEMCG_KMEM -+ kset_unregister(s->memcg_kset); -+#endif -+ kobject_uevent(&s->kobj, KOBJ_REMOVE); -+ kobject_del(&s->kobj); -+ kobject_put(&s->kobj); -+} -+ -+/* -+ * Need to buffer aliases during bootup until sysfs becomes -+ * available lest we lose that information. -+ */ -+struct saved_alias { -+ struct kmem_cache *s; -+ const char *name; -+ struct saved_alias *next; -+}; -+ -+static struct saved_alias *alias_list; -+ -+static int sysfs_slab_alias(struct kmem_cache *s, const char *name) -+{ -+ struct saved_alias *al; -+ -+ if (slab_state == FULL) { -+ /* -+ * If we have a leftover link then remove it. -+ */ -+ sysfs_remove_link(&slab_kset->kobj, name); -+ return sysfs_create_link(&slab_kset->kobj, &s->kobj, name); -+ } -+ -+ al = kmalloc(sizeof(struct saved_alias), GFP_KERNEL); -+ if (!al) -+ return -ENOMEM; -+ -+ al->s = s; -+ al->name = name; -+ al->next = alias_list; -+ alias_list = al; -+ return 0; -+} -+ -+static int __init slab_sysfs_init(void) -+{ -+ struct kmem_cache *s; -+ int err; -+ -+ mutex_lock(&slab_mutex); -+ -+ slab_kset = kset_create_and_add("slab", &slab_uevent_ops, kernel_kobj); -+ if (!slab_kset) { -+ mutex_unlock(&slab_mutex); -+ pr_err("Cannot register slab subsystem.\n"); -+ return -ENOSYS; -+ } -+ -+ slab_state = FULL; -+ -+ list_for_each_entry(s, &slab_caches, list) { -+ err = sysfs_slab_add(s); -+ if (err) -+ pr_err("SLUB: Unable to add boot slab %s to sysfs\n", -+ s->name); -+ } -+ -+ while (alias_list) { -+ struct saved_alias *al = alias_list; -+ -+ alias_list = alias_list->next; -+ err = sysfs_slab_alias(al->s, al->name); -+ if (err) -+ pr_err("SLUB: Unable to add boot slab alias %s to sysfs\n", -+ al->name); -+ kfree(al); -+ } -+ -+ mutex_unlock(&slab_mutex); -+ resiliency_test(); -+ return 0; -+} -+ -+__initcall(slab_sysfs_init); -+#endif /* CONFIG_SYSFS */ -+ -+/* -+ * The /proc/slabinfo ABI -+ */ -+#ifdef CONFIG_SLABINFO -+void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo) -+{ -+ unsigned long nr_slabs = 0; -+ unsigned long nr_objs = 0; -+ unsigned long nr_free = 0; -+ int node; -+ struct kmem_cache_node *n; -+ -+ for_each_kmem_cache_node(s, node, n) { -+ nr_slabs += node_nr_slabs(n); -+ nr_objs += node_nr_objs(n); -+ nr_free += count_partial(n, count_free); -+ } -+ -+ sinfo->active_objs = nr_objs - nr_free; -+ sinfo->num_objs = nr_objs; -+ sinfo->active_slabs = nr_slabs; -+ sinfo->num_slabs = nr_slabs; -+ sinfo->objects_per_slab = oo_objects(s->oo); -+ sinfo->cache_order = oo_order(s->oo); -+} -+ -+void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *s) -+{ -+} -+ -+ssize_t slabinfo_write(struct file *file, const char __user *buffer, -+ size_t count, loff_t *ppos) -+{ -+ return -EIO; -+} -+#endif /* CONFIG_SLABINFO */ -diff -Nur linux-4.1.10.orig/mm/swap.c linux-4.1.10/mm/swap.c ---- linux-4.1.10.orig/mm/swap.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/mm/swap.c 2015-10-07 18:00:08.000000000 +0200 -@@ -32,6 +32,7 @@ - #include - #include - #include -+#include - - #include "internal.h" - -@@ -45,6 +46,9 @@ - static DEFINE_PER_CPU(struct pagevec, lru_rotate_pvecs); - static DEFINE_PER_CPU(struct pagevec, lru_deactivate_file_pvecs); - -+static DEFINE_LOCAL_IRQ_LOCK(rotate_lock); -+DEFINE_LOCAL_IRQ_LOCK(swapvec_lock); -+ - /* - * This path almost never happens for VM activity - pages are normally - * freed via pagevecs. But it gets used by networking. -@@ -481,11 +485,11 @@ - unsigned long flags; - - page_cache_get(page); -- local_irq_save(flags); -+ local_lock_irqsave(rotate_lock, flags); - pvec = this_cpu_ptr(&lru_rotate_pvecs); - if (!pagevec_add(pvec, page)) - pagevec_move_tail(pvec); -- local_irq_restore(flags); -+ local_unlock_irqrestore(rotate_lock, flags); - } - } - -@@ -536,12 +540,13 @@ - void activate_page(struct page *page) - { - if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) { -- struct pagevec *pvec = &get_cpu_var(activate_page_pvecs); -+ struct pagevec *pvec = &get_locked_var(swapvec_lock, -+ activate_page_pvecs); - - page_cache_get(page); - if (!pagevec_add(pvec, page)) - pagevec_lru_move_fn(pvec, __activate_page, NULL); -- put_cpu_var(activate_page_pvecs); -+ put_locked_var(swapvec_lock, activate_page_pvecs); - } - } - -@@ -567,7 +572,7 @@ - - static void __lru_cache_activate_page(struct page *page) - { -- struct pagevec *pvec = &get_cpu_var(lru_add_pvec); -+ struct pagevec *pvec = &get_locked_var(swapvec_lock, lru_add_pvec); - int i; - - /* -@@ -589,7 +594,7 @@ - } - } - -- put_cpu_var(lru_add_pvec); -+ put_locked_var(swapvec_lock, lru_add_pvec); - } - - /* -@@ -628,13 +633,13 @@ - - static void __lru_cache_add(struct page *page) - { -- struct pagevec *pvec = &get_cpu_var(lru_add_pvec); -+ struct pagevec *pvec = &get_locked_var(swapvec_lock, lru_add_pvec); - - page_cache_get(page); - if (!pagevec_space(pvec)) - __pagevec_lru_add(pvec); - pagevec_add(pvec, page); -- put_cpu_var(lru_add_pvec); -+ put_locked_var(swapvec_lock, lru_add_pvec); - } - - /** -@@ -814,9 +819,9 @@ - unsigned long flags; - - /* No harm done if a racing interrupt already did this */ -- local_irq_save(flags); -+ local_lock_irqsave(rotate_lock, flags); - pagevec_move_tail(pvec); -- local_irq_restore(flags); -+ local_unlock_irqrestore(rotate_lock, flags); - } - - pvec = &per_cpu(lru_deactivate_file_pvecs, cpu); -@@ -844,18 +849,19 @@ - return; - - if (likely(get_page_unless_zero(page))) { -- struct pagevec *pvec = &get_cpu_var(lru_deactivate_file_pvecs); -+ struct pagevec *pvec = &get_locked_var(swapvec_lock, -+ lru_deactivate_file_pvecs); - - if (!pagevec_add(pvec, page)) - pagevec_lru_move_fn(pvec, lru_deactivate_file_fn, NULL); -- put_cpu_var(lru_deactivate_file_pvecs); -+ put_locked_var(swapvec_lock, lru_deactivate_file_pvecs); - } - } - - void lru_add_drain(void) - { -- lru_add_drain_cpu(get_cpu()); -- put_cpu(); -+ lru_add_drain_cpu(local_lock_cpu(swapvec_lock)); -+ local_unlock_cpu(swapvec_lock); - } - - static void lru_add_drain_per_cpu(struct work_struct *dummy) -diff -Nur linux-4.1.10.orig/mm/truncate.c linux-4.1.10/mm/truncate.c ---- linux-4.1.10.orig/mm/truncate.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/mm/truncate.c 2015-10-07 18:00:08.000000000 +0200 -@@ -56,8 +56,11 @@ - * protected by mapping->tree_lock. - */ - if (!workingset_node_shadows(node) && -- !list_empty(&node->private_list)) -- list_lru_del(&workingset_shadow_nodes, &node->private_list); -+ !list_empty(&node->private_list)) { -+ local_lock(workingset_shadow_lock); -+ list_lru_del(&__workingset_shadow_nodes, &node->private_list); -+ local_unlock(workingset_shadow_lock); -+ } - __radix_tree_delete_node(&mapping->page_tree, node); - unlock: - spin_unlock_irq(&mapping->tree_lock); -diff -Nur linux-4.1.10.orig/mm/vmalloc.c linux-4.1.10/mm/vmalloc.c ---- linux-4.1.10.orig/mm/vmalloc.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/mm/vmalloc.c 2015-10-07 18:00:08.000000000 +0200 -@@ -819,7 +819,7 @@ - struct vmap_block *vb; - struct vmap_area *va; - unsigned long vb_idx; -- int node, err; -+ int node, err, cpu; - void *vaddr; - - node = numa_node_id(); -@@ -862,11 +862,12 @@ - BUG_ON(err); - radix_tree_preload_end(); - -- vbq = &get_cpu_var(vmap_block_queue); -+ cpu = get_cpu_light(); -+ vbq = this_cpu_ptr(&vmap_block_queue); - spin_lock(&vbq->lock); - list_add_tail_rcu(&vb->free_list, &vbq->free); - spin_unlock(&vbq->lock); -- put_cpu_var(vmap_block_queue); -+ put_cpu_light(); - - return vaddr; - } -@@ -935,6 +936,7 @@ - struct vmap_block *vb; - void *vaddr = NULL; - unsigned int order; -+ int cpu; - - BUG_ON(size & ~PAGE_MASK); - BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC); -@@ -949,7 +951,8 @@ - order = get_order(size); - - rcu_read_lock(); -- vbq = &get_cpu_var(vmap_block_queue); -+ cpu = get_cpu_light(); -+ vbq = this_cpu_ptr(&vmap_block_queue); - list_for_each_entry_rcu(vb, &vbq->free, free_list) { - unsigned long pages_off; - -@@ -972,7 +975,7 @@ - break; - } - -- put_cpu_var(vmap_block_queue); -+ put_cpu_light(); - rcu_read_unlock(); - - /* Allocate new block if nothing was found */ -diff -Nur linux-4.1.10.orig/mm/vmstat.c linux-4.1.10/mm/vmstat.c ---- linux-4.1.10.orig/mm/vmstat.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/mm/vmstat.c 2015-10-07 18:00:08.000000000 +0200 -@@ -226,6 +226,7 @@ - long x; - long t; - -+ preempt_disable_rt(); - x = delta + __this_cpu_read(*p); - - t = __this_cpu_read(pcp->stat_threshold); -@@ -235,6 +236,7 @@ - x = 0; - } - __this_cpu_write(*p, x); -+ preempt_enable_rt(); - } - EXPORT_SYMBOL(__mod_zone_page_state); - -@@ -267,6 +269,7 @@ - s8 __percpu *p = pcp->vm_stat_diff + item; - s8 v, t; - -+ preempt_disable_rt(); - v = __this_cpu_inc_return(*p); - t = __this_cpu_read(pcp->stat_threshold); - if (unlikely(v > t)) { -@@ -275,6 +278,7 @@ - zone_page_state_add(v + overstep, zone, item); - __this_cpu_write(*p, -overstep); - } -+ preempt_enable_rt(); - } - - void __inc_zone_page_state(struct page *page, enum zone_stat_item item) -@@ -289,6 +293,7 @@ - s8 __percpu *p = pcp->vm_stat_diff + item; - s8 v, t; - -+ preempt_disable_rt(); - v = __this_cpu_dec_return(*p); - t = __this_cpu_read(pcp->stat_threshold); - if (unlikely(v < - t)) { -@@ -297,6 +302,7 @@ - zone_page_state_add(v - overstep, zone, item); - __this_cpu_write(*p, overstep); - } -+ preempt_enable_rt(); - } - - void __dec_zone_page_state(struct page *page, enum zone_stat_item item) -diff -Nur linux-4.1.10.orig/mm/workingset.c linux-4.1.10/mm/workingset.c ---- linux-4.1.10.orig/mm/workingset.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/mm/workingset.c 2015-10-07 18:00:08.000000000 +0200 -@@ -264,7 +264,8 @@ - * point where they would still be useful. - */ - --struct list_lru workingset_shadow_nodes; -+struct list_lru __workingset_shadow_nodes; -+DEFINE_LOCAL_IRQ_LOCK(workingset_shadow_lock); - - static unsigned long count_shadow_nodes(struct shrinker *shrinker, - struct shrink_control *sc) -@@ -274,9 +275,9 @@ - unsigned long pages; - - /* list_lru lock nests inside IRQ-safe mapping->tree_lock */ -- local_irq_disable(); -- shadow_nodes = list_lru_shrink_count(&workingset_shadow_nodes, sc); -- local_irq_enable(); -+ local_lock_irq(workingset_shadow_lock); -+ shadow_nodes = list_lru_shrink_count(&__workingset_shadow_nodes, sc); -+ local_unlock_irq(workingset_shadow_lock); - - pages = node_present_pages(sc->nid); - /* -@@ -363,9 +364,9 @@ - spin_unlock(&mapping->tree_lock); - ret = LRU_REMOVED_RETRY; - out: -- local_irq_enable(); -+ local_unlock_irq(workingset_shadow_lock); - cond_resched(); -- local_irq_disable(); -+ local_lock_irq(workingset_shadow_lock); - spin_lock(lru_lock); - return ret; - } -@@ -376,10 +377,10 @@ - unsigned long ret; - - /* list_lru lock nests inside IRQ-safe mapping->tree_lock */ -- local_irq_disable(); -- ret = list_lru_shrink_walk(&workingset_shadow_nodes, sc, -+ local_lock_irq(workingset_shadow_lock); -+ ret = list_lru_shrink_walk(&__workingset_shadow_nodes, sc, - shadow_lru_isolate, NULL); -- local_irq_enable(); -+ local_unlock_irq(workingset_shadow_lock); - return ret; - } - -@@ -400,7 +401,7 @@ - { - int ret; - -- ret = list_lru_init_key(&workingset_shadow_nodes, &shadow_nodes_key); -+ ret = list_lru_init_key(&__workingset_shadow_nodes, &shadow_nodes_key); - if (ret) - goto err; - ret = register_shrinker(&workingset_shadow_shrinker); -@@ -408,7 +409,7 @@ - goto err_list_lru; - return 0; - err_list_lru: -- list_lru_destroy(&workingset_shadow_nodes); -+ list_lru_destroy(&__workingset_shadow_nodes); - err: - return ret; - } -diff -Nur linux-4.1.10.orig/net/core/dev.c linux-4.1.10/net/core/dev.c ---- linux-4.1.10.orig/net/core/dev.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/net/core/dev.c 2015-10-07 18:00:08.000000000 +0200 -@@ -184,6 +184,7 @@ - static DEFINE_HASHTABLE(napi_hash, 8); - - static seqcount_t devnet_rename_seq; -+static DEFINE_MUTEX(devnet_rename_mutex); - - static inline void dev_base_seq_inc(struct net *net) - { -@@ -205,14 +206,14 @@ - static inline void rps_lock(struct softnet_data *sd) - { - #ifdef CONFIG_RPS -- spin_lock(&sd->input_pkt_queue.lock); -+ raw_spin_lock(&sd->input_pkt_queue.raw_lock); - #endif - } - - static inline void rps_unlock(struct softnet_data *sd) - { - #ifdef CONFIG_RPS -- spin_unlock(&sd->input_pkt_queue.lock); -+ raw_spin_unlock(&sd->input_pkt_queue.raw_lock); - #endif - } - -@@ -852,7 +853,8 @@ - strcpy(name, dev->name); - rcu_read_unlock(); - if (read_seqcount_retry(&devnet_rename_seq, seq)) { -- cond_resched(); -+ mutex_lock(&devnet_rename_mutex); -+ mutex_unlock(&devnet_rename_mutex); - goto retry; - } - -@@ -1121,20 +1123,17 @@ - if (dev->flags & IFF_UP) - return -EBUSY; - -- write_seqcount_begin(&devnet_rename_seq); -+ mutex_lock(&devnet_rename_mutex); -+ __raw_write_seqcount_begin(&devnet_rename_seq); - -- if (strncmp(newname, dev->name, IFNAMSIZ) == 0) { -- write_seqcount_end(&devnet_rename_seq); -- return 0; -- } -+ if (strncmp(newname, dev->name, IFNAMSIZ) == 0) -+ goto outunlock; - - memcpy(oldname, dev->name, IFNAMSIZ); - - err = dev_get_valid_name(net, dev, newname); -- if (err < 0) { -- write_seqcount_end(&devnet_rename_seq); -- return err; -- } -+ if (err < 0) -+ goto outunlock; - - if (oldname[0] && !strchr(oldname, '%')) - netdev_info(dev, "renamed from %s\n", oldname); -@@ -1147,11 +1146,12 @@ - if (ret) { - memcpy(dev->name, oldname, IFNAMSIZ); - dev->name_assign_type = old_assign_type; -- write_seqcount_end(&devnet_rename_seq); -- return ret; -+ err = ret; -+ goto outunlock; - } - -- write_seqcount_end(&devnet_rename_seq); -+ __raw_write_seqcount_end(&devnet_rename_seq); -+ mutex_unlock(&devnet_rename_mutex); - - netdev_adjacent_rename_links(dev, oldname); - -@@ -1172,7 +1172,8 @@ - /* err >= 0 after dev_alloc_name() or stores the first errno */ - if (err >= 0) { - err = ret; -- write_seqcount_begin(&devnet_rename_seq); -+ mutex_lock(&devnet_rename_mutex); -+ __raw_write_seqcount_begin(&devnet_rename_seq); - memcpy(dev->name, oldname, IFNAMSIZ); - memcpy(oldname, newname, IFNAMSIZ); - dev->name_assign_type = old_assign_type; -@@ -1185,6 +1186,11 @@ - } - - return err; -+ -+outunlock: -+ __raw_write_seqcount_end(&devnet_rename_seq); -+ mutex_unlock(&devnet_rename_mutex); -+ return err; - } - - /** -@@ -2214,6 +2220,7 @@ - sd->output_queue_tailp = &q->next_sched; - raise_softirq_irqoff(NET_TX_SOFTIRQ); - local_irq_restore(flags); -+ preempt_check_resched_rt(); - } - - void __netif_schedule(struct Qdisc *q) -@@ -2295,6 +2302,7 @@ - __this_cpu_write(softnet_data.completion_queue, skb); - raise_softirq_irqoff(NET_TX_SOFTIRQ); - local_irq_restore(flags); -+ preempt_check_resched_rt(); - } - EXPORT_SYMBOL(__dev_kfree_skb_irq); - -@@ -3365,6 +3373,7 @@ - rps_unlock(sd); - - local_irq_restore(flags); -+ preempt_check_resched_rt(); - - atomic_long_inc(&skb->dev->rx_dropped); - kfree_skb(skb); -@@ -3383,7 +3392,7 @@ - struct rps_dev_flow voidflow, *rflow = &voidflow; - int cpu; - -- preempt_disable(); -+ migrate_disable(); - rcu_read_lock(); - - cpu = get_rps_cpu(skb->dev, skb, &rflow); -@@ -3393,13 +3402,13 @@ - ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail); - - rcu_read_unlock(); -- preempt_enable(); -+ migrate_enable(); - } else - #endif - { - unsigned int qtail; -- ret = enqueue_to_backlog(skb, get_cpu(), &qtail); -- put_cpu(); -+ ret = enqueue_to_backlog(skb, get_cpu_light(), &qtail); -+ put_cpu_light(); - } - return ret; - } -@@ -3433,16 +3442,44 @@ - - trace_netif_rx_ni_entry(skb); - -- preempt_disable(); -+ local_bh_disable(); - err = netif_rx_internal(skb); -- if (local_softirq_pending()) -- do_softirq(); -- preempt_enable(); -+ local_bh_enable(); - - return err; - } - EXPORT_SYMBOL(netif_rx_ni); - -+#ifdef CONFIG_PREEMPT_RT_FULL -+/* -+ * RT runs ksoftirqd as a real time thread and the root_lock is a -+ * "sleeping spinlock". If the trylock fails then we can go into an -+ * infinite loop when ksoftirqd preempted the task which actually -+ * holds the lock, because we requeue q and raise NET_TX softirq -+ * causing ksoftirqd to loop forever. -+ * -+ * It's safe to use spin_lock on RT here as softirqs run in thread -+ * context and cannot deadlock against the thread which is holding -+ * root_lock. -+ * -+ * On !RT the trylock might fail, but there we bail out from the -+ * softirq loop after 10 attempts which we can't do on RT. And the -+ * task holding root_lock cannot be preempted, so the only downside of -+ * that trylock is that we need 10 loops to decide that we should have -+ * given up in the first one :) -+ */ -+static inline int take_root_lock(spinlock_t *lock) -+{ -+ spin_lock(lock); -+ return 1; -+} -+#else -+static inline int take_root_lock(spinlock_t *lock) -+{ -+ return spin_trylock(lock); -+} -+#endif -+ - static void net_tx_action(struct softirq_action *h) - { - struct softnet_data *sd = this_cpu_ptr(&softnet_data); -@@ -3484,7 +3521,7 @@ - head = head->next_sched; - - root_lock = qdisc_lock(q); -- if (spin_trylock(root_lock)) { -+ if (take_root_lock(root_lock)) { - smp_mb__before_atomic(); - clear_bit(__QDISC_STATE_SCHED, - &q->state); -@@ -3881,7 +3918,7 @@ - skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) { - if (skb->dev == dev) { - __skb_unlink(skb, &sd->input_pkt_queue); -- kfree_skb(skb); -+ __skb_queue_tail(&sd->tofree_queue, skb); - input_queue_head_incr(sd); - } - } -@@ -3890,10 +3927,13 @@ - skb_queue_walk_safe(&sd->process_queue, skb, tmp) { - if (skb->dev == dev) { - __skb_unlink(skb, &sd->process_queue); -- kfree_skb(skb); -+ __skb_queue_tail(&sd->tofree_queue, skb); - input_queue_head_incr(sd); - } - } -+ -+ if (!skb_queue_empty(&sd->tofree_queue)) -+ raise_softirq_irqoff(NET_RX_SOFTIRQ); - } - - static int napi_gro_complete(struct sk_buff *skb) -@@ -4344,6 +4384,7 @@ - sd->rps_ipi_list = NULL; - - local_irq_enable(); -+ preempt_check_resched_rt(); - - /* Send pending IPI's to kick RPS processing on remote cpus. */ - while (remsd) { -@@ -4357,6 +4398,7 @@ - } else - #endif - local_irq_enable(); -+ preempt_check_resched_rt(); - } - - static bool sd_has_rps_ipi_waiting(struct softnet_data *sd) -@@ -4438,6 +4480,7 @@ - local_irq_save(flags); - ____napi_schedule(this_cpu_ptr(&softnet_data), n); - local_irq_restore(flags); -+ preempt_check_resched_rt(); - } - EXPORT_SYMBOL(__napi_schedule); - -@@ -7167,6 +7210,7 @@ - - raise_softirq_irqoff(NET_TX_SOFTIRQ); - local_irq_enable(); -+ preempt_check_resched_rt(); - - /* Process offline CPU's input_pkt_queue */ - while ((skb = __skb_dequeue(&oldsd->process_queue))) { -@@ -7177,6 +7221,9 @@ - netif_rx_ni(skb); - input_queue_head_incr(oldsd); - } -+ while ((skb = __skb_dequeue(&oldsd->tofree_queue))) { -+ kfree_skb(skb); -+ } - - return NOTIFY_OK; - } -@@ -7478,8 +7525,9 @@ - for_each_possible_cpu(i) { - struct softnet_data *sd = &per_cpu(softnet_data, i); - -- skb_queue_head_init(&sd->input_pkt_queue); -- skb_queue_head_init(&sd->process_queue); -+ skb_queue_head_init_raw(&sd->input_pkt_queue); -+ skb_queue_head_init_raw(&sd->process_queue); -+ skb_queue_head_init_raw(&sd->tofree_queue); - INIT_LIST_HEAD(&sd->poll_list); - sd->output_queue_tailp = &sd->output_queue; - #ifdef CONFIG_RPS -diff -Nur linux-4.1.10.orig/net/core/dev.c.orig linux-4.1.10/net/core/dev.c.orig ---- linux-4.1.10.orig/net/core/dev.c.orig 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.10/net/core/dev.c.orig 2015-10-03 13:49:38.000000000 +0200 -@@ -0,0 +1,7522 @@ -+/* -+ * NET3 Protocol independent device support routines. -+ * -+ * This program is free software; you can redistribute it and/or -+ * modify it under the terms of the GNU General Public License -+ * as published by the Free Software Foundation; either version -+ * 2 of the License, or (at your option) any later version. -+ * -+ * Derived from the non IP parts of dev.c 1.0.19 -+ * Authors: Ross Biro -+ * Fred N. van Kempen, -+ * Mark Evans, -+ * -+ * Additional Authors: -+ * Florian la Roche -+ * Alan Cox -+ * David Hinds -+ * Alexey Kuznetsov -+ * Adam Sulmicki -+ * Pekka Riikonen -+ * -+ * Changes: -+ * D.J. Barrow : Fixed bug where dev->refcnt gets set -+ * to 2 if register_netdev gets called -+ * before net_dev_init & also removed a -+ * few lines of code in the process. -+ * Alan Cox : device private ioctl copies fields back. -+ * Alan Cox : Transmit queue code does relevant -+ * stunts to keep the queue safe. -+ * Alan Cox : Fixed double lock. -+ * Alan Cox : Fixed promisc NULL pointer trap -+ * ???????? : Support the full private ioctl range -+ * Alan Cox : Moved ioctl permission check into -+ * drivers -+ * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI -+ * Alan Cox : 100 backlog just doesn't cut it when -+ * you start doing multicast video 8) -+ * Alan Cox : Rewrote net_bh and list manager. -+ * Alan Cox : Fix ETH_P_ALL echoback lengths. -+ * Alan Cox : Took out transmit every packet pass -+ * Saved a few bytes in the ioctl handler -+ * Alan Cox : Network driver sets packet type before -+ * calling netif_rx. Saves a function -+ * call a packet. -+ * Alan Cox : Hashed net_bh() -+ * Richard Kooijman: Timestamp fixes. -+ * Alan Cox : Wrong field in SIOCGIFDSTADDR -+ * Alan Cox : Device lock protection. -+ * Alan Cox : Fixed nasty side effect of device close -+ * changes. -+ * Rudi Cilibrasi : Pass the right thing to -+ * set_mac_address() -+ * Dave Miller : 32bit quantity for the device lock to -+ * make it work out on a Sparc. -+ * Bjorn Ekwall : Added KERNELD hack. -+ * Alan Cox : Cleaned up the backlog initialise. -+ * Craig Metz : SIOCGIFCONF fix if space for under -+ * 1 device. -+ * Thomas Bogendoerfer : Return ENODEV for dev_open, if there -+ * is no device open function. -+ * Andi Kleen : Fix error reporting for SIOCGIFCONF -+ * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF -+ * Cyrus Durgin : Cleaned for KMOD -+ * Adam Sulmicki : Bug Fix : Network Device Unload -+ * A network device unload needs to purge -+ * the backlog queue. -+ * Paul Rusty Russell : SIOCSIFNAME -+ * Pekka Riikonen : Netdev boot-time settings code -+ * Andrew Morton : Make unregister_netdevice wait -+ * indefinitely on dev->refcnt -+ * J Hadi Salim : - Backlog queue sampling -+ * - netif_rx() feedback -+ */ -+ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+ -+#include "net-sysfs.h" -+ -+/* Instead of increasing this, you should create a hash table. */ -+#define MAX_GRO_SKBS 8 -+ -+/* This should be increased if a protocol with a bigger head is added. */ -+#define GRO_MAX_HEAD (MAX_HEADER + 128) -+ -+static DEFINE_SPINLOCK(ptype_lock); -+static DEFINE_SPINLOCK(offload_lock); -+struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly; -+struct list_head ptype_all __read_mostly; /* Taps */ -+static struct list_head offload_base __read_mostly; -+ -+static int netif_rx_internal(struct sk_buff *skb); -+static int call_netdevice_notifiers_info(unsigned long val, -+ struct net_device *dev, -+ struct netdev_notifier_info *info); -+ -+/* -+ * The @dev_base_head list is protected by @dev_base_lock and the rtnl -+ * semaphore. -+ * -+ * Pure readers hold dev_base_lock for reading, or rcu_read_lock() -+ * -+ * Writers must hold the rtnl semaphore while they loop through the -+ * dev_base_head list, and hold dev_base_lock for writing when they do the -+ * actual updates. This allows pure readers to access the list even -+ * while a writer is preparing to update it. -+ * -+ * To put it another way, dev_base_lock is held for writing only to -+ * protect against pure readers; the rtnl semaphore provides the -+ * protection against other writers. -+ * -+ * See, for example usages, register_netdevice() and -+ * unregister_netdevice(), which must be called with the rtnl -+ * semaphore held. -+ */ -+DEFINE_RWLOCK(dev_base_lock); -+EXPORT_SYMBOL(dev_base_lock); -+ -+/* protects napi_hash addition/deletion and napi_gen_id */ -+static DEFINE_SPINLOCK(napi_hash_lock); -+ -+static unsigned int napi_gen_id; -+static DEFINE_HASHTABLE(napi_hash, 8); -+ -+static seqcount_t devnet_rename_seq; -+ -+static inline void dev_base_seq_inc(struct net *net) -+{ -+ while (++net->dev_base_seq == 0); -+} -+ -+static inline struct hlist_head *dev_name_hash(struct net *net, const char *name) -+{ -+ unsigned int hash = full_name_hash(name, strnlen(name, IFNAMSIZ)); -+ -+ return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)]; -+} -+ -+static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex) -+{ -+ return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)]; -+} -+ -+static inline void rps_lock(struct softnet_data *sd) -+{ -+#ifdef CONFIG_RPS -+ spin_lock(&sd->input_pkt_queue.lock); -+#endif -+} -+ -+static inline void rps_unlock(struct softnet_data *sd) -+{ -+#ifdef CONFIG_RPS -+ spin_unlock(&sd->input_pkt_queue.lock); -+#endif -+} -+ -+/* Device list insertion */ -+static void list_netdevice(struct net_device *dev) -+{ -+ struct net *net = dev_net(dev); -+ -+ ASSERT_RTNL(); -+ -+ write_lock_bh(&dev_base_lock); -+ list_add_tail_rcu(&dev->dev_list, &net->dev_base_head); -+ hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name)); -+ hlist_add_head_rcu(&dev->index_hlist, -+ dev_index_hash(net, dev->ifindex)); -+ write_unlock_bh(&dev_base_lock); -+ -+ dev_base_seq_inc(net); -+} -+ -+/* Device list removal -+ * caller must respect a RCU grace period before freeing/reusing dev -+ */ -+static void unlist_netdevice(struct net_device *dev) -+{ -+ ASSERT_RTNL(); -+ -+ /* Unlink dev from the device chain */ -+ write_lock_bh(&dev_base_lock); -+ list_del_rcu(&dev->dev_list); -+ hlist_del_rcu(&dev->name_hlist); -+ hlist_del_rcu(&dev->index_hlist); -+ write_unlock_bh(&dev_base_lock); -+ -+ dev_base_seq_inc(dev_net(dev)); -+} -+ -+/* -+ * Our notifier list -+ */ -+ -+static RAW_NOTIFIER_HEAD(netdev_chain); -+ -+/* -+ * Device drivers call our routines to queue packets here. We empty the -+ * queue in the local softnet handler. -+ */ -+ -+DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data); -+EXPORT_PER_CPU_SYMBOL(softnet_data); -+ -+#ifdef CONFIG_LOCKDEP -+/* -+ * register_netdevice() inits txq->_xmit_lock and sets lockdep class -+ * according to dev->type -+ */ -+static const unsigned short netdev_lock_type[] = -+ {ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25, -+ ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET, -+ ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM, -+ ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP, -+ ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD, -+ ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25, -+ ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP, -+ ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD, -+ ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI, -+ ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE, -+ ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET, -+ ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL, -+ ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM, -+ ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE, -+ ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE}; -+ -+static const char *const netdev_lock_name[] = -+ {"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25", -+ "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET", -+ "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM", -+ "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP", -+ "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD", -+ "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25", -+ "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP", -+ "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD", -+ "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI", -+ "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE", -+ "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET", -+ "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL", -+ "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM", -+ "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE", -+ "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"}; -+ -+static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)]; -+static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)]; -+ -+static inline unsigned short netdev_lock_pos(unsigned short dev_type) -+{ -+ int i; -+ -+ for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++) -+ if (netdev_lock_type[i] == dev_type) -+ return i; -+ /* the last key is used by default */ -+ return ARRAY_SIZE(netdev_lock_type) - 1; -+} -+ -+static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock, -+ unsigned short dev_type) -+{ -+ int i; -+ -+ i = netdev_lock_pos(dev_type); -+ lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i], -+ netdev_lock_name[i]); -+} -+ -+static inline void netdev_set_addr_lockdep_class(struct net_device *dev) -+{ -+ int i; -+ -+ i = netdev_lock_pos(dev->type); -+ lockdep_set_class_and_name(&dev->addr_list_lock, -+ &netdev_addr_lock_key[i], -+ netdev_lock_name[i]); -+} -+#else -+static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock, -+ unsigned short dev_type) -+{ -+} -+static inline void netdev_set_addr_lockdep_class(struct net_device *dev) -+{ -+} -+#endif -+ -+/******************************************************************************* -+ -+ Protocol management and registration routines -+ -+*******************************************************************************/ -+ -+/* -+ * Add a protocol ID to the list. Now that the input handler is -+ * smarter we can dispense with all the messy stuff that used to be -+ * here. -+ * -+ * BEWARE!!! Protocol handlers, mangling input packets, -+ * MUST BE last in hash buckets and checking protocol handlers -+ * MUST start from promiscuous ptype_all chain in net_bh. -+ * It is true now, do not change it. -+ * Explanation follows: if protocol handler, mangling packet, will -+ * be the first on list, it is not able to sense, that packet -+ * is cloned and should be copied-on-write, so that it will -+ * change it and subsequent readers will get broken packet. -+ * --ANK (980803) -+ */ -+ -+static inline struct list_head *ptype_head(const struct packet_type *pt) -+{ -+ if (pt->type == htons(ETH_P_ALL)) -+ return pt->dev ? &pt->dev->ptype_all : &ptype_all; -+ else -+ return pt->dev ? &pt->dev->ptype_specific : -+ &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK]; -+} -+ -+/** -+ * dev_add_pack - add packet handler -+ * @pt: packet type declaration -+ * -+ * Add a protocol handler to the networking stack. The passed &packet_type -+ * is linked into kernel lists and may not be freed until it has been -+ * removed from the kernel lists. -+ * -+ * This call does not sleep therefore it can not -+ * guarantee all CPU's that are in middle of receiving packets -+ * will see the new packet type (until the next received packet). -+ */ -+ -+void dev_add_pack(struct packet_type *pt) -+{ -+ struct list_head *head = ptype_head(pt); -+ -+ spin_lock(&ptype_lock); -+ list_add_rcu(&pt->list, head); -+ spin_unlock(&ptype_lock); -+} -+EXPORT_SYMBOL(dev_add_pack); -+ -+/** -+ * __dev_remove_pack - remove packet handler -+ * @pt: packet type declaration -+ * -+ * Remove a protocol handler that was previously added to the kernel -+ * protocol handlers by dev_add_pack(). The passed &packet_type is removed -+ * from the kernel lists and can be freed or reused once this function -+ * returns. -+ * -+ * The packet type might still be in use by receivers -+ * and must not be freed until after all the CPU's have gone -+ * through a quiescent state. -+ */ -+void __dev_remove_pack(struct packet_type *pt) -+{ -+ struct list_head *head = ptype_head(pt); -+ struct packet_type *pt1; -+ -+ spin_lock(&ptype_lock); -+ -+ list_for_each_entry(pt1, head, list) { -+ if (pt == pt1) { -+ list_del_rcu(&pt->list); -+ goto out; -+ } -+ } -+ -+ pr_warn("dev_remove_pack: %p not found\n", pt); -+out: -+ spin_unlock(&ptype_lock); -+} -+EXPORT_SYMBOL(__dev_remove_pack); -+ -+/** -+ * dev_remove_pack - remove packet handler -+ * @pt: packet type declaration -+ * -+ * Remove a protocol handler that was previously added to the kernel -+ * protocol handlers by dev_add_pack(). The passed &packet_type is removed -+ * from the kernel lists and can be freed or reused once this function -+ * returns. -+ * -+ * This call sleeps to guarantee that no CPU is looking at the packet -+ * type after return. -+ */ -+void dev_remove_pack(struct packet_type *pt) -+{ -+ __dev_remove_pack(pt); -+ -+ synchronize_net(); -+} -+EXPORT_SYMBOL(dev_remove_pack); -+ -+ -+/** -+ * dev_add_offload - register offload handlers -+ * @po: protocol offload declaration -+ * -+ * Add protocol offload handlers to the networking stack. The passed -+ * &proto_offload is linked into kernel lists and may not be freed until -+ * it has been removed from the kernel lists. -+ * -+ * This call does not sleep therefore it can not -+ * guarantee all CPU's that are in middle of receiving packets -+ * will see the new offload handlers (until the next received packet). -+ */ -+void dev_add_offload(struct packet_offload *po) -+{ -+ struct list_head *head = &offload_base; -+ -+ spin_lock(&offload_lock); -+ list_add_rcu(&po->list, head); -+ spin_unlock(&offload_lock); -+} -+EXPORT_SYMBOL(dev_add_offload); -+ -+/** -+ * __dev_remove_offload - remove offload handler -+ * @po: packet offload declaration -+ * -+ * Remove a protocol offload handler that was previously added to the -+ * kernel offload handlers by dev_add_offload(). The passed &offload_type -+ * is removed from the kernel lists and can be freed or reused once this -+ * function returns. -+ * -+ * The packet type might still be in use by receivers -+ * and must not be freed until after all the CPU's have gone -+ * through a quiescent state. -+ */ -+static void __dev_remove_offload(struct packet_offload *po) -+{ -+ struct list_head *head = &offload_base; -+ struct packet_offload *po1; -+ -+ spin_lock(&offload_lock); -+ -+ list_for_each_entry(po1, head, list) { -+ if (po == po1) { -+ list_del_rcu(&po->list); -+ goto out; -+ } -+ } -+ -+ pr_warn("dev_remove_offload: %p not found\n", po); -+out: -+ spin_unlock(&offload_lock); -+} -+ -+/** -+ * dev_remove_offload - remove packet offload handler -+ * @po: packet offload declaration -+ * -+ * Remove a packet offload handler that was previously added to the kernel -+ * offload handlers by dev_add_offload(). The passed &offload_type is -+ * removed from the kernel lists and can be freed or reused once this -+ * function returns. -+ * -+ * This call sleeps to guarantee that no CPU is looking at the packet -+ * type after return. -+ */ -+void dev_remove_offload(struct packet_offload *po) -+{ -+ __dev_remove_offload(po); -+ -+ synchronize_net(); -+} -+EXPORT_SYMBOL(dev_remove_offload); -+ -+/****************************************************************************** -+ -+ Device Boot-time Settings Routines -+ -+*******************************************************************************/ -+ -+/* Boot time configuration table */ -+static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX]; -+ -+/** -+ * netdev_boot_setup_add - add new setup entry -+ * @name: name of the device -+ * @map: configured settings for the device -+ * -+ * Adds new setup entry to the dev_boot_setup list. The function -+ * returns 0 on error and 1 on success. This is a generic routine to -+ * all netdevices. -+ */ -+static int netdev_boot_setup_add(char *name, struct ifmap *map) -+{ -+ struct netdev_boot_setup *s; -+ int i; -+ -+ s = dev_boot_setup; -+ for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) { -+ if (s[i].name[0] == '\0' || s[i].name[0] == ' ') { -+ memset(s[i].name, 0, sizeof(s[i].name)); -+ strlcpy(s[i].name, name, IFNAMSIZ); -+ memcpy(&s[i].map, map, sizeof(s[i].map)); -+ break; -+ } -+ } -+ -+ return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1; -+} -+ -+/** -+ * netdev_boot_setup_check - check boot time settings -+ * @dev: the netdevice -+ * -+ * Check boot time settings for the device. -+ * The found settings are set for the device to be used -+ * later in the device probing. -+ * Returns 0 if no settings found, 1 if they are. -+ */ -+int netdev_boot_setup_check(struct net_device *dev) -+{ -+ struct netdev_boot_setup *s = dev_boot_setup; -+ int i; -+ -+ for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) { -+ if (s[i].name[0] != '\0' && s[i].name[0] != ' ' && -+ !strcmp(dev->name, s[i].name)) { -+ dev->irq = s[i].map.irq; -+ dev->base_addr = s[i].map.base_addr; -+ dev->mem_start = s[i].map.mem_start; -+ dev->mem_end = s[i].map.mem_end; -+ return 1; -+ } -+ } -+ return 0; -+} -+EXPORT_SYMBOL(netdev_boot_setup_check); -+ -+ -+/** -+ * netdev_boot_base - get address from boot time settings -+ * @prefix: prefix for network device -+ * @unit: id for network device -+ * -+ * Check boot time settings for the base address of device. -+ * The found settings are set for the device to be used -+ * later in the device probing. -+ * Returns 0 if no settings found. -+ */ -+unsigned long netdev_boot_base(const char *prefix, int unit) -+{ -+ const struct netdev_boot_setup *s = dev_boot_setup; -+ char name[IFNAMSIZ]; -+ int i; -+ -+ sprintf(name, "%s%d", prefix, unit); -+ -+ /* -+ * If device already registered then return base of 1 -+ * to indicate not to probe for this interface -+ */ -+ if (__dev_get_by_name(&init_net, name)) -+ return 1; -+ -+ for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) -+ if (!strcmp(name, s[i].name)) -+ return s[i].map.base_addr; -+ return 0; -+} -+ -+/* -+ * Saves at boot time configured settings for any netdevice. -+ */ -+int __init netdev_boot_setup(char *str) -+{ -+ int ints[5]; -+ struct ifmap map; -+ -+ str = get_options(str, ARRAY_SIZE(ints), ints); -+ if (!str || !*str) -+ return 0; -+ -+ /* Save settings */ -+ memset(&map, 0, sizeof(map)); -+ if (ints[0] > 0) -+ map.irq = ints[1]; -+ if (ints[0] > 1) -+ map.base_addr = ints[2]; -+ if (ints[0] > 2) -+ map.mem_start = ints[3]; -+ if (ints[0] > 3) -+ map.mem_end = ints[4]; -+ -+ /* Add new entry to the list */ -+ return netdev_boot_setup_add(str, &map); -+} -+ -+__setup("netdev=", netdev_boot_setup); -+ -+/******************************************************************************* -+ -+ Device Interface Subroutines -+ -+*******************************************************************************/ -+ -+/** -+ * dev_get_iflink - get 'iflink' value of a interface -+ * @dev: targeted interface -+ * -+ * Indicates the ifindex the interface is linked to. -+ * Physical interfaces have the same 'ifindex' and 'iflink' values. -+ */ -+ -+int dev_get_iflink(const struct net_device *dev) -+{ -+ if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink) -+ return dev->netdev_ops->ndo_get_iflink(dev); -+ -+ return dev->ifindex; -+} -+EXPORT_SYMBOL(dev_get_iflink); -+ -+/** -+ * __dev_get_by_name - find a device by its name -+ * @net: the applicable net namespace -+ * @name: name to find -+ * -+ * Find an interface by name. Must be called under RTNL semaphore -+ * or @dev_base_lock. If the name is found a pointer to the device -+ * is returned. If the name is not found then %NULL is returned. The -+ * reference counters are not incremented so the caller must be -+ * careful with locks. -+ */ -+ -+struct net_device *__dev_get_by_name(struct net *net, const char *name) -+{ -+ struct net_device *dev; -+ struct hlist_head *head = dev_name_hash(net, name); -+ -+ hlist_for_each_entry(dev, head, name_hlist) -+ if (!strncmp(dev->name, name, IFNAMSIZ)) -+ return dev; -+ -+ return NULL; -+} -+EXPORT_SYMBOL(__dev_get_by_name); -+ -+/** -+ * dev_get_by_name_rcu - find a device by its name -+ * @net: the applicable net namespace -+ * @name: name to find -+ * -+ * Find an interface by name. -+ * If the name is found a pointer to the device is returned. -+ * If the name is not found then %NULL is returned. -+ * The reference counters are not incremented so the caller must be -+ * careful with locks. The caller must hold RCU lock. -+ */ -+ -+struct net_device *dev_get_by_name_rcu(struct net *net, const char *name) -+{ -+ struct net_device *dev; -+ struct hlist_head *head = dev_name_hash(net, name); -+ -+ hlist_for_each_entry_rcu(dev, head, name_hlist) -+ if (!strncmp(dev->name, name, IFNAMSIZ)) -+ return dev; -+ -+ return NULL; -+} -+EXPORT_SYMBOL(dev_get_by_name_rcu); -+ -+/** -+ * dev_get_by_name - find a device by its name -+ * @net: the applicable net namespace -+ * @name: name to find -+ * -+ * Find an interface by name. This can be called from any -+ * context and does its own locking. The returned handle has -+ * the usage count incremented and the caller must use dev_put() to -+ * release it when it is no longer needed. %NULL is returned if no -+ * matching device is found. -+ */ -+ -+struct net_device *dev_get_by_name(struct net *net, const char *name) -+{ -+ struct net_device *dev; -+ -+ rcu_read_lock(); -+ dev = dev_get_by_name_rcu(net, name); -+ if (dev) -+ dev_hold(dev); -+ rcu_read_unlock(); -+ return dev; -+} -+EXPORT_SYMBOL(dev_get_by_name); -+ -+/** -+ * __dev_get_by_index - find a device by its ifindex -+ * @net: the applicable net namespace -+ * @ifindex: index of device -+ * -+ * Search for an interface by index. Returns %NULL if the device -+ * is not found or a pointer to the device. The device has not -+ * had its reference counter increased so the caller must be careful -+ * about locking. The caller must hold either the RTNL semaphore -+ * or @dev_base_lock. -+ */ -+ -+struct net_device *__dev_get_by_index(struct net *net, int ifindex) -+{ -+ struct net_device *dev; -+ struct hlist_head *head = dev_index_hash(net, ifindex); -+ -+ hlist_for_each_entry(dev, head, index_hlist) -+ if (dev->ifindex == ifindex) -+ return dev; -+ -+ return NULL; -+} -+EXPORT_SYMBOL(__dev_get_by_index); -+ -+/** -+ * dev_get_by_index_rcu - find a device by its ifindex -+ * @net: the applicable net namespace -+ * @ifindex: index of device -+ * -+ * Search for an interface by index. Returns %NULL if the device -+ * is not found or a pointer to the device. The device has not -+ * had its reference counter increased so the caller must be careful -+ * about locking. The caller must hold RCU lock. -+ */ -+ -+struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex) -+{ -+ struct net_device *dev; -+ struct hlist_head *head = dev_index_hash(net, ifindex); -+ -+ hlist_for_each_entry_rcu(dev, head, index_hlist) -+ if (dev->ifindex == ifindex) -+ return dev; -+ -+ return NULL; -+} -+EXPORT_SYMBOL(dev_get_by_index_rcu); -+ -+ -+/** -+ * dev_get_by_index - find a device by its ifindex -+ * @net: the applicable net namespace -+ * @ifindex: index of device -+ * -+ * Search for an interface by index. Returns NULL if the device -+ * is not found or a pointer to the device. The device returned has -+ * had a reference added and the pointer is safe until the user calls -+ * dev_put to indicate they have finished with it. -+ */ -+ -+struct net_device *dev_get_by_index(struct net *net, int ifindex) -+{ -+ struct net_device *dev; -+ -+ rcu_read_lock(); -+ dev = dev_get_by_index_rcu(net, ifindex); -+ if (dev) -+ dev_hold(dev); -+ rcu_read_unlock(); -+ return dev; -+} -+EXPORT_SYMBOL(dev_get_by_index); -+ -+/** -+ * netdev_get_name - get a netdevice name, knowing its ifindex. -+ * @net: network namespace -+ * @name: a pointer to the buffer where the name will be stored. -+ * @ifindex: the ifindex of the interface to get the name from. -+ * -+ * The use of raw_seqcount_begin() and cond_resched() before -+ * retrying is required as we want to give the writers a chance -+ * to complete when CONFIG_PREEMPT is not set. -+ */ -+int netdev_get_name(struct net *net, char *name, int ifindex) -+{ -+ struct net_device *dev; -+ unsigned int seq; -+ -+retry: -+ seq = raw_seqcount_begin(&devnet_rename_seq); -+ rcu_read_lock(); -+ dev = dev_get_by_index_rcu(net, ifindex); -+ if (!dev) { -+ rcu_read_unlock(); -+ return -ENODEV; -+ } -+ -+ strcpy(name, dev->name); -+ rcu_read_unlock(); -+ if (read_seqcount_retry(&devnet_rename_seq, seq)) { -+ cond_resched(); -+ goto retry; -+ } -+ -+ return 0; -+} -+ -+/** -+ * dev_getbyhwaddr_rcu - find a device by its hardware address -+ * @net: the applicable net namespace -+ * @type: media type of device -+ * @ha: hardware address -+ * -+ * Search for an interface by MAC address. Returns NULL if the device -+ * is not found or a pointer to the device. -+ * The caller must hold RCU or RTNL. -+ * The returned device has not had its ref count increased -+ * and the caller must therefore be careful about locking -+ * -+ */ -+ -+struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type, -+ const char *ha) -+{ -+ struct net_device *dev; -+ -+ for_each_netdev_rcu(net, dev) -+ if (dev->type == type && -+ !memcmp(dev->dev_addr, ha, dev->addr_len)) -+ return dev; -+ -+ return NULL; -+} -+EXPORT_SYMBOL(dev_getbyhwaddr_rcu); -+ -+struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type) -+{ -+ struct net_device *dev; -+ -+ ASSERT_RTNL(); -+ for_each_netdev(net, dev) -+ if (dev->type == type) -+ return dev; -+ -+ return NULL; -+} -+EXPORT_SYMBOL(__dev_getfirstbyhwtype); -+ -+struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type) -+{ -+ struct net_device *dev, *ret = NULL; -+ -+ rcu_read_lock(); -+ for_each_netdev_rcu(net, dev) -+ if (dev->type == type) { -+ dev_hold(dev); -+ ret = dev; -+ break; -+ } -+ rcu_read_unlock(); -+ return ret; -+} -+EXPORT_SYMBOL(dev_getfirstbyhwtype); -+ -+/** -+ * __dev_get_by_flags - find any device with given flags -+ * @net: the applicable net namespace -+ * @if_flags: IFF_* values -+ * @mask: bitmask of bits in if_flags to check -+ * -+ * Search for any interface with the given flags. Returns NULL if a device -+ * is not found or a pointer to the device. Must be called inside -+ * rtnl_lock(), and result refcount is unchanged. -+ */ -+ -+struct net_device *__dev_get_by_flags(struct net *net, unsigned short if_flags, -+ unsigned short mask) -+{ -+ struct net_device *dev, *ret; -+ -+ ASSERT_RTNL(); -+ -+ ret = NULL; -+ for_each_netdev(net, dev) { -+ if (((dev->flags ^ if_flags) & mask) == 0) { -+ ret = dev; -+ break; -+ } -+ } -+ return ret; -+} -+EXPORT_SYMBOL(__dev_get_by_flags); -+ -+/** -+ * dev_valid_name - check if name is okay for network device -+ * @name: name string -+ * -+ * Network device names need to be valid file names to -+ * to allow sysfs to work. We also disallow any kind of -+ * whitespace. -+ */ -+bool dev_valid_name(const char *name) -+{ -+ if (*name == '\0') -+ return false; -+ if (strlen(name) >= IFNAMSIZ) -+ return false; -+ if (!strcmp(name, ".") || !strcmp(name, "..")) -+ return false; -+ -+ while (*name) { -+ if (*name == '/' || *name == ':' || isspace(*name)) -+ return false; -+ name++; -+ } -+ return true; -+} -+EXPORT_SYMBOL(dev_valid_name); -+ -+/** -+ * __dev_alloc_name - allocate a name for a device -+ * @net: network namespace to allocate the device name in -+ * @name: name format string -+ * @buf: scratch buffer and result name string -+ * -+ * Passed a format string - eg "lt%d" it will try and find a suitable -+ * id. It scans list of devices to build up a free map, then chooses -+ * the first empty slot. The caller must hold the dev_base or rtnl lock -+ * while allocating the name and adding the device in order to avoid -+ * duplicates. -+ * Limited to bits_per_byte * page size devices (ie 32K on most platforms). -+ * Returns the number of the unit assigned or a negative errno code. -+ */ -+ -+static int __dev_alloc_name(struct net *net, const char *name, char *buf) -+{ -+ int i = 0; -+ const char *p; -+ const int max_netdevices = 8*PAGE_SIZE; -+ unsigned long *inuse; -+ struct net_device *d; -+ -+ p = strnchr(name, IFNAMSIZ-1, '%'); -+ if (p) { -+ /* -+ * Verify the string as this thing may have come from -+ * the user. There must be either one "%d" and no other "%" -+ * characters. -+ */ -+ if (p[1] != 'd' || strchr(p + 2, '%')) -+ return -EINVAL; -+ -+ /* Use one page as a bit array of possible slots */ -+ inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC); -+ if (!inuse) -+ return -ENOMEM; -+ -+ for_each_netdev(net, d) { -+ if (!sscanf(d->name, name, &i)) -+ continue; -+ if (i < 0 || i >= max_netdevices) -+ continue; -+ -+ /* avoid cases where sscanf is not exact inverse of printf */ -+ snprintf(buf, IFNAMSIZ, name, i); -+ if (!strncmp(buf, d->name, IFNAMSIZ)) -+ set_bit(i, inuse); -+ } -+ -+ i = find_first_zero_bit(inuse, max_netdevices); -+ free_page((unsigned long) inuse); -+ } -+ -+ if (buf != name) -+ snprintf(buf, IFNAMSIZ, name, i); -+ if (!__dev_get_by_name(net, buf)) -+ return i; -+ -+ /* It is possible to run out of possible slots -+ * when the name is long and there isn't enough space left -+ * for the digits, or if all bits are used. -+ */ -+ return -ENFILE; -+} -+ -+/** -+ * dev_alloc_name - allocate a name for a device -+ * @dev: device -+ * @name: name format string -+ * -+ * Passed a format string - eg "lt%d" it will try and find a suitable -+ * id. It scans list of devices to build up a free map, then chooses -+ * the first empty slot. The caller must hold the dev_base or rtnl lock -+ * while allocating the name and adding the device in order to avoid -+ * duplicates. -+ * Limited to bits_per_byte * page size devices (ie 32K on most platforms). -+ * Returns the number of the unit assigned or a negative errno code. -+ */ -+ -+int dev_alloc_name(struct net_device *dev, const char *name) -+{ -+ char buf[IFNAMSIZ]; -+ struct net *net; -+ int ret; -+ -+ BUG_ON(!dev_net(dev)); -+ net = dev_net(dev); -+ ret = __dev_alloc_name(net, name, buf); -+ if (ret >= 0) -+ strlcpy(dev->name, buf, IFNAMSIZ); -+ return ret; -+} -+EXPORT_SYMBOL(dev_alloc_name); -+ -+static int dev_alloc_name_ns(struct net *net, -+ struct net_device *dev, -+ const char *name) -+{ -+ char buf[IFNAMSIZ]; -+ int ret; -+ -+ ret = __dev_alloc_name(net, name, buf); -+ if (ret >= 0) -+ strlcpy(dev->name, buf, IFNAMSIZ); -+ return ret; -+} -+ -+static int dev_get_valid_name(struct net *net, -+ struct net_device *dev, -+ const char *name) -+{ -+ BUG_ON(!net); -+ -+ if (!dev_valid_name(name)) -+ return -EINVAL; -+ -+ if (strchr(name, '%')) -+ return dev_alloc_name_ns(net, dev, name); -+ else if (__dev_get_by_name(net, name)) -+ return -EEXIST; -+ else if (dev->name != name) -+ strlcpy(dev->name, name, IFNAMSIZ); -+ -+ return 0; -+} -+ -+/** -+ * dev_change_name - change name of a device -+ * @dev: device -+ * @newname: name (or format string) must be at least IFNAMSIZ -+ * -+ * Change name of a device, can pass format strings "eth%d". -+ * for wildcarding. -+ */ -+int dev_change_name(struct net_device *dev, const char *newname) -+{ -+ unsigned char old_assign_type; -+ char oldname[IFNAMSIZ]; -+ int err = 0; -+ int ret; -+ struct net *net; -+ -+ ASSERT_RTNL(); -+ BUG_ON(!dev_net(dev)); -+ -+ net = dev_net(dev); -+ if (dev->flags & IFF_UP) -+ return -EBUSY; -+ -+ write_seqcount_begin(&devnet_rename_seq); -+ -+ if (strncmp(newname, dev->name, IFNAMSIZ) == 0) { -+ write_seqcount_end(&devnet_rename_seq); -+ return 0; -+ } -+ -+ memcpy(oldname, dev->name, IFNAMSIZ); -+ -+ err = dev_get_valid_name(net, dev, newname); -+ if (err < 0) { -+ write_seqcount_end(&devnet_rename_seq); -+ return err; -+ } -+ -+ if (oldname[0] && !strchr(oldname, '%')) -+ netdev_info(dev, "renamed from %s\n", oldname); -+ -+ old_assign_type = dev->name_assign_type; -+ dev->name_assign_type = NET_NAME_RENAMED; -+ -+rollback: -+ ret = device_rename(&dev->dev, dev->name); -+ if (ret) { -+ memcpy(dev->name, oldname, IFNAMSIZ); -+ dev->name_assign_type = old_assign_type; -+ write_seqcount_end(&devnet_rename_seq); -+ return ret; -+ } -+ -+ write_seqcount_end(&devnet_rename_seq); -+ -+ netdev_adjacent_rename_links(dev, oldname); -+ -+ write_lock_bh(&dev_base_lock); -+ hlist_del_rcu(&dev->name_hlist); -+ write_unlock_bh(&dev_base_lock); -+ -+ synchronize_rcu(); -+ -+ write_lock_bh(&dev_base_lock); -+ hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name)); -+ write_unlock_bh(&dev_base_lock); -+ -+ ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev); -+ ret = notifier_to_errno(ret); -+ -+ if (ret) { -+ /* err >= 0 after dev_alloc_name() or stores the first errno */ -+ if (err >= 0) { -+ err = ret; -+ write_seqcount_begin(&devnet_rename_seq); -+ memcpy(dev->name, oldname, IFNAMSIZ); -+ memcpy(oldname, newname, IFNAMSIZ); -+ dev->name_assign_type = old_assign_type; -+ old_assign_type = NET_NAME_RENAMED; -+ goto rollback; -+ } else { -+ pr_err("%s: name change rollback failed: %d\n", -+ dev->name, ret); -+ } -+ } -+ -+ return err; -+} -+ -+/** -+ * dev_set_alias - change ifalias of a device -+ * @dev: device -+ * @alias: name up to IFALIASZ -+ * @len: limit of bytes to copy from info -+ * -+ * Set ifalias for a device, -+ */ -+int dev_set_alias(struct net_device *dev, const char *alias, size_t len) -+{ -+ char *new_ifalias; -+ -+ ASSERT_RTNL(); -+ -+ if (len >= IFALIASZ) -+ return -EINVAL; -+ -+ if (!len) { -+ kfree(dev->ifalias); -+ dev->ifalias = NULL; -+ return 0; -+ } -+ -+ new_ifalias = krealloc(dev->ifalias, len + 1, GFP_KERNEL); -+ if (!new_ifalias) -+ return -ENOMEM; -+ dev->ifalias = new_ifalias; -+ -+ strlcpy(dev->ifalias, alias, len+1); -+ return len; -+} -+ -+ -+/** -+ * netdev_features_change - device changes features -+ * @dev: device to cause notification -+ * -+ * Called to indicate a device has changed features. -+ */ -+void netdev_features_change(struct net_device *dev) -+{ -+ call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev); -+} -+EXPORT_SYMBOL(netdev_features_change); -+ -+/** -+ * netdev_state_change - device changes state -+ * @dev: device to cause notification -+ * -+ * Called to indicate a device has changed state. This function calls -+ * the notifier chains for netdev_chain and sends a NEWLINK message -+ * to the routing socket. -+ */ -+void netdev_state_change(struct net_device *dev) -+{ -+ if (dev->flags & IFF_UP) { -+ struct netdev_notifier_change_info change_info; -+ -+ change_info.flags_changed = 0; -+ call_netdevice_notifiers_info(NETDEV_CHANGE, dev, -+ &change_info.info); -+ rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL); -+ } -+} -+EXPORT_SYMBOL(netdev_state_change); -+ -+/** -+ * netdev_notify_peers - notify network peers about existence of @dev -+ * @dev: network device -+ * -+ * Generate traffic such that interested network peers are aware of -+ * @dev, such as by generating a gratuitous ARP. This may be used when -+ * a device wants to inform the rest of the network about some sort of -+ * reconfiguration such as a failover event or virtual machine -+ * migration. -+ */ -+void netdev_notify_peers(struct net_device *dev) -+{ -+ rtnl_lock(); -+ call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev); -+ rtnl_unlock(); -+} -+EXPORT_SYMBOL(netdev_notify_peers); -+ -+static int __dev_open(struct net_device *dev) -+{ -+ const struct net_device_ops *ops = dev->netdev_ops; -+ int ret; -+ -+ ASSERT_RTNL(); -+ -+ if (!netif_device_present(dev)) -+ return -ENODEV; -+ -+ /* Block netpoll from trying to do any rx path servicing. -+ * If we don't do this there is a chance ndo_poll_controller -+ * or ndo_poll may be running while we open the device -+ */ -+ netpoll_poll_disable(dev); -+ -+ ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev); -+ ret = notifier_to_errno(ret); -+ if (ret) -+ return ret; -+ -+ set_bit(__LINK_STATE_START, &dev->state); -+ -+ if (ops->ndo_validate_addr) -+ ret = ops->ndo_validate_addr(dev); -+ -+ if (!ret && ops->ndo_open) -+ ret = ops->ndo_open(dev); -+ -+ netpoll_poll_enable(dev); -+ -+ if (ret) -+ clear_bit(__LINK_STATE_START, &dev->state); -+ else { -+ dev->flags |= IFF_UP; -+ dev_set_rx_mode(dev); -+ dev_activate(dev); -+ add_device_randomness(dev->dev_addr, dev->addr_len); -+ } -+ -+ return ret; -+} -+ -+/** -+ * dev_open - prepare an interface for use. -+ * @dev: device to open -+ * -+ * Takes a device from down to up state. The device's private open -+ * function is invoked and then the multicast lists are loaded. Finally -+ * the device is moved into the up state and a %NETDEV_UP message is -+ * sent to the netdev notifier chain. -+ * -+ * Calling this function on an active interface is a nop. On a failure -+ * a negative errno code is returned. -+ */ -+int dev_open(struct net_device *dev) -+{ -+ int ret; -+ -+ if (dev->flags & IFF_UP) -+ return 0; -+ -+ ret = __dev_open(dev); -+ if (ret < 0) -+ return ret; -+ -+ rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL); -+ call_netdevice_notifiers(NETDEV_UP, dev); -+ -+ return ret; -+} -+EXPORT_SYMBOL(dev_open); -+ -+static int __dev_close_many(struct list_head *head) -+{ -+ struct net_device *dev; -+ -+ ASSERT_RTNL(); -+ might_sleep(); -+ -+ list_for_each_entry(dev, head, close_list) { -+ /* Temporarily disable netpoll until the interface is down */ -+ netpoll_poll_disable(dev); -+ -+ call_netdevice_notifiers(NETDEV_GOING_DOWN, dev); -+ -+ clear_bit(__LINK_STATE_START, &dev->state); -+ -+ /* Synchronize to scheduled poll. We cannot touch poll list, it -+ * can be even on different cpu. So just clear netif_running(). -+ * -+ * dev->stop() will invoke napi_disable() on all of it's -+ * napi_struct instances on this device. -+ */ -+ smp_mb__after_atomic(); /* Commit netif_running(). */ -+ } -+ -+ dev_deactivate_many(head); -+ -+ list_for_each_entry(dev, head, close_list) { -+ const struct net_device_ops *ops = dev->netdev_ops; -+ -+ /* -+ * Call the device specific close. This cannot fail. -+ * Only if device is UP -+ * -+ * We allow it to be called even after a DETACH hot-plug -+ * event. -+ */ -+ if (ops->ndo_stop) -+ ops->ndo_stop(dev); -+ -+ dev->flags &= ~IFF_UP; -+ netpoll_poll_enable(dev); -+ } -+ -+ return 0; -+} -+ -+static int __dev_close(struct net_device *dev) -+{ -+ int retval; -+ LIST_HEAD(single); -+ -+ list_add(&dev->close_list, &single); -+ retval = __dev_close_many(&single); -+ list_del(&single); -+ -+ return retval; -+} -+ -+int dev_close_many(struct list_head *head, bool unlink) -+{ -+ struct net_device *dev, *tmp; -+ -+ /* Remove the devices that don't need to be closed */ -+ list_for_each_entry_safe(dev, tmp, head, close_list) -+ if (!(dev->flags & IFF_UP)) -+ list_del_init(&dev->close_list); -+ -+ __dev_close_many(head); -+ -+ list_for_each_entry_safe(dev, tmp, head, close_list) { -+ rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL); -+ call_netdevice_notifiers(NETDEV_DOWN, dev); -+ if (unlink) -+ list_del_init(&dev->close_list); -+ } -+ -+ return 0; -+} -+EXPORT_SYMBOL(dev_close_many); -+ -+/** -+ * dev_close - shutdown an interface. -+ * @dev: device to shutdown -+ * -+ * This function moves an active device into down state. A -+ * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device -+ * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier -+ * chain. -+ */ -+int dev_close(struct net_device *dev) -+{ -+ if (dev->flags & IFF_UP) { -+ LIST_HEAD(single); -+ -+ list_add(&dev->close_list, &single); -+ dev_close_many(&single, true); -+ list_del(&single); -+ } -+ return 0; -+} -+EXPORT_SYMBOL(dev_close); -+ -+ -+/** -+ * dev_disable_lro - disable Large Receive Offload on a device -+ * @dev: device -+ * -+ * Disable Large Receive Offload (LRO) on a net device. Must be -+ * called under RTNL. This is needed if received packets may be -+ * forwarded to another interface. -+ */ -+void dev_disable_lro(struct net_device *dev) -+{ -+ struct net_device *lower_dev; -+ struct list_head *iter; -+ -+ dev->wanted_features &= ~NETIF_F_LRO; -+ netdev_update_features(dev); -+ -+ if (unlikely(dev->features & NETIF_F_LRO)) -+ netdev_WARN(dev, "failed to disable LRO!\n"); -+ -+ netdev_for_each_lower_dev(dev, lower_dev, iter) -+ dev_disable_lro(lower_dev); -+} -+EXPORT_SYMBOL(dev_disable_lro); -+ -+static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val, -+ struct net_device *dev) -+{ -+ struct netdev_notifier_info info; -+ -+ netdev_notifier_info_init(&info, dev); -+ return nb->notifier_call(nb, val, &info); -+} -+ -+static int dev_boot_phase = 1; -+ -+/** -+ * register_netdevice_notifier - register a network notifier block -+ * @nb: notifier -+ * -+ * Register a notifier to be called when network device events occur. -+ * The notifier passed is linked into the kernel structures and must -+ * not be reused until it has been unregistered. A negative errno code -+ * is returned on a failure. -+ * -+ * When registered all registration and up events are replayed -+ * to the new notifier to allow device to have a race free -+ * view of the network device list. -+ */ -+ -+int register_netdevice_notifier(struct notifier_block *nb) -+{ -+ struct net_device *dev; -+ struct net_device *last; -+ struct net *net; -+ int err; -+ -+ rtnl_lock(); -+ err = raw_notifier_chain_register(&netdev_chain, nb); -+ if (err) -+ goto unlock; -+ if (dev_boot_phase) -+ goto unlock; -+ for_each_net(net) { -+ for_each_netdev(net, dev) { -+ err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev); -+ err = notifier_to_errno(err); -+ if (err) -+ goto rollback; -+ -+ if (!(dev->flags & IFF_UP)) -+ continue; -+ -+ call_netdevice_notifier(nb, NETDEV_UP, dev); -+ } -+ } -+ -+unlock: -+ rtnl_unlock(); -+ return err; -+ -+rollback: -+ last = dev; -+ for_each_net(net) { -+ for_each_netdev(net, dev) { -+ if (dev == last) -+ goto outroll; -+ -+ if (dev->flags & IFF_UP) { -+ call_netdevice_notifier(nb, NETDEV_GOING_DOWN, -+ dev); -+ call_netdevice_notifier(nb, NETDEV_DOWN, dev); -+ } -+ call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev); -+ } -+ } -+ -+outroll: -+ raw_notifier_chain_unregister(&netdev_chain, nb); -+ goto unlock; -+} -+EXPORT_SYMBOL(register_netdevice_notifier); -+ -+/** -+ * unregister_netdevice_notifier - unregister a network notifier block -+ * @nb: notifier -+ * -+ * Unregister a notifier previously registered by -+ * register_netdevice_notifier(). The notifier is unlinked into the -+ * kernel structures and may then be reused. A negative errno code -+ * is returned on a failure. -+ * -+ * After unregistering unregister and down device events are synthesized -+ * for all devices on the device list to the removed notifier to remove -+ * the need for special case cleanup code. -+ */ -+ -+int unregister_netdevice_notifier(struct notifier_block *nb) -+{ -+ struct net_device *dev; -+ struct net *net; -+ int err; -+ -+ rtnl_lock(); -+ err = raw_notifier_chain_unregister(&netdev_chain, nb); -+ if (err) -+ goto unlock; -+ -+ for_each_net(net) { -+ for_each_netdev(net, dev) { -+ if (dev->flags & IFF_UP) { -+ call_netdevice_notifier(nb, NETDEV_GOING_DOWN, -+ dev); -+ call_netdevice_notifier(nb, NETDEV_DOWN, dev); -+ } -+ call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev); -+ } -+ } -+unlock: -+ rtnl_unlock(); -+ return err; -+} -+EXPORT_SYMBOL(unregister_netdevice_notifier); -+ -+/** -+ * call_netdevice_notifiers_info - call all network notifier blocks -+ * @val: value passed unmodified to notifier function -+ * @dev: net_device pointer passed unmodified to notifier function -+ * @info: notifier information data -+ * -+ * Call all network notifier blocks. Parameters and return value -+ * are as for raw_notifier_call_chain(). -+ */ -+ -+static int call_netdevice_notifiers_info(unsigned long val, -+ struct net_device *dev, -+ struct netdev_notifier_info *info) -+{ -+ ASSERT_RTNL(); -+ netdev_notifier_info_init(info, dev); -+ return raw_notifier_call_chain(&netdev_chain, val, info); -+} -+ -+/** -+ * call_netdevice_notifiers - call all network notifier blocks -+ * @val: value passed unmodified to notifier function -+ * @dev: net_device pointer passed unmodified to notifier function -+ * -+ * Call all network notifier blocks. Parameters and return value -+ * are as for raw_notifier_call_chain(). -+ */ -+ -+int call_netdevice_notifiers(unsigned long val, struct net_device *dev) -+{ -+ struct netdev_notifier_info info; -+ -+ return call_netdevice_notifiers_info(val, dev, &info); -+} -+EXPORT_SYMBOL(call_netdevice_notifiers); -+ -+#ifdef CONFIG_NET_CLS_ACT -+static struct static_key ingress_needed __read_mostly; -+ -+void net_inc_ingress_queue(void) -+{ -+ static_key_slow_inc(&ingress_needed); -+} -+EXPORT_SYMBOL_GPL(net_inc_ingress_queue); -+ -+void net_dec_ingress_queue(void) -+{ -+ static_key_slow_dec(&ingress_needed); -+} -+EXPORT_SYMBOL_GPL(net_dec_ingress_queue); -+#endif -+ -+static struct static_key netstamp_needed __read_mostly; -+#ifdef HAVE_JUMP_LABEL -+/* We are not allowed to call static_key_slow_dec() from irq context -+ * If net_disable_timestamp() is called from irq context, defer the -+ * static_key_slow_dec() calls. -+ */ -+static atomic_t netstamp_needed_deferred; -+#endif -+ -+void net_enable_timestamp(void) -+{ -+#ifdef HAVE_JUMP_LABEL -+ int deferred = atomic_xchg(&netstamp_needed_deferred, 0); -+ -+ if (deferred) { -+ while (--deferred) -+ static_key_slow_dec(&netstamp_needed); -+ return; -+ } -+#endif -+ static_key_slow_inc(&netstamp_needed); -+} -+EXPORT_SYMBOL(net_enable_timestamp); -+ -+void net_disable_timestamp(void) -+{ -+#ifdef HAVE_JUMP_LABEL -+ if (in_interrupt()) { -+ atomic_inc(&netstamp_needed_deferred); -+ return; -+ } -+#endif -+ static_key_slow_dec(&netstamp_needed); -+} -+EXPORT_SYMBOL(net_disable_timestamp); -+ -+static inline void net_timestamp_set(struct sk_buff *skb) -+{ -+ skb->tstamp.tv64 = 0; -+ if (static_key_false(&netstamp_needed)) -+ __net_timestamp(skb); -+} -+ -+#define net_timestamp_check(COND, SKB) \ -+ if (static_key_false(&netstamp_needed)) { \ -+ if ((COND) && !(SKB)->tstamp.tv64) \ -+ __net_timestamp(SKB); \ -+ } \ -+ -+bool is_skb_forwardable(struct net_device *dev, struct sk_buff *skb) -+{ -+ unsigned int len; -+ -+ if (!(dev->flags & IFF_UP)) -+ return false; -+ -+ len = dev->mtu + dev->hard_header_len + VLAN_HLEN; -+ if (skb->len <= len) -+ return true; -+ -+ /* if TSO is enabled, we don't care about the length as the packet -+ * could be forwarded without being segmented before -+ */ -+ if (skb_is_gso(skb)) -+ return true; -+ -+ return false; -+} -+EXPORT_SYMBOL_GPL(is_skb_forwardable); -+ -+int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb) -+{ -+ if (skb_orphan_frags(skb, GFP_ATOMIC) || -+ unlikely(!is_skb_forwardable(dev, skb))) { -+ atomic_long_inc(&dev->rx_dropped); -+ kfree_skb(skb); -+ return NET_RX_DROP; -+ } -+ -+ skb_scrub_packet(skb, true); -+ skb->priority = 0; -+ skb->protocol = eth_type_trans(skb, dev); -+ skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN); -+ -+ return 0; -+} -+EXPORT_SYMBOL_GPL(__dev_forward_skb); -+ -+/** -+ * dev_forward_skb - loopback an skb to another netif -+ * -+ * @dev: destination network device -+ * @skb: buffer to forward -+ * -+ * return values: -+ * NET_RX_SUCCESS (no congestion) -+ * NET_RX_DROP (packet was dropped, but freed) -+ * -+ * dev_forward_skb can be used for injecting an skb from the -+ * start_xmit function of one device into the receive queue -+ * of another device. -+ * -+ * The receiving device may be in another namespace, so -+ * we have to clear all information in the skb that could -+ * impact namespace isolation. -+ */ -+int dev_forward_skb(struct net_device *dev, struct sk_buff *skb) -+{ -+ return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb); -+} -+EXPORT_SYMBOL_GPL(dev_forward_skb); -+ -+static inline int deliver_skb(struct sk_buff *skb, -+ struct packet_type *pt_prev, -+ struct net_device *orig_dev) -+{ -+ if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC))) -+ return -ENOMEM; -+ atomic_inc(&skb->users); -+ return pt_prev->func(skb, skb->dev, pt_prev, orig_dev); -+} -+ -+static inline void deliver_ptype_list_skb(struct sk_buff *skb, -+ struct packet_type **pt, -+ struct net_device *orig_dev, -+ __be16 type, -+ struct list_head *ptype_list) -+{ -+ struct packet_type *ptype, *pt_prev = *pt; -+ -+ list_for_each_entry_rcu(ptype, ptype_list, list) { -+ if (ptype->type != type) -+ continue; -+ if (pt_prev) -+ deliver_skb(skb, pt_prev, orig_dev); -+ pt_prev = ptype; -+ } -+ *pt = pt_prev; -+} -+ -+static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb) -+{ -+ if (!ptype->af_packet_priv || !skb->sk) -+ return false; -+ -+ if (ptype->id_match) -+ return ptype->id_match(ptype, skb->sk); -+ else if ((struct sock *)ptype->af_packet_priv == skb->sk) -+ return true; -+ -+ return false; -+} -+ -+/* -+ * Support routine. Sends outgoing frames to any network -+ * taps currently in use. -+ */ -+ -+static void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev) -+{ -+ struct packet_type *ptype; -+ struct sk_buff *skb2 = NULL; -+ struct packet_type *pt_prev = NULL; -+ struct list_head *ptype_list = &ptype_all; -+ -+ rcu_read_lock(); -+again: -+ list_for_each_entry_rcu(ptype, ptype_list, list) { -+ /* Never send packets back to the socket -+ * they originated from - MvS (miquels@drinkel.ow.org) -+ */ -+ if (skb_loop_sk(ptype, skb)) -+ continue; -+ -+ if (pt_prev) { -+ deliver_skb(skb2, pt_prev, skb->dev); -+ pt_prev = ptype; -+ continue; -+ } -+ -+ /* need to clone skb, done only once */ -+ skb2 = skb_clone(skb, GFP_ATOMIC); -+ if (!skb2) -+ goto out_unlock; -+ -+ net_timestamp_set(skb2); -+ -+ /* skb->nh should be correctly -+ * set by sender, so that the second statement is -+ * just protection against buggy protocols. -+ */ -+ skb_reset_mac_header(skb2); -+ -+ if (skb_network_header(skb2) < skb2->data || -+ skb_network_header(skb2) > skb_tail_pointer(skb2)) { -+ net_crit_ratelimited("protocol %04x is buggy, dev %s\n", -+ ntohs(skb2->protocol), -+ dev->name); -+ skb_reset_network_header(skb2); -+ } -+ -+ skb2->transport_header = skb2->network_header; -+ skb2->pkt_type = PACKET_OUTGOING; -+ pt_prev = ptype; -+ } -+ -+ if (ptype_list == &ptype_all) { -+ ptype_list = &dev->ptype_all; -+ goto again; -+ } -+out_unlock: -+ if (pt_prev) -+ pt_prev->func(skb2, skb->dev, pt_prev, skb->dev); -+ rcu_read_unlock(); -+} -+ -+/** -+ * netif_setup_tc - Handle tc mappings on real_num_tx_queues change -+ * @dev: Network device -+ * @txq: number of queues available -+ * -+ * If real_num_tx_queues is changed the tc mappings may no longer be -+ * valid. To resolve this verify the tc mapping remains valid and if -+ * not NULL the mapping. With no priorities mapping to this -+ * offset/count pair it will no longer be used. In the worst case TC0 -+ * is invalid nothing can be done so disable priority mappings. If is -+ * expected that drivers will fix this mapping if they can before -+ * calling netif_set_real_num_tx_queues. -+ */ -+static void netif_setup_tc(struct net_device *dev, unsigned int txq) -+{ -+ int i; -+ struct netdev_tc_txq *tc = &dev->tc_to_txq[0]; -+ -+ /* If TC0 is invalidated disable TC mapping */ -+ if (tc->offset + tc->count > txq) { -+ pr_warn("Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n"); -+ dev->num_tc = 0; -+ return; -+ } -+ -+ /* Invalidated prio to tc mappings set to TC0 */ -+ for (i = 1; i < TC_BITMASK + 1; i++) { -+ int q = netdev_get_prio_tc_map(dev, i); -+ -+ tc = &dev->tc_to_txq[q]; -+ if (tc->offset + tc->count > txq) { -+ pr_warn("Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n", -+ i, q); -+ netdev_set_prio_tc_map(dev, i, 0); -+ } -+ } -+} -+ -+#ifdef CONFIG_XPS -+static DEFINE_MUTEX(xps_map_mutex); -+#define xmap_dereference(P) \ -+ rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex)) -+ -+static struct xps_map *remove_xps_queue(struct xps_dev_maps *dev_maps, -+ int cpu, u16 index) -+{ -+ struct xps_map *map = NULL; -+ int pos; -+ -+ if (dev_maps) -+ map = xmap_dereference(dev_maps->cpu_map[cpu]); -+ -+ for (pos = 0; map && pos < map->len; pos++) { -+ if (map->queues[pos] == index) { -+ if (map->len > 1) { -+ map->queues[pos] = map->queues[--map->len]; -+ } else { -+ RCU_INIT_POINTER(dev_maps->cpu_map[cpu], NULL); -+ kfree_rcu(map, rcu); -+ map = NULL; -+ } -+ break; -+ } -+ } -+ -+ return map; -+} -+ -+static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index) -+{ -+ struct xps_dev_maps *dev_maps; -+ int cpu, i; -+ bool active = false; -+ -+ mutex_lock(&xps_map_mutex); -+ dev_maps = xmap_dereference(dev->xps_maps); -+ -+ if (!dev_maps) -+ goto out_no_maps; -+ -+ for_each_possible_cpu(cpu) { -+ for (i = index; i < dev->num_tx_queues; i++) { -+ if (!remove_xps_queue(dev_maps, cpu, i)) -+ break; -+ } -+ if (i == dev->num_tx_queues) -+ active = true; -+ } -+ -+ if (!active) { -+ RCU_INIT_POINTER(dev->xps_maps, NULL); -+ kfree_rcu(dev_maps, rcu); -+ } -+ -+ for (i = index; i < dev->num_tx_queues; i++) -+ netdev_queue_numa_node_write(netdev_get_tx_queue(dev, i), -+ NUMA_NO_NODE); -+ -+out_no_maps: -+ mutex_unlock(&xps_map_mutex); -+} -+ -+static struct xps_map *expand_xps_map(struct xps_map *map, -+ int cpu, u16 index) -+{ -+ struct xps_map *new_map; -+ int alloc_len = XPS_MIN_MAP_ALLOC; -+ int i, pos; -+ -+ for (pos = 0; map && pos < map->len; pos++) { -+ if (map->queues[pos] != index) -+ continue; -+ return map; -+ } -+ -+ /* Need to add queue to this CPU's existing map */ -+ if (map) { -+ if (pos < map->alloc_len) -+ return map; -+ -+ alloc_len = map->alloc_len * 2; -+ } -+ -+ /* Need to allocate new map to store queue on this CPU's map */ -+ new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL, -+ cpu_to_node(cpu)); -+ if (!new_map) -+ return NULL; -+ -+ for (i = 0; i < pos; i++) -+ new_map->queues[i] = map->queues[i]; -+ new_map->alloc_len = alloc_len; -+ new_map->len = pos; -+ -+ return new_map; -+} -+ -+int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask, -+ u16 index) -+{ -+ struct xps_dev_maps *dev_maps, *new_dev_maps = NULL; -+ struct xps_map *map, *new_map; -+ int maps_sz = max_t(unsigned int, XPS_DEV_MAPS_SIZE, L1_CACHE_BYTES); -+ int cpu, numa_node_id = -2; -+ bool active = false; -+ -+ mutex_lock(&xps_map_mutex); -+ -+ dev_maps = xmap_dereference(dev->xps_maps); -+ -+ /* allocate memory for queue storage */ -+ for_each_online_cpu(cpu) { -+ if (!cpumask_test_cpu(cpu, mask)) -+ continue; -+ -+ if (!new_dev_maps) -+ new_dev_maps = kzalloc(maps_sz, GFP_KERNEL); -+ if (!new_dev_maps) { -+ mutex_unlock(&xps_map_mutex); -+ return -ENOMEM; -+ } -+ -+ map = dev_maps ? xmap_dereference(dev_maps->cpu_map[cpu]) : -+ NULL; -+ -+ map = expand_xps_map(map, cpu, index); -+ if (!map) -+ goto error; -+ -+ RCU_INIT_POINTER(new_dev_maps->cpu_map[cpu], map); -+ } -+ -+ if (!new_dev_maps) -+ goto out_no_new_maps; -+ -+ for_each_possible_cpu(cpu) { -+ if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu)) { -+ /* add queue to CPU maps */ -+ int pos = 0; -+ -+ map = xmap_dereference(new_dev_maps->cpu_map[cpu]); -+ while ((pos < map->len) && (map->queues[pos] != index)) -+ pos++; -+ -+ if (pos == map->len) -+ map->queues[map->len++] = index; -+#ifdef CONFIG_NUMA -+ if (numa_node_id == -2) -+ numa_node_id = cpu_to_node(cpu); -+ else if (numa_node_id != cpu_to_node(cpu)) -+ numa_node_id = -1; -+#endif -+ } else if (dev_maps) { -+ /* fill in the new device map from the old device map */ -+ map = xmap_dereference(dev_maps->cpu_map[cpu]); -+ RCU_INIT_POINTER(new_dev_maps->cpu_map[cpu], map); -+ } -+ -+ } -+ -+ rcu_assign_pointer(dev->xps_maps, new_dev_maps); -+ -+ /* Cleanup old maps */ -+ if (dev_maps) { -+ for_each_possible_cpu(cpu) { -+ new_map = xmap_dereference(new_dev_maps->cpu_map[cpu]); -+ map = xmap_dereference(dev_maps->cpu_map[cpu]); -+ if (map && map != new_map) -+ kfree_rcu(map, rcu); -+ } -+ -+ kfree_rcu(dev_maps, rcu); -+ } -+ -+ dev_maps = new_dev_maps; -+ active = true; -+ -+out_no_new_maps: -+ /* update Tx queue numa node */ -+ netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index), -+ (numa_node_id >= 0) ? numa_node_id : -+ NUMA_NO_NODE); -+ -+ if (!dev_maps) -+ goto out_no_maps; -+ -+ /* removes queue from unused CPUs */ -+ for_each_possible_cpu(cpu) { -+ if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu)) -+ continue; -+ -+ if (remove_xps_queue(dev_maps, cpu, index)) -+ active = true; -+ } -+ -+ /* free map if not active */ -+ if (!active) { -+ RCU_INIT_POINTER(dev->xps_maps, NULL); -+ kfree_rcu(dev_maps, rcu); -+ } -+ -+out_no_maps: -+ mutex_unlock(&xps_map_mutex); -+ -+ return 0; -+error: -+ /* remove any maps that we added */ -+ for_each_possible_cpu(cpu) { -+ new_map = xmap_dereference(new_dev_maps->cpu_map[cpu]); -+ map = dev_maps ? xmap_dereference(dev_maps->cpu_map[cpu]) : -+ NULL; -+ if (new_map && new_map != map) -+ kfree(new_map); -+ } -+ -+ mutex_unlock(&xps_map_mutex); -+ -+ kfree(new_dev_maps); -+ return -ENOMEM; -+} -+EXPORT_SYMBOL(netif_set_xps_queue); -+ -+#endif -+/* -+ * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues -+ * greater then real_num_tx_queues stale skbs on the qdisc must be flushed. -+ */ -+int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq) -+{ -+ int rc; -+ -+ if (txq < 1 || txq > dev->num_tx_queues) -+ return -EINVAL; -+ -+ if (dev->reg_state == NETREG_REGISTERED || -+ dev->reg_state == NETREG_UNREGISTERING) { -+ ASSERT_RTNL(); -+ -+ rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues, -+ txq); -+ if (rc) -+ return rc; -+ -+ if (dev->num_tc) -+ netif_setup_tc(dev, txq); -+ -+ if (txq < dev->real_num_tx_queues) { -+ qdisc_reset_all_tx_gt(dev, txq); -+#ifdef CONFIG_XPS -+ netif_reset_xps_queues_gt(dev, txq); -+#endif -+ } -+ } -+ -+ dev->real_num_tx_queues = txq; -+ return 0; -+} -+EXPORT_SYMBOL(netif_set_real_num_tx_queues); -+ -+#ifdef CONFIG_SYSFS -+/** -+ * netif_set_real_num_rx_queues - set actual number of RX queues used -+ * @dev: Network device -+ * @rxq: Actual number of RX queues -+ * -+ * This must be called either with the rtnl_lock held or before -+ * registration of the net device. Returns 0 on success, or a -+ * negative error code. If called before registration, it always -+ * succeeds. -+ */ -+int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq) -+{ -+ int rc; -+ -+ if (rxq < 1 || rxq > dev->num_rx_queues) -+ return -EINVAL; -+ -+ if (dev->reg_state == NETREG_REGISTERED) { -+ ASSERT_RTNL(); -+ -+ rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues, -+ rxq); -+ if (rc) -+ return rc; -+ } -+ -+ dev->real_num_rx_queues = rxq; -+ return 0; -+} -+EXPORT_SYMBOL(netif_set_real_num_rx_queues); -+#endif -+ -+/** -+ * netif_get_num_default_rss_queues - default number of RSS queues -+ * -+ * This routine should set an upper limit on the number of RSS queues -+ * used by default by multiqueue devices. -+ */ -+int netif_get_num_default_rss_queues(void) -+{ -+ return min_t(int, DEFAULT_MAX_NUM_RSS_QUEUES, num_online_cpus()); -+} -+EXPORT_SYMBOL(netif_get_num_default_rss_queues); -+ -+static inline void __netif_reschedule(struct Qdisc *q) -+{ -+ struct softnet_data *sd; -+ unsigned long flags; -+ -+ local_irq_save(flags); -+ sd = this_cpu_ptr(&softnet_data); -+ q->next_sched = NULL; -+ *sd->output_queue_tailp = q; -+ sd->output_queue_tailp = &q->next_sched; -+ raise_softirq_irqoff(NET_TX_SOFTIRQ); -+ local_irq_restore(flags); -+} -+ -+void __netif_schedule(struct Qdisc *q) -+{ -+ if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state)) -+ __netif_reschedule(q); -+} -+EXPORT_SYMBOL(__netif_schedule); -+ -+struct dev_kfree_skb_cb { -+ enum skb_free_reason reason; -+}; -+ -+static struct dev_kfree_skb_cb *get_kfree_skb_cb(const struct sk_buff *skb) -+{ -+ return (struct dev_kfree_skb_cb *)skb->cb; -+} -+ -+void netif_schedule_queue(struct netdev_queue *txq) -+{ -+ rcu_read_lock(); -+ if (!(txq->state & QUEUE_STATE_ANY_XOFF)) { -+ struct Qdisc *q = rcu_dereference(txq->qdisc); -+ -+ __netif_schedule(q); -+ } -+ rcu_read_unlock(); -+} -+EXPORT_SYMBOL(netif_schedule_queue); -+ -+/** -+ * netif_wake_subqueue - allow sending packets on subqueue -+ * @dev: network device -+ * @queue_index: sub queue index -+ * -+ * Resume individual transmit queue of a device with multiple transmit queues. -+ */ -+void netif_wake_subqueue(struct net_device *dev, u16 queue_index) -+{ -+ struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index); -+ -+ if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &txq->state)) { -+ struct Qdisc *q; -+ -+ rcu_read_lock(); -+ q = rcu_dereference(txq->qdisc); -+ __netif_schedule(q); -+ rcu_read_unlock(); -+ } -+} -+EXPORT_SYMBOL(netif_wake_subqueue); -+ -+void netif_tx_wake_queue(struct netdev_queue *dev_queue) -+{ -+ if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) { -+ struct Qdisc *q; -+ -+ rcu_read_lock(); -+ q = rcu_dereference(dev_queue->qdisc); -+ __netif_schedule(q); -+ rcu_read_unlock(); -+ } -+} -+EXPORT_SYMBOL(netif_tx_wake_queue); -+ -+void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason) -+{ -+ unsigned long flags; -+ -+ if (likely(atomic_read(&skb->users) == 1)) { -+ smp_rmb(); -+ atomic_set(&skb->users, 0); -+ } else if (likely(!atomic_dec_and_test(&skb->users))) { -+ return; -+ } -+ get_kfree_skb_cb(skb)->reason = reason; -+ local_irq_save(flags); -+ skb->next = __this_cpu_read(softnet_data.completion_queue); -+ __this_cpu_write(softnet_data.completion_queue, skb); -+ raise_softirq_irqoff(NET_TX_SOFTIRQ); -+ local_irq_restore(flags); -+} -+EXPORT_SYMBOL(__dev_kfree_skb_irq); -+ -+void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason) -+{ -+ if (in_irq() || irqs_disabled()) -+ __dev_kfree_skb_irq(skb, reason); -+ else -+ dev_kfree_skb(skb); -+} -+EXPORT_SYMBOL(__dev_kfree_skb_any); -+ -+ -+/** -+ * netif_device_detach - mark device as removed -+ * @dev: network device -+ * -+ * Mark device as removed from system and therefore no longer available. -+ */ -+void netif_device_detach(struct net_device *dev) -+{ -+ if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) && -+ netif_running(dev)) { -+ netif_tx_stop_all_queues(dev); -+ } -+} -+EXPORT_SYMBOL(netif_device_detach); -+ -+/** -+ * netif_device_attach - mark device as attached -+ * @dev: network device -+ * -+ * Mark device as attached from system and restart if needed. -+ */ -+void netif_device_attach(struct net_device *dev) -+{ -+ if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) && -+ netif_running(dev)) { -+ netif_tx_wake_all_queues(dev); -+ __netdev_watchdog_up(dev); -+ } -+} -+EXPORT_SYMBOL(netif_device_attach); -+ -+static void skb_warn_bad_offload(const struct sk_buff *skb) -+{ -+ static const netdev_features_t null_features = 0; -+ struct net_device *dev = skb->dev; -+ const char *driver = ""; -+ -+ if (!net_ratelimit()) -+ return; -+ -+ if (dev && dev->dev.parent) -+ driver = dev_driver_string(dev->dev.parent); -+ -+ WARN(1, "%s: caps=(%pNF, %pNF) len=%d data_len=%d gso_size=%d " -+ "gso_type=%d ip_summed=%d\n", -+ driver, dev ? &dev->features : &null_features, -+ skb->sk ? &skb->sk->sk_route_caps : &null_features, -+ skb->len, skb->data_len, skb_shinfo(skb)->gso_size, -+ skb_shinfo(skb)->gso_type, skb->ip_summed); -+} -+ -+/* -+ * Invalidate hardware checksum when packet is to be mangled, and -+ * complete checksum manually on outgoing path. -+ */ -+int skb_checksum_help(struct sk_buff *skb) -+{ -+ __wsum csum; -+ int ret = 0, offset; -+ -+ if (skb->ip_summed == CHECKSUM_COMPLETE) -+ goto out_set_summed; -+ -+ if (unlikely(skb_shinfo(skb)->gso_size)) { -+ skb_warn_bad_offload(skb); -+ return -EINVAL; -+ } -+ -+ /* Before computing a checksum, we should make sure no frag could -+ * be modified by an external entity : checksum could be wrong. -+ */ -+ if (skb_has_shared_frag(skb)) { -+ ret = __skb_linearize(skb); -+ if (ret) -+ goto out; -+ } -+ -+ offset = skb_checksum_start_offset(skb); -+ BUG_ON(offset >= skb_headlen(skb)); -+ csum = skb_checksum(skb, offset, skb->len - offset, 0); -+ -+ offset += skb->csum_offset; -+ BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb)); -+ -+ if (skb_cloned(skb) && -+ !skb_clone_writable(skb, offset + sizeof(__sum16))) { -+ ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC); -+ if (ret) -+ goto out; -+ } -+ -+ *(__sum16 *)(skb->data + offset) = csum_fold(csum); -+out_set_summed: -+ skb->ip_summed = CHECKSUM_NONE; -+out: -+ return ret; -+} -+EXPORT_SYMBOL(skb_checksum_help); -+ -+__be16 skb_network_protocol(struct sk_buff *skb, int *depth) -+{ -+ __be16 type = skb->protocol; -+ -+ /* Tunnel gso handlers can set protocol to ethernet. */ -+ if (type == htons(ETH_P_TEB)) { -+ struct ethhdr *eth; -+ -+ if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr)))) -+ return 0; -+ -+ eth = (struct ethhdr *)skb_mac_header(skb); -+ type = eth->h_proto; -+ } -+ -+ return __vlan_get_protocol(skb, type, depth); -+} -+ -+/** -+ * skb_mac_gso_segment - mac layer segmentation handler. -+ * @skb: buffer to segment -+ * @features: features for the output path (see dev->features) -+ */ -+struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb, -+ netdev_features_t features) -+{ -+ struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT); -+ struct packet_offload *ptype; -+ int vlan_depth = skb->mac_len; -+ __be16 type = skb_network_protocol(skb, &vlan_depth); -+ -+ if (unlikely(!type)) -+ return ERR_PTR(-EINVAL); -+ -+ __skb_pull(skb, vlan_depth); -+ -+ rcu_read_lock(); -+ list_for_each_entry_rcu(ptype, &offload_base, list) { -+ if (ptype->type == type && ptype->callbacks.gso_segment) { -+ segs = ptype->callbacks.gso_segment(skb, features); -+ break; -+ } -+ } -+ rcu_read_unlock(); -+ -+ __skb_push(skb, skb->data - skb_mac_header(skb)); -+ -+ return segs; -+} -+EXPORT_SYMBOL(skb_mac_gso_segment); -+ -+ -+/* openvswitch calls this on rx path, so we need a different check. -+ */ -+static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path) -+{ -+ if (tx_path) -+ return skb->ip_summed != CHECKSUM_PARTIAL; -+ else -+ return skb->ip_summed == CHECKSUM_NONE; -+} -+ -+/** -+ * __skb_gso_segment - Perform segmentation on skb. -+ * @skb: buffer to segment -+ * @features: features for the output path (see dev->features) -+ * @tx_path: whether it is called in TX path -+ * -+ * This function segments the given skb and returns a list of segments. -+ * -+ * It may return NULL if the skb requires no segmentation. This is -+ * only possible when GSO is used for verifying header integrity. -+ */ -+struct sk_buff *__skb_gso_segment(struct sk_buff *skb, -+ netdev_features_t features, bool tx_path) -+{ -+ if (unlikely(skb_needs_check(skb, tx_path))) { -+ int err; -+ -+ skb_warn_bad_offload(skb); -+ -+ err = skb_cow_head(skb, 0); -+ if (err < 0) -+ return ERR_PTR(err); -+ } -+ -+ SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb); -+ SKB_GSO_CB(skb)->encap_level = 0; -+ -+ skb_reset_mac_header(skb); -+ skb_reset_mac_len(skb); -+ -+ return skb_mac_gso_segment(skb, features); -+} -+EXPORT_SYMBOL(__skb_gso_segment); -+ -+/* Take action when hardware reception checksum errors are detected. */ -+#ifdef CONFIG_BUG -+void netdev_rx_csum_fault(struct net_device *dev) -+{ -+ if (net_ratelimit()) { -+ pr_err("%s: hw csum failure\n", dev ? dev->name : ""); -+ dump_stack(); -+ } -+} -+EXPORT_SYMBOL(netdev_rx_csum_fault); -+#endif -+ -+/* Actually, we should eliminate this check as soon as we know, that: -+ * 1. IOMMU is present and allows to map all the memory. -+ * 2. No high memory really exists on this machine. -+ */ -+ -+static int illegal_highdma(struct net_device *dev, struct sk_buff *skb) -+{ -+#ifdef CONFIG_HIGHMEM -+ int i; -+ if (!(dev->features & NETIF_F_HIGHDMA)) { -+ for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { -+ skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; -+ if (PageHighMem(skb_frag_page(frag))) -+ return 1; -+ } -+ } -+ -+ if (PCI_DMA_BUS_IS_PHYS) { -+ struct device *pdev = dev->dev.parent; -+ -+ if (!pdev) -+ return 0; -+ for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { -+ skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; -+ dma_addr_t addr = page_to_phys(skb_frag_page(frag)); -+ if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask) -+ return 1; -+ } -+ } -+#endif -+ return 0; -+} -+ -+/* If MPLS offload request, verify we are testing hardware MPLS features -+ * instead of standard features for the netdev. -+ */ -+#if IS_ENABLED(CONFIG_NET_MPLS_GSO) -+static netdev_features_t net_mpls_features(struct sk_buff *skb, -+ netdev_features_t features, -+ __be16 type) -+{ -+ if (eth_p_mpls(type)) -+ features &= skb->dev->mpls_features; -+ -+ return features; -+} -+#else -+static netdev_features_t net_mpls_features(struct sk_buff *skb, -+ netdev_features_t features, -+ __be16 type) -+{ -+ return features; -+} -+#endif -+ -+static netdev_features_t harmonize_features(struct sk_buff *skb, -+ netdev_features_t features) -+{ -+ int tmp; -+ __be16 type; -+ -+ type = skb_network_protocol(skb, &tmp); -+ features = net_mpls_features(skb, features, type); -+ -+ if (skb->ip_summed != CHECKSUM_NONE && -+ !can_checksum_protocol(features, type)) { -+ features &= ~NETIF_F_ALL_CSUM; -+ } else if (illegal_highdma(skb->dev, skb)) { -+ features &= ~NETIF_F_SG; -+ } -+ -+ return features; -+} -+ -+netdev_features_t passthru_features_check(struct sk_buff *skb, -+ struct net_device *dev, -+ netdev_features_t features) -+{ -+ return features; -+} -+EXPORT_SYMBOL(passthru_features_check); -+ -+static netdev_features_t dflt_features_check(const struct sk_buff *skb, -+ struct net_device *dev, -+ netdev_features_t features) -+{ -+ return vlan_features_check(skb, features); -+} -+ -+netdev_features_t netif_skb_features(struct sk_buff *skb) -+{ -+ struct net_device *dev = skb->dev; -+ netdev_features_t features = dev->features; -+ u16 gso_segs = skb_shinfo(skb)->gso_segs; -+ -+ if (gso_segs > dev->gso_max_segs || gso_segs < dev->gso_min_segs) -+ features &= ~NETIF_F_GSO_MASK; -+ -+ /* If encapsulation offload request, verify we are testing -+ * hardware encapsulation features instead of standard -+ * features for the netdev -+ */ -+ if (skb->encapsulation) -+ features &= dev->hw_enc_features; -+ -+ if (skb_vlan_tagged(skb)) -+ features = netdev_intersect_features(features, -+ dev->vlan_features | -+ NETIF_F_HW_VLAN_CTAG_TX | -+ NETIF_F_HW_VLAN_STAG_TX); -+ -+ if (dev->netdev_ops->ndo_features_check) -+ features &= dev->netdev_ops->ndo_features_check(skb, dev, -+ features); -+ else -+ features &= dflt_features_check(skb, dev, features); -+ -+ return harmonize_features(skb, features); -+} -+EXPORT_SYMBOL(netif_skb_features); -+ -+static int xmit_one(struct sk_buff *skb, struct net_device *dev, -+ struct netdev_queue *txq, bool more) -+{ -+ unsigned int len; -+ int rc; -+ -+ if (!list_empty(&ptype_all) || !list_empty(&dev->ptype_all)) -+ dev_queue_xmit_nit(skb, dev); -+ -+ len = skb->len; -+ trace_net_dev_start_xmit(skb, dev); -+ rc = netdev_start_xmit(skb, dev, txq, more); -+ trace_net_dev_xmit(skb, rc, dev, len); -+ -+ return rc; -+} -+ -+struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev, -+ struct netdev_queue *txq, int *ret) -+{ -+ struct sk_buff *skb = first; -+ int rc = NETDEV_TX_OK; -+ -+ while (skb) { -+ struct sk_buff *next = skb->next; -+ -+ skb->next = NULL; -+ rc = xmit_one(skb, dev, txq, next != NULL); -+ if (unlikely(!dev_xmit_complete(rc))) { -+ skb->next = next; -+ goto out; -+ } -+ -+ skb = next; -+ if (netif_xmit_stopped(txq) && skb) { -+ rc = NETDEV_TX_BUSY; -+ break; -+ } -+ } -+ -+out: -+ *ret = rc; -+ return skb; -+} -+ -+static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb, -+ netdev_features_t features) -+{ -+ if (skb_vlan_tag_present(skb) && -+ !vlan_hw_offload_capable(features, skb->vlan_proto)) -+ skb = __vlan_hwaccel_push_inside(skb); -+ return skb; -+} -+ -+static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev) -+{ -+ netdev_features_t features; -+ -+ if (skb->next) -+ return skb; -+ -+ features = netif_skb_features(skb); -+ skb = validate_xmit_vlan(skb, features); -+ if (unlikely(!skb)) -+ goto out_null; -+ -+ if (netif_needs_gso(skb, features)) { -+ struct sk_buff *segs; -+ -+ segs = skb_gso_segment(skb, features); -+ if (IS_ERR(segs)) { -+ goto out_kfree_skb; -+ } else if (segs) { -+ consume_skb(skb); -+ skb = segs; -+ } -+ } else { -+ if (skb_needs_linearize(skb, features) && -+ __skb_linearize(skb)) -+ goto out_kfree_skb; -+ -+ /* If packet is not checksummed and device does not -+ * support checksumming for this protocol, complete -+ * checksumming here. -+ */ -+ if (skb->ip_summed == CHECKSUM_PARTIAL) { -+ if (skb->encapsulation) -+ skb_set_inner_transport_header(skb, -+ skb_checksum_start_offset(skb)); -+ else -+ skb_set_transport_header(skb, -+ skb_checksum_start_offset(skb)); -+ if (!(features & NETIF_F_ALL_CSUM) && -+ skb_checksum_help(skb)) -+ goto out_kfree_skb; -+ } -+ } -+ -+ return skb; -+ -+out_kfree_skb: -+ kfree_skb(skb); -+out_null: -+ return NULL; -+} -+ -+struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev) -+{ -+ struct sk_buff *next, *head = NULL, *tail; -+ -+ for (; skb != NULL; skb = next) { -+ next = skb->next; -+ skb->next = NULL; -+ -+ /* in case skb wont be segmented, point to itself */ -+ skb->prev = skb; -+ -+ skb = validate_xmit_skb(skb, dev); -+ if (!skb) -+ continue; -+ -+ if (!head) -+ head = skb; -+ else -+ tail->next = skb; -+ /* If skb was segmented, skb->prev points to -+ * the last segment. If not, it still contains skb. -+ */ -+ tail = skb->prev; -+ } -+ return head; -+} -+ -+static void qdisc_pkt_len_init(struct sk_buff *skb) -+{ -+ const struct skb_shared_info *shinfo = skb_shinfo(skb); -+ -+ qdisc_skb_cb(skb)->pkt_len = skb->len; -+ -+ /* To get more precise estimation of bytes sent on wire, -+ * we add to pkt_len the headers size of all segments -+ */ -+ if (shinfo->gso_size) { -+ unsigned int hdr_len; -+ u16 gso_segs = shinfo->gso_segs; -+ -+ /* mac layer + network layer */ -+ hdr_len = skb_transport_header(skb) - skb_mac_header(skb); -+ -+ /* + transport layer */ -+ if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) -+ hdr_len += tcp_hdrlen(skb); -+ else -+ hdr_len += sizeof(struct udphdr); -+ -+ if (shinfo->gso_type & SKB_GSO_DODGY) -+ gso_segs = DIV_ROUND_UP(skb->len - hdr_len, -+ shinfo->gso_size); -+ -+ qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len; -+ } -+} -+ -+static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q, -+ struct net_device *dev, -+ struct netdev_queue *txq) -+{ -+ spinlock_t *root_lock = qdisc_lock(q); -+ bool contended; -+ int rc; -+ -+ qdisc_pkt_len_init(skb); -+ qdisc_calculate_pkt_len(skb, q); -+ /* -+ * Heuristic to force contended enqueues to serialize on a -+ * separate lock before trying to get qdisc main lock. -+ * This permits __QDISC___STATE_RUNNING owner to get the lock more -+ * often and dequeue packets faster. -+ */ -+ contended = qdisc_is_running(q); -+ if (unlikely(contended)) -+ spin_lock(&q->busylock); -+ -+ spin_lock(root_lock); -+ if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) { -+ kfree_skb(skb); -+ rc = NET_XMIT_DROP; -+ } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) && -+ qdisc_run_begin(q)) { -+ /* -+ * This is a work-conserving queue; there are no old skbs -+ * waiting to be sent out; and the qdisc is not running - -+ * xmit the skb directly. -+ */ -+ -+ qdisc_bstats_update(q, skb); -+ -+ if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) { -+ if (unlikely(contended)) { -+ spin_unlock(&q->busylock); -+ contended = false; -+ } -+ __qdisc_run(q); -+ } else -+ qdisc_run_end(q); -+ -+ rc = NET_XMIT_SUCCESS; -+ } else { -+ rc = q->enqueue(skb, q) & NET_XMIT_MASK; -+ if (qdisc_run_begin(q)) { -+ if (unlikely(contended)) { -+ spin_unlock(&q->busylock); -+ contended = false; -+ } -+ __qdisc_run(q); -+ } -+ } -+ spin_unlock(root_lock); -+ if (unlikely(contended)) -+ spin_unlock(&q->busylock); -+ return rc; -+} -+ -+#if IS_ENABLED(CONFIG_CGROUP_NET_PRIO) -+static void skb_update_prio(struct sk_buff *skb) -+{ -+ struct netprio_map *map = rcu_dereference_bh(skb->dev->priomap); -+ -+ if (!skb->priority && skb->sk && map) { -+ unsigned int prioidx = skb->sk->sk_cgrp_prioidx; -+ -+ if (prioidx < map->priomap_len) -+ skb->priority = map->priomap[prioidx]; -+ } -+} -+#else -+#define skb_update_prio(skb) -+#endif -+ -+DEFINE_PER_CPU(int, xmit_recursion); -+EXPORT_SYMBOL(xmit_recursion); -+ -+#define RECURSION_LIMIT 10 -+ -+/** -+ * dev_loopback_xmit - loop back @skb -+ * @skb: buffer to transmit -+ */ -+int dev_loopback_xmit(struct sock *sk, struct sk_buff *skb) -+{ -+ skb_reset_mac_header(skb); -+ __skb_pull(skb, skb_network_offset(skb)); -+ skb->pkt_type = PACKET_LOOPBACK; -+ skb->ip_summed = CHECKSUM_UNNECESSARY; -+ WARN_ON(!skb_dst(skb)); -+ skb_dst_force(skb); -+ netif_rx_ni(skb); -+ return 0; -+} -+EXPORT_SYMBOL(dev_loopback_xmit); -+ -+/** -+ * __dev_queue_xmit - transmit a buffer -+ * @skb: buffer to transmit -+ * @accel_priv: private data used for L2 forwarding offload -+ * -+ * Queue a buffer for transmission to a network device. The caller must -+ * have set the device and priority and built the buffer before calling -+ * this function. The function can be called from an interrupt. -+ * -+ * A negative errno code is returned on a failure. A success does not -+ * guarantee the frame will be transmitted as it may be dropped due -+ * to congestion or traffic shaping. -+ * -+ * ----------------------------------------------------------------------------------- -+ * I notice this method can also return errors from the queue disciplines, -+ * including NET_XMIT_DROP, which is a positive value. So, errors can also -+ * be positive. -+ * -+ * Regardless of the return value, the skb is consumed, so it is currently -+ * difficult to retry a send to this method. (You can bump the ref count -+ * before sending to hold a reference for retry if you are careful.) -+ * -+ * When calling this method, interrupts MUST be enabled. This is because -+ * the BH enable code must have IRQs enabled so that it will not deadlock. -+ * --BLG -+ */ -+static int __dev_queue_xmit(struct sk_buff *skb, void *accel_priv) -+{ -+ struct net_device *dev = skb->dev; -+ struct netdev_queue *txq; -+ struct Qdisc *q; -+ int rc = -ENOMEM; -+ -+ skb_reset_mac_header(skb); -+ -+ if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP)) -+ __skb_tstamp_tx(skb, NULL, skb->sk, SCM_TSTAMP_SCHED); -+ -+ /* Disable soft irqs for various locks below. Also -+ * stops preemption for RCU. -+ */ -+ rcu_read_lock_bh(); -+ -+ skb_update_prio(skb); -+ -+ /* If device/qdisc don't need skb->dst, release it right now while -+ * its hot in this cpu cache. -+ */ -+ if (dev->priv_flags & IFF_XMIT_DST_RELEASE) -+ skb_dst_drop(skb); -+ else -+ skb_dst_force(skb); -+ -+ txq = netdev_pick_tx(dev, skb, accel_priv); -+ q = rcu_dereference_bh(txq->qdisc); -+ -+#ifdef CONFIG_NET_CLS_ACT -+ skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_EGRESS); -+#endif -+ trace_net_dev_queue(skb); -+ if (q->enqueue) { -+ rc = __dev_xmit_skb(skb, q, dev, txq); -+ goto out; -+ } -+ -+ /* The device has no queue. Common case for software devices: -+ loopback, all the sorts of tunnels... -+ -+ Really, it is unlikely that netif_tx_lock protection is necessary -+ here. (f.e. loopback and IP tunnels are clean ignoring statistics -+ counters.) -+ However, it is possible, that they rely on protection -+ made by us here. -+ -+ Check this and shot the lock. It is not prone from deadlocks. -+ Either shot noqueue qdisc, it is even simpler 8) -+ */ -+ if (dev->flags & IFF_UP) { -+ int cpu = smp_processor_id(); /* ok because BHs are off */ -+ -+ if (txq->xmit_lock_owner != cpu) { -+ -+ if (__this_cpu_read(xmit_recursion) > RECURSION_LIMIT) -+ goto recursion_alert; -+ -+ skb = validate_xmit_skb(skb, dev); -+ if (!skb) -+ goto drop; -+ -+ HARD_TX_LOCK(dev, txq, cpu); -+ -+ if (!netif_xmit_stopped(txq)) { -+ __this_cpu_inc(xmit_recursion); -+ skb = dev_hard_start_xmit(skb, dev, txq, &rc); -+ __this_cpu_dec(xmit_recursion); -+ if (dev_xmit_complete(rc)) { -+ HARD_TX_UNLOCK(dev, txq); -+ goto out; -+ } -+ } -+ HARD_TX_UNLOCK(dev, txq); -+ net_crit_ratelimited("Virtual device %s asks to queue packet!\n", -+ dev->name); -+ } else { -+ /* Recursion is detected! It is possible, -+ * unfortunately -+ */ -+recursion_alert: -+ net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n", -+ dev->name); -+ } -+ } -+ -+ rc = -ENETDOWN; -+drop: -+ rcu_read_unlock_bh(); -+ -+ atomic_long_inc(&dev->tx_dropped); -+ kfree_skb_list(skb); -+ return rc; -+out: -+ rcu_read_unlock_bh(); -+ return rc; -+} -+ -+int dev_queue_xmit_sk(struct sock *sk, struct sk_buff *skb) -+{ -+ return __dev_queue_xmit(skb, NULL); -+} -+EXPORT_SYMBOL(dev_queue_xmit_sk); -+ -+int dev_queue_xmit_accel(struct sk_buff *skb, void *accel_priv) -+{ -+ return __dev_queue_xmit(skb, accel_priv); -+} -+EXPORT_SYMBOL(dev_queue_xmit_accel); -+ -+ -+/*======================================================================= -+ Receiver routines -+ =======================================================================*/ -+ -+int netdev_max_backlog __read_mostly = 1000; -+EXPORT_SYMBOL(netdev_max_backlog); -+ -+int netdev_tstamp_prequeue __read_mostly = 1; -+int netdev_budget __read_mostly = 300; -+int weight_p __read_mostly = 64; /* old backlog weight */ -+ -+/* Called with irq disabled */ -+static inline void ____napi_schedule(struct softnet_data *sd, -+ struct napi_struct *napi) -+{ -+ list_add_tail(&napi->poll_list, &sd->poll_list); -+ __raise_softirq_irqoff(NET_RX_SOFTIRQ); -+} -+ -+#ifdef CONFIG_RPS -+ -+/* One global table that all flow-based protocols share. */ -+struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly; -+EXPORT_SYMBOL(rps_sock_flow_table); -+u32 rps_cpu_mask __read_mostly; -+EXPORT_SYMBOL(rps_cpu_mask); -+ -+struct static_key rps_needed __read_mostly; -+ -+static struct rps_dev_flow * -+set_rps_cpu(struct net_device *dev, struct sk_buff *skb, -+ struct rps_dev_flow *rflow, u16 next_cpu) -+{ -+ if (next_cpu < nr_cpu_ids) { -+#ifdef CONFIG_RFS_ACCEL -+ struct netdev_rx_queue *rxqueue; -+ struct rps_dev_flow_table *flow_table; -+ struct rps_dev_flow *old_rflow; -+ u32 flow_id; -+ u16 rxq_index; -+ int rc; -+ -+ /* Should we steer this flow to a different hardware queue? */ -+ if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap || -+ !(dev->features & NETIF_F_NTUPLE)) -+ goto out; -+ rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu); -+ if (rxq_index == skb_get_rx_queue(skb)) -+ goto out; -+ -+ rxqueue = dev->_rx + rxq_index; -+ flow_table = rcu_dereference(rxqueue->rps_flow_table); -+ if (!flow_table) -+ goto out; -+ flow_id = skb_get_hash(skb) & flow_table->mask; -+ rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb, -+ rxq_index, flow_id); -+ if (rc < 0) -+ goto out; -+ old_rflow = rflow; -+ rflow = &flow_table->flows[flow_id]; -+ rflow->filter = rc; -+ if (old_rflow->filter == rflow->filter) -+ old_rflow->filter = RPS_NO_FILTER; -+ out: -+#endif -+ rflow->last_qtail = -+ per_cpu(softnet_data, next_cpu).input_queue_head; -+ } -+ -+ rflow->cpu = next_cpu; -+ return rflow; -+} -+ -+/* -+ * get_rps_cpu is called from netif_receive_skb and returns the target -+ * CPU from the RPS map of the receiving queue for a given skb. -+ * rcu_read_lock must be held on entry. -+ */ -+static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb, -+ struct rps_dev_flow **rflowp) -+{ -+ const struct rps_sock_flow_table *sock_flow_table; -+ struct netdev_rx_queue *rxqueue = dev->_rx; -+ struct rps_dev_flow_table *flow_table; -+ struct rps_map *map; -+ int cpu = -1; -+ u32 tcpu; -+ u32 hash; -+ -+ if (skb_rx_queue_recorded(skb)) { -+ u16 index = skb_get_rx_queue(skb); -+ -+ if (unlikely(index >= dev->real_num_rx_queues)) { -+ WARN_ONCE(dev->real_num_rx_queues > 1, -+ "%s received packet on queue %u, but number " -+ "of RX queues is %u\n", -+ dev->name, index, dev->real_num_rx_queues); -+ goto done; -+ } -+ rxqueue += index; -+ } -+ -+ /* Avoid computing hash if RFS/RPS is not active for this rxqueue */ -+ -+ flow_table = rcu_dereference(rxqueue->rps_flow_table); -+ map = rcu_dereference(rxqueue->rps_map); -+ if (!flow_table && !map) -+ goto done; -+ -+ skb_reset_network_header(skb); -+ hash = skb_get_hash(skb); -+ if (!hash) -+ goto done; -+ -+ sock_flow_table = rcu_dereference(rps_sock_flow_table); -+ if (flow_table && sock_flow_table) { -+ struct rps_dev_flow *rflow; -+ u32 next_cpu; -+ u32 ident; -+ -+ /* First check into global flow table if there is a match */ -+ ident = sock_flow_table->ents[hash & sock_flow_table->mask]; -+ if ((ident ^ hash) & ~rps_cpu_mask) -+ goto try_rps; -+ -+ next_cpu = ident & rps_cpu_mask; -+ -+ /* OK, now we know there is a match, -+ * we can look at the local (per receive queue) flow table -+ */ -+ rflow = &flow_table->flows[hash & flow_table->mask]; -+ tcpu = rflow->cpu; -+ -+ /* -+ * If the desired CPU (where last recvmsg was done) is -+ * different from current CPU (one in the rx-queue flow -+ * table entry), switch if one of the following holds: -+ * - Current CPU is unset (>= nr_cpu_ids). -+ * - Current CPU is offline. -+ * - The current CPU's queue tail has advanced beyond the -+ * last packet that was enqueued using this table entry. -+ * This guarantees that all previous packets for the flow -+ * have been dequeued, thus preserving in order delivery. -+ */ -+ if (unlikely(tcpu != next_cpu) && -+ (tcpu >= nr_cpu_ids || !cpu_online(tcpu) || -+ ((int)(per_cpu(softnet_data, tcpu).input_queue_head - -+ rflow->last_qtail)) >= 0)) { -+ tcpu = next_cpu; -+ rflow = set_rps_cpu(dev, skb, rflow, next_cpu); -+ } -+ -+ if (tcpu < nr_cpu_ids && cpu_online(tcpu)) { -+ *rflowp = rflow; -+ cpu = tcpu; -+ goto done; -+ } -+ } -+ -+try_rps: -+ -+ if (map) { -+ tcpu = map->cpus[reciprocal_scale(hash, map->len)]; -+ if (cpu_online(tcpu)) { -+ cpu = tcpu; -+ goto done; -+ } -+ } -+ -+done: -+ return cpu; -+} -+ -+#ifdef CONFIG_RFS_ACCEL -+ -+/** -+ * rps_may_expire_flow - check whether an RFS hardware filter may be removed -+ * @dev: Device on which the filter was set -+ * @rxq_index: RX queue index -+ * @flow_id: Flow ID passed to ndo_rx_flow_steer() -+ * @filter_id: Filter ID returned by ndo_rx_flow_steer() -+ * -+ * Drivers that implement ndo_rx_flow_steer() should periodically call -+ * this function for each installed filter and remove the filters for -+ * which it returns %true. -+ */ -+bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index, -+ u32 flow_id, u16 filter_id) -+{ -+ struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index; -+ struct rps_dev_flow_table *flow_table; -+ struct rps_dev_flow *rflow; -+ bool expire = true; -+ unsigned int cpu; -+ -+ rcu_read_lock(); -+ flow_table = rcu_dereference(rxqueue->rps_flow_table); -+ if (flow_table && flow_id <= flow_table->mask) { -+ rflow = &flow_table->flows[flow_id]; -+ cpu = ACCESS_ONCE(rflow->cpu); -+ if (rflow->filter == filter_id && cpu < nr_cpu_ids && -+ ((int)(per_cpu(softnet_data, cpu).input_queue_head - -+ rflow->last_qtail) < -+ (int)(10 * flow_table->mask))) -+ expire = false; -+ } -+ rcu_read_unlock(); -+ return expire; -+} -+EXPORT_SYMBOL(rps_may_expire_flow); -+ -+#endif /* CONFIG_RFS_ACCEL */ -+ -+/* Called from hardirq (IPI) context */ -+static void rps_trigger_softirq(void *data) -+{ -+ struct softnet_data *sd = data; -+ -+ ____napi_schedule(sd, &sd->backlog); -+ sd->received_rps++; -+} -+ -+#endif /* CONFIG_RPS */ -+ -+/* -+ * Check if this softnet_data structure is another cpu one -+ * If yes, queue it to our IPI list and return 1 -+ * If no, return 0 -+ */ -+static int rps_ipi_queued(struct softnet_data *sd) -+{ -+#ifdef CONFIG_RPS -+ struct softnet_data *mysd = this_cpu_ptr(&softnet_data); -+ -+ if (sd != mysd) { -+ sd->rps_ipi_next = mysd->rps_ipi_list; -+ mysd->rps_ipi_list = sd; -+ -+ __raise_softirq_irqoff(NET_RX_SOFTIRQ); -+ return 1; -+ } -+#endif /* CONFIG_RPS */ -+ return 0; -+} -+ -+#ifdef CONFIG_NET_FLOW_LIMIT -+int netdev_flow_limit_table_len __read_mostly = (1 << 12); -+#endif -+ -+static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen) -+{ -+#ifdef CONFIG_NET_FLOW_LIMIT -+ struct sd_flow_limit *fl; -+ struct softnet_data *sd; -+ unsigned int old_flow, new_flow; -+ -+ if (qlen < (netdev_max_backlog >> 1)) -+ return false; -+ -+ sd = this_cpu_ptr(&softnet_data); -+ -+ rcu_read_lock(); -+ fl = rcu_dereference(sd->flow_limit); -+ if (fl) { -+ new_flow = skb_get_hash(skb) & (fl->num_buckets - 1); -+ old_flow = fl->history[fl->history_head]; -+ fl->history[fl->history_head] = new_flow; -+ -+ fl->history_head++; -+ fl->history_head &= FLOW_LIMIT_HISTORY - 1; -+ -+ if (likely(fl->buckets[old_flow])) -+ fl->buckets[old_flow]--; -+ -+ if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) { -+ fl->count++; -+ rcu_read_unlock(); -+ return true; -+ } -+ } -+ rcu_read_unlock(); -+#endif -+ return false; -+} -+ -+/* -+ * enqueue_to_backlog is called to queue an skb to a per CPU backlog -+ * queue (may be a remote CPU queue). -+ */ -+static int enqueue_to_backlog(struct sk_buff *skb, int cpu, -+ unsigned int *qtail) -+{ -+ struct softnet_data *sd; -+ unsigned long flags; -+ unsigned int qlen; -+ -+ sd = &per_cpu(softnet_data, cpu); -+ -+ local_irq_save(flags); -+ -+ rps_lock(sd); -+ if (!netif_running(skb->dev)) -+ goto drop; -+ qlen = skb_queue_len(&sd->input_pkt_queue); -+ if (qlen <= netdev_max_backlog && !skb_flow_limit(skb, qlen)) { -+ if (qlen) { -+enqueue: -+ __skb_queue_tail(&sd->input_pkt_queue, skb); -+ input_queue_tail_incr_save(sd, qtail); -+ rps_unlock(sd); -+ local_irq_restore(flags); -+ return NET_RX_SUCCESS; -+ } -+ -+ /* Schedule NAPI for backlog device -+ * We can use non atomic operation since we own the queue lock -+ */ -+ if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) { -+ if (!rps_ipi_queued(sd)) -+ ____napi_schedule(sd, &sd->backlog); -+ } -+ goto enqueue; -+ } -+ -+drop: -+ sd->dropped++; -+ rps_unlock(sd); -+ -+ local_irq_restore(flags); -+ -+ atomic_long_inc(&skb->dev->rx_dropped); -+ kfree_skb(skb); -+ return NET_RX_DROP; -+} -+ -+static int netif_rx_internal(struct sk_buff *skb) -+{ -+ int ret; -+ -+ net_timestamp_check(netdev_tstamp_prequeue, skb); -+ -+ trace_netif_rx(skb); -+#ifdef CONFIG_RPS -+ if (static_key_false(&rps_needed)) { -+ struct rps_dev_flow voidflow, *rflow = &voidflow; -+ int cpu; -+ -+ preempt_disable(); -+ rcu_read_lock(); -+ -+ cpu = get_rps_cpu(skb->dev, skb, &rflow); -+ if (cpu < 0) -+ cpu = smp_processor_id(); -+ -+ ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail); -+ -+ rcu_read_unlock(); -+ preempt_enable(); -+ } else -+#endif -+ { -+ unsigned int qtail; -+ ret = enqueue_to_backlog(skb, get_cpu(), &qtail); -+ put_cpu(); -+ } -+ return ret; -+} -+ -+/** -+ * netif_rx - post buffer to the network code -+ * @skb: buffer to post -+ * -+ * This function receives a packet from a device driver and queues it for -+ * the upper (protocol) levels to process. It always succeeds. The buffer -+ * may be dropped during processing for congestion control or by the -+ * protocol layers. -+ * -+ * return values: -+ * NET_RX_SUCCESS (no congestion) -+ * NET_RX_DROP (packet was dropped) -+ * -+ */ -+ -+int netif_rx(struct sk_buff *skb) -+{ -+ trace_netif_rx_entry(skb); -+ -+ return netif_rx_internal(skb); -+} -+EXPORT_SYMBOL(netif_rx); -+ -+int netif_rx_ni(struct sk_buff *skb) -+{ -+ int err; -+ -+ trace_netif_rx_ni_entry(skb); -+ -+ preempt_disable(); -+ err = netif_rx_internal(skb); -+ if (local_softirq_pending()) -+ do_softirq(); -+ preempt_enable(); -+ -+ return err; -+} -+EXPORT_SYMBOL(netif_rx_ni); -+ -+static void net_tx_action(struct softirq_action *h) -+{ -+ struct softnet_data *sd = this_cpu_ptr(&softnet_data); -+ -+ if (sd->completion_queue) { -+ struct sk_buff *clist; -+ -+ local_irq_disable(); -+ clist = sd->completion_queue; -+ sd->completion_queue = NULL; -+ local_irq_enable(); -+ -+ while (clist) { -+ struct sk_buff *skb = clist; -+ clist = clist->next; -+ -+ WARN_ON(atomic_read(&skb->users)); -+ if (likely(get_kfree_skb_cb(skb)->reason == SKB_REASON_CONSUMED)) -+ trace_consume_skb(skb); -+ else -+ trace_kfree_skb(skb, net_tx_action); -+ __kfree_skb(skb); -+ } -+ } -+ -+ if (sd->output_queue) { -+ struct Qdisc *head; -+ -+ local_irq_disable(); -+ head = sd->output_queue; -+ sd->output_queue = NULL; -+ sd->output_queue_tailp = &sd->output_queue; -+ local_irq_enable(); -+ -+ while (head) { -+ struct Qdisc *q = head; -+ spinlock_t *root_lock; -+ -+ head = head->next_sched; -+ -+ root_lock = qdisc_lock(q); -+ if (spin_trylock(root_lock)) { -+ smp_mb__before_atomic(); -+ clear_bit(__QDISC_STATE_SCHED, -+ &q->state); -+ qdisc_run(q); -+ spin_unlock(root_lock); -+ } else { -+ if (!test_bit(__QDISC_STATE_DEACTIVATED, -+ &q->state)) { -+ __netif_reschedule(q); -+ } else { -+ smp_mb__before_atomic(); -+ clear_bit(__QDISC_STATE_SCHED, -+ &q->state); -+ } -+ } -+ } -+ } -+} -+ -+#if (defined(CONFIG_BRIDGE) || defined(CONFIG_BRIDGE_MODULE)) && \ -+ (defined(CONFIG_ATM_LANE) || defined(CONFIG_ATM_LANE_MODULE)) -+/* This hook is defined here for ATM LANE */ -+int (*br_fdb_test_addr_hook)(struct net_device *dev, -+ unsigned char *addr) __read_mostly; -+EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook); -+#endif -+ -+#ifdef CONFIG_NET_CLS_ACT -+/* TODO: Maybe we should just force sch_ingress to be compiled in -+ * when CONFIG_NET_CLS_ACT is? otherwise some useless instructions -+ * a compare and 2 stores extra right now if we dont have it on -+ * but have CONFIG_NET_CLS_ACT -+ * NOTE: This doesn't stop any functionality; if you dont have -+ * the ingress scheduler, you just can't add policies on ingress. -+ * -+ */ -+static int ing_filter(struct sk_buff *skb, struct netdev_queue *rxq) -+{ -+ struct net_device *dev = skb->dev; -+ u32 ttl = G_TC_RTTL(skb->tc_verd); -+ int result = TC_ACT_OK; -+ struct Qdisc *q; -+ -+ if (unlikely(MAX_RED_LOOP < ttl++)) { -+ net_warn_ratelimited("Redir loop detected Dropping packet (%d->%d)\n", -+ skb->skb_iif, dev->ifindex); -+ return TC_ACT_SHOT; -+ } -+ -+ skb->tc_verd = SET_TC_RTTL(skb->tc_verd, ttl); -+ skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_INGRESS); -+ -+ q = rcu_dereference(rxq->qdisc); -+ if (q != &noop_qdisc) { -+ spin_lock(qdisc_lock(q)); -+ if (likely(!test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) -+ result = qdisc_enqueue_root(skb, q); -+ spin_unlock(qdisc_lock(q)); -+ } -+ -+ return result; -+} -+ -+static inline struct sk_buff *handle_ing(struct sk_buff *skb, -+ struct packet_type **pt_prev, -+ int *ret, struct net_device *orig_dev) -+{ -+ struct netdev_queue *rxq = rcu_dereference(skb->dev->ingress_queue); -+ -+ if (!rxq || rcu_access_pointer(rxq->qdisc) == &noop_qdisc) -+ return skb; -+ -+ if (*pt_prev) { -+ *ret = deliver_skb(skb, *pt_prev, orig_dev); -+ *pt_prev = NULL; -+ } -+ -+ switch (ing_filter(skb, rxq)) { -+ case TC_ACT_SHOT: -+ case TC_ACT_STOLEN: -+ kfree_skb(skb); -+ return NULL; -+ } -+ -+ return skb; -+} -+#endif -+ -+/** -+ * netdev_rx_handler_register - register receive handler -+ * @dev: device to register a handler for -+ * @rx_handler: receive handler to register -+ * @rx_handler_data: data pointer that is used by rx handler -+ * -+ * Register a receive handler for a device. This handler will then be -+ * called from __netif_receive_skb. A negative errno code is returned -+ * on a failure. -+ * -+ * The caller must hold the rtnl_mutex. -+ * -+ * For a general description of rx_handler, see enum rx_handler_result. -+ */ -+int netdev_rx_handler_register(struct net_device *dev, -+ rx_handler_func_t *rx_handler, -+ void *rx_handler_data) -+{ -+ ASSERT_RTNL(); -+ -+ if (dev->rx_handler) -+ return -EBUSY; -+ -+ /* Note: rx_handler_data must be set before rx_handler */ -+ rcu_assign_pointer(dev->rx_handler_data, rx_handler_data); -+ rcu_assign_pointer(dev->rx_handler, rx_handler); -+ -+ return 0; -+} -+EXPORT_SYMBOL_GPL(netdev_rx_handler_register); -+ -+/** -+ * netdev_rx_handler_unregister - unregister receive handler -+ * @dev: device to unregister a handler from -+ * -+ * Unregister a receive handler from a device. -+ * -+ * The caller must hold the rtnl_mutex. -+ */ -+void netdev_rx_handler_unregister(struct net_device *dev) -+{ -+ -+ ASSERT_RTNL(); -+ RCU_INIT_POINTER(dev->rx_handler, NULL); -+ /* a reader seeing a non NULL rx_handler in a rcu_read_lock() -+ * section has a guarantee to see a non NULL rx_handler_data -+ * as well. -+ */ -+ synchronize_net(); -+ RCU_INIT_POINTER(dev->rx_handler_data, NULL); -+} -+EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister); -+ -+/* -+ * Limit the use of PFMEMALLOC reserves to those protocols that implement -+ * the special handling of PFMEMALLOC skbs. -+ */ -+static bool skb_pfmemalloc_protocol(struct sk_buff *skb) -+{ -+ switch (skb->protocol) { -+ case htons(ETH_P_ARP): -+ case htons(ETH_P_IP): -+ case htons(ETH_P_IPV6): -+ case htons(ETH_P_8021Q): -+ case htons(ETH_P_8021AD): -+ return true; -+ default: -+ return false; -+ } -+} -+ -+static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc) -+{ -+ struct packet_type *ptype, *pt_prev; -+ rx_handler_func_t *rx_handler; -+ struct net_device *orig_dev; -+ bool deliver_exact = false; -+ int ret = NET_RX_DROP; -+ __be16 type; -+ -+ net_timestamp_check(!netdev_tstamp_prequeue, skb); -+ -+ trace_netif_receive_skb(skb); -+ -+ orig_dev = skb->dev; -+ -+ skb_reset_network_header(skb); -+ if (!skb_transport_header_was_set(skb)) -+ skb_reset_transport_header(skb); -+ skb_reset_mac_len(skb); -+ -+ pt_prev = NULL; -+ -+another_round: -+ skb->skb_iif = skb->dev->ifindex; -+ -+ __this_cpu_inc(softnet_data.processed); -+ -+ if (skb->protocol == cpu_to_be16(ETH_P_8021Q) || -+ skb->protocol == cpu_to_be16(ETH_P_8021AD)) { -+ skb = skb_vlan_untag(skb); -+ if (unlikely(!skb)) -+ goto out; -+ } -+ -+#ifdef CONFIG_NET_CLS_ACT -+ if (skb->tc_verd & TC_NCLS) { -+ skb->tc_verd = CLR_TC_NCLS(skb->tc_verd); -+ goto ncls; -+ } -+#endif -+ -+ if (pfmemalloc) -+ goto skip_taps; -+ -+ list_for_each_entry_rcu(ptype, &ptype_all, list) { -+ if (pt_prev) -+ ret = deliver_skb(skb, pt_prev, orig_dev); -+ pt_prev = ptype; -+ } -+ -+ list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) { -+ if (pt_prev) -+ ret = deliver_skb(skb, pt_prev, orig_dev); -+ pt_prev = ptype; -+ } -+ -+skip_taps: -+#ifdef CONFIG_NET_CLS_ACT -+ if (static_key_false(&ingress_needed)) { -+ skb = handle_ing(skb, &pt_prev, &ret, orig_dev); -+ if (!skb) -+ goto out; -+ } -+ -+ skb->tc_verd = 0; -+ncls: -+#endif -+ if (pfmemalloc && !skb_pfmemalloc_protocol(skb)) -+ goto drop; -+ -+ if (skb_vlan_tag_present(skb)) { -+ if (pt_prev) { -+ ret = deliver_skb(skb, pt_prev, orig_dev); -+ pt_prev = NULL; -+ } -+ if (vlan_do_receive(&skb)) -+ goto another_round; -+ else if (unlikely(!skb)) -+ goto out; -+ } -+ -+ rx_handler = rcu_dereference(skb->dev->rx_handler); -+ if (rx_handler) { -+ if (pt_prev) { -+ ret = deliver_skb(skb, pt_prev, orig_dev); -+ pt_prev = NULL; -+ } -+ switch (rx_handler(&skb)) { -+ case RX_HANDLER_CONSUMED: -+ ret = NET_RX_SUCCESS; -+ goto out; -+ case RX_HANDLER_ANOTHER: -+ goto another_round; -+ case RX_HANDLER_EXACT: -+ deliver_exact = true; -+ case RX_HANDLER_PASS: -+ break; -+ default: -+ BUG(); -+ } -+ } -+ -+ if (unlikely(skb_vlan_tag_present(skb))) { -+ if (skb_vlan_tag_get_id(skb)) -+ skb->pkt_type = PACKET_OTHERHOST; -+ /* Note: we might in the future use prio bits -+ * and set skb->priority like in vlan_do_receive() -+ * For the time being, just ignore Priority Code Point -+ */ -+ skb->vlan_tci = 0; -+ } -+ -+ type = skb->protocol; -+ -+ /* deliver only exact match when indicated */ -+ if (likely(!deliver_exact)) { -+ deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type, -+ &ptype_base[ntohs(type) & -+ PTYPE_HASH_MASK]); -+ } -+ -+ deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type, -+ &orig_dev->ptype_specific); -+ -+ if (unlikely(skb->dev != orig_dev)) { -+ deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type, -+ &skb->dev->ptype_specific); -+ } -+ -+ if (pt_prev) { -+ if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC))) -+ goto drop; -+ else -+ ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev); -+ } else { -+drop: -+ atomic_long_inc(&skb->dev->rx_dropped); -+ kfree_skb(skb); -+ /* Jamal, now you will not able to escape explaining -+ * me how you were going to use this. :-) -+ */ -+ ret = NET_RX_DROP; -+ } -+ -+out: -+ return ret; -+} -+ -+static int __netif_receive_skb(struct sk_buff *skb) -+{ -+ int ret; -+ -+ if (sk_memalloc_socks() && skb_pfmemalloc(skb)) { -+ unsigned long pflags = current->flags; -+ -+ /* -+ * PFMEMALLOC skbs are special, they should -+ * - be delivered to SOCK_MEMALLOC sockets only -+ * - stay away from userspace -+ * - have bounded memory usage -+ * -+ * Use PF_MEMALLOC as this saves us from propagating the allocation -+ * context down to all allocation sites. -+ */ -+ current->flags |= PF_MEMALLOC; -+ ret = __netif_receive_skb_core(skb, true); -+ tsk_restore_flags(current, pflags, PF_MEMALLOC); -+ } else -+ ret = __netif_receive_skb_core(skb, false); -+ -+ return ret; -+} -+ -+static int netif_receive_skb_internal(struct sk_buff *skb) -+{ -+ int ret; -+ -+ net_timestamp_check(netdev_tstamp_prequeue, skb); -+ -+ if (skb_defer_rx_timestamp(skb)) -+ return NET_RX_SUCCESS; -+ -+ rcu_read_lock(); -+ -+#ifdef CONFIG_RPS -+ if (static_key_false(&rps_needed)) { -+ struct rps_dev_flow voidflow, *rflow = &voidflow; -+ int cpu = get_rps_cpu(skb->dev, skb, &rflow); -+ -+ if (cpu >= 0) { -+ ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail); -+ rcu_read_unlock(); -+ return ret; -+ } -+ } -+#endif -+ ret = __netif_receive_skb(skb); -+ rcu_read_unlock(); -+ return ret; -+} -+ -+/** -+ * netif_receive_skb - process receive buffer from network -+ * @skb: buffer to process -+ * -+ * netif_receive_skb() is the main receive data processing function. -+ * It always succeeds. The buffer may be dropped during processing -+ * for congestion control or by the protocol layers. -+ * -+ * This function may only be called from softirq context and interrupts -+ * should be enabled. -+ * -+ * Return values (usually ignored): -+ * NET_RX_SUCCESS: no congestion -+ * NET_RX_DROP: packet was dropped -+ */ -+int netif_receive_skb_sk(struct sock *sk, struct sk_buff *skb) -+{ -+ trace_netif_receive_skb_entry(skb); -+ -+ return netif_receive_skb_internal(skb); -+} -+EXPORT_SYMBOL(netif_receive_skb_sk); -+ -+/* Network device is going away, flush any packets still pending -+ * Called with irqs disabled. -+ */ -+static void flush_backlog(void *arg) -+{ -+ struct net_device *dev = arg; -+ struct softnet_data *sd = this_cpu_ptr(&softnet_data); -+ struct sk_buff *skb, *tmp; -+ -+ rps_lock(sd); -+ skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) { -+ if (skb->dev == dev) { -+ __skb_unlink(skb, &sd->input_pkt_queue); -+ kfree_skb(skb); -+ input_queue_head_incr(sd); -+ } -+ } -+ rps_unlock(sd); -+ -+ skb_queue_walk_safe(&sd->process_queue, skb, tmp) { -+ if (skb->dev == dev) { -+ __skb_unlink(skb, &sd->process_queue); -+ kfree_skb(skb); -+ input_queue_head_incr(sd); -+ } -+ } -+} -+ -+static int napi_gro_complete(struct sk_buff *skb) -+{ -+ struct packet_offload *ptype; -+ __be16 type = skb->protocol; -+ struct list_head *head = &offload_base; -+ int err = -ENOENT; -+ -+ BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb)); -+ -+ if (NAPI_GRO_CB(skb)->count == 1) { -+ skb_shinfo(skb)->gso_size = 0; -+ goto out; -+ } -+ -+ rcu_read_lock(); -+ list_for_each_entry_rcu(ptype, head, list) { -+ if (ptype->type != type || !ptype->callbacks.gro_complete) -+ continue; -+ -+ err = ptype->callbacks.gro_complete(skb, 0); -+ break; -+ } -+ rcu_read_unlock(); -+ -+ if (err) { -+ WARN_ON(&ptype->list == head); -+ kfree_skb(skb); -+ return NET_RX_SUCCESS; -+ } -+ -+out: -+ return netif_receive_skb_internal(skb); -+} -+ -+/* napi->gro_list contains packets ordered by age. -+ * youngest packets at the head of it. -+ * Complete skbs in reverse order to reduce latencies. -+ */ -+void napi_gro_flush(struct napi_struct *napi, bool flush_old) -+{ -+ struct sk_buff *skb, *prev = NULL; -+ -+ /* scan list and build reverse chain */ -+ for (skb = napi->gro_list; skb != NULL; skb = skb->next) { -+ skb->prev = prev; -+ prev = skb; -+ } -+ -+ for (skb = prev; skb; skb = prev) { -+ skb->next = NULL; -+ -+ if (flush_old && NAPI_GRO_CB(skb)->age == jiffies) -+ return; -+ -+ prev = skb->prev; -+ napi_gro_complete(skb); -+ napi->gro_count--; -+ } -+ -+ napi->gro_list = NULL; -+} -+EXPORT_SYMBOL(napi_gro_flush); -+ -+static void gro_list_prepare(struct napi_struct *napi, struct sk_buff *skb) -+{ -+ struct sk_buff *p; -+ unsigned int maclen = skb->dev->hard_header_len; -+ u32 hash = skb_get_hash_raw(skb); -+ -+ for (p = napi->gro_list; p; p = p->next) { -+ unsigned long diffs; -+ -+ NAPI_GRO_CB(p)->flush = 0; -+ -+ if (hash != skb_get_hash_raw(p)) { -+ NAPI_GRO_CB(p)->same_flow = 0; -+ continue; -+ } -+ -+ diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev; -+ diffs |= p->vlan_tci ^ skb->vlan_tci; -+ if (maclen == ETH_HLEN) -+ diffs |= compare_ether_header(skb_mac_header(p), -+ skb_mac_header(skb)); -+ else if (!diffs) -+ diffs = memcmp(skb_mac_header(p), -+ skb_mac_header(skb), -+ maclen); -+ NAPI_GRO_CB(p)->same_flow = !diffs; -+ } -+} -+ -+static void skb_gro_reset_offset(struct sk_buff *skb) -+{ -+ const struct skb_shared_info *pinfo = skb_shinfo(skb); -+ const skb_frag_t *frag0 = &pinfo->frags[0]; -+ -+ NAPI_GRO_CB(skb)->data_offset = 0; -+ NAPI_GRO_CB(skb)->frag0 = NULL; -+ NAPI_GRO_CB(skb)->frag0_len = 0; -+ -+ if (skb_mac_header(skb) == skb_tail_pointer(skb) && -+ pinfo->nr_frags && -+ !PageHighMem(skb_frag_page(frag0))) { -+ NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0); -+ NAPI_GRO_CB(skb)->frag0_len = skb_frag_size(frag0); -+ } -+} -+ -+static void gro_pull_from_frag0(struct sk_buff *skb, int grow) -+{ -+ struct skb_shared_info *pinfo = skb_shinfo(skb); -+ -+ BUG_ON(skb->end - skb->tail < grow); -+ -+ memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow); -+ -+ skb->data_len -= grow; -+ skb->tail += grow; -+ -+ pinfo->frags[0].page_offset += grow; -+ skb_frag_size_sub(&pinfo->frags[0], grow); -+ -+ if (unlikely(!skb_frag_size(&pinfo->frags[0]))) { -+ skb_frag_unref(skb, 0); -+ memmove(pinfo->frags, pinfo->frags + 1, -+ --pinfo->nr_frags * sizeof(pinfo->frags[0])); -+ } -+} -+ -+static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb) -+{ -+ struct sk_buff **pp = NULL; -+ struct packet_offload *ptype; -+ __be16 type = skb->protocol; -+ struct list_head *head = &offload_base; -+ int same_flow; -+ enum gro_result ret; -+ int grow; -+ -+ if (!(skb->dev->features & NETIF_F_GRO)) -+ goto normal; -+ -+ if (skb_is_gso(skb) || skb_has_frag_list(skb) || skb->csum_bad) -+ goto normal; -+ -+ gro_list_prepare(napi, skb); -+ -+ rcu_read_lock(); -+ list_for_each_entry_rcu(ptype, head, list) { -+ if (ptype->type != type || !ptype->callbacks.gro_receive) -+ continue; -+ -+ skb_set_network_header(skb, skb_gro_offset(skb)); -+ skb_reset_mac_len(skb); -+ NAPI_GRO_CB(skb)->same_flow = 0; -+ NAPI_GRO_CB(skb)->flush = 0; -+ NAPI_GRO_CB(skb)->free = 0; -+ NAPI_GRO_CB(skb)->udp_mark = 0; -+ NAPI_GRO_CB(skb)->gro_remcsum_start = 0; -+ -+ /* Setup for GRO checksum validation */ -+ switch (skb->ip_summed) { -+ case CHECKSUM_COMPLETE: -+ NAPI_GRO_CB(skb)->csum = skb->csum; -+ NAPI_GRO_CB(skb)->csum_valid = 1; -+ NAPI_GRO_CB(skb)->csum_cnt = 0; -+ break; -+ case CHECKSUM_UNNECESSARY: -+ NAPI_GRO_CB(skb)->csum_cnt = skb->csum_level + 1; -+ NAPI_GRO_CB(skb)->csum_valid = 0; -+ break; -+ default: -+ NAPI_GRO_CB(skb)->csum_cnt = 0; -+ NAPI_GRO_CB(skb)->csum_valid = 0; -+ } -+ -+ pp = ptype->callbacks.gro_receive(&napi->gro_list, skb); -+ break; -+ } -+ rcu_read_unlock(); -+ -+ if (&ptype->list == head) -+ goto normal; -+ -+ same_flow = NAPI_GRO_CB(skb)->same_flow; -+ ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED; -+ -+ if (pp) { -+ struct sk_buff *nskb = *pp; -+ -+ *pp = nskb->next; -+ nskb->next = NULL; -+ napi_gro_complete(nskb); -+ napi->gro_count--; -+ } -+ -+ if (same_flow) -+ goto ok; -+ -+ if (NAPI_GRO_CB(skb)->flush) -+ goto normal; -+ -+ if (unlikely(napi->gro_count >= MAX_GRO_SKBS)) { -+ struct sk_buff *nskb = napi->gro_list; -+ -+ /* locate the end of the list to select the 'oldest' flow */ -+ while (nskb->next) { -+ pp = &nskb->next; -+ nskb = *pp; -+ } -+ *pp = NULL; -+ nskb->next = NULL; -+ napi_gro_complete(nskb); -+ } else { -+ napi->gro_count++; -+ } -+ NAPI_GRO_CB(skb)->count = 1; -+ NAPI_GRO_CB(skb)->age = jiffies; -+ NAPI_GRO_CB(skb)->last = skb; -+ skb_shinfo(skb)->gso_size = skb_gro_len(skb); -+ skb->next = napi->gro_list; -+ napi->gro_list = skb; -+ ret = GRO_HELD; -+ -+pull: -+ grow = skb_gro_offset(skb) - skb_headlen(skb); -+ if (grow > 0) -+ gro_pull_from_frag0(skb, grow); -+ok: -+ return ret; -+ -+normal: -+ ret = GRO_NORMAL; -+ goto pull; -+} -+ -+struct packet_offload *gro_find_receive_by_type(__be16 type) -+{ -+ struct list_head *offload_head = &offload_base; -+ struct packet_offload *ptype; -+ -+ list_for_each_entry_rcu(ptype, offload_head, list) { -+ if (ptype->type != type || !ptype->callbacks.gro_receive) -+ continue; -+ return ptype; -+ } -+ return NULL; -+} -+EXPORT_SYMBOL(gro_find_receive_by_type); -+ -+struct packet_offload *gro_find_complete_by_type(__be16 type) -+{ -+ struct list_head *offload_head = &offload_base; -+ struct packet_offload *ptype; -+ -+ list_for_each_entry_rcu(ptype, offload_head, list) { -+ if (ptype->type != type || !ptype->callbacks.gro_complete) -+ continue; -+ return ptype; -+ } -+ return NULL; -+} -+EXPORT_SYMBOL(gro_find_complete_by_type); -+ -+static gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb) -+{ -+ switch (ret) { -+ case GRO_NORMAL: -+ if (netif_receive_skb_internal(skb)) -+ ret = GRO_DROP; -+ break; -+ -+ case GRO_DROP: -+ kfree_skb(skb); -+ break; -+ -+ case GRO_MERGED_FREE: -+ if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD) -+ kmem_cache_free(skbuff_head_cache, skb); -+ else -+ __kfree_skb(skb); -+ break; -+ -+ case GRO_HELD: -+ case GRO_MERGED: -+ break; -+ } -+ -+ return ret; -+} -+ -+gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb) -+{ -+ trace_napi_gro_receive_entry(skb); -+ -+ skb_gro_reset_offset(skb); -+ -+ return napi_skb_finish(dev_gro_receive(napi, skb), skb); -+} -+EXPORT_SYMBOL(napi_gro_receive); -+ -+static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb) -+{ -+ if (unlikely(skb->pfmemalloc)) { -+ consume_skb(skb); -+ return; -+ } -+ __skb_pull(skb, skb_headlen(skb)); -+ /* restore the reserve we had after netdev_alloc_skb_ip_align() */ -+ skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb)); -+ skb->vlan_tci = 0; -+ skb->dev = napi->dev; -+ skb->skb_iif = 0; -+ skb->encapsulation = 0; -+ skb_shinfo(skb)->gso_type = 0; -+ skb->truesize = SKB_TRUESIZE(skb_end_offset(skb)); -+ -+ napi->skb = skb; -+} -+ -+struct sk_buff *napi_get_frags(struct napi_struct *napi) -+{ -+ struct sk_buff *skb = napi->skb; -+ -+ if (!skb) { -+ skb = napi_alloc_skb(napi, GRO_MAX_HEAD); -+ napi->skb = skb; -+ } -+ return skb; -+} -+EXPORT_SYMBOL(napi_get_frags); -+ -+static gro_result_t napi_frags_finish(struct napi_struct *napi, -+ struct sk_buff *skb, -+ gro_result_t ret) -+{ -+ switch (ret) { -+ case GRO_NORMAL: -+ case GRO_HELD: -+ __skb_push(skb, ETH_HLEN); -+ skb->protocol = eth_type_trans(skb, skb->dev); -+ if (ret == GRO_NORMAL && netif_receive_skb_internal(skb)) -+ ret = GRO_DROP; -+ break; -+ -+ case GRO_DROP: -+ case GRO_MERGED_FREE: -+ napi_reuse_skb(napi, skb); -+ break; -+ -+ case GRO_MERGED: -+ break; -+ } -+ -+ return ret; -+} -+ -+/* Upper GRO stack assumes network header starts at gro_offset=0 -+ * Drivers could call both napi_gro_frags() and napi_gro_receive() -+ * We copy ethernet header into skb->data to have a common layout. -+ */ -+static struct sk_buff *napi_frags_skb(struct napi_struct *napi) -+{ -+ struct sk_buff *skb = napi->skb; -+ const struct ethhdr *eth; -+ unsigned int hlen = sizeof(*eth); -+ -+ napi->skb = NULL; -+ -+ skb_reset_mac_header(skb); -+ skb_gro_reset_offset(skb); -+ -+ eth = skb_gro_header_fast(skb, 0); -+ if (unlikely(skb_gro_header_hard(skb, hlen))) { -+ eth = skb_gro_header_slow(skb, hlen, 0); -+ if (unlikely(!eth)) { -+ napi_reuse_skb(napi, skb); -+ return NULL; -+ } -+ } else { -+ gro_pull_from_frag0(skb, hlen); -+ NAPI_GRO_CB(skb)->frag0 += hlen; -+ NAPI_GRO_CB(skb)->frag0_len -= hlen; -+ } -+ __skb_pull(skb, hlen); -+ -+ /* -+ * This works because the only protocols we care about don't require -+ * special handling. -+ * We'll fix it up properly in napi_frags_finish() -+ */ -+ skb->protocol = eth->h_proto; -+ -+ return skb; -+} -+ -+gro_result_t napi_gro_frags(struct napi_struct *napi) -+{ -+ struct sk_buff *skb = napi_frags_skb(napi); -+ -+ if (!skb) -+ return GRO_DROP; -+ -+ trace_napi_gro_frags_entry(skb); -+ -+ return napi_frags_finish(napi, skb, dev_gro_receive(napi, skb)); -+} -+EXPORT_SYMBOL(napi_gro_frags); -+ -+/* Compute the checksum from gro_offset and return the folded value -+ * after adding in any pseudo checksum. -+ */ -+__sum16 __skb_gro_checksum_complete(struct sk_buff *skb) -+{ -+ __wsum wsum; -+ __sum16 sum; -+ -+ wsum = skb_checksum(skb, skb_gro_offset(skb), skb_gro_len(skb), 0); -+ -+ /* NAPI_GRO_CB(skb)->csum holds pseudo checksum */ -+ sum = csum_fold(csum_add(NAPI_GRO_CB(skb)->csum, wsum)); -+ if (likely(!sum)) { -+ if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) && -+ !skb->csum_complete_sw) -+ netdev_rx_csum_fault(skb->dev); -+ } -+ -+ NAPI_GRO_CB(skb)->csum = wsum; -+ NAPI_GRO_CB(skb)->csum_valid = 1; -+ -+ return sum; -+} -+EXPORT_SYMBOL(__skb_gro_checksum_complete); -+ -+/* -+ * net_rps_action_and_irq_enable sends any pending IPI's for rps. -+ * Note: called with local irq disabled, but exits with local irq enabled. -+ */ -+static void net_rps_action_and_irq_enable(struct softnet_data *sd) -+{ -+#ifdef CONFIG_RPS -+ struct softnet_data *remsd = sd->rps_ipi_list; -+ -+ if (remsd) { -+ sd->rps_ipi_list = NULL; -+ -+ local_irq_enable(); -+ -+ /* Send pending IPI's to kick RPS processing on remote cpus. */ -+ while (remsd) { -+ struct softnet_data *next = remsd->rps_ipi_next; -+ -+ if (cpu_online(remsd->cpu)) -+ smp_call_function_single_async(remsd->cpu, -+ &remsd->csd); -+ remsd = next; -+ } -+ } else -+#endif -+ local_irq_enable(); -+} -+ -+static bool sd_has_rps_ipi_waiting(struct softnet_data *sd) -+{ -+#ifdef CONFIG_RPS -+ return sd->rps_ipi_list != NULL; -+#else -+ return false; -+#endif -+} -+ -+static int process_backlog(struct napi_struct *napi, int quota) -+{ -+ int work = 0; -+ struct softnet_data *sd = container_of(napi, struct softnet_data, backlog); -+ -+ /* Check if we have pending ipi, its better to send them now, -+ * not waiting net_rx_action() end. -+ */ -+ if (sd_has_rps_ipi_waiting(sd)) { -+ local_irq_disable(); -+ net_rps_action_and_irq_enable(sd); -+ } -+ -+ napi->weight = weight_p; -+ local_irq_disable(); -+ while (1) { -+ struct sk_buff *skb; -+ -+ while ((skb = __skb_dequeue(&sd->process_queue))) { -+ rcu_read_lock(); -+ local_irq_enable(); -+ __netif_receive_skb(skb); -+ rcu_read_unlock(); -+ local_irq_disable(); -+ input_queue_head_incr(sd); -+ if (++work >= quota) { -+ local_irq_enable(); -+ return work; -+ } -+ } -+ -+ rps_lock(sd); -+ if (skb_queue_empty(&sd->input_pkt_queue)) { -+ /* -+ * Inline a custom version of __napi_complete(). -+ * only current cpu owns and manipulates this napi, -+ * and NAPI_STATE_SCHED is the only possible flag set -+ * on backlog. -+ * We can use a plain write instead of clear_bit(), -+ * and we dont need an smp_mb() memory barrier. -+ */ -+ napi->state = 0; -+ rps_unlock(sd); -+ -+ break; -+ } -+ -+ skb_queue_splice_tail_init(&sd->input_pkt_queue, -+ &sd->process_queue); -+ rps_unlock(sd); -+ } -+ local_irq_enable(); -+ -+ return work; -+} -+ -+/** -+ * __napi_schedule - schedule for receive -+ * @n: entry to schedule -+ * -+ * The entry's receive function will be scheduled to run. -+ * Consider using __napi_schedule_irqoff() if hard irqs are masked. -+ */ -+void __napi_schedule(struct napi_struct *n) -+{ -+ unsigned long flags; -+ -+ local_irq_save(flags); -+ ____napi_schedule(this_cpu_ptr(&softnet_data), n); -+ local_irq_restore(flags); -+} -+EXPORT_SYMBOL(__napi_schedule); -+ -+/** -+ * __napi_schedule_irqoff - schedule for receive -+ * @n: entry to schedule -+ * -+ * Variant of __napi_schedule() assuming hard irqs are masked -+ */ -+void __napi_schedule_irqoff(struct napi_struct *n) -+{ -+ ____napi_schedule(this_cpu_ptr(&softnet_data), n); -+} -+EXPORT_SYMBOL(__napi_schedule_irqoff); -+ -+void __napi_complete(struct napi_struct *n) -+{ -+ BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state)); -+ -+ list_del_init(&n->poll_list); -+ smp_mb__before_atomic(); -+ clear_bit(NAPI_STATE_SCHED, &n->state); -+} -+EXPORT_SYMBOL(__napi_complete); -+ -+void napi_complete_done(struct napi_struct *n, int work_done) -+{ -+ unsigned long flags; -+ -+ /* -+ * don't let napi dequeue from the cpu poll list -+ * just in case its running on a different cpu -+ */ -+ if (unlikely(test_bit(NAPI_STATE_NPSVC, &n->state))) -+ return; -+ -+ if (n->gro_list) { -+ unsigned long timeout = 0; -+ -+ if (work_done) -+ timeout = n->dev->gro_flush_timeout; -+ -+ if (timeout) -+ hrtimer_start(&n->timer, ns_to_ktime(timeout), -+ HRTIMER_MODE_REL_PINNED); -+ else -+ napi_gro_flush(n, false); -+ } -+ if (likely(list_empty(&n->poll_list))) { -+ WARN_ON_ONCE(!test_and_clear_bit(NAPI_STATE_SCHED, &n->state)); -+ } else { -+ /* If n->poll_list is not empty, we need to mask irqs */ -+ local_irq_save(flags); -+ __napi_complete(n); -+ local_irq_restore(flags); -+ } -+} -+EXPORT_SYMBOL(napi_complete_done); -+ -+/* must be called under rcu_read_lock(), as we dont take a reference */ -+struct napi_struct *napi_by_id(unsigned int napi_id) -+{ -+ unsigned int hash = napi_id % HASH_SIZE(napi_hash); -+ struct napi_struct *napi; -+ -+ hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node) -+ if (napi->napi_id == napi_id) -+ return napi; -+ -+ return NULL; -+} -+EXPORT_SYMBOL_GPL(napi_by_id); -+ -+void napi_hash_add(struct napi_struct *napi) -+{ -+ if (!test_and_set_bit(NAPI_STATE_HASHED, &napi->state)) { -+ -+ spin_lock(&napi_hash_lock); -+ -+ /* 0 is not a valid id, we also skip an id that is taken -+ * we expect both events to be extremely rare -+ */ -+ napi->napi_id = 0; -+ while (!napi->napi_id) { -+ napi->napi_id = ++napi_gen_id; -+ if (napi_by_id(napi->napi_id)) -+ napi->napi_id = 0; -+ } -+ -+ hlist_add_head_rcu(&napi->napi_hash_node, -+ &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]); -+ -+ spin_unlock(&napi_hash_lock); -+ } -+} -+EXPORT_SYMBOL_GPL(napi_hash_add); -+ -+/* Warning : caller is responsible to make sure rcu grace period -+ * is respected before freeing memory containing @napi -+ */ -+void napi_hash_del(struct napi_struct *napi) -+{ -+ spin_lock(&napi_hash_lock); -+ -+ if (test_and_clear_bit(NAPI_STATE_HASHED, &napi->state)) -+ hlist_del_rcu(&napi->napi_hash_node); -+ -+ spin_unlock(&napi_hash_lock); -+} -+EXPORT_SYMBOL_GPL(napi_hash_del); -+ -+static enum hrtimer_restart napi_watchdog(struct hrtimer *timer) -+{ -+ struct napi_struct *napi; -+ -+ napi = container_of(timer, struct napi_struct, timer); -+ if (napi->gro_list) -+ napi_schedule(napi); -+ -+ return HRTIMER_NORESTART; -+} -+ -+void netif_napi_add(struct net_device *dev, struct napi_struct *napi, -+ int (*poll)(struct napi_struct *, int), int weight) -+{ -+ INIT_LIST_HEAD(&napi->poll_list); -+ hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED); -+ napi->timer.function = napi_watchdog; -+ napi->gro_count = 0; -+ napi->gro_list = NULL; -+ napi->skb = NULL; -+ napi->poll = poll; -+ if (weight > NAPI_POLL_WEIGHT) -+ pr_err_once("netif_napi_add() called with weight %d on device %s\n", -+ weight, dev->name); -+ napi->weight = weight; -+ list_add(&napi->dev_list, &dev->napi_list); -+ napi->dev = dev; -+#ifdef CONFIG_NETPOLL -+ spin_lock_init(&napi->poll_lock); -+ napi->poll_owner = -1; -+#endif -+ set_bit(NAPI_STATE_SCHED, &napi->state); -+} -+EXPORT_SYMBOL(netif_napi_add); -+ -+void napi_disable(struct napi_struct *n) -+{ -+ might_sleep(); -+ set_bit(NAPI_STATE_DISABLE, &n->state); -+ -+ while (test_and_set_bit(NAPI_STATE_SCHED, &n->state)) -+ msleep(1); -+ -+ hrtimer_cancel(&n->timer); -+ -+ clear_bit(NAPI_STATE_DISABLE, &n->state); -+} -+EXPORT_SYMBOL(napi_disable); -+ -+void netif_napi_del(struct napi_struct *napi) -+{ -+ list_del_init(&napi->dev_list); -+ napi_free_frags(napi); -+ -+ kfree_skb_list(napi->gro_list); -+ napi->gro_list = NULL; -+ napi->gro_count = 0; -+} -+EXPORT_SYMBOL(netif_napi_del); -+ -+static int napi_poll(struct napi_struct *n, struct list_head *repoll) -+{ -+ void *have; -+ int work, weight; -+ -+ list_del_init(&n->poll_list); -+ -+ have = netpoll_poll_lock(n); -+ -+ weight = n->weight; -+ -+ /* This NAPI_STATE_SCHED test is for avoiding a race -+ * with netpoll's poll_napi(). Only the entity which -+ * obtains the lock and sees NAPI_STATE_SCHED set will -+ * actually make the ->poll() call. Therefore we avoid -+ * accidentally calling ->poll() when NAPI is not scheduled. -+ */ -+ work = 0; -+ if (test_bit(NAPI_STATE_SCHED, &n->state)) { -+ work = n->poll(n, weight); -+ trace_napi_poll(n); -+ } -+ -+ WARN_ON_ONCE(work > weight); -+ -+ if (likely(work < weight)) -+ goto out_unlock; -+ -+ /* Drivers must not modify the NAPI state if they -+ * consume the entire weight. In such cases this code -+ * still "owns" the NAPI instance and therefore can -+ * move the instance around on the list at-will. -+ */ -+ if (unlikely(napi_disable_pending(n))) { -+ napi_complete(n); -+ goto out_unlock; -+ } -+ -+ if (n->gro_list) { -+ /* flush too old packets -+ * If HZ < 1000, flush all packets. -+ */ -+ napi_gro_flush(n, HZ >= 1000); -+ } -+ -+ /* Some drivers may have called napi_schedule -+ * prior to exhausting their budget. -+ */ -+ if (unlikely(!list_empty(&n->poll_list))) { -+ pr_warn_once("%s: Budget exhausted after napi rescheduled\n", -+ n->dev ? n->dev->name : "backlog"); -+ goto out_unlock; -+ } -+ -+ list_add_tail(&n->poll_list, repoll); -+ -+out_unlock: -+ netpoll_poll_unlock(have); -+ -+ return work; -+} -+ -+static void net_rx_action(struct softirq_action *h) -+{ -+ struct softnet_data *sd = this_cpu_ptr(&softnet_data); -+ unsigned long time_limit = jiffies + 2; -+ int budget = netdev_budget; -+ LIST_HEAD(list); -+ LIST_HEAD(repoll); -+ -+ local_irq_disable(); -+ list_splice_init(&sd->poll_list, &list); -+ local_irq_enable(); -+ -+ for (;;) { -+ struct napi_struct *n; -+ -+ if (list_empty(&list)) { -+ if (!sd_has_rps_ipi_waiting(sd) && list_empty(&repoll)) -+ return; -+ break; -+ } -+ -+ n = list_first_entry(&list, struct napi_struct, poll_list); -+ budget -= napi_poll(n, &repoll); -+ -+ /* If softirq window is exhausted then punt. -+ * Allow this to run for 2 jiffies since which will allow -+ * an average latency of 1.5/HZ. -+ */ -+ if (unlikely(budget <= 0 || -+ time_after_eq(jiffies, time_limit))) { -+ sd->time_squeeze++; -+ break; -+ } -+ } -+ -+ local_irq_disable(); -+ -+ list_splice_tail_init(&sd->poll_list, &list); -+ list_splice_tail(&repoll, &list); -+ list_splice(&list, &sd->poll_list); -+ if (!list_empty(&sd->poll_list)) -+ __raise_softirq_irqoff(NET_RX_SOFTIRQ); -+ -+ net_rps_action_and_irq_enable(sd); -+} -+ -+struct netdev_adjacent { -+ struct net_device *dev; -+ -+ /* upper master flag, there can only be one master device per list */ -+ bool master; -+ -+ /* counter for the number of times this device was added to us */ -+ u16 ref_nr; -+ -+ /* private field for the users */ -+ void *private; -+ -+ struct list_head list; -+ struct rcu_head rcu; -+}; -+ -+static struct netdev_adjacent *__netdev_find_adj(struct net_device *dev, -+ struct net_device *adj_dev, -+ struct list_head *adj_list) -+{ -+ struct netdev_adjacent *adj; -+ -+ list_for_each_entry(adj, adj_list, list) { -+ if (adj->dev == adj_dev) -+ return adj; -+ } -+ return NULL; -+} -+ -+/** -+ * netdev_has_upper_dev - Check if device is linked to an upper device -+ * @dev: device -+ * @upper_dev: upper device to check -+ * -+ * Find out if a device is linked to specified upper device and return true -+ * in case it is. Note that this checks only immediate upper device, -+ * not through a complete stack of devices. The caller must hold the RTNL lock. -+ */ -+bool netdev_has_upper_dev(struct net_device *dev, -+ struct net_device *upper_dev) -+{ -+ ASSERT_RTNL(); -+ -+ return __netdev_find_adj(dev, upper_dev, &dev->all_adj_list.upper); -+} -+EXPORT_SYMBOL(netdev_has_upper_dev); -+ -+/** -+ * netdev_has_any_upper_dev - Check if device is linked to some device -+ * @dev: device -+ * -+ * Find out if a device is linked to an upper device and return true in case -+ * it is. The caller must hold the RTNL lock. -+ */ -+static bool netdev_has_any_upper_dev(struct net_device *dev) -+{ -+ ASSERT_RTNL(); -+ -+ return !list_empty(&dev->all_adj_list.upper); -+} -+ -+/** -+ * netdev_master_upper_dev_get - Get master upper device -+ * @dev: device -+ * -+ * Find a master upper device and return pointer to it or NULL in case -+ * it's not there. The caller must hold the RTNL lock. -+ */ -+struct net_device *netdev_master_upper_dev_get(struct net_device *dev) -+{ -+ struct netdev_adjacent *upper; -+ -+ ASSERT_RTNL(); -+ -+ if (list_empty(&dev->adj_list.upper)) -+ return NULL; -+ -+ upper = list_first_entry(&dev->adj_list.upper, -+ struct netdev_adjacent, list); -+ if (likely(upper->master)) -+ return upper->dev; -+ return NULL; -+} -+EXPORT_SYMBOL(netdev_master_upper_dev_get); -+ -+void *netdev_adjacent_get_private(struct list_head *adj_list) -+{ -+ struct netdev_adjacent *adj; -+ -+ adj = list_entry(adj_list, struct netdev_adjacent, list); -+ -+ return adj->private; -+} -+EXPORT_SYMBOL(netdev_adjacent_get_private); -+ -+/** -+ * netdev_upper_get_next_dev_rcu - Get the next dev from upper list -+ * @dev: device -+ * @iter: list_head ** of the current position -+ * -+ * Gets the next device from the dev's upper list, starting from iter -+ * position. The caller must hold RCU read lock. -+ */ -+struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev, -+ struct list_head **iter) -+{ -+ struct netdev_adjacent *upper; -+ -+ WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held()); -+ -+ upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); -+ -+ if (&upper->list == &dev->adj_list.upper) -+ return NULL; -+ -+ *iter = &upper->list; -+ -+ return upper->dev; -+} -+EXPORT_SYMBOL(netdev_upper_get_next_dev_rcu); -+ -+/** -+ * netdev_all_upper_get_next_dev_rcu - Get the next dev from upper list -+ * @dev: device -+ * @iter: list_head ** of the current position -+ * -+ * Gets the next device from the dev's upper list, starting from iter -+ * position. The caller must hold RCU read lock. -+ */ -+struct net_device *netdev_all_upper_get_next_dev_rcu(struct net_device *dev, -+ struct list_head **iter) -+{ -+ struct netdev_adjacent *upper; -+ -+ WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held()); -+ -+ upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); -+ -+ if (&upper->list == &dev->all_adj_list.upper) -+ return NULL; -+ -+ *iter = &upper->list; -+ -+ return upper->dev; -+} -+EXPORT_SYMBOL(netdev_all_upper_get_next_dev_rcu); -+ -+/** -+ * netdev_lower_get_next_private - Get the next ->private from the -+ * lower neighbour list -+ * @dev: device -+ * @iter: list_head ** of the current position -+ * -+ * Gets the next netdev_adjacent->private from the dev's lower neighbour -+ * list, starting from iter position. The caller must hold either hold the -+ * RTNL lock or its own locking that guarantees that the neighbour lower -+ * list will remain unchainged. -+ */ -+void *netdev_lower_get_next_private(struct net_device *dev, -+ struct list_head **iter) -+{ -+ struct netdev_adjacent *lower; -+ -+ lower = list_entry(*iter, struct netdev_adjacent, list); -+ -+ if (&lower->list == &dev->adj_list.lower) -+ return NULL; -+ -+ *iter = lower->list.next; -+ -+ return lower->private; -+} -+EXPORT_SYMBOL(netdev_lower_get_next_private); -+ -+/** -+ * netdev_lower_get_next_private_rcu - Get the next ->private from the -+ * lower neighbour list, RCU -+ * variant -+ * @dev: device -+ * @iter: list_head ** of the current position -+ * -+ * Gets the next netdev_adjacent->private from the dev's lower neighbour -+ * list, starting from iter position. The caller must hold RCU read lock. -+ */ -+void *netdev_lower_get_next_private_rcu(struct net_device *dev, -+ struct list_head **iter) -+{ -+ struct netdev_adjacent *lower; -+ -+ WARN_ON_ONCE(!rcu_read_lock_held()); -+ -+ lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); -+ -+ if (&lower->list == &dev->adj_list.lower) -+ return NULL; -+ -+ *iter = &lower->list; -+ -+ return lower->private; -+} -+EXPORT_SYMBOL(netdev_lower_get_next_private_rcu); -+ -+/** -+ * netdev_lower_get_next - Get the next device from the lower neighbour -+ * list -+ * @dev: device -+ * @iter: list_head ** of the current position -+ * -+ * Gets the next netdev_adjacent from the dev's lower neighbour -+ * list, starting from iter position. The caller must hold RTNL lock or -+ * its own locking that guarantees that the neighbour lower -+ * list will remain unchainged. -+ */ -+void *netdev_lower_get_next(struct net_device *dev, struct list_head **iter) -+{ -+ struct netdev_adjacent *lower; -+ -+ lower = list_entry((*iter)->next, struct netdev_adjacent, list); -+ -+ if (&lower->list == &dev->adj_list.lower) -+ return NULL; -+ -+ *iter = &lower->list; -+ -+ return lower->dev; -+} -+EXPORT_SYMBOL(netdev_lower_get_next); -+ -+/** -+ * netdev_lower_get_first_private_rcu - Get the first ->private from the -+ * lower neighbour list, RCU -+ * variant -+ * @dev: device -+ * -+ * Gets the first netdev_adjacent->private from the dev's lower neighbour -+ * list. The caller must hold RCU read lock. -+ */ -+void *netdev_lower_get_first_private_rcu(struct net_device *dev) -+{ -+ struct netdev_adjacent *lower; -+ -+ lower = list_first_or_null_rcu(&dev->adj_list.lower, -+ struct netdev_adjacent, list); -+ if (lower) -+ return lower->private; -+ return NULL; -+} -+EXPORT_SYMBOL(netdev_lower_get_first_private_rcu); -+ -+/** -+ * netdev_master_upper_dev_get_rcu - Get master upper device -+ * @dev: device -+ * -+ * Find a master upper device and return pointer to it or NULL in case -+ * it's not there. The caller must hold the RCU read lock. -+ */ -+struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev) -+{ -+ struct netdev_adjacent *upper; -+ -+ upper = list_first_or_null_rcu(&dev->adj_list.upper, -+ struct netdev_adjacent, list); -+ if (upper && likely(upper->master)) -+ return upper->dev; -+ return NULL; -+} -+EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu); -+ -+static int netdev_adjacent_sysfs_add(struct net_device *dev, -+ struct net_device *adj_dev, -+ struct list_head *dev_list) -+{ -+ char linkname[IFNAMSIZ+7]; -+ sprintf(linkname, dev_list == &dev->adj_list.upper ? -+ "upper_%s" : "lower_%s", adj_dev->name); -+ return sysfs_create_link(&(dev->dev.kobj), &(adj_dev->dev.kobj), -+ linkname); -+} -+static void netdev_adjacent_sysfs_del(struct net_device *dev, -+ char *name, -+ struct list_head *dev_list) -+{ -+ char linkname[IFNAMSIZ+7]; -+ sprintf(linkname, dev_list == &dev->adj_list.upper ? -+ "upper_%s" : "lower_%s", name); -+ sysfs_remove_link(&(dev->dev.kobj), linkname); -+} -+ -+static inline bool netdev_adjacent_is_neigh_list(struct net_device *dev, -+ struct net_device *adj_dev, -+ struct list_head *dev_list) -+{ -+ return (dev_list == &dev->adj_list.upper || -+ dev_list == &dev->adj_list.lower) && -+ net_eq(dev_net(dev), dev_net(adj_dev)); -+} -+ -+static int __netdev_adjacent_dev_insert(struct net_device *dev, -+ struct net_device *adj_dev, -+ struct list_head *dev_list, -+ void *private, bool master) -+{ -+ struct netdev_adjacent *adj; -+ int ret; -+ -+ adj = __netdev_find_adj(dev, adj_dev, dev_list); -+ -+ if (adj) { -+ adj->ref_nr++; -+ return 0; -+ } -+ -+ adj = kmalloc(sizeof(*adj), GFP_KERNEL); -+ if (!adj) -+ return -ENOMEM; -+ -+ adj->dev = adj_dev; -+ adj->master = master; -+ adj->ref_nr = 1; -+ adj->private = private; -+ dev_hold(adj_dev); -+ -+ pr_debug("dev_hold for %s, because of link added from %s to %s\n", -+ adj_dev->name, dev->name, adj_dev->name); -+ -+ if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) { -+ ret = netdev_adjacent_sysfs_add(dev, adj_dev, dev_list); -+ if (ret) -+ goto free_adj; -+ } -+ -+ /* Ensure that master link is always the first item in list. */ -+ if (master) { -+ ret = sysfs_create_link(&(dev->dev.kobj), -+ &(adj_dev->dev.kobj), "master"); -+ if (ret) -+ goto remove_symlinks; -+ -+ list_add_rcu(&adj->list, dev_list); -+ } else { -+ list_add_tail_rcu(&adj->list, dev_list); -+ } -+ -+ return 0; -+ -+remove_symlinks: -+ if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) -+ netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list); -+free_adj: -+ kfree(adj); -+ dev_put(adj_dev); -+ -+ return ret; -+} -+ -+static void __netdev_adjacent_dev_remove(struct net_device *dev, -+ struct net_device *adj_dev, -+ struct list_head *dev_list) -+{ -+ struct netdev_adjacent *adj; -+ -+ adj = __netdev_find_adj(dev, adj_dev, dev_list); -+ -+ if (!adj) { -+ pr_err("tried to remove device %s from %s\n", -+ dev->name, adj_dev->name); -+ BUG(); -+ } -+ -+ if (adj->ref_nr > 1) { -+ pr_debug("%s to %s ref_nr-- = %d\n", dev->name, adj_dev->name, -+ adj->ref_nr-1); -+ adj->ref_nr--; -+ return; -+ } -+ -+ if (adj->master) -+ sysfs_remove_link(&(dev->dev.kobj), "master"); -+ -+ if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) -+ netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list); -+ -+ list_del_rcu(&adj->list); -+ pr_debug("dev_put for %s, because link removed from %s to %s\n", -+ adj_dev->name, dev->name, adj_dev->name); -+ dev_put(adj_dev); -+ kfree_rcu(adj, rcu); -+} -+ -+static int __netdev_adjacent_dev_link_lists(struct net_device *dev, -+ struct net_device *upper_dev, -+ struct list_head *up_list, -+ struct list_head *down_list, -+ void *private, bool master) -+{ -+ int ret; -+ -+ ret = __netdev_adjacent_dev_insert(dev, upper_dev, up_list, private, -+ master); -+ if (ret) -+ return ret; -+ -+ ret = __netdev_adjacent_dev_insert(upper_dev, dev, down_list, private, -+ false); -+ if (ret) { -+ __netdev_adjacent_dev_remove(dev, upper_dev, up_list); -+ return ret; -+ } -+ -+ return 0; -+} -+ -+static int __netdev_adjacent_dev_link(struct net_device *dev, -+ struct net_device *upper_dev) -+{ -+ return __netdev_adjacent_dev_link_lists(dev, upper_dev, -+ &dev->all_adj_list.upper, -+ &upper_dev->all_adj_list.lower, -+ NULL, false); -+} -+ -+static void __netdev_adjacent_dev_unlink_lists(struct net_device *dev, -+ struct net_device *upper_dev, -+ struct list_head *up_list, -+ struct list_head *down_list) -+{ -+ __netdev_adjacent_dev_remove(dev, upper_dev, up_list); -+ __netdev_adjacent_dev_remove(upper_dev, dev, down_list); -+} -+ -+static void __netdev_adjacent_dev_unlink(struct net_device *dev, -+ struct net_device *upper_dev) -+{ -+ __netdev_adjacent_dev_unlink_lists(dev, upper_dev, -+ &dev->all_adj_list.upper, -+ &upper_dev->all_adj_list.lower); -+} -+ -+static int __netdev_adjacent_dev_link_neighbour(struct net_device *dev, -+ struct net_device *upper_dev, -+ void *private, bool master) -+{ -+ int ret = __netdev_adjacent_dev_link(dev, upper_dev); -+ -+ if (ret) -+ return ret; -+ -+ ret = __netdev_adjacent_dev_link_lists(dev, upper_dev, -+ &dev->adj_list.upper, -+ &upper_dev->adj_list.lower, -+ private, master); -+ if (ret) { -+ __netdev_adjacent_dev_unlink(dev, upper_dev); -+ return ret; -+ } -+ -+ return 0; -+} -+ -+static void __netdev_adjacent_dev_unlink_neighbour(struct net_device *dev, -+ struct net_device *upper_dev) -+{ -+ __netdev_adjacent_dev_unlink(dev, upper_dev); -+ __netdev_adjacent_dev_unlink_lists(dev, upper_dev, -+ &dev->adj_list.upper, -+ &upper_dev->adj_list.lower); -+} -+ -+static int __netdev_upper_dev_link(struct net_device *dev, -+ struct net_device *upper_dev, bool master, -+ void *private) -+{ -+ struct netdev_adjacent *i, *j, *to_i, *to_j; -+ int ret = 0; -+ -+ ASSERT_RTNL(); -+ -+ if (dev == upper_dev) -+ return -EBUSY; -+ -+ /* To prevent loops, check if dev is not upper device to upper_dev. */ -+ if (__netdev_find_adj(upper_dev, dev, &upper_dev->all_adj_list.upper)) -+ return -EBUSY; -+ -+ if (__netdev_find_adj(dev, upper_dev, &dev->adj_list.upper)) -+ return -EEXIST; -+ -+ if (master && netdev_master_upper_dev_get(dev)) -+ return -EBUSY; -+ -+ ret = __netdev_adjacent_dev_link_neighbour(dev, upper_dev, private, -+ master); -+ if (ret) -+ return ret; -+ -+ /* Now that we linked these devs, make all the upper_dev's -+ * all_adj_list.upper visible to every dev's all_adj_list.lower an -+ * versa, and don't forget the devices itself. All of these -+ * links are non-neighbours. -+ */ -+ list_for_each_entry(i, &dev->all_adj_list.lower, list) { -+ list_for_each_entry(j, &upper_dev->all_adj_list.upper, list) { -+ pr_debug("Interlinking %s with %s, non-neighbour\n", -+ i->dev->name, j->dev->name); -+ ret = __netdev_adjacent_dev_link(i->dev, j->dev); -+ if (ret) -+ goto rollback_mesh; -+ } -+ } -+ -+ /* add dev to every upper_dev's upper device */ -+ list_for_each_entry(i, &upper_dev->all_adj_list.upper, list) { -+ pr_debug("linking %s's upper device %s with %s\n", -+ upper_dev->name, i->dev->name, dev->name); -+ ret = __netdev_adjacent_dev_link(dev, i->dev); -+ if (ret) -+ goto rollback_upper_mesh; -+ } -+ -+ /* add upper_dev to every dev's lower device */ -+ list_for_each_entry(i, &dev->all_adj_list.lower, list) { -+ pr_debug("linking %s's lower device %s with %s\n", dev->name, -+ i->dev->name, upper_dev->name); -+ ret = __netdev_adjacent_dev_link(i->dev, upper_dev); -+ if (ret) -+ goto rollback_lower_mesh; -+ } -+ -+ call_netdevice_notifiers(NETDEV_CHANGEUPPER, dev); -+ return 0; -+ -+rollback_lower_mesh: -+ to_i = i; -+ list_for_each_entry(i, &dev->all_adj_list.lower, list) { -+ if (i == to_i) -+ break; -+ __netdev_adjacent_dev_unlink(i->dev, upper_dev); -+ } -+ -+ i = NULL; -+ -+rollback_upper_mesh: -+ to_i = i; -+ list_for_each_entry(i, &upper_dev->all_adj_list.upper, list) { -+ if (i == to_i) -+ break; -+ __netdev_adjacent_dev_unlink(dev, i->dev); -+ } -+ -+ i = j = NULL; -+ -+rollback_mesh: -+ to_i = i; -+ to_j = j; -+ list_for_each_entry(i, &dev->all_adj_list.lower, list) { -+ list_for_each_entry(j, &upper_dev->all_adj_list.upper, list) { -+ if (i == to_i && j == to_j) -+ break; -+ __netdev_adjacent_dev_unlink(i->dev, j->dev); -+ } -+ if (i == to_i) -+ break; -+ } -+ -+ __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev); -+ -+ return ret; -+} -+ -+/** -+ * netdev_upper_dev_link - Add a link to the upper device -+ * @dev: device -+ * @upper_dev: new upper device -+ * -+ * Adds a link to device which is upper to this one. The caller must hold -+ * the RTNL lock. On a failure a negative errno code is returned. -+ * On success the reference counts are adjusted and the function -+ * returns zero. -+ */ -+int netdev_upper_dev_link(struct net_device *dev, -+ struct net_device *upper_dev) -+{ -+ return __netdev_upper_dev_link(dev, upper_dev, false, NULL); -+} -+EXPORT_SYMBOL(netdev_upper_dev_link); -+ -+/** -+ * netdev_master_upper_dev_link - Add a master link to the upper device -+ * @dev: device -+ * @upper_dev: new upper device -+ * -+ * Adds a link to device which is upper to this one. In this case, only -+ * one master upper device can be linked, although other non-master devices -+ * might be linked as well. The caller must hold the RTNL lock. -+ * On a failure a negative errno code is returned. On success the reference -+ * counts are adjusted and the function returns zero. -+ */ -+int netdev_master_upper_dev_link(struct net_device *dev, -+ struct net_device *upper_dev) -+{ -+ return __netdev_upper_dev_link(dev, upper_dev, true, NULL); -+} -+EXPORT_SYMBOL(netdev_master_upper_dev_link); -+ -+int netdev_master_upper_dev_link_private(struct net_device *dev, -+ struct net_device *upper_dev, -+ void *private) -+{ -+ return __netdev_upper_dev_link(dev, upper_dev, true, private); -+} -+EXPORT_SYMBOL(netdev_master_upper_dev_link_private); -+ -+/** -+ * netdev_upper_dev_unlink - Removes a link to upper device -+ * @dev: device -+ * @upper_dev: new upper device -+ * -+ * Removes a link to device which is upper to this one. The caller must hold -+ * the RTNL lock. -+ */ -+void netdev_upper_dev_unlink(struct net_device *dev, -+ struct net_device *upper_dev) -+{ -+ struct netdev_adjacent *i, *j; -+ ASSERT_RTNL(); -+ -+ __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev); -+ -+ /* Here is the tricky part. We must remove all dev's lower -+ * devices from all upper_dev's upper devices and vice -+ * versa, to maintain the graph relationship. -+ */ -+ list_for_each_entry(i, &dev->all_adj_list.lower, list) -+ list_for_each_entry(j, &upper_dev->all_adj_list.upper, list) -+ __netdev_adjacent_dev_unlink(i->dev, j->dev); -+ -+ /* remove also the devices itself from lower/upper device -+ * list -+ */ -+ list_for_each_entry(i, &dev->all_adj_list.lower, list) -+ __netdev_adjacent_dev_unlink(i->dev, upper_dev); -+ -+ list_for_each_entry(i, &upper_dev->all_adj_list.upper, list) -+ __netdev_adjacent_dev_unlink(dev, i->dev); -+ -+ call_netdevice_notifiers(NETDEV_CHANGEUPPER, dev); -+} -+EXPORT_SYMBOL(netdev_upper_dev_unlink); -+ -+/** -+ * netdev_bonding_info_change - Dispatch event about slave change -+ * @dev: device -+ * @bonding_info: info to dispatch -+ * -+ * Send NETDEV_BONDING_INFO to netdev notifiers with info. -+ * The caller must hold the RTNL lock. -+ */ -+void netdev_bonding_info_change(struct net_device *dev, -+ struct netdev_bonding_info *bonding_info) -+{ -+ struct netdev_notifier_bonding_info info; -+ -+ memcpy(&info.bonding_info, bonding_info, -+ sizeof(struct netdev_bonding_info)); -+ call_netdevice_notifiers_info(NETDEV_BONDING_INFO, dev, -+ &info.info); -+} -+EXPORT_SYMBOL(netdev_bonding_info_change); -+ -+static void netdev_adjacent_add_links(struct net_device *dev) -+{ -+ struct netdev_adjacent *iter; -+ -+ struct net *net = dev_net(dev); -+ -+ list_for_each_entry(iter, &dev->adj_list.upper, list) { -+ if (!net_eq(net,dev_net(iter->dev))) -+ continue; -+ netdev_adjacent_sysfs_add(iter->dev, dev, -+ &iter->dev->adj_list.lower); -+ netdev_adjacent_sysfs_add(dev, iter->dev, -+ &dev->adj_list.upper); -+ } -+ -+ list_for_each_entry(iter, &dev->adj_list.lower, list) { -+ if (!net_eq(net,dev_net(iter->dev))) -+ continue; -+ netdev_adjacent_sysfs_add(iter->dev, dev, -+ &iter->dev->adj_list.upper); -+ netdev_adjacent_sysfs_add(dev, iter->dev, -+ &dev->adj_list.lower); -+ } -+} -+ -+static void netdev_adjacent_del_links(struct net_device *dev) -+{ -+ struct netdev_adjacent *iter; -+ -+ struct net *net = dev_net(dev); -+ -+ list_for_each_entry(iter, &dev->adj_list.upper, list) { -+ if (!net_eq(net,dev_net(iter->dev))) -+ continue; -+ netdev_adjacent_sysfs_del(iter->dev, dev->name, -+ &iter->dev->adj_list.lower); -+ netdev_adjacent_sysfs_del(dev, iter->dev->name, -+ &dev->adj_list.upper); -+ } -+ -+ list_for_each_entry(iter, &dev->adj_list.lower, list) { -+ if (!net_eq(net,dev_net(iter->dev))) -+ continue; -+ netdev_adjacent_sysfs_del(iter->dev, dev->name, -+ &iter->dev->adj_list.upper); -+ netdev_adjacent_sysfs_del(dev, iter->dev->name, -+ &dev->adj_list.lower); -+ } -+} -+ -+void netdev_adjacent_rename_links(struct net_device *dev, char *oldname) -+{ -+ struct netdev_adjacent *iter; -+ -+ struct net *net = dev_net(dev); -+ -+ list_for_each_entry(iter, &dev->adj_list.upper, list) { -+ if (!net_eq(net,dev_net(iter->dev))) -+ continue; -+ netdev_adjacent_sysfs_del(iter->dev, oldname, -+ &iter->dev->adj_list.lower); -+ netdev_adjacent_sysfs_add(iter->dev, dev, -+ &iter->dev->adj_list.lower); -+ } -+ -+ list_for_each_entry(iter, &dev->adj_list.lower, list) { -+ if (!net_eq(net,dev_net(iter->dev))) -+ continue; -+ netdev_adjacent_sysfs_del(iter->dev, oldname, -+ &iter->dev->adj_list.upper); -+ netdev_adjacent_sysfs_add(iter->dev, dev, -+ &iter->dev->adj_list.upper); -+ } -+} -+ -+void *netdev_lower_dev_get_private(struct net_device *dev, -+ struct net_device *lower_dev) -+{ -+ struct netdev_adjacent *lower; -+ -+ if (!lower_dev) -+ return NULL; -+ lower = __netdev_find_adj(dev, lower_dev, &dev->adj_list.lower); -+ if (!lower) -+ return NULL; -+ -+ return lower->private; -+} -+EXPORT_SYMBOL(netdev_lower_dev_get_private); -+ -+ -+int dev_get_nest_level(struct net_device *dev, -+ bool (*type_check)(struct net_device *dev)) -+{ -+ struct net_device *lower = NULL; -+ struct list_head *iter; -+ int max_nest = -1; -+ int nest; -+ -+ ASSERT_RTNL(); -+ -+ netdev_for_each_lower_dev(dev, lower, iter) { -+ nest = dev_get_nest_level(lower, type_check); -+ if (max_nest < nest) -+ max_nest = nest; -+ } -+ -+ if (type_check(dev)) -+ max_nest++; -+ -+ return max_nest; -+} -+EXPORT_SYMBOL(dev_get_nest_level); -+ -+static void dev_change_rx_flags(struct net_device *dev, int flags) -+{ -+ const struct net_device_ops *ops = dev->netdev_ops; -+ -+ if (ops->ndo_change_rx_flags) -+ ops->ndo_change_rx_flags(dev, flags); -+} -+ -+static int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify) -+{ -+ unsigned int old_flags = dev->flags; -+ kuid_t uid; -+ kgid_t gid; -+ -+ ASSERT_RTNL(); -+ -+ dev->flags |= IFF_PROMISC; -+ dev->promiscuity += inc; -+ if (dev->promiscuity == 0) { -+ /* -+ * Avoid overflow. -+ * If inc causes overflow, untouch promisc and return error. -+ */ -+ if (inc < 0) -+ dev->flags &= ~IFF_PROMISC; -+ else { -+ dev->promiscuity -= inc; -+ pr_warn("%s: promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n", -+ dev->name); -+ return -EOVERFLOW; -+ } -+ } -+ if (dev->flags != old_flags) { -+ pr_info("device %s %s promiscuous mode\n", -+ dev->name, -+ dev->flags & IFF_PROMISC ? "entered" : "left"); -+ if (audit_enabled) { -+ current_uid_gid(&uid, &gid); -+ audit_log(current->audit_context, GFP_ATOMIC, -+ AUDIT_ANOM_PROMISCUOUS, -+ "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u", -+ dev->name, (dev->flags & IFF_PROMISC), -+ (old_flags & IFF_PROMISC), -+ from_kuid(&init_user_ns, audit_get_loginuid(current)), -+ from_kuid(&init_user_ns, uid), -+ from_kgid(&init_user_ns, gid), -+ audit_get_sessionid(current)); -+ } -+ -+ dev_change_rx_flags(dev, IFF_PROMISC); -+ } -+ if (notify) -+ __dev_notify_flags(dev, old_flags, IFF_PROMISC); -+ return 0; -+} -+ -+/** -+ * dev_set_promiscuity - update promiscuity count on a device -+ * @dev: device -+ * @inc: modifier -+ * -+ * Add or remove promiscuity from a device. While the count in the device -+ * remains above zero the interface remains promiscuous. Once it hits zero -+ * the device reverts back to normal filtering operation. A negative inc -+ * value is used to drop promiscuity on the device. -+ * Return 0 if successful or a negative errno code on error. -+ */ -+int dev_set_promiscuity(struct net_device *dev, int inc) -+{ -+ unsigned int old_flags = dev->flags; -+ int err; -+ -+ err = __dev_set_promiscuity(dev, inc, true); + err = dev_get_valid_name(net, dev, newname); +- if (err < 0) { +- write_seqcount_end(&devnet_rename_seq); +- return err; +- } + if (err < 0) -+ return err; -+ if (dev->flags != old_flags) -+ dev_set_rx_mode(dev); -+ return err; -+} -+EXPORT_SYMBOL(dev_set_promiscuity); -+ -+static int __dev_set_allmulti(struct net_device *dev, int inc, bool notify) -+{ -+ unsigned int old_flags = dev->flags, old_gflags = dev->gflags; -+ -+ ASSERT_RTNL(); -+ -+ dev->flags |= IFF_ALLMULTI; -+ dev->allmulti += inc; -+ if (dev->allmulti == 0) { -+ /* -+ * Avoid overflow. -+ * If inc causes overflow, untouch allmulti and return error. -+ */ -+ if (inc < 0) -+ dev->flags &= ~IFF_ALLMULTI; -+ else { -+ dev->allmulti -= inc; -+ pr_warn("%s: allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n", -+ dev->name); -+ return -EOVERFLOW; -+ } -+ } -+ if (dev->flags ^ old_flags) { -+ dev_change_rx_flags(dev, IFF_ALLMULTI); -+ dev_set_rx_mode(dev); -+ if (notify) -+ __dev_notify_flags(dev, old_flags, -+ dev->gflags ^ old_gflags); -+ } -+ return 0; -+} -+ -+/** -+ * dev_set_allmulti - update allmulti count on a device -+ * @dev: device -+ * @inc: modifier -+ * -+ * Add or remove reception of all multicast frames to a device. While the -+ * count in the device remains above zero the interface remains listening -+ * to all interfaces. Once it hits zero the device reverts back to normal -+ * filtering operation. A negative @inc value is used to drop the counter -+ * when releasing a resource needing all multicasts. -+ * Return 0 if successful or a negative errno code on error. -+ */ -+ -+int dev_set_allmulti(struct net_device *dev, int inc) -+{ -+ return __dev_set_allmulti(dev, inc, true); -+} -+EXPORT_SYMBOL(dev_set_allmulti); -+ -+/* -+ * Upload unicast and multicast address lists to device and -+ * configure RX filtering. When the device doesn't support unicast -+ * filtering it is put in promiscuous mode while unicast addresses -+ * are present. -+ */ -+void __dev_set_rx_mode(struct net_device *dev) -+{ -+ const struct net_device_ops *ops = dev->netdev_ops; -+ -+ /* dev_open will call this function so the list will stay sane. */ -+ if (!(dev->flags&IFF_UP)) -+ return; -+ -+ if (!netif_device_present(dev)) -+ return; -+ -+ if (!(dev->priv_flags & IFF_UNICAST_FLT)) { -+ /* Unicast addresses changes may only happen under the rtnl, -+ * therefore calling __dev_set_promiscuity here is safe. -+ */ -+ if (!netdev_uc_empty(dev) && !dev->uc_promisc) { -+ __dev_set_promiscuity(dev, 1, false); -+ dev->uc_promisc = true; -+ } else if (netdev_uc_empty(dev) && dev->uc_promisc) { -+ __dev_set_promiscuity(dev, -1, false); -+ dev->uc_promisc = false; -+ } -+ } -+ -+ if (ops->ndo_set_rx_mode) -+ ops->ndo_set_rx_mode(dev); -+} -+ -+void dev_set_rx_mode(struct net_device *dev) -+{ -+ netif_addr_lock_bh(dev); -+ __dev_set_rx_mode(dev); -+ netif_addr_unlock_bh(dev); -+} -+ -+/** -+ * dev_get_flags - get flags reported to userspace -+ * @dev: device -+ * -+ * Get the combination of flag bits exported through APIs to userspace. -+ */ -+unsigned int dev_get_flags(const struct net_device *dev) -+{ -+ unsigned int flags; -+ -+ flags = (dev->flags & ~(IFF_PROMISC | -+ IFF_ALLMULTI | -+ IFF_RUNNING | -+ IFF_LOWER_UP | -+ IFF_DORMANT)) | -+ (dev->gflags & (IFF_PROMISC | -+ IFF_ALLMULTI)); -+ -+ if (netif_running(dev)) { -+ if (netif_oper_up(dev)) -+ flags |= IFF_RUNNING; -+ if (netif_carrier_ok(dev)) -+ flags |= IFF_LOWER_UP; -+ if (netif_dormant(dev)) -+ flags |= IFF_DORMANT; -+ } -+ -+ return flags; -+} -+EXPORT_SYMBOL(dev_get_flags); -+ -+int __dev_change_flags(struct net_device *dev, unsigned int flags) -+{ -+ unsigned int old_flags = dev->flags; -+ int ret; -+ -+ ASSERT_RTNL(); -+ -+ /* -+ * Set the flags on our device. -+ */ -+ -+ dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP | -+ IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL | -+ IFF_AUTOMEDIA)) | -+ (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC | -+ IFF_ALLMULTI)); -+ -+ /* -+ * Load in the correct multicast list now the flags have changed. -+ */ -+ -+ if ((old_flags ^ flags) & IFF_MULTICAST) -+ dev_change_rx_flags(dev, IFF_MULTICAST); -+ -+ dev_set_rx_mode(dev); -+ -+ /* -+ * Have we downed the interface. We handle IFF_UP ourselves -+ * according to user attempts to set it, rather than blindly -+ * setting it. -+ */ -+ -+ ret = 0; -+ if ((old_flags ^ flags) & IFF_UP) -+ ret = ((old_flags & IFF_UP) ? __dev_close : __dev_open)(dev); -+ -+ if ((flags ^ dev->gflags) & IFF_PROMISC) { -+ int inc = (flags & IFF_PROMISC) ? 1 : -1; -+ unsigned int old_flags = dev->flags; -+ -+ dev->gflags ^= IFF_PROMISC; -+ -+ if (__dev_set_promiscuity(dev, inc, false) >= 0) -+ if (dev->flags != old_flags) -+ dev_set_rx_mode(dev); -+ } -+ -+ /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI -+ is important. Some (broken) drivers set IFF_PROMISC, when -+ IFF_ALLMULTI is requested not asking us and not reporting. -+ */ -+ if ((flags ^ dev->gflags) & IFF_ALLMULTI) { -+ int inc = (flags & IFF_ALLMULTI) ? 1 : -1; -+ -+ dev->gflags ^= IFF_ALLMULTI; -+ __dev_set_allmulti(dev, inc, false); -+ } -+ -+ return ret; -+} -+ -+void __dev_notify_flags(struct net_device *dev, unsigned int old_flags, -+ unsigned int gchanges) -+{ -+ unsigned int changes = dev->flags ^ old_flags; -+ -+ if (gchanges) -+ rtmsg_ifinfo(RTM_NEWLINK, dev, gchanges, GFP_ATOMIC); -+ -+ if (changes & IFF_UP) { -+ if (dev->flags & IFF_UP) -+ call_netdevice_notifiers(NETDEV_UP, dev); -+ else -+ call_netdevice_notifiers(NETDEV_DOWN, dev); -+ } -+ -+ if (dev->flags & IFF_UP && -+ (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE))) { -+ struct netdev_notifier_change_info change_info; -+ -+ change_info.flags_changed = changes; -+ call_netdevice_notifiers_info(NETDEV_CHANGE, dev, -+ &change_info.info); -+ } -+} -+ -+/** -+ * dev_change_flags - change device settings -+ * @dev: device -+ * @flags: device state flags -+ * -+ * Change settings on device based state flags. The flags are -+ * in the userspace exported format. -+ */ -+int dev_change_flags(struct net_device *dev, unsigned int flags) -+{ -+ int ret; -+ unsigned int changes, old_flags = dev->flags, old_gflags = dev->gflags; -+ -+ ret = __dev_change_flags(dev, flags); -+ if (ret < 0) -+ return ret; -+ -+ changes = (old_flags ^ dev->flags) | (old_gflags ^ dev->gflags); -+ __dev_notify_flags(dev, old_flags, changes); -+ return ret; -+} -+EXPORT_SYMBOL(dev_change_flags); -+ -+static int __dev_set_mtu(struct net_device *dev, int new_mtu) -+{ -+ const struct net_device_ops *ops = dev->netdev_ops; -+ -+ if (ops->ndo_change_mtu) -+ return ops->ndo_change_mtu(dev, new_mtu); -+ -+ dev->mtu = new_mtu; -+ return 0; -+} -+ -+/** -+ * dev_set_mtu - Change maximum transfer unit -+ * @dev: device -+ * @new_mtu: new transfer unit -+ * -+ * Change the maximum transfer size of the network device. -+ */ -+int dev_set_mtu(struct net_device *dev, int new_mtu) -+{ -+ int err, orig_mtu; -+ -+ if (new_mtu == dev->mtu) -+ return 0; -+ -+ /* MTU must be positive. */ -+ if (new_mtu < 0) -+ return -EINVAL; -+ -+ if (!netif_device_present(dev)) -+ return -ENODEV; -+ -+ err = call_netdevice_notifiers(NETDEV_PRECHANGEMTU, dev); -+ err = notifier_to_errno(err); -+ if (err) -+ return err; -+ -+ orig_mtu = dev->mtu; -+ err = __dev_set_mtu(dev, new_mtu); -+ -+ if (!err) { -+ err = call_netdevice_notifiers(NETDEV_CHANGEMTU, dev); -+ err = notifier_to_errno(err); -+ if (err) { -+ /* setting mtu back and notifying everyone again, -+ * so that they have a chance to revert changes. -+ */ -+ __dev_set_mtu(dev, orig_mtu); -+ call_netdevice_notifiers(NETDEV_CHANGEMTU, dev); -+ } -+ } -+ return err; -+} -+EXPORT_SYMBOL(dev_set_mtu); -+ -+/** -+ * dev_set_group - Change group this device belongs to -+ * @dev: device -+ * @new_group: group this device should belong to -+ */ -+void dev_set_group(struct net_device *dev, int new_group) -+{ -+ dev->group = new_group; -+} -+EXPORT_SYMBOL(dev_set_group); -+ -+/** -+ * dev_set_mac_address - Change Media Access Control Address -+ * @dev: device -+ * @sa: new address -+ * -+ * Change the hardware (MAC) address of the device -+ */ -+int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa) -+{ -+ const struct net_device_ops *ops = dev->netdev_ops; -+ int err; -+ -+ if (!ops->ndo_set_mac_address) -+ return -EOPNOTSUPP; -+ if (sa->sa_family != dev->type) -+ return -EINVAL; -+ if (!netif_device_present(dev)) -+ return -ENODEV; -+ err = ops->ndo_set_mac_address(dev, sa); -+ if (err) -+ return err; -+ dev->addr_assign_type = NET_ADDR_SET; -+ call_netdevice_notifiers(NETDEV_CHANGEADDR, dev); -+ add_device_randomness(dev->dev_addr, dev->addr_len); -+ return 0; -+} -+EXPORT_SYMBOL(dev_set_mac_address); -+ -+/** -+ * dev_change_carrier - Change device carrier -+ * @dev: device -+ * @new_carrier: new value -+ * -+ * Change device carrier -+ */ -+int dev_change_carrier(struct net_device *dev, bool new_carrier) -+{ -+ const struct net_device_ops *ops = dev->netdev_ops; -+ -+ if (!ops->ndo_change_carrier) -+ return -EOPNOTSUPP; -+ if (!netif_device_present(dev)) -+ return -ENODEV; -+ return ops->ndo_change_carrier(dev, new_carrier); -+} -+EXPORT_SYMBOL(dev_change_carrier); -+ -+/** -+ * dev_get_phys_port_id - Get device physical port ID -+ * @dev: device -+ * @ppid: port ID -+ * -+ * Get device physical port ID -+ */ -+int dev_get_phys_port_id(struct net_device *dev, -+ struct netdev_phys_item_id *ppid) -+{ -+ const struct net_device_ops *ops = dev->netdev_ops; -+ -+ if (!ops->ndo_get_phys_port_id) -+ return -EOPNOTSUPP; -+ return ops->ndo_get_phys_port_id(dev, ppid); -+} -+EXPORT_SYMBOL(dev_get_phys_port_id); -+ -+/** -+ * dev_get_phys_port_name - Get device physical port name -+ * @dev: device -+ * @name: port name -+ * -+ * Get device physical port name -+ */ -+int dev_get_phys_port_name(struct net_device *dev, -+ char *name, size_t len) -+{ -+ const struct net_device_ops *ops = dev->netdev_ops; -+ -+ if (!ops->ndo_get_phys_port_name) -+ return -EOPNOTSUPP; -+ return ops->ndo_get_phys_port_name(dev, name, len); -+} -+EXPORT_SYMBOL(dev_get_phys_port_name); -+ -+/** -+ * dev_new_index - allocate an ifindex -+ * @net: the applicable net namespace -+ * -+ * Returns a suitable unique value for a new device interface -+ * number. The caller must hold the rtnl semaphore or the -+ * dev_base_lock to be sure it remains unique. -+ */ -+static int dev_new_index(struct net *net) -+{ -+ int ifindex = net->ifindex; -+ for (;;) { -+ if (++ifindex <= 0) -+ ifindex = 1; -+ if (!__dev_get_by_index(net, ifindex)) -+ return net->ifindex = ifindex; -+ } -+} -+ -+/* Delayed registration/unregisteration */ -+static LIST_HEAD(net_todo_list); -+DECLARE_WAIT_QUEUE_HEAD(netdev_unregistering_wq); -+ -+static void net_set_todo(struct net_device *dev) -+{ -+ list_add_tail(&dev->todo_list, &net_todo_list); -+ dev_net(dev)->dev_unreg_count++; -+} -+ -+static void rollback_registered_many(struct list_head *head) -+{ -+ struct net_device *dev, *tmp; -+ LIST_HEAD(close_head); -+ -+ BUG_ON(dev_boot_phase); -+ ASSERT_RTNL(); -+ -+ list_for_each_entry_safe(dev, tmp, head, unreg_list) { -+ /* Some devices call without registering -+ * for initialization unwind. Remove those -+ * devices and proceed with the remaining. -+ */ -+ if (dev->reg_state == NETREG_UNINITIALIZED) { -+ pr_debug("unregister_netdevice: device %s/%p never was registered\n", -+ dev->name, dev); -+ -+ WARN_ON(1); -+ list_del(&dev->unreg_list); -+ continue; -+ } -+ dev->dismantle = true; -+ BUG_ON(dev->reg_state != NETREG_REGISTERED); -+ } -+ -+ /* If device is running, close it first. */ -+ list_for_each_entry(dev, head, unreg_list) -+ list_add_tail(&dev->close_list, &close_head); -+ dev_close_many(&close_head, true); -+ -+ list_for_each_entry(dev, head, unreg_list) { -+ /* And unlink it from device chain. */ -+ unlist_netdevice(dev); -+ -+ dev->reg_state = NETREG_UNREGISTERING; -+ on_each_cpu(flush_backlog, dev, 1); -+ } -+ -+ synchronize_net(); -+ -+ list_for_each_entry(dev, head, unreg_list) { -+ struct sk_buff *skb = NULL; -+ -+ /* Shutdown queueing discipline. */ -+ dev_shutdown(dev); -+ -+ -+ /* Notify protocols, that we are about to destroy -+ this device. They should clean all the things. -+ */ -+ call_netdevice_notifiers(NETDEV_UNREGISTER, dev); -+ -+ if (!dev->rtnl_link_ops || -+ dev->rtnl_link_state == RTNL_LINK_INITIALIZED) -+ skb = rtmsg_ifinfo_build_skb(RTM_DELLINK, dev, ~0U, -+ GFP_KERNEL); -+ -+ /* -+ * Flush the unicast and multicast chains -+ */ -+ dev_uc_flush(dev); -+ dev_mc_flush(dev); -+ -+ if (dev->netdev_ops->ndo_uninit) -+ dev->netdev_ops->ndo_uninit(dev); -+ -+ if (skb) -+ rtmsg_ifinfo_send(skb, dev, GFP_KERNEL); -+ -+ /* Notifier chain MUST detach us all upper devices. */ -+ WARN_ON(netdev_has_any_upper_dev(dev)); -+ -+ /* Remove entries from kobject tree */ -+ netdev_unregister_kobject(dev); -+#ifdef CONFIG_XPS -+ /* Remove XPS queueing entries */ -+ netif_reset_xps_queues_gt(dev, 0); -+#endif -+ } -+ -+ synchronize_net(); -+ -+ list_for_each_entry(dev, head, unreg_list) -+ dev_put(dev); -+} -+ -+static void rollback_registered(struct net_device *dev) -+{ -+ LIST_HEAD(single); -+ -+ list_add(&dev->unreg_list, &single); -+ rollback_registered_many(&single); -+ list_del(&single); -+} -+ -+static netdev_features_t netdev_fix_features(struct net_device *dev, -+ netdev_features_t features) -+{ -+ /* Fix illegal checksum combinations */ -+ if ((features & NETIF_F_HW_CSUM) && -+ (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) { -+ netdev_warn(dev, "mixed HW and IP checksum settings.\n"); -+ features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM); -+ } -+ -+ /* TSO requires that SG is present as well. */ -+ if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) { -+ netdev_dbg(dev, "Dropping TSO features since no SG feature.\n"); -+ features &= ~NETIF_F_ALL_TSO; -+ } -+ -+ if ((features & NETIF_F_TSO) && !(features & NETIF_F_HW_CSUM) && -+ !(features & NETIF_F_IP_CSUM)) { -+ netdev_dbg(dev, "Dropping TSO features since no CSUM feature.\n"); -+ features &= ~NETIF_F_TSO; -+ features &= ~NETIF_F_TSO_ECN; -+ } -+ -+ if ((features & NETIF_F_TSO6) && !(features & NETIF_F_HW_CSUM) && -+ !(features & NETIF_F_IPV6_CSUM)) { -+ netdev_dbg(dev, "Dropping TSO6 features since no CSUM feature.\n"); -+ features &= ~NETIF_F_TSO6; -+ } -+ -+ /* TSO ECN requires that TSO is present as well. */ -+ if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN) -+ features &= ~NETIF_F_TSO_ECN; -+ -+ /* Software GSO depends on SG. */ -+ if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) { -+ netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n"); -+ features &= ~NETIF_F_GSO; -+ } -+ -+ /* UFO needs SG and checksumming */ -+ if (features & NETIF_F_UFO) { -+ /* maybe split UFO into V4 and V6? */ -+ if (!((features & NETIF_F_GEN_CSUM) || -+ (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM)) -+ == (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) { -+ netdev_dbg(dev, -+ "Dropping NETIF_F_UFO since no checksum offload features.\n"); -+ features &= ~NETIF_F_UFO; -+ } -+ -+ if (!(features & NETIF_F_SG)) { -+ netdev_dbg(dev, -+ "Dropping NETIF_F_UFO since no NETIF_F_SG feature.\n"); -+ features &= ~NETIF_F_UFO; -+ } -+ } -+ -+#ifdef CONFIG_NET_RX_BUSY_POLL -+ if (dev->netdev_ops->ndo_busy_poll) -+ features |= NETIF_F_BUSY_POLL; -+ else -+#endif -+ features &= ~NETIF_F_BUSY_POLL; -+ -+ return features; -+} -+ -+int __netdev_update_features(struct net_device *dev) -+{ -+ netdev_features_t features; -+ int err = 0; -+ -+ ASSERT_RTNL(); -+ -+ features = netdev_get_wanted_features(dev); -+ -+ if (dev->netdev_ops->ndo_fix_features) -+ features = dev->netdev_ops->ndo_fix_features(dev, features); -+ -+ /* driver might be less strict about feature dependencies */ -+ features = netdev_fix_features(dev, features); -+ -+ if (dev->features == features) -+ return 0; -+ -+ netdev_dbg(dev, "Features changed: %pNF -> %pNF\n", -+ &dev->features, &features); -+ -+ if (dev->netdev_ops->ndo_set_features) -+ err = dev->netdev_ops->ndo_set_features(dev, features); -+ -+ if (unlikely(err < 0)) { -+ netdev_err(dev, -+ "set_features() failed (%d); wanted %pNF, left %pNF\n", -+ err, &features, &dev->features); -+ return -1; -+ } -+ -+ if (!err) -+ dev->features = features; -+ -+ return 1; -+} -+ -+/** -+ * netdev_update_features - recalculate device features -+ * @dev: the device to check -+ * -+ * Recalculate dev->features set and send notifications if it -+ * has changed. Should be called after driver or hardware dependent -+ * conditions might have changed that influence the features. -+ */ -+void netdev_update_features(struct net_device *dev) -+{ -+ if (__netdev_update_features(dev)) -+ netdev_features_change(dev); -+} -+EXPORT_SYMBOL(netdev_update_features); -+ -+/** -+ * netdev_change_features - recalculate device features -+ * @dev: the device to check -+ * -+ * Recalculate dev->features set and send notifications even -+ * if they have not changed. Should be called instead of -+ * netdev_update_features() if also dev->vlan_features might -+ * have changed to allow the changes to be propagated to stacked -+ * VLAN devices. -+ */ -+void netdev_change_features(struct net_device *dev) -+{ -+ __netdev_update_features(dev); -+ netdev_features_change(dev); -+} -+EXPORT_SYMBOL(netdev_change_features); -+ -+/** -+ * netif_stacked_transfer_operstate - transfer operstate -+ * @rootdev: the root or lower level device to transfer state from -+ * @dev: the device to transfer operstate to -+ * -+ * Transfer operational state from root to device. This is normally -+ * called when a stacking relationship exists between the root -+ * device and the device(a leaf device). -+ */ -+void netif_stacked_transfer_operstate(const struct net_device *rootdev, -+ struct net_device *dev) -+{ -+ if (rootdev->operstate == IF_OPER_DORMANT) -+ netif_dormant_on(dev); -+ else -+ netif_dormant_off(dev); -+ -+ if (netif_carrier_ok(rootdev)) { -+ if (!netif_carrier_ok(dev)) -+ netif_carrier_on(dev); -+ } else { -+ if (netif_carrier_ok(dev)) -+ netif_carrier_off(dev); -+ } -+} -+EXPORT_SYMBOL(netif_stacked_transfer_operstate); -+ -+#ifdef CONFIG_SYSFS -+static int netif_alloc_rx_queues(struct net_device *dev) -+{ -+ unsigned int i, count = dev->num_rx_queues; -+ struct netdev_rx_queue *rx; -+ size_t sz = count * sizeof(*rx); -+ -+ BUG_ON(count < 1); -+ -+ rx = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT); -+ if (!rx) { -+ rx = vzalloc(sz); -+ if (!rx) -+ return -ENOMEM; -+ } -+ dev->_rx = rx; -+ -+ for (i = 0; i < count; i++) -+ rx[i].dev = dev; -+ return 0; -+} -+#endif -+ -+static void netdev_init_one_queue(struct net_device *dev, -+ struct netdev_queue *queue, void *_unused) -+{ -+ /* Initialize queue lock */ -+ spin_lock_init(&queue->_xmit_lock); -+ netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type); -+ queue->xmit_lock_owner = -1; -+ netdev_queue_numa_node_write(queue, NUMA_NO_NODE); -+ queue->dev = dev; -+#ifdef CONFIG_BQL -+ dql_init(&queue->dql, HZ); -+#endif -+} -+ -+static void netif_free_tx_queues(struct net_device *dev) -+{ -+ kvfree(dev->_tx); -+} -+ -+static int netif_alloc_netdev_queues(struct net_device *dev) -+{ -+ unsigned int count = dev->num_tx_queues; -+ struct netdev_queue *tx; -+ size_t sz = count * sizeof(*tx); -+ -+ if (count < 1 || count > 0xffff) -+ return -EINVAL; -+ -+ tx = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT); -+ if (!tx) { -+ tx = vzalloc(sz); -+ if (!tx) -+ return -ENOMEM; -+ } -+ dev->_tx = tx; -+ -+ netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL); -+ spin_lock_init(&dev->tx_global_lock); -+ -+ return 0; -+} -+ -+/** -+ * register_netdevice - register a network device -+ * @dev: device to register -+ * -+ * Take a completed network device structure and add it to the kernel -+ * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier -+ * chain. 0 is returned on success. A negative errno code is returned -+ * on a failure to set up the device, or if the name is a duplicate. -+ * -+ * Callers must hold the rtnl semaphore. You may want -+ * register_netdev() instead of this. -+ * -+ * BUGS: -+ * The locking appears insufficient to guarantee two parallel registers -+ * will not get the same name. -+ */ -+ -+int register_netdevice(struct net_device *dev) -+{ -+ int ret; -+ struct net *net = dev_net(dev); -+ -+ BUG_ON(dev_boot_phase); -+ ASSERT_RTNL(); -+ -+ might_sleep(); -+ -+ /* When net_device's are persistent, this will be fatal. */ -+ BUG_ON(dev->reg_state != NETREG_UNINITIALIZED); -+ BUG_ON(!net); -+ -+ spin_lock_init(&dev->addr_list_lock); -+ netdev_set_addr_lockdep_class(dev); -+ -+ ret = dev_get_valid_name(net, dev, dev->name); -+ if (ret < 0) -+ goto out; -+ -+ /* Init, if this function is available */ -+ if (dev->netdev_ops->ndo_init) { -+ ret = dev->netdev_ops->ndo_init(dev); -+ if (ret) { -+ if (ret > 0) -+ ret = -EIO; -+ goto out; -+ } -+ } -+ -+ if (((dev->hw_features | dev->features) & -+ NETIF_F_HW_VLAN_CTAG_FILTER) && -+ (!dev->netdev_ops->ndo_vlan_rx_add_vid || -+ !dev->netdev_ops->ndo_vlan_rx_kill_vid)) { -+ netdev_WARN(dev, "Buggy VLAN acceleration in driver!\n"); -+ ret = -EINVAL; -+ goto err_uninit; -+ } -+ -+ ret = -EBUSY; -+ if (!dev->ifindex) -+ dev->ifindex = dev_new_index(net); -+ else if (__dev_get_by_index(net, dev->ifindex)) -+ goto err_uninit; -+ -+ /* Transfer changeable features to wanted_features and enable -+ * software offloads (GSO and GRO). -+ */ -+ dev->hw_features |= NETIF_F_SOFT_FEATURES; -+ dev->features |= NETIF_F_SOFT_FEATURES; -+ dev->wanted_features = dev->features & dev->hw_features; -+ -+ if (!(dev->flags & IFF_LOOPBACK)) { -+ dev->hw_features |= NETIF_F_NOCACHE_COPY; -+ } -+ -+ /* Make NETIF_F_HIGHDMA inheritable to VLAN devices. -+ */ -+ dev->vlan_features |= NETIF_F_HIGHDMA; -+ -+ /* Make NETIF_F_SG inheritable to tunnel devices. -+ */ -+ dev->hw_enc_features |= NETIF_F_SG; -+ -+ /* Make NETIF_F_SG inheritable to MPLS. -+ */ -+ dev->mpls_features |= NETIF_F_SG; -+ -+ ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev); -+ ret = notifier_to_errno(ret); -+ if (ret) -+ goto err_uninit; -+ -+ ret = netdev_register_kobject(dev); -+ if (ret) -+ goto err_uninit; -+ dev->reg_state = NETREG_REGISTERED; -+ -+ __netdev_update_features(dev); -+ -+ /* -+ * Default initial state at registry is that the -+ * device is present. -+ */ -+ -+ set_bit(__LINK_STATE_PRESENT, &dev->state); -+ -+ linkwatch_init_dev(dev); -+ -+ dev_init_scheduler(dev); -+ dev_hold(dev); -+ list_netdevice(dev); -+ add_device_randomness(dev->dev_addr, dev->addr_len); -+ -+ /* If the device has permanent device address, driver should -+ * set dev_addr and also addr_assign_type should be set to -+ * NET_ADDR_PERM (default value). -+ */ -+ if (dev->addr_assign_type == NET_ADDR_PERM) -+ memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len); -+ -+ /* Notify protocols, that a new device appeared. */ -+ ret = call_netdevice_notifiers(NETDEV_REGISTER, dev); -+ ret = notifier_to_errno(ret); -+ if (ret) { -+ rollback_registered(dev); -+ dev->reg_state = NETREG_UNREGISTERED; -+ } -+ /* -+ * Prevent userspace races by waiting until the network -+ * device is fully setup before sending notifications. -+ */ -+ if (!dev->rtnl_link_ops || -+ dev->rtnl_link_state == RTNL_LINK_INITIALIZED) -+ rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL); -+ -+out: -+ return ret; -+ -+err_uninit: -+ if (dev->netdev_ops->ndo_uninit) -+ dev->netdev_ops->ndo_uninit(dev); -+ goto out; -+} -+EXPORT_SYMBOL(register_netdevice); -+ -+/** -+ * init_dummy_netdev - init a dummy network device for NAPI -+ * @dev: device to init -+ * -+ * This takes a network device structure and initialize the minimum -+ * amount of fields so it can be used to schedule NAPI polls without -+ * registering a full blown interface. This is to be used by drivers -+ * that need to tie several hardware interfaces to a single NAPI -+ * poll scheduler due to HW limitations. -+ */ -+int init_dummy_netdev(struct net_device *dev) -+{ -+ /* Clear everything. Note we don't initialize spinlocks -+ * are they aren't supposed to be taken by any of the -+ * NAPI code and this dummy netdev is supposed to be -+ * only ever used for NAPI polls -+ */ -+ memset(dev, 0, sizeof(struct net_device)); -+ -+ /* make sure we BUG if trying to hit standard -+ * register/unregister code path -+ */ -+ dev->reg_state = NETREG_DUMMY; -+ -+ /* NAPI wants this */ -+ INIT_LIST_HEAD(&dev->napi_list); -+ -+ /* a dummy interface is started by default */ -+ set_bit(__LINK_STATE_PRESENT, &dev->state); -+ set_bit(__LINK_STATE_START, &dev->state); -+ -+ /* Note : We dont allocate pcpu_refcnt for dummy devices, -+ * because users of this 'device' dont need to change -+ * its refcount. -+ */ -+ -+ return 0; -+} -+EXPORT_SYMBOL_GPL(init_dummy_netdev); -+ -+ -+/** -+ * register_netdev - register a network device -+ * @dev: device to register -+ * -+ * Take a completed network device structure and add it to the kernel -+ * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier -+ * chain. 0 is returned on success. A negative errno code is returned -+ * on a failure to set up the device, or if the name is a duplicate. -+ * -+ * This is a wrapper around register_netdevice that takes the rtnl semaphore -+ * and expands the device name if you passed a format string to -+ * alloc_netdev. -+ */ -+int register_netdev(struct net_device *dev) -+{ -+ int err; -+ -+ rtnl_lock(); -+ err = register_netdevice(dev); -+ rtnl_unlock(); -+ return err; -+} -+EXPORT_SYMBOL(register_netdev); -+ -+int netdev_refcnt_read(const struct net_device *dev) -+{ -+ int i, refcnt = 0; -+ -+ for_each_possible_cpu(i) -+ refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i); -+ return refcnt; -+} -+EXPORT_SYMBOL(netdev_refcnt_read); -+ -+/** -+ * netdev_wait_allrefs - wait until all references are gone. -+ * @dev: target net_device -+ * -+ * This is called when unregistering network devices. -+ * -+ * Any protocol or device that holds a reference should register -+ * for netdevice notification, and cleanup and put back the -+ * reference if they receive an UNREGISTER event. -+ * We can get stuck here if buggy protocols don't correctly -+ * call dev_put. -+ */ -+static void netdev_wait_allrefs(struct net_device *dev) -+{ -+ unsigned long rebroadcast_time, warning_time; -+ int refcnt; -+ -+ linkwatch_forget_dev(dev); -+ -+ rebroadcast_time = warning_time = jiffies; -+ refcnt = netdev_refcnt_read(dev); -+ -+ while (refcnt != 0) { -+ if (time_after(jiffies, rebroadcast_time + 1 * HZ)) { -+ rtnl_lock(); -+ -+ /* Rebroadcast unregister notification */ -+ call_netdevice_notifiers(NETDEV_UNREGISTER, dev); -+ -+ __rtnl_unlock(); -+ rcu_barrier(); -+ rtnl_lock(); -+ -+ call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev); -+ if (test_bit(__LINK_STATE_LINKWATCH_PENDING, -+ &dev->state)) { -+ /* We must not have linkwatch events -+ * pending on unregister. If this -+ * happens, we simply run the queue -+ * unscheduled, resulting in a noop -+ * for this device. -+ */ -+ linkwatch_run_queue(); -+ } -+ -+ __rtnl_unlock(); -+ -+ rebroadcast_time = jiffies; -+ } -+ -+ msleep(250); -+ -+ refcnt = netdev_refcnt_read(dev); -+ -+ if (time_after(jiffies, warning_time + 10 * HZ)) { -+ pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n", -+ dev->name, refcnt); -+ warning_time = jiffies; -+ } -+ } -+} ++ goto outunlock; + + if (oldname[0] && !strchr(oldname, '%')) + netdev_info(dev, "renamed from %s\n", oldname); +@@ -1147,11 +1146,12 @@ + if (ret) { + memcpy(dev->name, oldname, IFNAMSIZ); + dev->name_assign_type = old_assign_type; +- write_seqcount_end(&devnet_rename_seq); +- return ret; ++ err = ret; ++ goto outunlock; + } + +- write_seqcount_end(&devnet_rename_seq); ++ __raw_write_seqcount_end(&devnet_rename_seq); ++ mutex_unlock(&devnet_rename_mutex); + + netdev_adjacent_rename_links(dev, oldname); + +@@ -1172,7 +1172,8 @@ + /* err >= 0 after dev_alloc_name() or stores the first errno */ + if (err >= 0) { + err = ret; +- write_seqcount_begin(&devnet_rename_seq); ++ mutex_lock(&devnet_rename_mutex); ++ __raw_write_seqcount_begin(&devnet_rename_seq); + memcpy(dev->name, oldname, IFNAMSIZ); + memcpy(oldname, newname, IFNAMSIZ); + dev->name_assign_type = old_assign_type; +@@ -1185,6 +1186,11 @@ + } + + return err; + -+/* The sequence is: -+ * -+ * rtnl_lock(); -+ * ... -+ * register_netdevice(x1); -+ * register_netdevice(x2); -+ * ... -+ * unregister_netdevice(y1); -+ * unregister_netdevice(y2); -+ * ... -+ * rtnl_unlock(); -+ * free_netdev(y1); -+ * free_netdev(y2); ++outunlock: ++ __raw_write_seqcount_end(&devnet_rename_seq); ++ mutex_unlock(&devnet_rename_mutex); ++ return err; + } + + /** +@@ -2214,6 +2220,7 @@ + sd->output_queue_tailp = &q->next_sched; + raise_softirq_irqoff(NET_TX_SOFTIRQ); + local_irq_restore(flags); ++ preempt_check_resched_rt(); + } + + void __netif_schedule(struct Qdisc *q) +@@ -2295,6 +2302,7 @@ + __this_cpu_write(softnet_data.completion_queue, skb); + raise_softirq_irqoff(NET_TX_SOFTIRQ); + local_irq_restore(flags); ++ preempt_check_resched_rt(); + } + EXPORT_SYMBOL(__dev_kfree_skb_irq); + +@@ -3365,6 +3373,7 @@ + rps_unlock(sd); + + local_irq_restore(flags); ++ preempt_check_resched_rt(); + + atomic_long_inc(&skb->dev->rx_dropped); + kfree_skb(skb); +@@ -3383,7 +3392,7 @@ + struct rps_dev_flow voidflow, *rflow = &voidflow; + int cpu; + +- preempt_disable(); ++ migrate_disable(); + rcu_read_lock(); + + cpu = get_rps_cpu(skb->dev, skb, &rflow); +@@ -3393,13 +3402,13 @@ + ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail); + + rcu_read_unlock(); +- preempt_enable(); ++ migrate_enable(); + } else + #endif + { + unsigned int qtail; +- ret = enqueue_to_backlog(skb, get_cpu(), &qtail); +- put_cpu(); ++ ret = enqueue_to_backlog(skb, get_cpu_light(), &qtail); ++ put_cpu_light(); + } + return ret; + } +@@ -3433,16 +3442,44 @@ + + trace_netif_rx_ni_entry(skb); + +- preempt_disable(); ++ local_bh_disable(); + err = netif_rx_internal(skb); +- if (local_softirq_pending()) +- do_softirq(); +- preempt_enable(); ++ local_bh_enable(); + + return err; + } + EXPORT_SYMBOL(netif_rx_ni); + ++#ifdef CONFIG_PREEMPT_RT_FULL ++/* ++ * RT runs ksoftirqd as a real time thread and the root_lock is a ++ * "sleeping spinlock". If the trylock fails then we can go into an ++ * infinite loop when ksoftirqd preempted the task which actually ++ * holds the lock, because we requeue q and raise NET_TX softirq ++ * causing ksoftirqd to loop forever. + * -+ * We are invoked by rtnl_unlock(). -+ * This allows us to deal with problems: -+ * 1) We can delete sysfs objects which invoke hotplug -+ * without deadlocking with linkwatch via keventd. -+ * 2) Since we run with the RTNL semaphore not held, we can sleep -+ * safely in order to wait for the netdev refcnt to drop to zero. ++ * It's safe to use spin_lock on RT here as softirqs run in thread ++ * context and cannot deadlock against the thread which is holding ++ * root_lock. + * -+ * We must not return until all unregister events added during -+ * the interval the lock was held have been completed. ++ * On !RT the trylock might fail, but there we bail out from the ++ * softirq loop after 10 attempts which we can't do on RT. And the ++ * task holding root_lock cannot be preempted, so the only downside of ++ * that trylock is that we need 10 loops to decide that we should have ++ * given up in the first one :) + */ -+void netdev_run_todo(void) ++static inline int take_root_lock(spinlock_t *lock) +{ -+ struct list_head list; -+ -+ /* Snapshot list, allow later requests */ -+ list_replace_init(&net_todo_list, &list); -+ -+ __rtnl_unlock(); -+ -+ -+ /* Wait for rcu callbacks to finish before next phase */ -+ if (!list_empty(&list)) -+ rcu_barrier(); -+ -+ while (!list_empty(&list)) { -+ struct net_device *dev -+ = list_first_entry(&list, struct net_device, todo_list); -+ list_del(&dev->todo_list); -+ -+ rtnl_lock(); -+ call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev); -+ __rtnl_unlock(); -+ -+ if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) { -+ pr_err("network todo '%s' but state %d\n", -+ dev->name, dev->reg_state); -+ dump_stack(); -+ continue; -+ } -+ -+ dev->reg_state = NETREG_UNREGISTERED; -+ -+ netdev_wait_allrefs(dev); -+ -+ /* paranoia */ -+ BUG_ON(netdev_refcnt_read(dev)); -+ BUG_ON(!list_empty(&dev->ptype_all)); -+ BUG_ON(!list_empty(&dev->ptype_specific)); -+ WARN_ON(rcu_access_pointer(dev->ip_ptr)); -+ WARN_ON(rcu_access_pointer(dev->ip6_ptr)); -+ WARN_ON(dev->dn_ptr); -+ -+ if (dev->destructor) -+ dev->destructor(dev); -+ -+ /* Report a network device has been unregistered */ -+ rtnl_lock(); -+ dev_net(dev)->dev_unreg_count--; -+ __rtnl_unlock(); -+ wake_up(&netdev_unregistering_wq); -+ -+ /* Free network device */ -+ kobject_put(&dev->dev.kobj); -+ } ++ spin_lock(lock); ++ return 1; +} -+ -+/* Convert net_device_stats to rtnl_link_stats64. They have the same -+ * fields in the same order, with only the type differing. -+ */ -+void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64, -+ const struct net_device_stats *netdev_stats) -+{ -+#if BITS_PER_LONG == 64 -+ BUILD_BUG_ON(sizeof(*stats64) != sizeof(*netdev_stats)); -+ memcpy(stats64, netdev_stats, sizeof(*stats64)); +#else -+ size_t i, n = sizeof(*stats64) / sizeof(u64); -+ const unsigned long *src = (const unsigned long *)netdev_stats; -+ u64 *dst = (u64 *)stats64; -+ -+ BUILD_BUG_ON(sizeof(*netdev_stats) / sizeof(unsigned long) != -+ sizeof(*stats64) / sizeof(u64)); -+ for (i = 0; i < n; i++) -+ dst[i] = src[i]; -+#endif -+} -+EXPORT_SYMBOL(netdev_stats_to_stats64); -+ -+/** -+ * dev_get_stats - get network device statistics -+ * @dev: device to get statistics from -+ * @storage: place to store stats -+ * -+ * Get network statistics from device. Return @storage. -+ * The device driver may provide its own method by setting -+ * dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats; -+ * otherwise the internal statistics structure is used. -+ */ -+struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev, -+ struct rtnl_link_stats64 *storage) -+{ -+ const struct net_device_ops *ops = dev->netdev_ops; -+ -+ if (ops->ndo_get_stats64) { -+ memset(storage, 0, sizeof(*storage)); -+ ops->ndo_get_stats64(dev, storage); -+ } else if (ops->ndo_get_stats) { -+ netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev)); -+ } else { -+ netdev_stats_to_stats64(storage, &dev->stats); -+ } -+ storage->rx_dropped += atomic_long_read(&dev->rx_dropped); -+ storage->tx_dropped += atomic_long_read(&dev->tx_dropped); -+ return storage; -+} -+EXPORT_SYMBOL(dev_get_stats); -+ -+struct netdev_queue *dev_ingress_queue_create(struct net_device *dev) -+{ -+ struct netdev_queue *queue = dev_ingress_queue(dev); -+ -+#ifdef CONFIG_NET_CLS_ACT -+ if (queue) -+ return queue; -+ queue = kzalloc(sizeof(*queue), GFP_KERNEL); -+ if (!queue) -+ return NULL; -+ netdev_init_one_queue(dev, queue, NULL); -+ RCU_INIT_POINTER(queue->qdisc, &noop_qdisc); -+ queue->qdisc_sleeping = &noop_qdisc; -+ rcu_assign_pointer(dev->ingress_queue, queue); -+#endif -+ return queue; -+} -+ -+static const struct ethtool_ops default_ethtool_ops; -+ -+void netdev_set_default_ethtool_ops(struct net_device *dev, -+ const struct ethtool_ops *ops) -+{ -+ if (dev->ethtool_ops == &default_ethtool_ops) -+ dev->ethtool_ops = ops; -+} -+EXPORT_SYMBOL_GPL(netdev_set_default_ethtool_ops); -+ -+void netdev_freemem(struct net_device *dev) -+{ -+ char *addr = (char *)dev - dev->padded; -+ -+ kvfree(addr); -+} -+ -+/** -+ * alloc_netdev_mqs - allocate network device -+ * @sizeof_priv: size of private data to allocate space for -+ * @name: device name format string -+ * @name_assign_type: origin of device name -+ * @setup: callback to initialize device -+ * @txqs: the number of TX subqueues to allocate -+ * @rxqs: the number of RX subqueues to allocate -+ * -+ * Allocates a struct net_device with private data area for driver use -+ * and performs basic initialization. Also allocates subqueue structs -+ * for each queue on the device. -+ */ -+struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name, -+ unsigned char name_assign_type, -+ void (*setup)(struct net_device *), -+ unsigned int txqs, unsigned int rxqs) ++static inline int take_root_lock(spinlock_t *lock) +{ -+ struct net_device *dev; -+ size_t alloc_size; -+ struct net_device *p; -+ -+ BUG_ON(strlen(name) >= sizeof(dev->name)); -+ -+ if (txqs < 1) { -+ pr_err("alloc_netdev: Unable to allocate device with zero queues\n"); -+ return NULL; -+ } -+ -+#ifdef CONFIG_SYSFS -+ if (rxqs < 1) { -+ pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n"); -+ return NULL; -+ } -+#endif -+ -+ alloc_size = sizeof(struct net_device); -+ if (sizeof_priv) { -+ /* ensure 32-byte alignment of private area */ -+ alloc_size = ALIGN(alloc_size, NETDEV_ALIGN); -+ alloc_size += sizeof_priv; -+ } -+ /* ensure 32-byte alignment of whole construct */ -+ alloc_size += NETDEV_ALIGN - 1; -+ -+ p = kzalloc(alloc_size, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT); -+ if (!p) -+ p = vzalloc(alloc_size); -+ if (!p) -+ return NULL; -+ -+ dev = PTR_ALIGN(p, NETDEV_ALIGN); -+ dev->padded = (char *)dev - (char *)p; -+ -+ dev->pcpu_refcnt = alloc_percpu(int); -+ if (!dev->pcpu_refcnt) -+ goto free_dev; -+ -+ if (dev_addr_init(dev)) -+ goto free_pcpu; -+ -+ dev_mc_init(dev); -+ dev_uc_init(dev); -+ -+ dev_net_set(dev, &init_net); -+ -+ dev->gso_max_size = GSO_MAX_SIZE; -+ dev->gso_max_segs = GSO_MAX_SEGS; -+ dev->gso_min_segs = 0; -+ -+ INIT_LIST_HEAD(&dev->napi_list); -+ INIT_LIST_HEAD(&dev->unreg_list); -+ INIT_LIST_HEAD(&dev->close_list); -+ INIT_LIST_HEAD(&dev->link_watch_list); -+ INIT_LIST_HEAD(&dev->adj_list.upper); -+ INIT_LIST_HEAD(&dev->adj_list.lower); -+ INIT_LIST_HEAD(&dev->all_adj_list.upper); -+ INIT_LIST_HEAD(&dev->all_adj_list.lower); -+ INIT_LIST_HEAD(&dev->ptype_all); -+ INIT_LIST_HEAD(&dev->ptype_specific); -+ dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM; -+ setup(dev); -+ -+ dev->num_tx_queues = txqs; -+ dev->real_num_tx_queues = txqs; -+ if (netif_alloc_netdev_queues(dev)) -+ goto free_all; -+ -+#ifdef CONFIG_SYSFS -+ dev->num_rx_queues = rxqs; -+ dev->real_num_rx_queues = rxqs; -+ if (netif_alloc_rx_queues(dev)) -+ goto free_all; -+#endif -+ -+ strcpy(dev->name, name); -+ dev->name_assign_type = name_assign_type; -+ dev->group = INIT_NETDEV_GROUP; -+ if (!dev->ethtool_ops) -+ dev->ethtool_ops = &default_ethtool_ops; -+ return dev; -+ -+free_all: -+ free_netdev(dev); -+ return NULL; -+ -+free_pcpu: -+ free_percpu(dev->pcpu_refcnt); -+free_dev: -+ netdev_freemem(dev); -+ return NULL; ++ return spin_trylock(lock); +} -+EXPORT_SYMBOL(alloc_netdev_mqs); -+ -+/** -+ * free_netdev - free network device -+ * @dev: device -+ * -+ * This function does the last stage of destroying an allocated device -+ * interface. The reference to the device object is released. -+ * If this is the last reference then it will be freed. -+ */ -+void free_netdev(struct net_device *dev) -+{ -+ struct napi_struct *p, *n; -+ -+ netif_free_tx_queues(dev); -+#ifdef CONFIG_SYSFS -+ kvfree(dev->_rx); +#endif + -+ kfree(rcu_dereference_protected(dev->ingress_queue, 1)); -+ -+ /* Flush device addresses */ -+ dev_addr_flush(dev); -+ -+ list_for_each_entry_safe(p, n, &dev->napi_list, dev_list) -+ netif_napi_del(p); -+ -+ free_percpu(dev->pcpu_refcnt); -+ dev->pcpu_refcnt = NULL; -+ -+ /* Compatibility with error handling in drivers */ -+ if (dev->reg_state == NETREG_UNINITIALIZED) { -+ netdev_freemem(dev); -+ return; -+ } -+ -+ BUG_ON(dev->reg_state != NETREG_UNREGISTERED); -+ dev->reg_state = NETREG_RELEASED; -+ -+ /* will free via device release */ -+ put_device(&dev->dev); -+} -+EXPORT_SYMBOL(free_netdev); -+ -+/** -+ * synchronize_net - Synchronize with packet receive processing -+ * -+ * Wait for packets currently being received to be done. -+ * Does not block later packets from starting. -+ */ -+void synchronize_net(void) -+{ -+ might_sleep(); -+ if (rtnl_is_locked()) -+ synchronize_rcu_expedited(); -+ else -+ synchronize_rcu(); -+} -+EXPORT_SYMBOL(synchronize_net); -+ -+/** -+ * unregister_netdevice_queue - remove device from the kernel -+ * @dev: device -+ * @head: list -+ * -+ * This function shuts down a device interface and removes it -+ * from the kernel tables. -+ * If head not NULL, device is queued to be unregistered later. -+ * -+ * Callers must hold the rtnl semaphore. You may want -+ * unregister_netdev() instead of this. -+ */ -+ -+void unregister_netdevice_queue(struct net_device *dev, struct list_head *head) -+{ -+ ASSERT_RTNL(); -+ -+ if (head) { -+ list_move_tail(&dev->unreg_list, head); -+ } else { -+ rollback_registered(dev); -+ /* Finish processing unregister after unlock */ -+ net_set_todo(dev); -+ } -+} -+EXPORT_SYMBOL(unregister_netdevice_queue); -+ -+/** -+ * unregister_netdevice_many - unregister many devices -+ * @head: list of devices -+ * -+ * Note: As most callers use a stack allocated list_head, -+ * we force a list_del() to make sure stack wont be corrupted later. -+ */ -+void unregister_netdevice_many(struct list_head *head) -+{ -+ struct net_device *dev; -+ -+ if (!list_empty(head)) { -+ rollback_registered_many(head); -+ list_for_each_entry(dev, head, unreg_list) -+ net_set_todo(dev); -+ list_del(head); -+ } -+} -+EXPORT_SYMBOL(unregister_netdevice_many); -+ -+/** -+ * unregister_netdev - remove device from the kernel -+ * @dev: device -+ * -+ * This function shuts down a device interface and removes it -+ * from the kernel tables. -+ * -+ * This is just a wrapper for unregister_netdevice that takes -+ * the rtnl semaphore. In general you want to use this and not -+ * unregister_netdevice. -+ */ -+void unregister_netdev(struct net_device *dev) -+{ -+ rtnl_lock(); -+ unregister_netdevice(dev); -+ rtnl_unlock(); -+} -+EXPORT_SYMBOL(unregister_netdev); -+ -+/** -+ * dev_change_net_namespace - move device to different nethost namespace -+ * @dev: device -+ * @net: network namespace -+ * @pat: If not NULL name pattern to try if the current device name -+ * is already taken in the destination network namespace. -+ * -+ * This function shuts down a device interface and moves it -+ * to a new network namespace. On success 0 is returned, on -+ * a failure a netagive errno code is returned. -+ * -+ * Callers must hold the rtnl semaphore. -+ */ -+ -+int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat) -+{ -+ int err; -+ -+ ASSERT_RTNL(); -+ -+ /* Don't allow namespace local devices to be moved. */ -+ err = -EINVAL; -+ if (dev->features & NETIF_F_NETNS_LOCAL) -+ goto out; -+ -+ /* Ensure the device has been registrered */ -+ if (dev->reg_state != NETREG_REGISTERED) -+ goto out; -+ -+ /* Get out if there is nothing todo */ -+ err = 0; -+ if (net_eq(dev_net(dev), net)) -+ goto out; -+ -+ /* Pick the destination device name, and ensure -+ * we can use it in the destination network namespace. -+ */ -+ err = -EEXIST; -+ if (__dev_get_by_name(net, dev->name)) { -+ /* We get here if we can't use the current device name */ -+ if (!pat) -+ goto out; -+ if (dev_get_valid_name(net, dev, pat) < 0) -+ goto out; -+ } -+ -+ /* -+ * And now a mini version of register_netdevice unregister_netdevice. -+ */ -+ -+ /* If device is running close it first. */ -+ dev_close(dev); -+ -+ /* And unlink it from device chain */ -+ err = -ENODEV; -+ unlist_netdevice(dev); -+ -+ synchronize_net(); -+ -+ /* Shutdown queueing discipline. */ -+ dev_shutdown(dev); -+ -+ /* Notify protocols, that we are about to destroy -+ this device. They should clean all the things. -+ -+ Note that dev->reg_state stays at NETREG_REGISTERED. -+ This is wanted because this way 8021q and macvlan know -+ the device is just moving and can keep their slaves up. -+ */ -+ call_netdevice_notifiers(NETDEV_UNREGISTER, dev); -+ rcu_barrier(); -+ call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev); -+ rtmsg_ifinfo(RTM_DELLINK, dev, ~0U, GFP_KERNEL); -+ -+ /* -+ * Flush the unicast and multicast chains -+ */ -+ dev_uc_flush(dev); -+ dev_mc_flush(dev); -+ -+ /* Send a netdev-removed uevent to the old namespace */ -+ kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE); -+ netdev_adjacent_del_links(dev); -+ -+ /* Actually switch the network namespace */ -+ dev_net_set(dev, net); -+ -+ /* If there is an ifindex conflict assign a new one */ -+ if (__dev_get_by_index(net, dev->ifindex)) -+ dev->ifindex = dev_new_index(net); -+ -+ /* Send a netdev-add uevent to the new namespace */ -+ kobject_uevent(&dev->dev.kobj, KOBJ_ADD); -+ netdev_adjacent_add_links(dev); -+ -+ /* Fixup kobjects */ -+ err = device_rename(&dev->dev, dev->name); -+ WARN_ON(err); -+ -+ /* Add the device back in the hashes */ -+ list_netdevice(dev); -+ -+ /* Notify protocols, that a new device appeared. */ -+ call_netdevice_notifiers(NETDEV_REGISTER, dev); -+ -+ /* -+ * Prevent userspace races by waiting until the network -+ * device is fully setup before sending notifications. -+ */ -+ rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL); -+ -+ synchronize_net(); -+ err = 0; -+out: -+ return err; -+} -+EXPORT_SYMBOL_GPL(dev_change_net_namespace); -+ -+static int dev_cpu_callback(struct notifier_block *nfb, -+ unsigned long action, -+ void *ocpu) -+{ -+ struct sk_buff **list_skb; -+ struct sk_buff *skb; -+ unsigned int cpu, oldcpu = (unsigned long)ocpu; -+ struct softnet_data *sd, *oldsd; -+ -+ if (action != CPU_DEAD && action != CPU_DEAD_FROZEN) -+ return NOTIFY_OK; -+ -+ local_irq_disable(); -+ cpu = smp_processor_id(); -+ sd = &per_cpu(softnet_data, cpu); -+ oldsd = &per_cpu(softnet_data, oldcpu); -+ -+ /* Find end of our completion_queue. */ -+ list_skb = &sd->completion_queue; -+ while (*list_skb) -+ list_skb = &(*list_skb)->next; -+ /* Append completion queue from offline CPU. */ -+ *list_skb = oldsd->completion_queue; -+ oldsd->completion_queue = NULL; -+ -+ /* Append output queue from offline CPU. */ -+ if (oldsd->output_queue) { -+ *sd->output_queue_tailp = oldsd->output_queue; -+ sd->output_queue_tailp = oldsd->output_queue_tailp; -+ oldsd->output_queue = NULL; -+ oldsd->output_queue_tailp = &oldsd->output_queue; -+ } -+ /* Append NAPI poll list from offline CPU, with one exception : -+ * process_backlog() must be called by cpu owning percpu backlog. -+ * We properly handle process_queue & input_pkt_queue later. -+ */ -+ while (!list_empty(&oldsd->poll_list)) { -+ struct napi_struct *napi = list_first_entry(&oldsd->poll_list, -+ struct napi_struct, -+ poll_list); -+ -+ list_del_init(&napi->poll_list); -+ if (napi->poll == process_backlog) -+ napi->state = 0; -+ else -+ ____napi_schedule(sd, napi); -+ } -+ -+ raise_softirq_irqoff(NET_TX_SOFTIRQ); -+ local_irq_enable(); -+ -+ /* Process offline CPU's input_pkt_queue */ -+ while ((skb = __skb_dequeue(&oldsd->process_queue))) { -+ netif_rx_ni(skb); -+ input_queue_head_incr(oldsd); -+ } -+ while ((skb = skb_dequeue(&oldsd->input_pkt_queue))) { -+ netif_rx_ni(skb); -+ input_queue_head_incr(oldsd); -+ } -+ -+ return NOTIFY_OK; -+} -+ -+ -+/** -+ * netdev_increment_features - increment feature set by one -+ * @all: current feature set -+ * @one: new feature set -+ * @mask: mask feature set -+ * -+ * Computes a new feature set after adding a device with feature set -+ * @one to the master device with current feature set @all. Will not -+ * enable anything that is off in @mask. Returns the new feature set. -+ */ -+netdev_features_t netdev_increment_features(netdev_features_t all, -+ netdev_features_t one, netdev_features_t mask) -+{ -+ if (mask & NETIF_F_GEN_CSUM) -+ mask |= NETIF_F_ALL_CSUM; -+ mask |= NETIF_F_VLAN_CHALLENGED; -+ -+ all |= one & (NETIF_F_ONE_FOR_ALL|NETIF_F_ALL_CSUM) & mask; -+ all &= one | ~NETIF_F_ALL_FOR_ALL; -+ -+ /* If one device supports hw checksumming, set for all. */ -+ if (all & NETIF_F_GEN_CSUM) -+ all &= ~(NETIF_F_ALL_CSUM & ~NETIF_F_GEN_CSUM); -+ -+ return all; -+} -+EXPORT_SYMBOL(netdev_increment_features); -+ -+static struct hlist_head * __net_init netdev_create_hash(void) -+{ -+ int i; -+ struct hlist_head *hash; -+ -+ hash = kmalloc(sizeof(*hash) * NETDEV_HASHENTRIES, GFP_KERNEL); -+ if (hash != NULL) -+ for (i = 0; i < NETDEV_HASHENTRIES; i++) -+ INIT_HLIST_HEAD(&hash[i]); -+ -+ return hash; -+} -+ -+/* Initialize per network namespace state */ -+static int __net_init netdev_init(struct net *net) -+{ -+ if (net != &init_net) -+ INIT_LIST_HEAD(&net->dev_base_head); -+ -+ net->dev_name_head = netdev_create_hash(); -+ if (net->dev_name_head == NULL) -+ goto err_name; -+ -+ net->dev_index_head = netdev_create_hash(); -+ if (net->dev_index_head == NULL) -+ goto err_idx; -+ -+ return 0; -+ -+err_idx: -+ kfree(net->dev_name_head); -+err_name: -+ return -ENOMEM; -+} -+ -+/** -+ * netdev_drivername - network driver for the device -+ * @dev: network device -+ * -+ * Determine network driver for device. -+ */ -+const char *netdev_drivername(const struct net_device *dev) -+{ -+ const struct device_driver *driver; -+ const struct device *parent; -+ const char *empty = ""; -+ -+ parent = dev->dev.parent; -+ if (!parent) -+ return empty; -+ -+ driver = parent->driver; -+ if (driver && driver->name) -+ return driver->name; -+ return empty; -+} -+ -+static void __netdev_printk(const char *level, const struct net_device *dev, -+ struct va_format *vaf) -+{ -+ if (dev && dev->dev.parent) { -+ dev_printk_emit(level[1] - '0', -+ dev->dev.parent, -+ "%s %s %s%s: %pV", -+ dev_driver_string(dev->dev.parent), -+ dev_name(dev->dev.parent), -+ netdev_name(dev), netdev_reg_state(dev), -+ vaf); -+ } else if (dev) { -+ printk("%s%s%s: %pV", -+ level, netdev_name(dev), netdev_reg_state(dev), vaf); -+ } else { -+ printk("%s(NULL net_device): %pV", level, vaf); -+ } -+} -+ -+void netdev_printk(const char *level, const struct net_device *dev, -+ const char *format, ...) -+{ -+ struct va_format vaf; -+ va_list args; -+ -+ va_start(args, format); -+ -+ vaf.fmt = format; -+ vaf.va = &args; -+ -+ __netdev_printk(level, dev, &vaf); -+ -+ va_end(args); -+} -+EXPORT_SYMBOL(netdev_printk); -+ -+#define define_netdev_printk_level(func, level) \ -+void func(const struct net_device *dev, const char *fmt, ...) \ -+{ \ -+ struct va_format vaf; \ -+ va_list args; \ -+ \ -+ va_start(args, fmt); \ -+ \ -+ vaf.fmt = fmt; \ -+ vaf.va = &args; \ -+ \ -+ __netdev_printk(level, dev, &vaf); \ -+ \ -+ va_end(args); \ -+} \ -+EXPORT_SYMBOL(func); -+ -+define_netdev_printk_level(netdev_emerg, KERN_EMERG); -+define_netdev_printk_level(netdev_alert, KERN_ALERT); -+define_netdev_printk_level(netdev_crit, KERN_CRIT); -+define_netdev_printk_level(netdev_err, KERN_ERR); -+define_netdev_printk_level(netdev_warn, KERN_WARNING); -+define_netdev_printk_level(netdev_notice, KERN_NOTICE); -+define_netdev_printk_level(netdev_info, KERN_INFO); -+ -+static void __net_exit netdev_exit(struct net *net) -+{ -+ kfree(net->dev_name_head); -+ kfree(net->dev_index_head); -+} -+ -+static struct pernet_operations __net_initdata netdev_net_ops = { -+ .init = netdev_init, -+ .exit = netdev_exit, -+}; -+ -+static void __net_exit default_device_exit(struct net *net) -+{ -+ struct net_device *dev, *aux; -+ /* -+ * Push all migratable network devices back to the -+ * initial network namespace -+ */ -+ rtnl_lock(); -+ for_each_netdev_safe(net, dev, aux) { -+ int err; -+ char fb_name[IFNAMSIZ]; -+ -+ /* Ignore unmoveable devices (i.e. loopback) */ -+ if (dev->features & NETIF_F_NETNS_LOCAL) -+ continue; -+ -+ /* Leave virtual devices for the generic cleanup */ -+ if (dev->rtnl_link_ops) -+ continue; -+ -+ /* Push remaining network devices to init_net */ -+ snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex); -+ err = dev_change_net_namespace(dev, &init_net, fb_name); -+ if (err) { -+ pr_emerg("%s: failed to move %s to init_net: %d\n", -+ __func__, dev->name, err); -+ BUG(); -+ } -+ } -+ rtnl_unlock(); -+} -+ -+static void __net_exit rtnl_lock_unregistering(struct list_head *net_list) -+{ -+ /* Return with the rtnl_lock held when there are no network -+ * devices unregistering in any network namespace in net_list. -+ */ -+ struct net *net; -+ bool unregistering; -+ DEFINE_WAIT_FUNC(wait, woken_wake_function); -+ -+ add_wait_queue(&netdev_unregistering_wq, &wait); -+ for (;;) { -+ unregistering = false; -+ rtnl_lock(); -+ list_for_each_entry(net, net_list, exit_list) { -+ if (net->dev_unreg_count > 0) { -+ unregistering = true; -+ break; -+ } -+ } -+ if (!unregistering) -+ break; -+ __rtnl_unlock(); -+ -+ wait_woken(&wait, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT); -+ } -+ remove_wait_queue(&netdev_unregistering_wq, &wait); -+} -+ -+static void __net_exit default_device_exit_batch(struct list_head *net_list) -+{ -+ /* At exit all network devices most be removed from a network -+ * namespace. Do this in the reverse order of registration. -+ * Do this across as many network namespaces as possible to -+ * improve batching efficiency. -+ */ -+ struct net_device *dev; -+ struct net *net; -+ LIST_HEAD(dev_kill_list); -+ -+ /* To prevent network device cleanup code from dereferencing -+ * loopback devices or network devices that have been freed -+ * wait here for all pending unregistrations to complete, -+ * before unregistring the loopback device and allowing the -+ * network namespace be freed. -+ * -+ * The netdev todo list containing all network devices -+ * unregistrations that happen in default_device_exit_batch -+ * will run in the rtnl_unlock() at the end of -+ * default_device_exit_batch. -+ */ -+ rtnl_lock_unregistering(net_list); -+ list_for_each_entry(net, net_list, exit_list) { -+ for_each_netdev_reverse(net, dev) { -+ if (dev->rtnl_link_ops && dev->rtnl_link_ops->dellink) -+ dev->rtnl_link_ops->dellink(dev, &dev_kill_list); -+ else -+ unregister_netdevice_queue(dev, &dev_kill_list); -+ } -+ } -+ unregister_netdevice_many(&dev_kill_list); -+ rtnl_unlock(); -+} -+ -+static struct pernet_operations __net_initdata default_device_ops = { -+ .exit = default_device_exit, -+ .exit_batch = default_device_exit_batch, -+}; -+ -+/* -+ * Initialize the DEV module. At boot time this walks the device list and -+ * unhooks any devices that fail to initialise (normally hardware not -+ * present) and leaves us with a valid list of present and active devices. -+ * -+ */ -+ -+/* -+ * This is called single threaded during boot, so no need -+ * to take the rtnl semaphore. -+ */ -+static int __init net_dev_init(void) -+{ -+ int i, rc = -ENOMEM; -+ -+ BUG_ON(!dev_boot_phase); -+ -+ if (dev_proc_init()) -+ goto out; -+ -+ if (netdev_kobject_init()) -+ goto out; -+ -+ INIT_LIST_HEAD(&ptype_all); -+ for (i = 0; i < PTYPE_HASH_SIZE; i++) -+ INIT_LIST_HEAD(&ptype_base[i]); -+ -+ INIT_LIST_HEAD(&offload_base); -+ -+ if (register_pernet_subsys(&netdev_net_ops)) -+ goto out; -+ -+ /* -+ * Initialise the packet receive queues. -+ */ -+ -+ for_each_possible_cpu(i) { -+ struct softnet_data *sd = &per_cpu(softnet_data, i); -+ -+ skb_queue_head_init(&sd->input_pkt_queue); -+ skb_queue_head_init(&sd->process_queue); -+ INIT_LIST_HEAD(&sd->poll_list); -+ sd->output_queue_tailp = &sd->output_queue; -+#ifdef CONFIG_RPS -+ sd->csd.func = rps_trigger_softirq; -+ sd->csd.info = sd; -+ sd->cpu = i; -+#endif + static void net_tx_action(struct softirq_action *h) + { + struct softnet_data *sd = this_cpu_ptr(&softnet_data); +@@ -3484,7 +3521,7 @@ + head = head->next_sched; + + root_lock = qdisc_lock(q); +- if (spin_trylock(root_lock)) { ++ if (take_root_lock(root_lock)) { + smp_mb__before_atomic(); + clear_bit(__QDISC_STATE_SCHED, + &q->state); +@@ -3881,7 +3918,7 @@ + skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) { + if (skb->dev == dev) { + __skb_unlink(skb, &sd->input_pkt_queue); +- kfree_skb(skb); ++ __skb_queue_tail(&sd->tofree_queue, skb); + input_queue_head_incr(sd); + } + } +@@ -3890,10 +3927,13 @@ + skb_queue_walk_safe(&sd->process_queue, skb, tmp) { + if (skb->dev == dev) { + __skb_unlink(skb, &sd->process_queue); +- kfree_skb(skb); ++ __skb_queue_tail(&sd->tofree_queue, skb); + input_queue_head_incr(sd); + } + } + -+ sd->backlog.poll = process_backlog; -+ sd->backlog.weight = weight_p; ++ if (!skb_queue_empty(&sd->tofree_queue)) ++ raise_softirq_irqoff(NET_RX_SOFTIRQ); + } + + static int napi_gro_complete(struct sk_buff *skb) +@@ -4344,6 +4384,7 @@ + sd->rps_ipi_list = NULL; + + local_irq_enable(); ++ preempt_check_resched_rt(); + + /* Send pending IPI's to kick RPS processing on remote cpus. */ + while (remsd) { +@@ -4357,6 +4398,7 @@ + } else + #endif + local_irq_enable(); ++ preempt_check_resched_rt(); + } + + static bool sd_has_rps_ipi_waiting(struct softnet_data *sd) +@@ -4438,6 +4480,7 @@ + local_irq_save(flags); + ____napi_schedule(this_cpu_ptr(&softnet_data), n); + local_irq_restore(flags); ++ preempt_check_resched_rt(); + } + EXPORT_SYMBOL(__napi_schedule); + +@@ -7167,6 +7210,7 @@ + + raise_softirq_irqoff(NET_TX_SOFTIRQ); + local_irq_enable(); ++ preempt_check_resched_rt(); + + /* Process offline CPU's input_pkt_queue */ + while ((skb = __skb_dequeue(&oldsd->process_queue))) { +@@ -7177,6 +7221,9 @@ + netif_rx_ni(skb); + input_queue_head_incr(oldsd); + } ++ while ((skb = __skb_dequeue(&oldsd->tofree_queue))) { ++ kfree_skb(skb); + } -+ -+ dev_boot_phase = 0; -+ -+ /* The loopback device is special if any other network devices -+ * is present in a network namespace the loopback device must -+ * be present. Since we now dynamically allocate and free the -+ * loopback device ensure this invariant is maintained by -+ * keeping the loopback device as the first device on the -+ * list of network devices. Ensuring the loopback devices -+ * is the first device that appears and the last network device -+ * that disappears. -+ */ -+ if (register_pernet_device(&loopback_net_ops)) -+ goto out; -+ -+ if (register_pernet_device(&default_device_ops)) -+ goto out; -+ -+ open_softirq(NET_TX_SOFTIRQ, net_tx_action); -+ open_softirq(NET_RX_SOFTIRQ, net_rx_action); -+ -+ hotcpu_notifier(dev_cpu_callback, 0); -+ dst_init(); -+ rc = 0; -+out: -+ return rc; -+} -+ -+subsys_initcall(net_dev_init); + + return NOTIFY_OK; + } +@@ -7478,8 +7525,9 @@ + for_each_possible_cpu(i) { + struct softnet_data *sd = &per_cpu(softnet_data, i); + +- skb_queue_head_init(&sd->input_pkt_queue); +- skb_queue_head_init(&sd->process_queue); ++ skb_queue_head_init_raw(&sd->input_pkt_queue); ++ skb_queue_head_init_raw(&sd->process_queue); ++ skb_queue_head_init_raw(&sd->tofree_queue); + INIT_LIST_HEAD(&sd->poll_list); + sd->output_queue_tailp = &sd->output_queue; + #ifdef CONFIG_RPS diff -Nur linux-4.1.10.orig/net/core/skbuff.c linux-4.1.10/net/core/skbuff.c --- linux-4.1.10.orig/net/core/skbuff.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/net/core/skbuff.c 2015-10-07 18:00:08.000000000 +0200 ++++ linux-4.1.10/net/core/skbuff.c 2015-10-12 22:33:32.344672261 +0200 @@ -63,6 +63,7 @@ #include #include @@ -70085,7 +26772,7 @@ diff -Nur linux-4.1.10.orig/net/core/skbuff.c linux-4.1.10/net/core/skbuff.c diff -Nur linux-4.1.10.orig/net/core/sock.c linux-4.1.10/net/core/sock.c --- linux-4.1.10.orig/net/core/sock.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/net/core/sock.c 2015-10-07 18:00:08.000000000 +0200 ++++ linux-4.1.10/net/core/sock.c 2015-10-12 22:33:32.344672261 +0200 @@ -2370,12 +2370,11 @@ if (sk->sk_lock.owned) __lock_sock(sk); @@ -70102,7 +26789,7 @@ diff -Nur linux-4.1.10.orig/net/core/sock.c linux-4.1.10/net/core/sock.c diff -Nur linux-4.1.10.orig/net/ipv4/icmp.c linux-4.1.10/net/ipv4/icmp.c --- linux-4.1.10.orig/net/ipv4/icmp.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/net/ipv4/icmp.c 2015-10-07 18:00:08.000000000 +0200 ++++ linux-4.1.10/net/ipv4/icmp.c 2015-10-12 22:33:32.344672261 +0200 @@ -69,6 +69,7 @@ #include #include @@ -70156,7 +26843,7 @@ diff -Nur linux-4.1.10.orig/net/ipv4/icmp.c linux-4.1.10/net/ipv4/icmp.c return true; diff -Nur linux-4.1.10.orig/net/ipv4/sysctl_net_ipv4.c linux-4.1.10/net/ipv4/sysctl_net_ipv4.c --- linux-4.1.10.orig/net/ipv4/sysctl_net_ipv4.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/net/ipv4/sysctl_net_ipv4.c 2015-10-07 18:00:08.000000000 +0200 ++++ linux-4.1.10/net/ipv4/sysctl_net_ipv4.c 2015-10-12 22:33:32.344672261 +0200 @@ -779,6 +779,13 @@ .proc_handler = proc_dointvec }, @@ -70173,7 +26860,7 @@ diff -Nur linux-4.1.10.orig/net/ipv4/sysctl_net_ipv4.c linux-4.1.10/net/ipv4/sys .maxlen = sizeof(int), diff -Nur linux-4.1.10.orig/net/mac80211/rx.c linux-4.1.10/net/mac80211/rx.c --- linux-4.1.10.orig/net/mac80211/rx.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/net/mac80211/rx.c 2015-10-07 18:00:08.000000000 +0200 ++++ linux-4.1.10/net/mac80211/rx.c 2015-10-12 22:33:32.344672261 +0200 @@ -3554,7 +3554,7 @@ struct ieee80211_supported_band *sband; struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb); @@ -70185,7 +26872,7 @@ diff -Nur linux-4.1.10.orig/net/mac80211/rx.c linux-4.1.10/net/mac80211/rx.c goto drop; diff -Nur linux-4.1.10.orig/net/netfilter/core.c linux-4.1.10/net/netfilter/core.c --- linux-4.1.10.orig/net/netfilter/core.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/net/netfilter/core.c 2015-10-07 18:00:08.000000000 +0200 ++++ linux-4.1.10/net/netfilter/core.c 2015-10-12 22:33:32.344672261 +0200 @@ -22,11 +22,17 @@ #include #include @@ -70206,7 +26893,7 @@ diff -Nur linux-4.1.10.orig/net/netfilter/core.c linux-4.1.10/net/netfilter/core const struct nf_afinfo __rcu *nf_afinfo[NFPROTO_NUMPROTO] __read_mostly; diff -Nur linux-4.1.10.orig/net/packet/af_packet.c linux-4.1.10/net/packet/af_packet.c --- linux-4.1.10.orig/net/packet/af_packet.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/net/packet/af_packet.c 2015-10-07 18:00:08.000000000 +0200 ++++ linux-4.1.10/net/packet/af_packet.c 2015-10-12 22:33:32.344672261 +0200 @@ -63,6 +63,7 @@ #include #include @@ -70235,7 +26922,7 @@ diff -Nur linux-4.1.10.orig/net/packet/af_packet.c linux-4.1.10/net/packet/af_pa prb_close_block(pkc, pbd, po, status); diff -Nur linux-4.1.10.orig/net/rds/ib_rdma.c linux-4.1.10/net/rds/ib_rdma.c --- linux-4.1.10.orig/net/rds/ib_rdma.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/net/rds/ib_rdma.c 2015-10-07 18:00:08.000000000 +0200 ++++ linux-4.1.10/net/rds/ib_rdma.c 2015-10-12 22:33:32.344672261 +0200 @@ -34,6 +34,7 @@ #include #include @@ -70255,7 +26942,7 @@ diff -Nur linux-4.1.10.orig/net/rds/ib_rdma.c linux-4.1.10/net/rds/ib_rdma.c diff -Nur linux-4.1.10.orig/net/sched/sch_generic.c linux-4.1.10/net/sched/sch_generic.c --- linux-4.1.10.orig/net/sched/sch_generic.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/net/sched/sch_generic.c 2015-10-07 18:00:08.000000000 +0200 ++++ linux-4.1.10/net/sched/sch_generic.c 2015-10-12 22:33:32.344672261 +0200 @@ -894,7 +894,7 @@ /* Wait for outstanding qdisc_run calls. */ list_for_each_entry(dev, head, close_list) @@ -70267,7 +26954,7 @@ diff -Nur linux-4.1.10.orig/net/sched/sch_generic.c linux-4.1.10/net/sched/sch_g void dev_deactivate(struct net_device *dev) diff -Nur linux-4.1.10.orig/net/sunrpc/svc_xprt.c linux-4.1.10/net/sunrpc/svc_xprt.c --- linux-4.1.10.orig/net/sunrpc/svc_xprt.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/net/sunrpc/svc_xprt.c 2015-10-07 18:00:08.000000000 +0200 ++++ linux-4.1.10/net/sunrpc/svc_xprt.c 2015-10-12 22:33:32.344672261 +0200 @@ -341,7 +341,7 @@ goto out; } @@ -70297,7 +26984,7 @@ diff -Nur linux-4.1.10.orig/net/sunrpc/svc_xprt.c linux-4.1.10/net/sunrpc/svc_xp } diff -Nur linux-4.1.10.orig/scripts/mkcompile_h linux-4.1.10/scripts/mkcompile_h --- linux-4.1.10.orig/scripts/mkcompile_h 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/scripts/mkcompile_h 2015-10-07 18:00:08.000000000 +0200 ++++ linux-4.1.10/scripts/mkcompile_h 2015-10-12 22:33:32.344672261 +0200 @@ -4,7 +4,8 @@ ARCH=$2 SMP=$3 @@ -70318,7 +27005,7 @@ diff -Nur linux-4.1.10.orig/scripts/mkcompile_h linux-4.1.10/scripts/mkcompile_h # Truncate to maximum length diff -Nur linux-4.1.10.orig/sound/core/pcm_native.c linux-4.1.10/sound/core/pcm_native.c --- linux-4.1.10.orig/sound/core/pcm_native.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/sound/core/pcm_native.c 2015-10-07 18:00:08.000000000 +0200 ++++ linux-4.1.10/sound/core/pcm_native.c 2015-10-12 22:33:32.348671997 +0200 @@ -123,7 +123,7 @@ void snd_pcm_stream_lock_irq(struct snd_pcm_substream *substream) { @@ -70357,7 +27044,7 @@ diff -Nur linux-4.1.10.orig/sound/core/pcm_native.c linux-4.1.10/sound/core/pcm_ diff -Nur linux-4.1.10.orig/sound/soc/intel/atom/sst/sst.c linux-4.1.10/sound/soc/intel/atom/sst/sst.c --- linux-4.1.10.orig/sound/soc/intel/atom/sst/sst.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/sound/soc/intel/atom/sst/sst.c 2015-10-07 18:00:08.000000000 +0200 ++++ linux-4.1.10/sound/soc/intel/atom/sst/sst.c 2015-10-12 22:33:32.348671997 +0200 @@ -368,8 +368,8 @@ * initialize by FW or driver when firmware is loaded */ @@ -70371,7 +27058,7 @@ diff -Nur linux-4.1.10.orig/sound/soc/intel/atom/sst/sst.c linux-4.1.10/sound/so diff -Nur linux-4.1.10.orig/virt/kvm/async_pf.c linux-4.1.10/virt/kvm/async_pf.c --- linux-4.1.10.orig/virt/kvm/async_pf.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/virt/kvm/async_pf.c 2015-10-07 18:00:08.000000000 +0200 ++++ linux-4.1.10/virt/kvm/async_pf.c 2015-10-12 22:33:32.348671997 +0200 @@ -94,8 +94,8 @@ trace_kvm_async_pf_completed(addr, gva); @@ -70385,7 +27072,7 @@ diff -Nur linux-4.1.10.orig/virt/kvm/async_pf.c linux-4.1.10/virt/kvm/async_pf.c kvm_put_kvm(vcpu->kvm); diff -Nur linux-4.1.10.orig/virt/kvm/kvm_main.c linux-4.1.10/virt/kvm/kvm_main.c --- linux-4.1.10.orig/virt/kvm/kvm_main.c 2015-10-03 13:49:38.000000000 +0200 -+++ linux-4.1.10/virt/kvm/kvm_main.c 2015-10-07 18:00:08.000000000 +0200 ++++ linux-4.1.10/virt/kvm/kvm_main.c 2015-10-12 22:33:32.348671997 +0200 @@ -218,7 +218,7 @@ vcpu->kvm = kvm; vcpu->vcpu_id = id; -- cgit v1.2.3