From 1056ae1a7ef8a7019385296db7341a23110146b0 Mon Sep 17 00:00:00 2001 From: Waldemar Brodkorb Date: Wed, 7 Oct 2015 20:37:17 +0200 Subject: update rt patch --- target/linux/patches/4.1.10/realtime.patch | 80223 +++++++++++++++++++++------ 1 file changed, 62006 insertions(+), 18217 deletions(-) (limited to 'target/linux') diff --git a/target/linux/patches/4.1.10/realtime.patch b/target/linux/patches/4.1.10/realtime.patch index 9b5b92ee1..e9d7af67d 100644 --- a/target/linux/patches/4.1.10/realtime.patch +++ b/target/linux/patches/4.1.10/realtime.patch @@ -1,6 +1,6 @@ -diff -Nur linux-4.1.6.orig/arch/alpha/mm/fault.c linux-4.1.6/arch/alpha/mm/fault.c ---- linux-4.1.6.orig/arch/alpha/mm/fault.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/alpha/mm/fault.c 2015-09-08 23:49:03.498378314 +0200 +diff -Nur linux-4.1.10.orig/arch/alpha/mm/fault.c linux-4.1.10/arch/alpha/mm/fault.c +--- linux-4.1.10.orig/arch/alpha/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/alpha/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 @@ -23,8 +23,7 @@ #include #include @@ -20,9 +20,9 @@ diff -Nur linux-4.1.6.orig/arch/alpha/mm/fault.c linux-4.1.6/arch/alpha/mm/fault goto no_context; #ifdef CONFIG_ALPHA_LARGE_VMALLOC -diff -Nur linux-4.1.6.orig/arch/arc/include/asm/futex.h linux-4.1.6/arch/arc/include/asm/futex.h ---- linux-4.1.6.orig/arch/arc/include/asm/futex.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arc/include/asm/futex.h 2015-09-08 23:49:03.498378314 +0200 +diff -Nur linux-4.1.10.orig/arch/arc/include/asm/futex.h linux-4.1.10/arch/arc/include/asm/futex.h +--- linux-4.1.10.orig/arch/arc/include/asm/futex.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arc/include/asm/futex.h 2015-10-07 18:00:07.000000000 +0200 @@ -53,7 +53,7 @@ if (!access_ok(VERIFY_WRITE, uaddr, sizeof(int))) return -EFAULT; @@ -68,9 +68,9 @@ diff -Nur linux-4.1.6.orig/arch/arc/include/asm/futex.h linux-4.1.6/arch/arc/inc *uval = val; return val; -diff -Nur linux-4.1.6.orig/arch/arc/mm/fault.c linux-4.1.6/arch/arc/mm/fault.c ---- linux-4.1.6.orig/arch/arc/mm/fault.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arc/mm/fault.c 2015-09-08 23:49:03.498378314 +0200 +diff -Nur linux-4.1.10.orig/arch/arc/mm/fault.c linux-4.1.10/arch/arc/mm/fault.c +--- linux-4.1.10.orig/arch/arc/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arc/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 @@ -86,7 +86,7 @@ * If we're in an interrupt or have no user * context, we must not take the fault.. @@ -80,9 +80,9 @@ diff -Nur linux-4.1.6.orig/arch/arc/mm/fault.c linux-4.1.6/arch/arc/mm/fault.c goto no_context; if (user_mode(regs)) -diff -Nur linux-4.1.6.orig/arch/arm/include/asm/cmpxchg.h linux-4.1.6/arch/arm/include/asm/cmpxchg.h ---- linux-4.1.6.orig/arch/arm/include/asm/cmpxchg.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm/include/asm/cmpxchg.h 2015-09-08 23:49:03.498378314 +0200 +diff -Nur linux-4.1.10.orig/arch/arm/include/asm/cmpxchg.h linux-4.1.10/arch/arm/include/asm/cmpxchg.h +--- linux-4.1.10.orig/arch/arm/include/asm/cmpxchg.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm/include/asm/cmpxchg.h 2015-10-07 18:00:07.000000000 +0200 @@ -129,6 +129,8 @@ #else /* min ARCH >= ARMv6 */ @@ -92,9 +92,9 @@ diff -Nur linux-4.1.6.orig/arch/arm/include/asm/cmpxchg.h linux-4.1.6/arch/arm/i extern void __bad_cmpxchg(volatile void *ptr, int size); /* -diff -Nur linux-4.1.6.orig/arch/arm/include/asm/futex.h linux-4.1.6/arch/arm/include/asm/futex.h ---- linux-4.1.6.orig/arch/arm/include/asm/futex.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm/include/asm/futex.h 2015-09-08 23:49:03.498378314 +0200 +diff -Nur linux-4.1.10.orig/arch/arm/include/asm/futex.h linux-4.1.10/arch/arm/include/asm/futex.h +--- linux-4.1.10.orig/arch/arm/include/asm/futex.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm/include/asm/futex.h 2015-10-07 18:00:07.000000000 +0200 @@ -93,6 +93,7 @@ if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32))) return -EFAULT; @@ -136,9 +136,9 @@ diff -Nur linux-4.1.6.orig/arch/arm/include/asm/futex.h linux-4.1.6/arch/arm/inc if (!ret) { switch (cmp) { -diff -Nur linux-4.1.6.orig/arch/arm/include/asm/switch_to.h linux-4.1.6/arch/arm/include/asm/switch_to.h ---- linux-4.1.6.orig/arch/arm/include/asm/switch_to.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm/include/asm/switch_to.h 2015-09-08 23:49:03.498378314 +0200 +diff -Nur linux-4.1.10.orig/arch/arm/include/asm/switch_to.h linux-4.1.10/arch/arm/include/asm/switch_to.h +--- linux-4.1.10.orig/arch/arm/include/asm/switch_to.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm/include/asm/switch_to.h 2015-10-07 18:00:07.000000000 +0200 @@ -3,6 +3,13 @@ #include @@ -161,9 +161,9 @@ diff -Nur linux-4.1.6.orig/arch/arm/include/asm/switch_to.h linux-4.1.6/arch/arm last = __switch_to(prev,task_thread_info(prev), task_thread_info(next)); \ } while (0) -diff -Nur linux-4.1.6.orig/arch/arm/include/asm/thread_info.h linux-4.1.6/arch/arm/include/asm/thread_info.h ---- linux-4.1.6.orig/arch/arm/include/asm/thread_info.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm/include/asm/thread_info.h 2015-09-08 23:49:03.498378314 +0200 +diff -Nur linux-4.1.10.orig/arch/arm/include/asm/thread_info.h linux-4.1.10/arch/arm/include/asm/thread_info.h +--- linux-4.1.10.orig/arch/arm/include/asm/thread_info.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm/include/asm/thread_info.h 2015-10-07 18:00:07.000000000 +0200 @@ -50,6 +50,7 @@ struct thread_info { unsigned long flags; /* low level flags */ @@ -188,9 +188,9 @@ diff -Nur linux-4.1.6.orig/arch/arm/include/asm/thread_info.h linux-4.1.6/arch/a #define _TIF_UPROBE (1 << TIF_UPROBE) #define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE) #define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT) -diff -Nur linux-4.1.6.orig/arch/arm/Kconfig linux-4.1.6/arch/arm/Kconfig ---- linux-4.1.6.orig/arch/arm/Kconfig 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm/Kconfig 2015-09-08 23:49:03.498378314 +0200 +diff -Nur linux-4.1.10.orig/arch/arm/Kconfig linux-4.1.10/arch/arm/Kconfig +--- linux-4.1.10.orig/arch/arm/Kconfig 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm/Kconfig 2015-10-07 18:00:07.000000000 +0200 @@ -31,7 +31,7 @@ select HARDIRQS_SW_RESEND select HAVE_ARCH_AUDITSYSCALL if (AEABI && !OABI_COMPAT) @@ -208,9 +208,9 @@ diff -Nur linux-4.1.6.orig/arch/arm/Kconfig linux-4.1.6/arch/arm/Kconfig select HAVE_RCU_TABLE_FREE if (SMP && ARM_LPAE) select HAVE_REGS_AND_STACK_ACCESS_API select HAVE_SYSCALL_TRACEPOINTS -diff -Nur linux-4.1.6.orig/arch/arm/kernel/asm-offsets.c linux-4.1.6/arch/arm/kernel/asm-offsets.c ---- linux-4.1.6.orig/arch/arm/kernel/asm-offsets.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm/kernel/asm-offsets.c 2015-09-08 23:49:03.498378314 +0200 +diff -Nur linux-4.1.10.orig/arch/arm/kernel/asm-offsets.c linux-4.1.10/arch/arm/kernel/asm-offsets.c +--- linux-4.1.10.orig/arch/arm/kernel/asm-offsets.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm/kernel/asm-offsets.c 2015-10-07 18:00:07.000000000 +0200 @@ -65,6 +65,7 @@ BLANK(); DEFINE(TI_FLAGS, offsetof(struct thread_info, flags)); @@ -219,9 +219,9 @@ diff -Nur linux-4.1.6.orig/arch/arm/kernel/asm-offsets.c linux-4.1.6/arch/arm/ke DEFINE(TI_ADDR_LIMIT, offsetof(struct thread_info, addr_limit)); DEFINE(TI_TASK, offsetof(struct thread_info, task)); DEFINE(TI_CPU, offsetof(struct thread_info, cpu)); -diff -Nur linux-4.1.6.orig/arch/arm/kernel/entry-armv.S linux-4.1.6/arch/arm/kernel/entry-armv.S ---- linux-4.1.6.orig/arch/arm/kernel/entry-armv.S 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm/kernel/entry-armv.S 2015-09-08 23:49:03.498378314 +0200 +diff -Nur linux-4.1.10.orig/arch/arm/kernel/entry-armv.S linux-4.1.10/arch/arm/kernel/entry-armv.S +--- linux-4.1.10.orig/arch/arm/kernel/entry-armv.S 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm/kernel/entry-armv.S 2015-10-07 18:00:07.000000000 +0200 @@ -208,11 +208,18 @@ #ifdef CONFIG_PREEMPT get_thread_info tsk @@ -252,9 +252,9 @@ diff -Nur linux-4.1.6.orig/arch/arm/kernel/entry-armv.S linux-4.1.6/arch/arm/ker reteq r8 @ go again b 1b #endif -diff -Nur linux-4.1.6.orig/arch/arm/kernel/process.c linux-4.1.6/arch/arm/kernel/process.c ---- linux-4.1.6.orig/arch/arm/kernel/process.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm/kernel/process.c 2015-09-08 23:49:03.498378314 +0200 +diff -Nur linux-4.1.10.orig/arch/arm/kernel/process.c linux-4.1.10/arch/arm/kernel/process.c +--- linux-4.1.10.orig/arch/arm/kernel/process.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm/kernel/process.c 2015-10-07 18:00:07.000000000 +0200 @@ -290,6 +290,30 @@ } @@ -286,9 +286,9 @@ diff -Nur linux-4.1.6.orig/arch/arm/kernel/process.c linux-4.1.6/arch/arm/kernel #ifdef CONFIG_KUSER_HELPERS /* * The vectors page is always readable from user space for the -diff -Nur linux-4.1.6.orig/arch/arm/kernel/signal.c linux-4.1.6/arch/arm/kernel/signal.c ---- linux-4.1.6.orig/arch/arm/kernel/signal.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm/kernel/signal.c 2015-09-08 23:49:03.498378314 +0200 +diff -Nur linux-4.1.10.orig/arch/arm/kernel/signal.c linux-4.1.10/arch/arm/kernel/signal.c +--- linux-4.1.10.orig/arch/arm/kernel/signal.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm/kernel/signal.c 2015-10-07 18:00:07.000000000 +0200 @@ -563,7 +563,8 @@ do_work_pending(struct pt_regs *regs, unsigned int thread_flags, int syscall) { @@ -299,9 +299,31 @@ diff -Nur linux-4.1.6.orig/arch/arm/kernel/signal.c linux-4.1.6/arch/arm/kernel/ schedule(); } else { if (unlikely(!user_mode(regs))) -diff -Nur linux-4.1.6.orig/arch/arm/kernel/unwind.c linux-4.1.6/arch/arm/kernel/unwind.c ---- linux-4.1.6.orig/arch/arm/kernel/unwind.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm/kernel/unwind.c 2015-09-08 23:49:03.498378314 +0200 +diff -Nur linux-4.1.10.orig/arch/arm/kernel/smp.c linux-4.1.10/arch/arm/kernel/smp.c +--- linux-4.1.10.orig/arch/arm/kernel/smp.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm/kernel/smp.c 2015-10-07 18:00:07.000000000 +0200 +@@ -213,8 +213,6 @@ + flush_cache_louis(); + local_flush_tlb_all(); + +- clear_tasks_mm_cpumask(cpu); +- + return 0; + } + +@@ -230,6 +228,9 @@ + pr_err("CPU%u: cpu didn't die\n", cpu); + return; + } ++ ++ clear_tasks_mm_cpumask(cpu); ++ + pr_notice("CPU%u: shutdown\n", cpu); + + /* +diff -Nur linux-4.1.10.orig/arch/arm/kernel/unwind.c linux-4.1.10/arch/arm/kernel/unwind.c +--- linux-4.1.10.orig/arch/arm/kernel/unwind.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm/kernel/unwind.c 2015-10-07 18:00:07.000000000 +0200 @@ -93,7 +93,7 @@ static const struct unwind_idx *__origin_unwind_idx; extern const struct unwind_idx __stop_unwind_idx[]; @@ -353,9 +375,9 @@ diff -Nur linux-4.1.6.orig/arch/arm/kernel/unwind.c linux-4.1.6/arch/arm/kernel/ kfree(tab); } -diff -Nur linux-4.1.6.orig/arch/arm/kvm/arm.c linux-4.1.6/arch/arm/kvm/arm.c ---- linux-4.1.6.orig/arch/arm/kvm/arm.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm/kvm/arm.c 2015-09-08 23:49:03.498378314 +0200 +diff -Nur linux-4.1.10.orig/arch/arm/kvm/arm.c linux-4.1.10/arch/arm/kvm/arm.c +--- linux-4.1.10.orig/arch/arm/kvm/arm.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm/kvm/arm.c 2015-10-07 18:00:07.000000000 +0200 @@ -474,9 +474,9 @@ static void vcpu_pause(struct kvm_vcpu *vcpu) @@ -368,9 +390,9 @@ diff -Nur linux-4.1.6.orig/arch/arm/kvm/arm.c linux-4.1.6/arch/arm/kvm/arm.c } static int kvm_vcpu_initialized(struct kvm_vcpu *vcpu) -diff -Nur linux-4.1.6.orig/arch/arm/kvm/psci.c linux-4.1.6/arch/arm/kvm/psci.c ---- linux-4.1.6.orig/arch/arm/kvm/psci.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm/kvm/psci.c 2015-09-08 23:49:03.498378314 +0200 +diff -Nur linux-4.1.10.orig/arch/arm/kvm/psci.c linux-4.1.10/arch/arm/kvm/psci.c +--- linux-4.1.10.orig/arch/arm/kvm/psci.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm/kvm/psci.c 2015-10-07 18:00:07.000000000 +0200 @@ -68,7 +68,7 @@ { struct kvm *kvm = source_vcpu->kvm; @@ -389,9 +411,9 @@ diff -Nur linux-4.1.6.orig/arch/arm/kvm/psci.c linux-4.1.6/arch/arm/kvm/psci.c return PSCI_RET_SUCCESS; } -diff -Nur linux-4.1.6.orig/arch/arm/mach-exynos/platsmp.c linux-4.1.6/arch/arm/mach-exynos/platsmp.c ---- linux-4.1.6.orig/arch/arm/mach-exynos/platsmp.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm/mach-exynos/platsmp.c 2015-09-08 23:49:03.498378314 +0200 +diff -Nur linux-4.1.10.orig/arch/arm/mach-exynos/platsmp.c linux-4.1.10/arch/arm/mach-exynos/platsmp.c +--- linux-4.1.10.orig/arch/arm/mach-exynos/platsmp.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm/mach-exynos/platsmp.c 2015-10-07 18:00:07.000000000 +0200 @@ -231,7 +231,7 @@ return (void __iomem *)(S5P_VA_SCU); } @@ -439,9 +461,9 @@ diff -Nur linux-4.1.6.orig/arch/arm/mach-exynos/platsmp.c linux-4.1.6/arch/arm/m return pen_release != -1 ? ret : 0; } -diff -Nur linux-4.1.6.orig/arch/arm/mach-hisi/platmcpm.c linux-4.1.6/arch/arm/mach-hisi/platmcpm.c ---- linux-4.1.6.orig/arch/arm/mach-hisi/platmcpm.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm/mach-hisi/platmcpm.c 2015-09-08 23:49:03.498378314 +0200 +diff -Nur linux-4.1.10.orig/arch/arm/mach-hisi/platmcpm.c linux-4.1.10/arch/arm/mach-hisi/platmcpm.c +--- linux-4.1.10.orig/arch/arm/mach-hisi/platmcpm.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm/mach-hisi/platmcpm.c 2015-10-07 18:00:07.000000000 +0200 @@ -57,7 +57,7 @@ static void __iomem *sysctrl, *fabric; @@ -544,9 +566,9 @@ diff -Nur linux-4.1.6.orig/arch/arm/mach-hisi/platmcpm.c linux-4.1.6/arch/arm/ma } static void __naked hip04_mcpm_power_up_setup(unsigned int affinity_level) -diff -Nur linux-4.1.6.orig/arch/arm/mach-omap2/omap-smp.c linux-4.1.6/arch/arm/mach-omap2/omap-smp.c ---- linux-4.1.6.orig/arch/arm/mach-omap2/omap-smp.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm/mach-omap2/omap-smp.c 2015-09-08 23:49:03.498378314 +0200 +diff -Nur linux-4.1.10.orig/arch/arm/mach-omap2/omap-smp.c linux-4.1.10/arch/arm/mach-omap2/omap-smp.c +--- linux-4.1.10.orig/arch/arm/mach-omap2/omap-smp.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm/mach-omap2/omap-smp.c 2015-10-07 18:00:07.000000000 +0200 @@ -43,7 +43,7 @@ /* SCU base address */ static void __iomem *scu_base; @@ -585,9 +607,9 @@ diff -Nur linux-4.1.6.orig/arch/arm/mach-omap2/omap-smp.c linux-4.1.6/arch/arm/m return 0; } -diff -Nur linux-4.1.6.orig/arch/arm/mach-prima2/platsmp.c linux-4.1.6/arch/arm/mach-prima2/platsmp.c ---- linux-4.1.6.orig/arch/arm/mach-prima2/platsmp.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm/mach-prima2/platsmp.c 2015-09-08 23:49:03.498378314 +0200 +diff -Nur linux-4.1.10.orig/arch/arm/mach-prima2/platsmp.c linux-4.1.10/arch/arm/mach-prima2/platsmp.c +--- linux-4.1.10.orig/arch/arm/mach-prima2/platsmp.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm/mach-prima2/platsmp.c 2015-10-07 18:00:07.000000000 +0200 @@ -22,7 +22,7 @@ static void __iomem *clk_base; @@ -626,9 +648,9 @@ diff -Nur linux-4.1.6.orig/arch/arm/mach-prima2/platsmp.c linux-4.1.6/arch/arm/m return pen_release != -1 ? -ENOSYS : 0; } -diff -Nur linux-4.1.6.orig/arch/arm/mach-qcom/platsmp.c linux-4.1.6/arch/arm/mach-qcom/platsmp.c ---- linux-4.1.6.orig/arch/arm/mach-qcom/platsmp.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm/mach-qcom/platsmp.c 2015-09-08 23:49:03.498378314 +0200 +diff -Nur linux-4.1.10.orig/arch/arm/mach-qcom/platsmp.c linux-4.1.10/arch/arm/mach-qcom/platsmp.c +--- linux-4.1.10.orig/arch/arm/mach-qcom/platsmp.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm/mach-qcom/platsmp.c 2015-10-07 18:00:07.000000000 +0200 @@ -46,7 +46,7 @@ extern void secondary_startup_arm(void); @@ -667,9 +689,9 @@ diff -Nur linux-4.1.6.orig/arch/arm/mach-qcom/platsmp.c linux-4.1.6/arch/arm/mac return ret; } -diff -Nur linux-4.1.6.orig/arch/arm/mach-spear/platsmp.c linux-4.1.6/arch/arm/mach-spear/platsmp.c ---- linux-4.1.6.orig/arch/arm/mach-spear/platsmp.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm/mach-spear/platsmp.c 2015-09-08 23:49:03.498378314 +0200 +diff -Nur linux-4.1.10.orig/arch/arm/mach-spear/platsmp.c linux-4.1.10/arch/arm/mach-spear/platsmp.c +--- linux-4.1.10.orig/arch/arm/mach-spear/platsmp.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm/mach-spear/platsmp.c 2015-10-07 18:00:07.000000000 +0200 @@ -32,7 +32,7 @@ sync_cache_w(&pen_release); } @@ -708,9 +730,9 @@ diff -Nur linux-4.1.6.orig/arch/arm/mach-spear/platsmp.c linux-4.1.6/arch/arm/ma return pen_release != -1 ? -ENOSYS : 0; } -diff -Nur linux-4.1.6.orig/arch/arm/mach-sti/platsmp.c linux-4.1.6/arch/arm/mach-sti/platsmp.c ---- linux-4.1.6.orig/arch/arm/mach-sti/platsmp.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm/mach-sti/platsmp.c 2015-09-08 23:49:03.498378314 +0200 +diff -Nur linux-4.1.10.orig/arch/arm/mach-sti/platsmp.c linux-4.1.10/arch/arm/mach-sti/platsmp.c +--- linux-4.1.10.orig/arch/arm/mach-sti/platsmp.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm/mach-sti/platsmp.c 2015-10-07 18:00:07.000000000 +0200 @@ -34,7 +34,7 @@ sync_cache_w(&pen_release); } @@ -749,9 +771,9 @@ diff -Nur linux-4.1.6.orig/arch/arm/mach-sti/platsmp.c linux-4.1.6/arch/arm/mach return pen_release != -1 ? -ENOSYS : 0; } -diff -Nur linux-4.1.6.orig/arch/arm/mach-ux500/platsmp.c linux-4.1.6/arch/arm/mach-ux500/platsmp.c ---- linux-4.1.6.orig/arch/arm/mach-ux500/platsmp.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm/mach-ux500/platsmp.c 2015-09-08 23:49:03.498378314 +0200 +diff -Nur linux-4.1.10.orig/arch/arm/mach-ux500/platsmp.c linux-4.1.10/arch/arm/mach-ux500/platsmp.c +--- linux-4.1.10.orig/arch/arm/mach-ux500/platsmp.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm/mach-ux500/platsmp.c 2015-10-07 18:00:07.000000000 +0200 @@ -51,7 +51,7 @@ return NULL; } @@ -790,9 +812,9 @@ diff -Nur linux-4.1.6.orig/arch/arm/mach-ux500/platsmp.c linux-4.1.6/arch/arm/ma return pen_release != -1 ? -ENOSYS : 0; } -diff -Nur linux-4.1.6.orig/arch/arm/mm/fault.c linux-4.1.6/arch/arm/mm/fault.c ---- linux-4.1.6.orig/arch/arm/mm/fault.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm/mm/fault.c 2015-09-08 23:49:03.498378314 +0200 +diff -Nur linux-4.1.10.orig/arch/arm/mm/fault.c linux-4.1.10/arch/arm/mm/fault.c +--- linux-4.1.10.orig/arch/arm/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 @@ -276,7 +276,7 @@ * If we're in an interrupt or have no user * context, we must not take the fault.. @@ -822,9 +844,9 @@ diff -Nur linux-4.1.6.orig/arch/arm/mm/fault.c linux-4.1.6/arch/arm/mm/fault.c do_bad_area(addr, fsr, regs); return 0; } -diff -Nur linux-4.1.6.orig/arch/arm/mm/highmem.c linux-4.1.6/arch/arm/mm/highmem.c ---- linux-4.1.6.orig/arch/arm/mm/highmem.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm/mm/highmem.c 2015-09-08 23:49:03.498378314 +0200 +diff -Nur linux-4.1.10.orig/arch/arm/mm/highmem.c linux-4.1.10/arch/arm/mm/highmem.c +--- linux-4.1.10.orig/arch/arm/mm/highmem.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm/mm/highmem.c 2015-10-07 18:00:07.000000000 +0200 @@ -54,11 +54,13 @@ void *kmap_atomic(struct page *page) @@ -927,9 +949,9 @@ diff -Nur linux-4.1.6.orig/arch/arm/mm/highmem.c linux-4.1.6/arch/arm/mm/highmem + } +} +#endif -diff -Nur linux-4.1.6.orig/arch/arm/plat-versatile/platsmp.c linux-4.1.6/arch/arm/plat-versatile/platsmp.c ---- linux-4.1.6.orig/arch/arm/plat-versatile/platsmp.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm/plat-versatile/platsmp.c 2015-09-08 23:49:03.502377871 +0200 +diff -Nur linux-4.1.10.orig/arch/arm/plat-versatile/platsmp.c linux-4.1.10/arch/arm/plat-versatile/platsmp.c +--- linux-4.1.10.orig/arch/arm/plat-versatile/platsmp.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm/plat-versatile/platsmp.c 2015-10-07 18:00:07.000000000 +0200 @@ -30,7 +30,7 @@ sync_cache_w(&pen_release); } @@ -968,9 +990,9 @@ diff -Nur linux-4.1.6.orig/arch/arm/plat-versatile/platsmp.c linux-4.1.6/arch/ar return pen_release != -1 ? -ENOSYS : 0; } -diff -Nur linux-4.1.6.orig/arch/arm64/include/asm/futex.h linux-4.1.6/arch/arm64/include/asm/futex.h ---- linux-4.1.6.orig/arch/arm64/include/asm/futex.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm64/include/asm/futex.h 2015-09-08 23:49:03.502377871 +0200 +diff -Nur linux-4.1.10.orig/arch/arm64/include/asm/futex.h linux-4.1.10/arch/arm64/include/asm/futex.h +--- linux-4.1.10.orig/arch/arm64/include/asm/futex.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm64/include/asm/futex.h 2015-10-07 18:00:07.000000000 +0200 @@ -58,7 +58,7 @@ if (!access_ok(VERIFY_WRITE, uaddr, sizeof(u32))) return -EFAULT; @@ -989,9 +1011,9 @@ diff -Nur linux-4.1.6.orig/arch/arm64/include/asm/futex.h linux-4.1.6/arch/arm64 if (!ret) { switch (cmp) { -diff -Nur linux-4.1.6.orig/arch/arm64/include/asm/thread_info.h linux-4.1.6/arch/arm64/include/asm/thread_info.h ---- linux-4.1.6.orig/arch/arm64/include/asm/thread_info.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm64/include/asm/thread_info.h 2015-09-08 23:49:03.502377871 +0200 +diff -Nur linux-4.1.10.orig/arch/arm64/include/asm/thread_info.h linux-4.1.10/arch/arm64/include/asm/thread_info.h +--- linux-4.1.10.orig/arch/arm64/include/asm/thread_info.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm64/include/asm/thread_info.h 2015-10-07 18:00:07.000000000 +0200 @@ -47,6 +47,7 @@ mm_segment_t addr_limit; /* address limit */ struct task_struct *task; /* main task structure */ @@ -1016,9 +1038,9 @@ diff -Nur linux-4.1.6.orig/arch/arm64/include/asm/thread_info.h linux-4.1.6/arch #define _TIF_NOHZ (1 << TIF_NOHZ) #define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE) #define _TIF_SYSCALL_AUDIT (1 << TIF_SYSCALL_AUDIT) -diff -Nur linux-4.1.6.orig/arch/arm64/Kconfig linux-4.1.6/arch/arm64/Kconfig ---- linux-4.1.6.orig/arch/arm64/Kconfig 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm64/Kconfig 2015-09-08 23:49:03.502377871 +0200 +diff -Nur linux-4.1.10.orig/arch/arm64/Kconfig linux-4.1.10/arch/arm64/Kconfig +--- linux-4.1.10.orig/arch/arm64/Kconfig 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm64/Kconfig 2015-10-07 18:00:07.000000000 +0200 @@ -69,8 +69,10 @@ select HAVE_PERF_REGS select HAVE_PERF_USER_STACK_DUMP @@ -1030,9 +1052,9 @@ diff -Nur linux-4.1.6.orig/arch/arm64/Kconfig linux-4.1.6/arch/arm64/Kconfig select MODULES_USE_ELF_RELA select NO_BOOTMEM select OF -diff -Nur linux-4.1.6.orig/arch/arm64/kernel/asm-offsets.c linux-4.1.6/arch/arm64/kernel/asm-offsets.c ---- linux-4.1.6.orig/arch/arm64/kernel/asm-offsets.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm64/kernel/asm-offsets.c 2015-09-08 23:49:03.502377871 +0200 +diff -Nur linux-4.1.10.orig/arch/arm64/kernel/asm-offsets.c linux-4.1.10/arch/arm64/kernel/asm-offsets.c +--- linux-4.1.10.orig/arch/arm64/kernel/asm-offsets.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm64/kernel/asm-offsets.c 2015-10-07 18:00:07.000000000 +0200 @@ -35,6 +35,7 @@ BLANK(); DEFINE(TI_FLAGS, offsetof(struct thread_info, flags)); @@ -1041,9 +1063,9 @@ diff -Nur linux-4.1.6.orig/arch/arm64/kernel/asm-offsets.c linux-4.1.6/arch/arm6 DEFINE(TI_ADDR_LIMIT, offsetof(struct thread_info, addr_limit)); DEFINE(TI_TASK, offsetof(struct thread_info, task)); DEFINE(TI_CPU, offsetof(struct thread_info, cpu)); -diff -Nur linux-4.1.6.orig/arch/arm64/kernel/entry.S linux-4.1.6/arch/arm64/kernel/entry.S ---- linux-4.1.6.orig/arch/arm64/kernel/entry.S 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm64/kernel/entry.S 2015-09-08 23:49:03.502377871 +0200 +diff -Nur linux-4.1.10.orig/arch/arm64/kernel/entry.S linux-4.1.10/arch/arm64/kernel/entry.S +--- linux-4.1.10.orig/arch/arm64/kernel/entry.S 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm64/kernel/entry.S 2015-10-07 18:00:07.000000000 +0200 @@ -367,11 +367,16 @@ #ifdef CONFIG_PREEMPT get_thread_info tsk @@ -1080,9 +1102,9 @@ diff -Nur linux-4.1.6.orig/arch/arm64/kernel/entry.S linux-4.1.6/arch/arm64/kern /* TIF_SIGPENDING, TIF_NOTIFY_RESUME or TIF_FOREIGN_FPSTATE case */ ldr x2, [sp, #S_PSTATE] mov x0, sp // 'regs' -diff -Nur linux-4.1.6.orig/arch/arm64/kernel/perf_event.c linux-4.1.6/arch/arm64/kernel/perf_event.c ---- linux-4.1.6.orig/arch/arm64/kernel/perf_event.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm64/kernel/perf_event.c 2015-09-08 23:49:03.502377871 +0200 +diff -Nur linux-4.1.10.orig/arch/arm64/kernel/perf_event.c linux-4.1.10/arch/arm64/kernel/perf_event.c +--- linux-4.1.10.orig/arch/arm64/kernel/perf_event.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm64/kernel/perf_event.c 2015-10-07 18:00:07.000000000 +0200 @@ -488,7 +488,7 @@ } @@ -1092,9 +1114,9 @@ diff -Nur linux-4.1.6.orig/arch/arm64/kernel/perf_event.c linux-4.1.6/arch/arm64 "arm-pmu", armpmu); if (err) { pr_err("unable to request IRQ%d for ARM PMU counters\n", -diff -Nur linux-4.1.6.orig/arch/arm64/mm/fault.c linux-4.1.6/arch/arm64/mm/fault.c ---- linux-4.1.6.orig/arch/arm64/mm/fault.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/arm64/mm/fault.c 2015-09-08 23:49:03.502377871 +0200 +diff -Nur linux-4.1.10.orig/arch/arm64/mm/fault.c linux-4.1.10/arch/arm64/mm/fault.c +--- linux-4.1.10.orig/arch/arm64/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/arm64/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 @@ -211,7 +211,7 @@ * If we're in an interrupt or have no user context, we must not take * the fault. @@ -1104,9 +1126,9 @@ diff -Nur linux-4.1.6.orig/arch/arm64/mm/fault.c linux-4.1.6/arch/arm64/mm/fault goto no_context; if (user_mode(regs)) -diff -Nur linux-4.1.6.orig/arch/avr32/include/asm/uaccess.h linux-4.1.6/arch/avr32/include/asm/uaccess.h ---- linux-4.1.6.orig/arch/avr32/include/asm/uaccess.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/avr32/include/asm/uaccess.h 2015-09-08 23:49:03.502377871 +0200 +diff -Nur linux-4.1.10.orig/arch/avr32/include/asm/uaccess.h linux-4.1.10/arch/avr32/include/asm/uaccess.h +--- linux-4.1.10.orig/arch/avr32/include/asm/uaccess.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/avr32/include/asm/uaccess.h 2015-10-07 18:00:07.000000000 +0200 @@ -97,7 +97,8 @@ * @x: Value to copy to user space. * @ptr: Destination address, in user space. @@ -1147,9 +1169,9 @@ diff -Nur linux-4.1.6.orig/arch/avr32/include/asm/uaccess.h linux-4.1.6/arch/avr * * This macro copies a single simple variable from user space to kernel * space. It supports simple types like char and int, but not larger -diff -Nur linux-4.1.6.orig/arch/avr32/mm/fault.c linux-4.1.6/arch/avr32/mm/fault.c ---- linux-4.1.6.orig/arch/avr32/mm/fault.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/avr32/mm/fault.c 2015-09-08 23:49:03.502377871 +0200 +diff -Nur linux-4.1.10.orig/arch/avr32/mm/fault.c linux-4.1.10/arch/avr32/mm/fault.c +--- linux-4.1.10.orig/arch/avr32/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/avr32/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 @@ -14,11 +14,11 @@ #include #include @@ -1172,9 +1194,9 @@ diff -Nur linux-4.1.6.orig/arch/avr32/mm/fault.c linux-4.1.6/arch/avr32/mm/fault goto no_context; local_irq_enable(); -diff -Nur linux-4.1.6.orig/arch/cris/mm/fault.c linux-4.1.6/arch/cris/mm/fault.c ---- linux-4.1.6.orig/arch/cris/mm/fault.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/cris/mm/fault.c 2015-09-08 23:49:03.502377871 +0200 +diff -Nur linux-4.1.10.orig/arch/cris/mm/fault.c linux-4.1.10/arch/cris/mm/fault.c +--- linux-4.1.10.orig/arch/cris/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/cris/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 @@ -8,7 +8,7 @@ #include #include @@ -1198,9 +1220,9 @@ diff -Nur linux-4.1.6.orig/arch/cris/mm/fault.c linux-4.1.6/arch/cris/mm/fault.c goto no_context; if (user_mode(regs)) -diff -Nur linux-4.1.6.orig/arch/frv/mm/fault.c linux-4.1.6/arch/frv/mm/fault.c ---- linux-4.1.6.orig/arch/frv/mm/fault.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/frv/mm/fault.c 2015-09-08 23:49:03.502377871 +0200 +diff -Nur linux-4.1.10.orig/arch/frv/mm/fault.c linux-4.1.10/arch/frv/mm/fault.c +--- linux-4.1.10.orig/arch/frv/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/frv/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 @@ -19,9 +19,9 @@ #include #include @@ -1221,9 +1243,9 @@ diff -Nur linux-4.1.6.orig/arch/frv/mm/fault.c linux-4.1.6/arch/frv/mm/fault.c goto no_context; if (user_mode(__frame)) -diff -Nur linux-4.1.6.orig/arch/frv/mm/highmem.c linux-4.1.6/arch/frv/mm/highmem.c ---- linux-4.1.6.orig/arch/frv/mm/highmem.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/frv/mm/highmem.c 2015-09-08 23:49:03.502377871 +0200 +diff -Nur linux-4.1.10.orig/arch/frv/mm/highmem.c linux-4.1.10/arch/frv/mm/highmem.c +--- linux-4.1.10.orig/arch/frv/mm/highmem.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/frv/mm/highmem.c 2015-10-07 18:00:07.000000000 +0200 @@ -42,6 +42,7 @@ unsigned long paddr; int type; @@ -1239,9 +1261,9 @@ diff -Nur linux-4.1.6.orig/arch/frv/mm/highmem.c linux-4.1.6/arch/frv/mm/highmem + preempt_enable(); } EXPORT_SYMBOL(__kunmap_atomic); -diff -Nur linux-4.1.6.orig/arch/hexagon/include/asm/uaccess.h linux-4.1.6/arch/hexagon/include/asm/uaccess.h ---- linux-4.1.6.orig/arch/hexagon/include/asm/uaccess.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/hexagon/include/asm/uaccess.h 2015-09-08 23:49:04.386279854 +0200 +diff -Nur linux-4.1.10.orig/arch/hexagon/include/asm/uaccess.h linux-4.1.10/arch/hexagon/include/asm/uaccess.h +--- linux-4.1.10.orig/arch/hexagon/include/asm/uaccess.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/hexagon/include/asm/uaccess.h 2015-10-07 18:00:07.000000000 +0200 @@ -36,7 +36,8 @@ * @addr: User space pointer to start of block to check * @size: Size of block to check @@ -1252,9 +1274,9 @@ diff -Nur linux-4.1.6.orig/arch/hexagon/include/asm/uaccess.h linux-4.1.6/arch/h * * Checks if a pointer to a block of memory in user space is valid. * -diff -Nur linux-4.1.6.orig/arch/ia64/mm/fault.c linux-4.1.6/arch/ia64/mm/fault.c ---- linux-4.1.6.orig/arch/ia64/mm/fault.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/ia64/mm/fault.c 2015-09-08 23:49:04.658249697 +0200 +diff -Nur linux-4.1.10.orig/arch/ia64/mm/fault.c linux-4.1.10/arch/ia64/mm/fault.c +--- linux-4.1.10.orig/arch/ia64/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/ia64/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 @@ -11,10 +11,10 @@ #include #include @@ -1276,9 +1298,9 @@ diff -Nur linux-4.1.6.orig/arch/ia64/mm/fault.c linux-4.1.6/arch/ia64/mm/fault.c goto no_context; #ifdef CONFIG_VIRTUAL_MEM_MAP -diff -Nur linux-4.1.6.orig/arch/Kconfig linux-4.1.6/arch/Kconfig ---- linux-4.1.6.orig/arch/Kconfig 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/Kconfig 2015-09-08 23:49:03.498378314 +0200 +diff -Nur linux-4.1.10.orig/arch/Kconfig linux-4.1.10/arch/Kconfig +--- linux-4.1.10.orig/arch/Kconfig 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/Kconfig 2015-10-07 18:00:07.000000000 +0200 @@ -6,6 +6,7 @@ tristate "OProfile system profiling" depends on PROFILING @@ -1287,9 +1309,9 @@ diff -Nur linux-4.1.6.orig/arch/Kconfig linux-4.1.6/arch/Kconfig select RING_BUFFER select RING_BUFFER_ALLOW_SWAP help -diff -Nur linux-4.1.6.orig/arch/m32r/include/asm/uaccess.h linux-4.1.6/arch/m32r/include/asm/uaccess.h ---- linux-4.1.6.orig/arch/m32r/include/asm/uaccess.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/m32r/include/asm/uaccess.h 2015-09-08 23:49:04.658249697 +0200 +diff -Nur linux-4.1.10.orig/arch/m32r/include/asm/uaccess.h linux-4.1.10/arch/m32r/include/asm/uaccess.h +--- linux-4.1.10.orig/arch/m32r/include/asm/uaccess.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/m32r/include/asm/uaccess.h 2015-10-07 18:00:07.000000000 +0200 @@ -91,7 +91,8 @@ * @addr: User space pointer to start of block to check * @size: Size of block to check @@ -1390,9 +1412,9 @@ diff -Nur linux-4.1.6.orig/arch/m32r/include/asm/uaccess.h linux-4.1.6/arch/m32r * * Get the size of a NUL-terminated string in user space. * -diff -Nur linux-4.1.6.orig/arch/m32r/mm/fault.c linux-4.1.6/arch/m32r/mm/fault.c ---- linux-4.1.6.orig/arch/m32r/mm/fault.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/m32r/mm/fault.c 2015-09-08 23:49:04.658249697 +0200 +diff -Nur linux-4.1.10.orig/arch/m32r/mm/fault.c linux-4.1.10/arch/m32r/mm/fault.c +--- linux-4.1.10.orig/arch/m32r/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/m32r/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 @@ -24,9 +24,9 @@ #include /* For unblank_screen() */ #include @@ -1418,9 +1440,9 @@ diff -Nur linux-4.1.6.orig/arch/m32r/mm/fault.c linux-4.1.6/arch/m32r/mm/fault.c goto bad_area_nosemaphore; if (error_code & ACE_USERMODE) -diff -Nur linux-4.1.6.orig/arch/m68k/mm/fault.c linux-4.1.6/arch/m68k/mm/fault.c ---- linux-4.1.6.orig/arch/m68k/mm/fault.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/m68k/mm/fault.c 2015-09-08 23:49:04.658249697 +0200 +diff -Nur linux-4.1.10.orig/arch/m68k/mm/fault.c linux-4.1.10/arch/m68k/mm/fault.c +--- linux-4.1.10.orig/arch/m68k/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/m68k/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 @@ -10,10 +10,10 @@ #include #include @@ -1442,9 +1464,9 @@ diff -Nur linux-4.1.6.orig/arch/m68k/mm/fault.c linux-4.1.6/arch/m68k/mm/fault.c goto no_context; if (user_mode(regs)) -diff -Nur linux-4.1.6.orig/arch/metag/mm/fault.c linux-4.1.6/arch/metag/mm/fault.c ---- linux-4.1.6.orig/arch/metag/mm/fault.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/metag/mm/fault.c 2015-09-08 23:49:04.658249697 +0200 +diff -Nur linux-4.1.10.orig/arch/metag/mm/fault.c linux-4.1.10/arch/metag/mm/fault.c +--- linux-4.1.10.orig/arch/metag/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/metag/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 @@ -105,7 +105,7 @@ mm = tsk->mm; @@ -1454,9 +1476,9 @@ diff -Nur linux-4.1.6.orig/arch/metag/mm/fault.c linux-4.1.6/arch/metag/mm/fault goto no_context; if (user_mode(regs)) -diff -Nur linux-4.1.6.orig/arch/metag/mm/highmem.c linux-4.1.6/arch/metag/mm/highmem.c ---- linux-4.1.6.orig/arch/metag/mm/highmem.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/metag/mm/highmem.c 2015-09-08 23:49:04.662249253 +0200 +diff -Nur linux-4.1.10.orig/arch/metag/mm/highmem.c linux-4.1.10/arch/metag/mm/highmem.c +--- linux-4.1.10.orig/arch/metag/mm/highmem.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/metag/mm/highmem.c 2015-10-07 18:00:07.000000000 +0200 @@ -43,7 +43,7 @@ unsigned long vaddr; int type; @@ -1482,9 +1504,9 @@ diff -Nur linux-4.1.6.orig/arch/metag/mm/highmem.c linux-4.1.6/arch/metag/mm/hig pagefault_disable(); type = kmap_atomic_idx_push(); -diff -Nur linux-4.1.6.orig/arch/microblaze/include/asm/uaccess.h linux-4.1.6/arch/microblaze/include/asm/uaccess.h ---- linux-4.1.6.orig/arch/microblaze/include/asm/uaccess.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/microblaze/include/asm/uaccess.h 2015-09-08 23:49:04.662249253 +0200 +diff -Nur linux-4.1.10.orig/arch/microblaze/include/asm/uaccess.h linux-4.1.10/arch/microblaze/include/asm/uaccess.h +--- linux-4.1.10.orig/arch/microblaze/include/asm/uaccess.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/microblaze/include/asm/uaccess.h 2015-10-07 18:00:07.000000000 +0200 @@ -178,7 +178,8 @@ * @x: Variable to store result. * @ptr: Source address, in user space. @@ -1505,9 +1527,9 @@ diff -Nur linux-4.1.6.orig/arch/microblaze/include/asm/uaccess.h linux-4.1.6/arc * * This macro copies a single simple value from kernel space to user * space. It supports simple types like char and int, but not larger -diff -Nur linux-4.1.6.orig/arch/microblaze/mm/fault.c linux-4.1.6/arch/microblaze/mm/fault.c ---- linux-4.1.6.orig/arch/microblaze/mm/fault.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/microblaze/mm/fault.c 2015-09-08 23:49:04.670248364 +0200 +diff -Nur linux-4.1.10.orig/arch/microblaze/mm/fault.c linux-4.1.10/arch/microblaze/mm/fault.c +--- linux-4.1.10.orig/arch/microblaze/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/microblaze/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 @@ -107,14 +107,14 @@ if ((error_code & 0x13) == 0x13 || (error_code & 0x11) == 0x11) is_write = 0; @@ -1527,9 +1549,9 @@ diff -Nur linux-4.1.6.orig/arch/microblaze/mm/fault.c linux-4.1.6/arch/microblaz pr_emerg("r15 = %lx MSR = %lx\n", regs->r15, regs->msr); die("Weird page fault", regs, SIGSEGV); -diff -Nur linux-4.1.6.orig/arch/microblaze/mm/highmem.c linux-4.1.6/arch/microblaze/mm/highmem.c ---- linux-4.1.6.orig/arch/microblaze/mm/highmem.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/microblaze/mm/highmem.c 2015-09-08 23:49:04.670248364 +0200 +diff -Nur linux-4.1.10.orig/arch/microblaze/mm/highmem.c linux-4.1.10/arch/microblaze/mm/highmem.c +--- linux-4.1.10.orig/arch/microblaze/mm/highmem.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/microblaze/mm/highmem.c 2015-10-07 18:00:07.000000000 +0200 @@ -37,7 +37,7 @@ unsigned long vaddr; int idx, type; @@ -1554,9 +1576,9 @@ diff -Nur linux-4.1.6.orig/arch/microblaze/mm/highmem.c linux-4.1.6/arch/microbl + preempt_enable(); } EXPORT_SYMBOL(__kunmap_atomic); -diff -Nur linux-4.1.6.orig/arch/mips/include/asm/uaccess.h linux-4.1.6/arch/mips/include/asm/uaccess.h ---- linux-4.1.6.orig/arch/mips/include/asm/uaccess.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/mips/include/asm/uaccess.h 2015-09-08 23:49:04.670248364 +0200 +diff -Nur linux-4.1.10.orig/arch/mips/include/asm/uaccess.h linux-4.1.10/arch/mips/include/asm/uaccess.h +--- linux-4.1.10.orig/arch/mips/include/asm/uaccess.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/mips/include/asm/uaccess.h 2015-10-07 18:00:07.000000000 +0200 @@ -103,7 +103,8 @@ * @addr: User space pointer to start of block to check * @size: Size of block to check @@ -1707,9 +1729,9 @@ diff -Nur linux-4.1.6.orig/arch/mips/include/asm/uaccess.h linux-4.1.6/arch/mips * * Get the size of a NUL-terminated string in user space. * -diff -Nur linux-4.1.6.orig/arch/mips/Kconfig linux-4.1.6/arch/mips/Kconfig ---- linux-4.1.6.orig/arch/mips/Kconfig 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/mips/Kconfig 2015-09-08 23:49:04.670248364 +0200 +diff -Nur linux-4.1.10.orig/arch/mips/Kconfig linux-4.1.10/arch/mips/Kconfig +--- linux-4.1.10.orig/arch/mips/Kconfig 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/mips/Kconfig 2015-10-07 18:00:07.000000000 +0200 @@ -2366,7 +2366,7 @@ # config HIGHMEM @@ -1719,9 +1741,9 @@ diff -Nur linux-4.1.6.orig/arch/mips/Kconfig linux-4.1.6/arch/mips/Kconfig config CPU_SUPPORTS_HIGHMEM bool -diff -Nur linux-4.1.6.orig/arch/mips/kernel/signal-common.h linux-4.1.6/arch/mips/kernel/signal-common.h ---- linux-4.1.6.orig/arch/mips/kernel/signal-common.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/mips/kernel/signal-common.h 2015-09-08 23:49:04.674247921 +0200 +diff -Nur linux-4.1.10.orig/arch/mips/kernel/signal-common.h linux-4.1.10/arch/mips/kernel/signal-common.h +--- linux-4.1.10.orig/arch/mips/kernel/signal-common.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/mips/kernel/signal-common.h 2015-10-07 18:00:07.000000000 +0200 @@ -28,12 +28,7 @@ extern int fpcsr_pending(unsigned int __user *fpcsr); @@ -1737,9 +1759,9 @@ diff -Nur linux-4.1.6.orig/arch/mips/kernel/signal-common.h linux-4.1.6/arch/mip +#define unlock_fpu_owner() ({ pagefault_enable(); preempt_enable(); }) #endif /* __SIGNAL_COMMON_H */ -diff -Nur linux-4.1.6.orig/arch/mips/mm/fault.c linux-4.1.6/arch/mips/mm/fault.c ---- linux-4.1.6.orig/arch/mips/mm/fault.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/mips/mm/fault.c 2015-09-08 23:49:04.674247921 +0200 +diff -Nur linux-4.1.10.orig/arch/mips/mm/fault.c linux-4.1.10/arch/mips/mm/fault.c +--- linux-4.1.10.orig/arch/mips/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/mips/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 @@ -21,10 +21,10 @@ #include #include @@ -1761,9 +1783,9 @@ diff -Nur linux-4.1.6.orig/arch/mips/mm/fault.c linux-4.1.6/arch/mips/mm/fault.c goto bad_area_nosemaphore; if (user_mode(regs)) -diff -Nur linux-4.1.6.orig/arch/mips/mm/highmem.c linux-4.1.6/arch/mips/mm/highmem.c ---- linux-4.1.6.orig/arch/mips/mm/highmem.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/mips/mm/highmem.c 2015-09-08 23:49:04.674247921 +0200 +diff -Nur linux-4.1.10.orig/arch/mips/mm/highmem.c linux-4.1.10/arch/mips/mm/highmem.c +--- linux-4.1.10.orig/arch/mips/mm/highmem.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/mips/mm/highmem.c 2015-10-07 18:00:07.000000000 +0200 @@ -47,7 +47,7 @@ unsigned long vaddr; int idx, type; @@ -1797,9 +1819,9 @@ diff -Nur linux-4.1.6.orig/arch/mips/mm/highmem.c linux-4.1.6/arch/mips/mm/highm pagefault_disable(); type = kmap_atomic_idx_push(); -diff -Nur linux-4.1.6.orig/arch/mips/mm/init.c linux-4.1.6/arch/mips/mm/init.c ---- linux-4.1.6.orig/arch/mips/mm/init.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/mips/mm/init.c 2015-09-08 23:49:04.674247921 +0200 +diff -Nur linux-4.1.10.orig/arch/mips/mm/init.c linux-4.1.10/arch/mips/mm/init.c +--- linux-4.1.10.orig/arch/mips/mm/init.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/mips/mm/init.c 2015-10-07 18:00:07.000000000 +0200 @@ -90,6 +90,7 @@ BUG_ON(Page_dcache_dirty(page)); @@ -1816,9 +1838,9 @@ diff -Nur linux-4.1.6.orig/arch/mips/mm/init.c linux-4.1.6/arch/mips/mm/init.c } void copy_user_highpage(struct page *to, struct page *from, -diff -Nur linux-4.1.6.orig/arch/mn10300/include/asm/highmem.h linux-4.1.6/arch/mn10300/include/asm/highmem.h ---- linux-4.1.6.orig/arch/mn10300/include/asm/highmem.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/mn10300/include/asm/highmem.h 2015-09-08 23:49:04.674247921 +0200 +diff -Nur linux-4.1.10.orig/arch/mn10300/include/asm/highmem.h linux-4.1.10/arch/mn10300/include/asm/highmem.h +--- linux-4.1.10.orig/arch/mn10300/include/asm/highmem.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/mn10300/include/asm/highmem.h 2015-10-07 18:00:07.000000000 +0200 @@ -75,6 +75,7 @@ unsigned long vaddr; int idx, type; @@ -1843,9 +1865,9 @@ diff -Nur linux-4.1.6.orig/arch/mn10300/include/asm/highmem.h linux-4.1.6/arch/m } #endif /* __KERNEL__ */ -diff -Nur linux-4.1.6.orig/arch/mn10300/mm/fault.c linux-4.1.6/arch/mn10300/mm/fault.c ---- linux-4.1.6.orig/arch/mn10300/mm/fault.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/mn10300/mm/fault.c 2015-09-08 23:49:04.674247921 +0200 +diff -Nur linux-4.1.10.orig/arch/mn10300/mm/fault.c linux-4.1.10/arch/mn10300/mm/fault.c +--- linux-4.1.10.orig/arch/mn10300/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/mn10300/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 @@ -23,8 +23,8 @@ #include #include @@ -1865,9 +1887,9 @@ diff -Nur linux-4.1.6.orig/arch/mn10300/mm/fault.c linux-4.1.6/arch/mn10300/mm/f goto no_context; if ((fault_code & MMUFCR_xFC_ACCESS) == MMUFCR_xFC_ACCESS_USR) -diff -Nur linux-4.1.6.orig/arch/nios2/mm/fault.c linux-4.1.6/arch/nios2/mm/fault.c ---- linux-4.1.6.orig/arch/nios2/mm/fault.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/nios2/mm/fault.c 2015-09-08 23:49:04.674247921 +0200 +diff -Nur linux-4.1.10.orig/arch/nios2/mm/fault.c linux-4.1.10/arch/nios2/mm/fault.c +--- linux-4.1.10.orig/arch/nios2/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/nios2/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 @@ -77,7 +77,7 @@ * If we're in an interrupt or have no user * context, we must not take the fault.. @@ -1877,9 +1899,9 @@ diff -Nur linux-4.1.6.orig/arch/nios2/mm/fault.c linux-4.1.6/arch/nios2/mm/fault goto bad_area_nosemaphore; if (user_mode(regs)) -diff -Nur linux-4.1.6.orig/arch/parisc/include/asm/cacheflush.h linux-4.1.6/arch/parisc/include/asm/cacheflush.h ---- linux-4.1.6.orig/arch/parisc/include/asm/cacheflush.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/parisc/include/asm/cacheflush.h 2015-09-08 23:49:04.674247921 +0200 +diff -Nur linux-4.1.10.orig/arch/parisc/include/asm/cacheflush.h linux-4.1.10/arch/parisc/include/asm/cacheflush.h +--- linux-4.1.10.orig/arch/parisc/include/asm/cacheflush.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/parisc/include/asm/cacheflush.h 2015-10-07 18:00:07.000000000 +0200 @@ -142,6 +142,7 @@ static inline void *kmap_atomic(struct page *page) @@ -1896,9 +1918,9 @@ diff -Nur linux-4.1.6.orig/arch/parisc/include/asm/cacheflush.h linux-4.1.6/arch } #define kmap_atomic_prot(page, prot) kmap_atomic(page) -diff -Nur linux-4.1.6.orig/arch/parisc/kernel/traps.c linux-4.1.6/arch/parisc/kernel/traps.c ---- linux-4.1.6.orig/arch/parisc/kernel/traps.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/parisc/kernel/traps.c 2015-09-08 23:49:04.678247479 +0200 +diff -Nur linux-4.1.10.orig/arch/parisc/kernel/traps.c linux-4.1.10/arch/parisc/kernel/traps.c +--- linux-4.1.10.orig/arch/parisc/kernel/traps.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/parisc/kernel/traps.c 2015-10-07 18:00:07.000000000 +0200 @@ -26,9 +26,9 @@ #include #include @@ -1919,9 +1941,9 @@ diff -Nur linux-4.1.6.orig/arch/parisc/kernel/traps.c linux-4.1.6/arch/parisc/ke { pdc_chassis_send_status(PDC_CHASSIS_DIRECT_PANIC); parisc_terminate("Kernel Fault", regs, code, fault_address); -diff -Nur linux-4.1.6.orig/arch/parisc/mm/fault.c linux-4.1.6/arch/parisc/mm/fault.c ---- linux-4.1.6.orig/arch/parisc/mm/fault.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/parisc/mm/fault.c 2015-09-08 23:49:04.678247479 +0200 +diff -Nur linux-4.1.10.orig/arch/parisc/mm/fault.c linux-4.1.10/arch/parisc/mm/fault.c +--- linux-4.1.10.orig/arch/parisc/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/parisc/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 @@ -15,8 +15,8 @@ #include #include @@ -1941,9 +1963,9 @@ diff -Nur linux-4.1.6.orig/arch/parisc/mm/fault.c linux-4.1.6/arch/parisc/mm/fau goto no_context; tsk = current; -diff -Nur linux-4.1.6.orig/arch/powerpc/include/asm/kvm_host.h linux-4.1.6/arch/powerpc/include/asm/kvm_host.h ---- linux-4.1.6.orig/arch/powerpc/include/asm/kvm_host.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/powerpc/include/asm/kvm_host.h 2015-09-08 23:49:04.810232841 +0200 +diff -Nur linux-4.1.10.orig/arch/powerpc/include/asm/kvm_host.h linux-4.1.10/arch/powerpc/include/asm/kvm_host.h +--- linux-4.1.10.orig/arch/powerpc/include/asm/kvm_host.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/powerpc/include/asm/kvm_host.h 2015-10-07 18:00:07.000000000 +0200 @@ -280,7 +280,7 @@ u8 in_guest; struct list_head runnable_threads; @@ -1962,9 +1984,9 @@ diff -Nur linux-4.1.6.orig/arch/powerpc/include/asm/kvm_host.h linux-4.1.6/arch/ struct kvmppc_vcore *vcore; int ret; int trap; -diff -Nur linux-4.1.6.orig/arch/powerpc/include/asm/thread_info.h linux-4.1.6/arch/powerpc/include/asm/thread_info.h ---- linux-4.1.6.orig/arch/powerpc/include/asm/thread_info.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/powerpc/include/asm/thread_info.h 2015-09-08 23:49:04.810232841 +0200 +diff -Nur linux-4.1.10.orig/arch/powerpc/include/asm/thread_info.h linux-4.1.10/arch/powerpc/include/asm/thread_info.h +--- linux-4.1.10.orig/arch/powerpc/include/asm/thread_info.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/powerpc/include/asm/thread_info.h 2015-10-07 18:00:07.000000000 +0200 @@ -42,6 +42,8 @@ int cpu; /* cpu we're on */ int preempt_count; /* 0 => preemptable, @@ -2011,9 +2033,9 @@ diff -Nur linux-4.1.6.orig/arch/powerpc/include/asm/thread_info.h linux-4.1.6/ar /* Bits in local_flags */ /* Don't move TLF_NAPPING without adjusting the code in entry_32.S */ -diff -Nur linux-4.1.6.orig/arch/powerpc/Kconfig linux-4.1.6/arch/powerpc/Kconfig ---- linux-4.1.6.orig/arch/powerpc/Kconfig 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/powerpc/Kconfig 2015-09-08 23:49:04.810232841 +0200 +diff -Nur linux-4.1.10.orig/arch/powerpc/Kconfig linux-4.1.10/arch/powerpc/Kconfig +--- linux-4.1.10.orig/arch/powerpc/Kconfig 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/powerpc/Kconfig 2015-10-07 18:00:07.000000000 +0200 @@ -60,10 +60,11 @@ config RWSEM_GENERIC_SPINLOCK @@ -2044,9 +2066,9 @@ diff -Nur linux-4.1.6.orig/arch/powerpc/Kconfig linux-4.1.6/arch/powerpc/Kconfig source kernel/Kconfig.hz source kernel/Kconfig.preempt -diff -Nur linux-4.1.6.orig/arch/powerpc/kernel/asm-offsets.c linux-4.1.6/arch/powerpc/kernel/asm-offsets.c ---- linux-4.1.6.orig/arch/powerpc/kernel/asm-offsets.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/powerpc/kernel/asm-offsets.c 2015-09-08 23:49:04.810232841 +0200 +diff -Nur linux-4.1.10.orig/arch/powerpc/kernel/asm-offsets.c linux-4.1.10/arch/powerpc/kernel/asm-offsets.c +--- linux-4.1.10.orig/arch/powerpc/kernel/asm-offsets.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/powerpc/kernel/asm-offsets.c 2015-10-07 18:00:07.000000000 +0200 @@ -160,6 +160,7 @@ DEFINE(TI_FLAGS, offsetof(struct thread_info, flags)); DEFINE(TI_LOCAL_FLAGS, offsetof(struct thread_info, local_flags)); @@ -2055,9 +2077,9 @@ diff -Nur linux-4.1.6.orig/arch/powerpc/kernel/asm-offsets.c linux-4.1.6/arch/po DEFINE(TI_TASK, offsetof(struct thread_info, task)); DEFINE(TI_CPU, offsetof(struct thread_info, cpu)); -diff -Nur linux-4.1.6.orig/arch/powerpc/kernel/entry_32.S linux-4.1.6/arch/powerpc/kernel/entry_32.S ---- linux-4.1.6.orig/arch/powerpc/kernel/entry_32.S 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/powerpc/kernel/entry_32.S 2015-09-08 23:49:04.810232841 +0200 +diff -Nur linux-4.1.10.orig/arch/powerpc/kernel/entry_32.S linux-4.1.10/arch/powerpc/kernel/entry_32.S +--- linux-4.1.10.orig/arch/powerpc/kernel/entry_32.S 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/powerpc/kernel/entry_32.S 2015-10-07 18:00:07.000000000 +0200 @@ -813,7 +813,14 @@ cmpwi 0,r0,0 /* if non-zero, just restore regs and return */ bne restore @@ -2106,9 +2128,9 @@ diff -Nur linux-4.1.6.orig/arch/powerpc/kernel/entry_32.S linux-4.1.6/arch/power bne- do_resched andi. r0,r9,_TIF_USER_WORK_MASK beq restore_user -diff -Nur linux-4.1.6.orig/arch/powerpc/kernel/entry_64.S linux-4.1.6/arch/powerpc/kernel/entry_64.S ---- linux-4.1.6.orig/arch/powerpc/kernel/entry_64.S 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/powerpc/kernel/entry_64.S 2015-09-08 23:49:04.818231956 +0200 +diff -Nur linux-4.1.10.orig/arch/powerpc/kernel/entry_64.S linux-4.1.10/arch/powerpc/kernel/entry_64.S +--- linux-4.1.10.orig/arch/powerpc/kernel/entry_64.S 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/powerpc/kernel/entry_64.S 2015-10-07 18:00:07.000000000 +0200 @@ -636,7 +636,7 @@ #else beq restore @@ -2147,9 +2169,9 @@ diff -Nur linux-4.1.6.orig/arch/powerpc/kernel/entry_64.S linux-4.1.6/arch/power bne 1b /* -diff -Nur linux-4.1.6.orig/arch/powerpc/kernel/irq.c linux-4.1.6/arch/powerpc/kernel/irq.c ---- linux-4.1.6.orig/arch/powerpc/kernel/irq.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/powerpc/kernel/irq.c 2015-09-08 23:49:04.818231956 +0200 +diff -Nur linux-4.1.10.orig/arch/powerpc/kernel/irq.c linux-4.1.10/arch/powerpc/kernel/irq.c +--- linux-4.1.10.orig/arch/powerpc/kernel/irq.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/powerpc/kernel/irq.c 2015-10-07 18:00:07.000000000 +0200 @@ -614,6 +614,7 @@ } } @@ -2166,9 +2188,9 @@ diff -Nur linux-4.1.6.orig/arch/powerpc/kernel/irq.c linux-4.1.6/arch/powerpc/ke irq_hw_number_t virq_to_hw(unsigned int virq) { -diff -Nur linux-4.1.6.orig/arch/powerpc/kernel/misc_32.S linux-4.1.6/arch/powerpc/kernel/misc_32.S ---- linux-4.1.6.orig/arch/powerpc/kernel/misc_32.S 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/powerpc/kernel/misc_32.S 2015-09-08 23:49:04.818231956 +0200 +diff -Nur linux-4.1.10.orig/arch/powerpc/kernel/misc_32.S linux-4.1.10/arch/powerpc/kernel/misc_32.S +--- linux-4.1.10.orig/arch/powerpc/kernel/misc_32.S 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/powerpc/kernel/misc_32.S 2015-10-07 18:00:07.000000000 +0200 @@ -40,6 +40,7 @@ * We store the saved ksp_limit in the unused part * of the STACK_FRAME_OVERHEAD @@ -2185,9 +2207,9 @@ diff -Nur linux-4.1.6.orig/arch/powerpc/kernel/misc_32.S linux-4.1.6/arch/powerp /* * void call_do_irq(struct pt_regs *regs, struct thread_info *irqtp); -diff -Nur linux-4.1.6.orig/arch/powerpc/kernel/misc_64.S linux-4.1.6/arch/powerpc/kernel/misc_64.S ---- linux-4.1.6.orig/arch/powerpc/kernel/misc_64.S 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/powerpc/kernel/misc_64.S 2015-09-08 23:49:04.822231513 +0200 +diff -Nur linux-4.1.10.orig/arch/powerpc/kernel/misc_64.S linux-4.1.10/arch/powerpc/kernel/misc_64.S +--- linux-4.1.10.orig/arch/powerpc/kernel/misc_64.S 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/powerpc/kernel/misc_64.S 2015-10-07 18:00:07.000000000 +0200 @@ -29,6 +29,7 @@ .text @@ -2204,9 +2226,9 @@ diff -Nur linux-4.1.6.orig/arch/powerpc/kernel/misc_64.S linux-4.1.6/arch/powerp _GLOBAL(call_do_irq) mflr r0 -diff -Nur linux-4.1.6.orig/arch/powerpc/kvm/book3s_hv.c linux-4.1.6/arch/powerpc/kvm/book3s_hv.c ---- linux-4.1.6.orig/arch/powerpc/kvm/book3s_hv.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/powerpc/kvm/book3s_hv.c 2015-09-08 23:49:04.826231069 +0200 +diff -Nur linux-4.1.10.orig/arch/powerpc/kvm/book3s_hv.c linux-4.1.10/arch/powerpc/kvm/book3s_hv.c +--- linux-4.1.10.orig/arch/powerpc/kvm/book3s_hv.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/powerpc/kvm/book3s_hv.c 2015-10-07 18:00:07.000000000 +0200 @@ -115,11 +115,11 @@ static void kvmppc_fast_vcpu_kick_hv(struct kvm_vcpu *vcpu) { @@ -2282,9 +2304,9 @@ diff -Nur linux-4.1.6.orig/arch/powerpc/kvm/book3s_hv.c linux-4.1.6/arch/powerpc } } -diff -Nur linux-4.1.6.orig/arch/powerpc/kvm/Kconfig linux-4.1.6/arch/powerpc/kvm/Kconfig ---- linux-4.1.6.orig/arch/powerpc/kvm/Kconfig 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/powerpc/kvm/Kconfig 2015-09-08 23:49:04.822231513 +0200 +diff -Nur linux-4.1.10.orig/arch/powerpc/kvm/Kconfig linux-4.1.10/arch/powerpc/kvm/Kconfig +--- linux-4.1.10.orig/arch/powerpc/kvm/Kconfig 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/powerpc/kvm/Kconfig 2015-10-07 18:00:07.000000000 +0200 @@ -172,6 +172,7 @@ config KVM_MPIC bool "KVM in-kernel MPIC emulation" @@ -2293,9 +2315,9 @@ diff -Nur linux-4.1.6.orig/arch/powerpc/kvm/Kconfig linux-4.1.6/arch/powerpc/kvm select HAVE_KVM_IRQCHIP select HAVE_KVM_IRQFD select HAVE_KVM_IRQ_ROUTING -diff -Nur linux-4.1.6.orig/arch/powerpc/mm/fault.c linux-4.1.6/arch/powerpc/mm/fault.c ---- linux-4.1.6.orig/arch/powerpc/mm/fault.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/powerpc/mm/fault.c 2015-09-08 23:49:05.246184500 +0200 +diff -Nur linux-4.1.10.orig/arch/powerpc/mm/fault.c linux-4.1.10/arch/powerpc/mm/fault.c +--- linux-4.1.10.orig/arch/powerpc/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/powerpc/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 @@ -33,13 +33,13 @@ #include #include @@ -2331,9 +2353,9 @@ diff -Nur linux-4.1.6.orig/arch/powerpc/mm/fault.c linux-4.1.6/arch/powerpc/mm/f printk(KERN_EMERG "NIP = %lx MSR = %lx\n", regs->nip, regs->msr); die("Weird page fault", regs, SIGSEGV); -diff -Nur linux-4.1.6.orig/arch/powerpc/mm/highmem.c linux-4.1.6/arch/powerpc/mm/highmem.c ---- linux-4.1.6.orig/arch/powerpc/mm/highmem.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/powerpc/mm/highmem.c 2015-09-08 23:49:05.250184055 +0200 +diff -Nur linux-4.1.10.orig/arch/powerpc/mm/highmem.c linux-4.1.10/arch/powerpc/mm/highmem.c +--- linux-4.1.10.orig/arch/powerpc/mm/highmem.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/powerpc/mm/highmem.c 2015-10-07 18:00:07.000000000 +0200 @@ -34,7 +34,7 @@ unsigned long vaddr; int idx, type; @@ -2358,9 +2380,9 @@ diff -Nur linux-4.1.6.orig/arch/powerpc/mm/highmem.c linux-4.1.6/arch/powerpc/mm + preempt_enable(); } EXPORT_SYMBOL(__kunmap_atomic); -diff -Nur linux-4.1.6.orig/arch/powerpc/platforms/ps3/device-init.c linux-4.1.6/arch/powerpc/platforms/ps3/device-init.c ---- linux-4.1.6.orig/arch/powerpc/platforms/ps3/device-init.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/powerpc/platforms/ps3/device-init.c 2015-09-08 23:49:05.250184055 +0200 +diff -Nur linux-4.1.10.orig/arch/powerpc/platforms/ps3/device-init.c linux-4.1.10/arch/powerpc/platforms/ps3/device-init.c +--- linux-4.1.10.orig/arch/powerpc/platforms/ps3/device-init.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/powerpc/platforms/ps3/device-init.c 2015-10-07 18:00:07.000000000 +0200 @@ -752,7 +752,7 @@ } pr_debug("%s:%u: notification %s issued\n", __func__, __LINE__, op); @@ -2370,9 +2392,9 @@ diff -Nur linux-4.1.6.orig/arch/powerpc/platforms/ps3/device-init.c linux-4.1.6/ dev->done.done || kthread_should_stop()); if (kthread_should_stop()) res = -EINTR; -diff -Nur linux-4.1.6.orig/arch/s390/include/asm/kvm_host.h linux-4.1.6/arch/s390/include/asm/kvm_host.h ---- linux-4.1.6.orig/arch/s390/include/asm/kvm_host.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/s390/include/asm/kvm_host.h 2015-09-08 23:49:05.250184055 +0200 +diff -Nur linux-4.1.10.orig/arch/s390/include/asm/kvm_host.h linux-4.1.10/arch/s390/include/asm/kvm_host.h +--- linux-4.1.10.orig/arch/s390/include/asm/kvm_host.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/s390/include/asm/kvm_host.h 2015-10-07 18:00:07.000000000 +0200 @@ -419,7 +419,7 @@ struct kvm_s390_local_interrupt { spinlock_t lock; @@ -2382,9 +2404,9 @@ diff -Nur linux-4.1.6.orig/arch/s390/include/asm/kvm_host.h linux-4.1.6/arch/s39 atomic_t *cpuflags; DECLARE_BITMAP(sigp_emerg_pending, KVM_MAX_VCPUS); struct kvm_s390_irq_payload irq; -diff -Nur linux-4.1.6.orig/arch/s390/include/asm/uaccess.h linux-4.1.6/arch/s390/include/asm/uaccess.h ---- linux-4.1.6.orig/arch/s390/include/asm/uaccess.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/s390/include/asm/uaccess.h 2015-09-08 23:49:05.250184055 +0200 +diff -Nur linux-4.1.10.orig/arch/s390/include/asm/uaccess.h linux-4.1.10/arch/s390/include/asm/uaccess.h +--- linux-4.1.10.orig/arch/s390/include/asm/uaccess.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/s390/include/asm/uaccess.h 2015-10-07 18:00:07.000000000 +0200 @@ -98,7 +98,8 @@ * @from: Source address, in user space. * @n: Number of bytes to copy. @@ -2435,9 +2457,9 @@ diff -Nur linux-4.1.6.orig/arch/s390/include/asm/uaccess.h linux-4.1.6/arch/s390 * * Get the size of a NUL-terminated string in user space. * -diff -Nur linux-4.1.6.orig/arch/s390/kvm/interrupt.c linux-4.1.6/arch/s390/kvm/interrupt.c ---- linux-4.1.6.orig/arch/s390/kvm/interrupt.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/s390/kvm/interrupt.c 2015-09-08 23:49:05.250184055 +0200 +diff -Nur linux-4.1.10.orig/arch/s390/kvm/interrupt.c linux-4.1.10/arch/s390/kvm/interrupt.c +--- linux-4.1.10.orig/arch/s390/kvm/interrupt.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/s390/kvm/interrupt.c 2015-10-07 18:00:07.000000000 +0200 @@ -875,13 +875,13 @@ void kvm_s390_vcpu_wakeup(struct kvm_vcpu *vcpu) @@ -2472,9 +2494,9 @@ diff -Nur linux-4.1.6.orig/arch/s390/kvm/interrupt.c linux-4.1.6/arch/s390/kvm/i spin_unlock(&li->lock); return rc; } -diff -Nur linux-4.1.6.orig/arch/s390/mm/fault.c linux-4.1.6/arch/s390/mm/fault.c ---- linux-4.1.6.orig/arch/s390/mm/fault.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/s390/mm/fault.c 2015-09-08 23:49:05.254183611 +0200 +diff -Nur linux-4.1.10.orig/arch/s390/mm/fault.c linux-4.1.10/arch/s390/mm/fault.c +--- linux-4.1.10.orig/arch/s390/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/s390/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 @@ -399,7 +399,7 @@ * user context. */ @@ -2484,9 +2506,9 @@ diff -Nur linux-4.1.6.orig/arch/s390/mm/fault.c linux-4.1.6/arch/s390/mm/fault.c goto out; address = trans_exc_code & __FAIL_ADDR_MASK; -diff -Nur linux-4.1.6.orig/arch/score/include/asm/uaccess.h linux-4.1.6/arch/score/include/asm/uaccess.h ---- linux-4.1.6.orig/arch/score/include/asm/uaccess.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/score/include/asm/uaccess.h 2015-09-08 23:49:05.254183611 +0200 +diff -Nur linux-4.1.10.orig/arch/score/include/asm/uaccess.h linux-4.1.10/arch/score/include/asm/uaccess.h +--- linux-4.1.10.orig/arch/score/include/asm/uaccess.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/score/include/asm/uaccess.h 2015-10-07 18:00:07.000000000 +0200 @@ -36,7 +36,8 @@ * @addr: User space pointer to start of block to check * @size: Size of block to check @@ -2537,9 +2559,9 @@ diff -Nur linux-4.1.6.orig/arch/score/include/asm/uaccess.h linux-4.1.6/arch/sco * * This macro copies a single simple variable from user space to kernel * space. It supports simple types like char and int, but not larger -diff -Nur linux-4.1.6.orig/arch/score/mm/fault.c linux-4.1.6/arch/score/mm/fault.c ---- linux-4.1.6.orig/arch/score/mm/fault.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/score/mm/fault.c 2015-09-08 23:49:05.254183611 +0200 +diff -Nur linux-4.1.10.orig/arch/score/mm/fault.c linux-4.1.10/arch/score/mm/fault.c +--- linux-4.1.10.orig/arch/score/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/score/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 @@ -34,6 +34,7 @@ #include #include @@ -2557,9 +2579,9 @@ diff -Nur linux-4.1.6.orig/arch/score/mm/fault.c linux-4.1.6/arch/score/mm/fault goto bad_area_nosemaphore; if (user_mode(regs)) -diff -Nur linux-4.1.6.orig/arch/sh/kernel/irq.c linux-4.1.6/arch/sh/kernel/irq.c ---- linux-4.1.6.orig/arch/sh/kernel/irq.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/sh/kernel/irq.c 2015-09-08 23:49:05.254183611 +0200 +diff -Nur linux-4.1.10.orig/arch/sh/kernel/irq.c linux-4.1.10/arch/sh/kernel/irq.c +--- linux-4.1.10.orig/arch/sh/kernel/irq.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/sh/kernel/irq.c 2015-10-07 18:00:07.000000000 +0200 @@ -147,6 +147,7 @@ hardirq_ctx[cpu] = NULL; } @@ -2576,9 +2598,9 @@ diff -Nur linux-4.1.6.orig/arch/sh/kernel/irq.c linux-4.1.6/arch/sh/kernel/irq.c #else static inline void handle_one_irq(unsigned int irq) { -diff -Nur linux-4.1.6.orig/arch/sh/mm/fault.c linux-4.1.6/arch/sh/mm/fault.c ---- linux-4.1.6.orig/arch/sh/mm/fault.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/sh/mm/fault.c 2015-09-08 23:49:05.258183169 +0200 +diff -Nur linux-4.1.10.orig/arch/sh/mm/fault.c linux-4.1.10/arch/sh/mm/fault.c +--- linux-4.1.10.orig/arch/sh/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/sh/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 @@ -17,6 +17,7 @@ #include #include @@ -2599,9 +2621,9 @@ diff -Nur linux-4.1.6.orig/arch/sh/mm/fault.c linux-4.1.6/arch/sh/mm/fault.c bad_area_nosemaphore(regs, error_code, address); return; } -diff -Nur linux-4.1.6.orig/arch/sparc/Kconfig linux-4.1.6/arch/sparc/Kconfig ---- linux-4.1.6.orig/arch/sparc/Kconfig 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/sparc/Kconfig 2015-09-08 23:49:05.258183169 +0200 +diff -Nur linux-4.1.10.orig/arch/sparc/Kconfig linux-4.1.10/arch/sparc/Kconfig +--- linux-4.1.10.orig/arch/sparc/Kconfig 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/sparc/Kconfig 2015-10-07 18:00:07.000000000 +0200 @@ -189,12 +189,10 @@ source kernel/Kconfig.hz @@ -2617,9 +2639,9 @@ diff -Nur linux-4.1.6.orig/arch/sparc/Kconfig linux-4.1.6/arch/sparc/Kconfig config GENERIC_HWEIGHT bool -diff -Nur linux-4.1.6.orig/arch/sparc/kernel/irq_64.c linux-4.1.6/arch/sparc/kernel/irq_64.c ---- linux-4.1.6.orig/arch/sparc/kernel/irq_64.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/sparc/kernel/irq_64.c 2015-09-08 23:49:05.258183169 +0200 +diff -Nur linux-4.1.10.orig/arch/sparc/kernel/irq_64.c linux-4.1.10/arch/sparc/kernel/irq_64.c +--- linux-4.1.10.orig/arch/sparc/kernel/irq_64.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/sparc/kernel/irq_64.c 2015-10-07 18:00:07.000000000 +0200 @@ -849,6 +849,7 @@ set_irq_regs(old_regs); } @@ -2636,9 +2658,9 @@ diff -Nur linux-4.1.6.orig/arch/sparc/kernel/irq_64.c linux-4.1.6/arch/sparc/ker #ifdef CONFIG_HOTPLUG_CPU void fixup_irqs(void) -diff -Nur linux-4.1.6.orig/arch/sparc/mm/fault_32.c linux-4.1.6/arch/sparc/mm/fault_32.c ---- linux-4.1.6.orig/arch/sparc/mm/fault_32.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/sparc/mm/fault_32.c 2015-09-08 23:49:05.258183169 +0200 +diff -Nur linux-4.1.10.orig/arch/sparc/mm/fault_32.c linux-4.1.10/arch/sparc/mm/fault_32.c +--- linux-4.1.10.orig/arch/sparc/mm/fault_32.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/sparc/mm/fault_32.c 2015-10-07 18:00:07.000000000 +0200 @@ -21,6 +21,7 @@ #include #include @@ -2664,9 +2686,9 @@ diff -Nur linux-4.1.6.orig/arch/sparc/mm/fault_32.c linux-4.1.6/arch/sparc/mm/fa goto no_context; perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); -diff -Nur linux-4.1.6.orig/arch/sparc/mm/fault_64.c linux-4.1.6/arch/sparc/mm/fault_64.c ---- linux-4.1.6.orig/arch/sparc/mm/fault_64.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/sparc/mm/fault_64.c 2015-09-08 23:49:05.262182726 +0200 +diff -Nur linux-4.1.10.orig/arch/sparc/mm/fault_64.c linux-4.1.10/arch/sparc/mm/fault_64.c +--- linux-4.1.10.orig/arch/sparc/mm/fault_64.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/sparc/mm/fault_64.c 2015-10-07 18:00:07.000000000 +0200 @@ -22,12 +22,12 @@ #include #include @@ -2690,9 +2712,9 @@ diff -Nur linux-4.1.6.orig/arch/sparc/mm/fault_64.c linux-4.1.6/arch/sparc/mm/fa goto intr_or_no_mm; perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, address); -diff -Nur linux-4.1.6.orig/arch/sparc/mm/highmem.c linux-4.1.6/arch/sparc/mm/highmem.c ---- linux-4.1.6.orig/arch/sparc/mm/highmem.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/sparc/mm/highmem.c 2015-09-08 23:49:05.262182726 +0200 +diff -Nur linux-4.1.10.orig/arch/sparc/mm/highmem.c linux-4.1.10/arch/sparc/mm/highmem.c +--- linux-4.1.10.orig/arch/sparc/mm/highmem.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/sparc/mm/highmem.c 2015-10-07 18:00:07.000000000 +0200 @@ -53,7 +53,7 @@ unsigned long vaddr; long idx, type; @@ -2717,9 +2739,9 @@ diff -Nur linux-4.1.6.orig/arch/sparc/mm/highmem.c linux-4.1.6/arch/sparc/mm/hig + preempt_enable(); } EXPORT_SYMBOL(__kunmap_atomic); -diff -Nur linux-4.1.6.orig/arch/sparc/mm/init_64.c linux-4.1.6/arch/sparc/mm/init_64.c ---- linux-4.1.6.orig/arch/sparc/mm/init_64.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/sparc/mm/init_64.c 2015-09-08 23:49:05.262182726 +0200 +diff -Nur linux-4.1.10.orig/arch/sparc/mm/init_64.c linux-4.1.10/arch/sparc/mm/init_64.c +--- linux-4.1.10.orig/arch/sparc/mm/init_64.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/sparc/mm/init_64.c 2015-10-07 18:00:07.000000000 +0200 @@ -2738,7 +2738,7 @@ struct mm_struct *mm = current->mm; struct tsb_config *tp; @@ -2729,9 +2751,9 @@ diff -Nur linux-4.1.6.orig/arch/sparc/mm/init_64.c linux-4.1.6/arch/sparc/mm/ini const struct exception_table_entry *entry; entry = search_exception_tables(regs->tpc); -diff -Nur linux-4.1.6.orig/arch/tile/include/asm/uaccess.h linux-4.1.6/arch/tile/include/asm/uaccess.h ---- linux-4.1.6.orig/arch/tile/include/asm/uaccess.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/tile/include/asm/uaccess.h 2015-09-08 23:49:05.266182281 +0200 +diff -Nur linux-4.1.10.orig/arch/tile/include/asm/uaccess.h linux-4.1.10/arch/tile/include/asm/uaccess.h +--- linux-4.1.10.orig/arch/tile/include/asm/uaccess.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/tile/include/asm/uaccess.h 2015-10-07 18:00:07.000000000 +0200 @@ -78,7 +78,8 @@ * @addr: User space pointer to start of block to check * @size: Size of block to check @@ -2792,9 +2814,9 @@ diff -Nur linux-4.1.6.orig/arch/tile/include/asm/uaccess.h linux-4.1.6/arch/tile * * Copy data from user space to user space. Caller must check * the specified blocks with access_ok() before calling this function. -diff -Nur linux-4.1.6.orig/arch/tile/mm/fault.c linux-4.1.6/arch/tile/mm/fault.c ---- linux-4.1.6.orig/arch/tile/mm/fault.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/tile/mm/fault.c 2015-09-08 23:49:05.266182281 +0200 +diff -Nur linux-4.1.10.orig/arch/tile/mm/fault.c linux-4.1.10/arch/tile/mm/fault.c +--- linux-4.1.10.orig/arch/tile/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/tile/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 @@ -354,9 +354,9 @@ /* @@ -2807,9 +2829,9 @@ diff -Nur linux-4.1.6.orig/arch/tile/mm/fault.c linux-4.1.6/arch/tile/mm/fault.c vma = NULL; /* happy compiler */ goto bad_area_nosemaphore; } -diff -Nur linux-4.1.6.orig/arch/tile/mm/highmem.c linux-4.1.6/arch/tile/mm/highmem.c ---- linux-4.1.6.orig/arch/tile/mm/highmem.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/tile/mm/highmem.c 2015-09-08 23:49:05.266182281 +0200 +diff -Nur linux-4.1.10.orig/arch/tile/mm/highmem.c linux-4.1.10/arch/tile/mm/highmem.c +--- linux-4.1.10.orig/arch/tile/mm/highmem.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/tile/mm/highmem.c 2015-10-07 18:00:07.000000000 +0200 @@ -201,7 +201,7 @@ int idx, type; pte_t *pte; @@ -2827,9 +2849,9 @@ diff -Nur linux-4.1.6.orig/arch/tile/mm/highmem.c linux-4.1.6/arch/tile/mm/highm } EXPORT_SYMBOL(__kunmap_atomic); -diff -Nur linux-4.1.6.orig/arch/um/kernel/trap.c linux-4.1.6/arch/um/kernel/trap.c ---- linux-4.1.6.orig/arch/um/kernel/trap.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/um/kernel/trap.c 2015-09-08 23:49:05.266182281 +0200 +diff -Nur linux-4.1.10.orig/arch/um/kernel/trap.c linux-4.1.10/arch/um/kernel/trap.c +--- linux-4.1.10.orig/arch/um/kernel/trap.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/um/kernel/trap.c 2015-10-07 18:00:07.000000000 +0200 @@ -35,10 +35,10 @@ *code_out = SEGV_MAPERR; @@ -2843,9 +2865,9 @@ diff -Nur linux-4.1.6.orig/arch/um/kernel/trap.c linux-4.1.6/arch/um/kernel/trap goto out_nosemaphore; if (is_user) -diff -Nur linux-4.1.6.orig/arch/unicore32/mm/fault.c linux-4.1.6/arch/unicore32/mm/fault.c ---- linux-4.1.6.orig/arch/unicore32/mm/fault.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/unicore32/mm/fault.c 2015-09-08 23:49:05.266182281 +0200 +diff -Nur linux-4.1.10.orig/arch/unicore32/mm/fault.c linux-4.1.10/arch/unicore32/mm/fault.c +--- linux-4.1.10.orig/arch/unicore32/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/unicore32/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 @@ -218,7 +218,7 @@ * If we're in an interrupt or have no user * context, we must not take the fault.. @@ -2855,9 +2877,9 @@ diff -Nur linux-4.1.6.orig/arch/unicore32/mm/fault.c linux-4.1.6/arch/unicore32/ goto no_context; if (user_mode(regs)) -diff -Nur linux-4.1.6.orig/arch/x86/crypto/aesni-intel_glue.c linux-4.1.6/arch/x86/crypto/aesni-intel_glue.c ---- linux-4.1.6.orig/arch/x86/crypto/aesni-intel_glue.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/crypto/aesni-intel_glue.c 2015-09-08 23:49:05.270181837 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/crypto/aesni-intel_glue.c linux-4.1.10/arch/x86/crypto/aesni-intel_glue.c +--- linux-4.1.10.orig/arch/x86/crypto/aesni-intel_glue.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/crypto/aesni-intel_glue.c 2015-10-07 18:00:07.000000000 +0200 @@ -382,14 +382,14 @@ err = blkcipher_walk_virt(desc, &walk); desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; @@ -2950,9 +2972,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/crypto/aesni-intel_glue.c linux-4.1.6/arch/x return err; } -diff -Nur linux-4.1.6.orig/arch/x86/crypto/cast5_avx_glue.c linux-4.1.6/arch/x86/crypto/cast5_avx_glue.c ---- linux-4.1.6.orig/arch/x86/crypto/cast5_avx_glue.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/crypto/cast5_avx_glue.c 2015-09-08 23:49:05.270181837 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/crypto/cast5_avx_glue.c linux-4.1.10/arch/x86/crypto/cast5_avx_glue.c +--- linux-4.1.10.orig/arch/x86/crypto/cast5_avx_glue.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/crypto/cast5_avx_glue.c 2015-10-07 18:00:07.000000000 +0200 @@ -60,7 +60,7 @@ static int ecb_crypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk, bool enc) @@ -3032,9 +3054,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/crypto/cast5_avx_glue.c linux-4.1.6/arch/x86 if (walk.nbytes) { ctr_crypt_final(desc, &walk); err = blkcipher_walk_done(desc, &walk, 0); -diff -Nur linux-4.1.6.orig/arch/x86/crypto/glue_helper.c linux-4.1.6/arch/x86/crypto/glue_helper.c ---- linux-4.1.6.orig/arch/x86/crypto/glue_helper.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/crypto/glue_helper.c 2015-09-08 23:49:05.270181837 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/crypto/glue_helper.c linux-4.1.10/arch/x86/crypto/glue_helper.c +--- linux-4.1.10.orig/arch/x86/crypto/glue_helper.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/crypto/glue_helper.c 2015-10-07 18:00:07.000000000 +0200 @@ -39,7 +39,7 @@ void *ctx = crypto_blkcipher_ctx(desc->tfm); const unsigned int bsize = 128 / 8; @@ -3150,9 +3172,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/crypto/glue_helper.c linux-4.1.6/arch/x86/cr return err; } EXPORT_SYMBOL_GPL(glue_xts_crypt_128bit); -diff -Nur linux-4.1.6.orig/arch/x86/include/asm/preempt.h linux-4.1.6/arch/x86/include/asm/preempt.h ---- linux-4.1.6.orig/arch/x86/include/asm/preempt.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/include/asm/preempt.h 2015-09-08 23:49:05.270181837 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/include/asm/preempt.h linux-4.1.10/arch/x86/include/asm/preempt.h +--- linux-4.1.10.orig/arch/x86/include/asm/preempt.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/include/asm/preempt.h 2015-10-07 18:00:07.000000000 +0200 @@ -82,17 +82,33 @@ * a decrement which hits zero means we have no preempt_count and should * reschedule. @@ -3188,9 +3210,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/include/asm/preempt.h linux-4.1.6/arch/x86/i } #ifdef CONFIG_PREEMPT -diff -Nur linux-4.1.6.orig/arch/x86/include/asm/signal.h linux-4.1.6/arch/x86/include/asm/signal.h ---- linux-4.1.6.orig/arch/x86/include/asm/signal.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/include/asm/signal.h 2015-09-08 23:49:05.270181837 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/include/asm/signal.h linux-4.1.10/arch/x86/include/asm/signal.h +--- linux-4.1.10.orig/arch/x86/include/asm/signal.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/include/asm/signal.h 2015-10-07 18:00:07.000000000 +0200 @@ -23,6 +23,19 @@ unsigned long sig[_NSIG_WORDS]; } sigset_t; @@ -3211,9 +3233,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/include/asm/signal.h linux-4.1.6/arch/x86/in #ifndef CONFIG_COMPAT typedef sigset_t compat_sigset_t; #endif -diff -Nur linux-4.1.6.orig/arch/x86/include/asm/stackprotector.h linux-4.1.6/arch/x86/include/asm/stackprotector.h ---- linux-4.1.6.orig/arch/x86/include/asm/stackprotector.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/include/asm/stackprotector.h 2015-09-08 23:49:05.298178732 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/include/asm/stackprotector.h linux-4.1.10/arch/x86/include/asm/stackprotector.h +--- linux-4.1.10.orig/arch/x86/include/asm/stackprotector.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/include/asm/stackprotector.h 2015-10-07 18:00:07.000000000 +0200 @@ -57,7 +57,7 @@ */ static __always_inline void boot_init_stack_canary(void) @@ -3240,9 +3262,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/include/asm/stackprotector.h linux-4.1.6/arc tsc = __native_read_tsc(); canary += tsc + (tsc << 32UL); -diff -Nur linux-4.1.6.orig/arch/x86/include/asm/thread_info.h linux-4.1.6/arch/x86/include/asm/thread_info.h ---- linux-4.1.6.orig/arch/x86/include/asm/thread_info.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/include/asm/thread_info.h 2015-09-08 23:49:05.514154783 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/include/asm/thread_info.h linux-4.1.10/arch/x86/include/asm/thread_info.h +--- linux-4.1.10.orig/arch/x86/include/asm/thread_info.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/include/asm/thread_info.h 2015-10-07 18:00:07.000000000 +0200 @@ -55,6 +55,8 @@ __u32 status; /* thread synchronous flags */ __u32 cpu; /* current CPU */ @@ -3277,9 +3299,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/include/asm/thread_info.h linux-4.1.6/arch/x #define STACK_WARN (THREAD_SIZE/8) /* -diff -Nur linux-4.1.6.orig/arch/x86/include/asm/uaccess_32.h linux-4.1.6/arch/x86/include/asm/uaccess_32.h ---- linux-4.1.6.orig/arch/x86/include/asm/uaccess_32.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/include/asm/uaccess_32.h 2015-09-08 23:49:05.514154783 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/include/asm/uaccess_32.h linux-4.1.10/arch/x86/include/asm/uaccess_32.h +--- linux-4.1.10.orig/arch/x86/include/asm/uaccess_32.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/include/asm/uaccess_32.h 2015-10-07 18:00:07.000000000 +0200 @@ -70,7 +70,8 @@ * @from: Source address, in kernel space. * @n: Number of bytes to copy. @@ -3300,9 +3322,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/include/asm/uaccess_32.h linux-4.1.6/arch/x8 * * Copy data from user space to kernel space. Caller must check * the specified block with access_ok() before calling this function. -diff -Nur linux-4.1.6.orig/arch/x86/include/asm/uaccess.h linux-4.1.6/arch/x86/include/asm/uaccess.h ---- linux-4.1.6.orig/arch/x86/include/asm/uaccess.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/include/asm/uaccess.h 2015-09-08 23:49:05.514154783 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/include/asm/uaccess.h linux-4.1.10/arch/x86/include/asm/uaccess.h +--- linux-4.1.10.orig/arch/x86/include/asm/uaccess.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/include/asm/uaccess.h 2015-10-07 18:00:07.000000000 +0200 @@ -74,7 +74,8 @@ * @addr: User space pointer to start of block to check * @size: Size of block to check @@ -3353,9 +3375,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/include/asm/uaccess.h linux-4.1.6/arch/x86/i * * This macro copies a single simple value from kernel space to user * space. It supports simple types like char and int, but not larger -diff -Nur linux-4.1.6.orig/arch/x86/include/asm/uv/uv_bau.h linux-4.1.6/arch/x86/include/asm/uv/uv_bau.h ---- linux-4.1.6.orig/arch/x86/include/asm/uv/uv_bau.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/include/asm/uv/uv_bau.h 2015-09-08 23:49:05.514154783 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/include/asm/uv/uv_bau.h linux-4.1.10/arch/x86/include/asm/uv/uv_bau.h +--- linux-4.1.10.orig/arch/x86/include/asm/uv/uv_bau.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/include/asm/uv/uv_bau.h 2015-10-07 18:00:07.000000000 +0200 @@ -615,9 +615,9 @@ cycles_t send_message; cycles_t period_end; @@ -3389,9 +3411,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/include/asm/uv/uv_bau.h linux-4.1.6/arch/x86 return 1; } -diff -Nur linux-4.1.6.orig/arch/x86/include/asm/uv/uv_hub.h linux-4.1.6/arch/x86/include/asm/uv/uv_hub.h ---- linux-4.1.6.orig/arch/x86/include/asm/uv/uv_hub.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/include/asm/uv/uv_hub.h 2015-09-08 23:49:05.518154339 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/include/asm/uv/uv_hub.h linux-4.1.10/arch/x86/include/asm/uv/uv_hub.h +--- linux-4.1.10.orig/arch/x86/include/asm/uv/uv_hub.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/include/asm/uv/uv_hub.h 2015-10-07 18:00:07.000000000 +0200 @@ -492,7 +492,7 @@ unsigned short nr_online_cpus; unsigned short pnode; @@ -3401,9 +3423,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/include/asm/uv/uv_hub.h linux-4.1.6/arch/x86 unsigned long nmi_count; /* obsolete, see uv_hub_nmi */ }; extern struct uv_blade_info *uv_blade_info; -diff -Nur linux-4.1.6.orig/arch/x86/Kconfig linux-4.1.6/arch/x86/Kconfig ---- linux-4.1.6.orig/arch/x86/Kconfig 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/Kconfig 2015-09-08 23:49:05.270181837 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/Kconfig linux-4.1.10/arch/x86/Kconfig +--- linux-4.1.10.orig/arch/x86/Kconfig 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/Kconfig 2015-10-07 18:00:07.000000000 +0200 @@ -22,6 +22,7 @@ ### Arch settings config X86 @@ -3434,9 +3456,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/Kconfig linux-4.1.6/arch/x86/Kconfig ---help--- Enable maximum number of CPUS and NUMA Nodes for this architecture. If unsure, say N. -diff -Nur linux-4.1.6.orig/arch/x86/kernel/apic/io_apic.c linux-4.1.6/arch/x86/kernel/apic/io_apic.c ---- linux-4.1.6.orig/arch/x86/kernel/apic/io_apic.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/kernel/apic/io_apic.c 2015-09-08 23:49:05.518154339 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/kernel/apic/io_apic.c linux-4.1.10/arch/x86/kernel/apic/io_apic.c +--- linux-4.1.10.orig/arch/x86/kernel/apic/io_apic.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/kernel/apic/io_apic.c 2015-10-07 18:00:07.000000000 +0200 @@ -1891,7 +1891,8 @@ static inline bool ioapic_irqd_mask(struct irq_data *data, struct irq_cfg *cfg) { @@ -3447,9 +3469,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/kernel/apic/io_apic.c linux-4.1.6/arch/x86/k mask_ioapic(cfg); return true; } -diff -Nur linux-4.1.6.orig/arch/x86/kernel/apic/x2apic_uv_x.c linux-4.1.6/arch/x86/kernel/apic/x2apic_uv_x.c ---- linux-4.1.6.orig/arch/x86/kernel/apic/x2apic_uv_x.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/kernel/apic/x2apic_uv_x.c 2015-09-08 23:49:05.518154339 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/kernel/apic/x2apic_uv_x.c linux-4.1.10/arch/x86/kernel/apic/x2apic_uv_x.c +--- linux-4.1.10.orig/arch/x86/kernel/apic/x2apic_uv_x.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/kernel/apic/x2apic_uv_x.c 2015-10-07 18:00:07.000000000 +0200 @@ -949,7 +949,7 @@ uv_blade_info[blade].pnode = pnode; uv_blade_info[blade].nr_possible_cpus = 0; @@ -3459,9 +3481,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/kernel/apic/x2apic_uv_x.c linux-4.1.6/arch/x min_pnode = min(pnode, min_pnode); max_pnode = max(pnode, max_pnode); blade++; -diff -Nur linux-4.1.6.orig/arch/x86/kernel/asm-offsets.c linux-4.1.6/arch/x86/kernel/asm-offsets.c ---- linux-4.1.6.orig/arch/x86/kernel/asm-offsets.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/kernel/asm-offsets.c 2015-09-08 23:49:05.522153895 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/kernel/asm-offsets.c linux-4.1.10/arch/x86/kernel/asm-offsets.c +--- linux-4.1.10.orig/arch/x86/kernel/asm-offsets.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/kernel/asm-offsets.c 2015-10-07 18:00:07.000000000 +0200 @@ -32,6 +32,7 @@ OFFSET(TI_flags, thread_info, flags); OFFSET(TI_status, thread_info, status); @@ -3476,9 +3498,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/kernel/asm-offsets.c linux-4.1.6/arch/x86/ke DEFINE(PTREGS_SIZE, sizeof(struct pt_regs)); + DEFINE(_PREEMPT_ENABLED, PREEMPT_ENABLED); } -diff -Nur linux-4.1.6.orig/arch/x86/kernel/cpu/mcheck/mce.c linux-4.1.6/arch/x86/kernel/cpu/mcheck/mce.c ---- linux-4.1.6.orig/arch/x86/kernel/cpu/mcheck/mce.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/kernel/cpu/mcheck/mce.c 2015-09-08 23:49:05.522153895 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/kernel/cpu/mcheck/mce.c linux-4.1.10/arch/x86/kernel/cpu/mcheck/mce.c +--- linux-4.1.10.orig/arch/x86/kernel/cpu/mcheck/mce.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/kernel/cpu/mcheck/mce.c 2015-10-07 18:00:07.000000000 +0200 @@ -41,6 +41,8 @@ #include #include @@ -3720,9 +3742,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/kernel/cpu/mcheck/mce.c linux-4.1.6/arch/x86 if (!zalloc_cpumask_var(&mce_device_initialized, GFP_KERNEL)) { err = -ENOMEM; goto err_out; -diff -Nur linux-4.1.6.orig/arch/x86/kernel/dumpstack_32.c linux-4.1.6/arch/x86/kernel/dumpstack_32.c ---- linux-4.1.6.orig/arch/x86/kernel/dumpstack_32.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/kernel/dumpstack_32.c 2015-09-08 23:49:05.526153452 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/kernel/dumpstack_32.c linux-4.1.10/arch/x86/kernel/dumpstack_32.c +--- linux-4.1.10.orig/arch/x86/kernel/dumpstack_32.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/kernel/dumpstack_32.c 2015-10-07 18:00:07.000000000 +0200 @@ -42,7 +42,7 @@ unsigned long *stack, unsigned long bp, const struct stacktrace_ops *ops, void *data) @@ -3741,9 +3763,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/kernel/dumpstack_32.c linux-4.1.6/arch/x86/k } EXPORT_SYMBOL(dump_trace); -diff -Nur linux-4.1.6.orig/arch/x86/kernel/dumpstack_64.c linux-4.1.6/arch/x86/kernel/dumpstack_64.c ---- linux-4.1.6.orig/arch/x86/kernel/dumpstack_64.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/kernel/dumpstack_64.c 2015-09-08 23:49:05.526153452 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/kernel/dumpstack_64.c linux-4.1.10/arch/x86/kernel/dumpstack_64.c +--- linux-4.1.10.orig/arch/x86/kernel/dumpstack_64.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/kernel/dumpstack_64.c 2015-10-07 18:00:07.000000000 +0200 @@ -152,7 +152,7 @@ unsigned long *stack, unsigned long bp, const struct stacktrace_ops *ops, void *data) @@ -3780,9 +3802,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/kernel/dumpstack_64.c linux-4.1.6/arch/x86/k pr_cont("\n"); show_trace_log_lvl(task, regs, sp, bp, log_lvl); -diff -Nur linux-4.1.6.orig/arch/x86/kernel/entry_32.S linux-4.1.6/arch/x86/kernel/entry_32.S ---- linux-4.1.6.orig/arch/x86/kernel/entry_32.S 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/kernel/entry_32.S 2015-09-08 23:49:05.526153452 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/kernel/entry_32.S linux-4.1.10/arch/x86/kernel/entry_32.S +--- linux-4.1.10.orig/arch/x86/kernel/entry_32.S 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/kernel/entry_32.S 2015-10-07 18:00:07.000000000 +0200 @@ -359,8 +359,24 @@ ENTRY(resume_kernel) DISABLE_INTERRUPTS(CLBR_ANY) @@ -3826,9 +3848,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/kernel/entry_32.S linux-4.1.6/arch/x86/kerne jnz work_resched work_notifysig: # deal with pending signals and -diff -Nur linux-4.1.6.orig/arch/x86/kernel/entry_64.S linux-4.1.6/arch/x86/kernel/entry_64.S ---- linux-4.1.6.orig/arch/x86/kernel/entry_64.S 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/kernel/entry_64.S 2015-09-08 23:49:05.530153010 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/kernel/entry_64.S linux-4.1.10/arch/x86/kernel/entry_64.S +--- linux-4.1.10.orig/arch/x86/kernel/entry_64.S 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/kernel/entry_64.S 2015-10-07 18:00:07.000000000 +0200 @@ -370,8 +370,8 @@ /* First do a reschedule test. */ /* edx: work, edi: workmask */ @@ -3891,9 +3913,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/kernel/entry_64.S linux-4.1.6/arch/x86/kerne #ifdef CONFIG_XEN idtentry xen_hypervisor_callback xen_do_hypervisor_callback has_error_code=0 -diff -Nur linux-4.1.6.orig/arch/x86/kernel/irq_32.c linux-4.1.6/arch/x86/kernel/irq_32.c ---- linux-4.1.6.orig/arch/x86/kernel/irq_32.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/kernel/irq_32.c 2015-09-08 23:49:05.530153010 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/kernel/irq_32.c linux-4.1.10/arch/x86/kernel/irq_32.c +--- linux-4.1.10.orig/arch/x86/kernel/irq_32.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/kernel/irq_32.c 2015-10-07 18:00:07.000000000 +0200 @@ -135,6 +135,7 @@ cpu, per_cpu(hardirq_stack, cpu), per_cpu(softirq_stack, cpu)); } @@ -3910,9 +3932,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/kernel/irq_32.c linux-4.1.6/arch/x86/kernel/ bool handle_irq(unsigned irq, struct pt_regs *regs) { -diff -Nur linux-4.1.6.orig/arch/x86/kernel/process_32.c linux-4.1.6/arch/x86/kernel/process_32.c ---- linux-4.1.6.orig/arch/x86/kernel/process_32.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/kernel/process_32.c 2015-09-08 23:49:05.530153010 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/kernel/process_32.c linux-4.1.10/arch/x86/kernel/process_32.c +--- linux-4.1.10.orig/arch/x86/kernel/process_32.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/kernel/process_32.c 2015-10-07 18:00:07.000000000 +0200 @@ -35,6 +35,7 @@ #include #include @@ -3966,10 +3988,10 @@ diff -Nur linux-4.1.6.orig/arch/x86/kernel/process_32.c linux-4.1.6/arch/x86/ker /* * Leave lazy mode, flushing any hypercalls made here. * This must be done before restoring TLS segments so -diff -Nur linux-4.1.6.orig/arch/x86/kernel/signal.c linux-4.1.6/arch/x86/kernel/signal.c ---- linux-4.1.6.orig/arch/x86/kernel/signal.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/kernel/signal.c 2015-09-08 23:49:05.534152567 +0200 -@@ -727,6 +727,14 @@ +diff -Nur linux-4.1.10.orig/arch/x86/kernel/signal.c linux-4.1.10/arch/x86/kernel/signal.c +--- linux-4.1.10.orig/arch/x86/kernel/signal.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/kernel/signal.c 2015-10-07 18:00:07.000000000 +0200 +@@ -723,6 +723,14 @@ { user_exit(); @@ -3984,9 +4006,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/kernel/signal.c linux-4.1.6/arch/x86/kernel/ if (thread_info_flags & _TIF_UPROBE) uprobe_notify_resume(regs); -diff -Nur linux-4.1.6.orig/arch/x86/kernel/traps.c linux-4.1.6/arch/x86/kernel/traps.c ---- linux-4.1.6.orig/arch/x86/kernel/traps.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/kernel/traps.c 2015-09-08 23:49:05.534152567 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/kernel/traps.c linux-4.1.10/arch/x86/kernel/traps.c +--- linux-4.1.10.orig/arch/x86/kernel/traps.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/kernel/traps.c 2015-10-07 18:00:07.000000000 +0200 @@ -88,9 +88,21 @@ local_irq_enable(); } @@ -4061,9 +4083,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/kernel/traps.c linux-4.1.6/arch/x86/kernel/t debug_stack_usage_dec(); exit: -diff -Nur linux-4.1.6.orig/arch/x86/kvm/lapic.c linux-4.1.6/arch/x86/kvm/lapic.c ---- linux-4.1.6.orig/arch/x86/kvm/lapic.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/kvm/lapic.c 2015-09-08 23:49:05.534152567 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/kvm/lapic.c linux-4.1.10/arch/x86/kvm/lapic.c +--- linux-4.1.10.orig/arch/x86/kvm/lapic.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/kvm/lapic.c 2015-10-07 18:00:07.000000000 +0200 @@ -1104,7 +1104,7 @@ static void apic_timer_expired(struct kvm_lapic *apic) { @@ -4164,9 +4186,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/kvm/lapic.c linux-4.1.6/arch/x86/kvm/lapic.c } /* -diff -Nur linux-4.1.6.orig/arch/x86/kvm/x86.c linux-4.1.6/arch/x86/kvm/x86.c ---- linux-4.1.6.orig/arch/x86/kvm/x86.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/kvm/x86.c 2015-09-08 23:49:05.538152123 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/kvm/x86.c linux-4.1.10/arch/x86/kvm/x86.c +--- linux-4.1.10.orig/arch/x86/kvm/x86.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/kvm/x86.c 2015-10-07 18:00:07.000000000 +0200 @@ -5813,6 +5813,13 @@ goto out; } @@ -4181,9 +4203,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/kvm/x86.c linux-4.1.6/arch/x86/kvm/x86.c r = kvm_mmu_module_init(); if (r) goto out_free_percpu; -diff -Nur linux-4.1.6.orig/arch/x86/lib/usercopy_32.c linux-4.1.6/arch/x86/lib/usercopy_32.c ---- linux-4.1.6.orig/arch/x86/lib/usercopy_32.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/lib/usercopy_32.c 2015-09-08 23:49:05.754128173 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/lib/usercopy_32.c linux-4.1.10/arch/x86/lib/usercopy_32.c +--- linux-4.1.10.orig/arch/x86/lib/usercopy_32.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/lib/usercopy_32.c 2015-10-07 18:00:07.000000000 +0200 @@ -647,7 +647,8 @@ * @from: Source address, in kernel space. * @n: Number of bytes to copy. @@ -4204,9 +4226,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/lib/usercopy_32.c linux-4.1.6/arch/x86/lib/u * * Copy data from user space to kernel space. * -diff -Nur linux-4.1.6.orig/arch/x86/mm/fault.c linux-4.1.6/arch/x86/mm/fault.c ---- linux-4.1.6.orig/arch/x86/mm/fault.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/mm/fault.c 2015-09-08 23:49:05.754128173 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/mm/fault.c linux-4.1.10/arch/x86/mm/fault.c +--- linux-4.1.10.orig/arch/x86/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 @@ -13,6 +13,7 @@ #include /* hstate_index_to_shift */ #include /* prefetchw */ @@ -4227,9 +4249,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/mm/fault.c linux-4.1.6/arch/x86/mm/fault.c bad_area_nosemaphore(regs, error_code, address); return; } -diff -Nur linux-4.1.6.orig/arch/x86/mm/highmem_32.c linux-4.1.6/arch/x86/mm/highmem_32.c ---- linux-4.1.6.orig/arch/x86/mm/highmem_32.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/mm/highmem_32.c 2015-09-08 23:49:05.754128173 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/mm/highmem_32.c linux-4.1.10/arch/x86/mm/highmem_32.c +--- linux-4.1.10.orig/arch/x86/mm/highmem_32.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/mm/highmem_32.c 2015-10-07 18:00:07.000000000 +0200 @@ -32,10 +32,11 @@ */ void *kmap_atomic_prot(struct page *page, pgprot_t prot) @@ -4273,9 +4295,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/mm/highmem_32.c linux-4.1.6/arch/x86/mm/high } EXPORT_SYMBOL(__kunmap_atomic); -diff -Nur linux-4.1.6.orig/arch/x86/mm/iomap_32.c linux-4.1.6/arch/x86/mm/iomap_32.c ---- linux-4.1.6.orig/arch/x86/mm/iomap_32.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/mm/iomap_32.c 2015-09-08 23:49:05.754128173 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/mm/iomap_32.c linux-4.1.10/arch/x86/mm/iomap_32.c +--- linux-4.1.10.orig/arch/x86/mm/iomap_32.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/mm/iomap_32.c 2015-10-07 18:00:07.000000000 +0200 @@ -56,15 +56,22 @@ void *kmap_atomic_prot_pfn(unsigned long pfn, pgprot_t prot) @@ -4315,9 +4337,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/mm/iomap_32.c linux-4.1.6/arch/x86/mm/iomap_ + preempt_enable(); } EXPORT_SYMBOL_GPL(iounmap_atomic); -diff -Nur linux-4.1.6.orig/arch/x86/platform/uv/tlb_uv.c linux-4.1.6/arch/x86/platform/uv/tlb_uv.c ---- linux-4.1.6.orig/arch/x86/platform/uv/tlb_uv.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/platform/uv/tlb_uv.c 2015-09-08 23:49:05.758127730 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/platform/uv/tlb_uv.c linux-4.1.10/arch/x86/platform/uv/tlb_uv.c +--- linux-4.1.10.orig/arch/x86/platform/uv/tlb_uv.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/platform/uv/tlb_uv.c 2015-10-07 18:00:07.000000000 +0200 @@ -714,9 +714,9 @@ quiesce_local_uvhub(hmaster); @@ -4404,9 +4426,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/platform/uv/tlb_uv.c linux-4.1.6/arch/x86/pl } } -diff -Nur linux-4.1.6.orig/arch/x86/platform/uv/uv_time.c linux-4.1.6/arch/x86/platform/uv/uv_time.c ---- linux-4.1.6.orig/arch/x86/platform/uv/uv_time.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/x86/platform/uv/uv_time.c 2015-09-08 23:49:05.758127730 +0200 +diff -Nur linux-4.1.10.orig/arch/x86/platform/uv/uv_time.c linux-4.1.10/arch/x86/platform/uv/uv_time.c +--- linux-4.1.10.orig/arch/x86/platform/uv/uv_time.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/x86/platform/uv/uv_time.c 2015-10-07 18:00:07.000000000 +0200 @@ -58,7 +58,7 @@ /* There is one of these allocated per node */ @@ -4487,9 +4509,9 @@ diff -Nur linux-4.1.6.orig/arch/x86/platform/uv/uv_time.c linux-4.1.6/arch/x86/p } /* -diff -Nur linux-4.1.6.orig/arch/xtensa/mm/fault.c linux-4.1.6/arch/xtensa/mm/fault.c ---- linux-4.1.6.orig/arch/xtensa/mm/fault.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/xtensa/mm/fault.c 2015-09-08 23:49:05.758127730 +0200 +diff -Nur linux-4.1.10.orig/arch/xtensa/mm/fault.c linux-4.1.10/arch/xtensa/mm/fault.c +--- linux-4.1.10.orig/arch/xtensa/mm/fault.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/xtensa/mm/fault.c 2015-10-07 18:00:07.000000000 +0200 @@ -15,10 +15,10 @@ #include #include @@ -4511,9 +4533,9 @@ diff -Nur linux-4.1.6.orig/arch/xtensa/mm/fault.c linux-4.1.6/arch/xtensa/mm/fau bad_page_fault(regs, address, SIGSEGV); return; } -diff -Nur linux-4.1.6.orig/arch/xtensa/mm/highmem.c linux-4.1.6/arch/xtensa/mm/highmem.c ---- linux-4.1.6.orig/arch/xtensa/mm/highmem.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/arch/xtensa/mm/highmem.c 2015-09-08 23:49:05.758127730 +0200 +diff -Nur linux-4.1.10.orig/arch/xtensa/mm/highmem.c linux-4.1.10/arch/xtensa/mm/highmem.c +--- linux-4.1.10.orig/arch/xtensa/mm/highmem.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/arch/xtensa/mm/highmem.c 2015-10-07 18:00:07.000000000 +0200 @@ -42,6 +42,7 @@ enum fixed_addresses idx; unsigned long vaddr; @@ -4530,9 +4552,9 @@ diff -Nur linux-4.1.6.orig/arch/xtensa/mm/highmem.c linux-4.1.6/arch/xtensa/mm/h } EXPORT_SYMBOL(__kunmap_atomic); -diff -Nur linux-4.1.6.orig/block/blk-core.c linux-4.1.6/block/blk-core.c ---- linux-4.1.6.orig/block/blk-core.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/block/blk-core.c 2015-09-08 23:49:05.762127285 +0200 +diff -Nur linux-4.1.10.orig/block/blk-core.c linux-4.1.10/block/blk-core.c +--- linux-4.1.10.orig/block/blk-core.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/block/blk-core.c 2015-10-07 18:00:07.000000000 +0200 @@ -100,6 +100,9 @@ INIT_LIST_HEAD(&rq->queuelist); @@ -4608,9 +4630,9 @@ diff -Nur linux-4.1.6.orig/block/blk-core.c linux-4.1.6/block/blk-core.c } void blk_finish_plug(struct blk_plug *plug) -diff -Nur linux-4.1.6.orig/block/blk-ioc.c linux-4.1.6/block/blk-ioc.c ---- linux-4.1.6.orig/block/blk-ioc.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/block/blk-ioc.c 2015-09-08 23:49:05.762127285 +0200 +diff -Nur linux-4.1.10.orig/block/blk-ioc.c linux-4.1.10/block/blk-ioc.c +--- linux-4.1.10.orig/block/blk-ioc.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/block/blk-ioc.c 2015-10-07 18:00:07.000000000 +0200 @@ -7,6 +7,7 @@ #include #include @@ -4637,9 +4659,9 @@ diff -Nur linux-4.1.6.orig/block/blk-ioc.c linux-4.1.6/block/blk-ioc.c goto retry; } } -diff -Nur linux-4.1.6.orig/block/blk-iopoll.c linux-4.1.6/block/blk-iopoll.c ---- linux-4.1.6.orig/block/blk-iopoll.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/block/blk-iopoll.c 2015-09-08 23:49:05.762127285 +0200 +diff -Nur linux-4.1.10.orig/block/blk-iopoll.c linux-4.1.10/block/blk-iopoll.c +--- linux-4.1.10.orig/block/blk-iopoll.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/block/blk-iopoll.c 2015-10-07 18:00:07.000000000 +0200 @@ -35,6 +35,7 @@ list_add_tail(&iop->list, this_cpu_ptr(&blk_cpu_iopoll)); __raise_softirq_irqoff(BLOCK_IOPOLL_SOFTIRQ); @@ -4664,9 +4686,9 @@ diff -Nur linux-4.1.6.orig/block/blk-iopoll.c linux-4.1.6/block/blk-iopoll.c } return NOTIFY_OK; -diff -Nur linux-4.1.6.orig/block/blk-mq.c linux-4.1.6/block/blk-mq.c ---- linux-4.1.6.orig/block/blk-mq.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/block/blk-mq.c 2015-09-08 23:49:05.762127285 +0200 +diff -Nur linux-4.1.10.orig/block/blk-mq.c linux-4.1.10/block/blk-mq.c +--- linux-4.1.10.orig/block/blk-mq.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/block/blk-mq.c 2015-10-07 18:00:07.000000000 +0200 @@ -88,7 +88,7 @@ if (!(gfp & __GFP_WAIT)) return -EBUSY; @@ -4802,9 +4824,9 @@ diff -Nur linux-4.1.6.orig/block/blk-mq.c linux-4.1.6/block/blk-mq.c return blk_mq_hctx_cpu_offline(hctx, cpu); /* -diff -Nur linux-4.1.6.orig/block/blk-mq-cpu.c linux-4.1.6/block/blk-mq-cpu.c ---- linux-4.1.6.orig/block/blk-mq-cpu.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/block/blk-mq-cpu.c 2015-09-08 23:49:05.762127285 +0200 +diff -Nur linux-4.1.10.orig/block/blk-mq-cpu.c linux-4.1.10/block/blk-mq-cpu.c +--- linux-4.1.10.orig/block/blk-mq-cpu.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/block/blk-mq-cpu.c 2015-10-07 18:00:07.000000000 +0200 @@ -16,7 +16,7 @@ #include "blk-mq.h" @@ -4856,9 +4878,9 @@ diff -Nur linux-4.1.6.orig/block/blk-mq-cpu.c linux-4.1.6/block/blk-mq-cpu.c } void blk_mq_init_cpu_notifier(struct blk_mq_cpu_notifier *notifier, -diff -Nur linux-4.1.6.orig/block/blk-mq.h linux-4.1.6/block/blk-mq.h ---- linux-4.1.6.orig/block/blk-mq.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/block/blk-mq.h 2015-09-08 23:49:05.762127285 +0200 +diff -Nur linux-4.1.10.orig/block/blk-mq.h linux-4.1.10/block/blk-mq.h +--- linux-4.1.10.orig/block/blk-mq.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/block/blk-mq.h 2015-10-07 18:00:07.000000000 +0200 @@ -76,7 +76,10 @@ static inline struct blk_mq_ctx *__blk_mq_get_ctx(struct request_queue *q, unsigned int cpu) @@ -4886,9 +4908,9 @@ diff -Nur linux-4.1.6.orig/block/blk-mq.h linux-4.1.6/block/blk-mq.h } struct blk_mq_alloc_data { -diff -Nur linux-4.1.6.orig/block/blk-softirq.c linux-4.1.6/block/blk-softirq.c ---- linux-4.1.6.orig/block/blk-softirq.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/block/blk-softirq.c 2015-09-08 23:49:05.762127285 +0200 +diff -Nur linux-4.1.10.orig/block/blk-softirq.c linux-4.1.10/block/blk-softirq.c +--- linux-4.1.10.orig/block/blk-softirq.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/block/blk-softirq.c 2015-10-07 18:00:07.000000000 +0200 @@ -51,6 +51,7 @@ raise_softirq_irqoff(BLOCK_SOFTIRQ); @@ -4913,9 +4935,9 @@ diff -Nur linux-4.1.6.orig/block/blk-softirq.c linux-4.1.6/block/blk-softirq.c } /** -diff -Nur linux-4.1.6.orig/block/bounce.c linux-4.1.6/block/bounce.c ---- linux-4.1.6.orig/block/bounce.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/block/bounce.c 2015-09-08 23:49:05.762127285 +0200 +diff -Nur linux-4.1.10.orig/block/bounce.c linux-4.1.10/block/bounce.c +--- linux-4.1.10.orig/block/bounce.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/block/bounce.c 2015-10-07 18:00:07.000000000 +0200 @@ -54,11 +54,11 @@ unsigned long flags; unsigned char *vto; @@ -4930,9 +4952,9 @@ diff -Nur linux-4.1.6.orig/block/bounce.c linux-4.1.6/block/bounce.c } #else /* CONFIG_HIGHMEM */ -diff -Nur linux-4.1.6.orig/crypto/algapi.c linux-4.1.6/crypto/algapi.c ---- linux-4.1.6.orig/crypto/algapi.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/crypto/algapi.c 2015-09-08 23:49:05.762127285 +0200 +diff -Nur linux-4.1.10.orig/crypto/algapi.c linux-4.1.10/crypto/algapi.c +--- linux-4.1.10.orig/crypto/algapi.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/crypto/algapi.c 2015-10-07 18:00:07.000000000 +0200 @@ -695,13 +695,13 @@ int crypto_register_notifier(struct notifier_block *nb) @@ -4949,9 +4971,9 @@ diff -Nur linux-4.1.6.orig/crypto/algapi.c linux-4.1.6/crypto/algapi.c } EXPORT_SYMBOL_GPL(crypto_unregister_notifier); -diff -Nur linux-4.1.6.orig/crypto/api.c linux-4.1.6/crypto/api.c ---- linux-4.1.6.orig/crypto/api.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/crypto/api.c 2015-09-08 23:49:05.762127285 +0200 +diff -Nur linux-4.1.10.orig/crypto/api.c linux-4.1.10/crypto/api.c +--- linux-4.1.10.orig/crypto/api.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/crypto/api.c 2015-10-07 18:00:07.000000000 +0200 @@ -31,7 +31,7 @@ DECLARE_RWSEM(crypto_alg_sem); EXPORT_SYMBOL_GPL(crypto_alg_sem); @@ -4974,9 +4996,9 @@ diff -Nur linux-4.1.6.orig/crypto/api.c linux-4.1.6/crypto/api.c } return ok; -diff -Nur linux-4.1.6.orig/crypto/internal.h linux-4.1.6/crypto/internal.h ---- linux-4.1.6.orig/crypto/internal.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/crypto/internal.h 2015-09-08 23:49:05.762127285 +0200 +diff -Nur linux-4.1.10.orig/crypto/internal.h linux-4.1.10/crypto/internal.h +--- linux-4.1.10.orig/crypto/internal.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/crypto/internal.h 2015-10-07 18:00:07.000000000 +0200 @@ -48,7 +48,7 @@ extern struct list_head crypto_alg_list; @@ -4995,9 +5017,9 @@ diff -Nur linux-4.1.6.orig/crypto/internal.h linux-4.1.6/crypto/internal.h } #endif /* _CRYPTO_INTERNAL_H */ -diff -Nur linux-4.1.6.orig/Documentation/hwlat_detector.txt linux-4.1.6/Documentation/hwlat_detector.txt ---- linux-4.1.6.orig/Documentation/hwlat_detector.txt 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.6/Documentation/hwlat_detector.txt 2015-09-08 23:49:03.494378758 +0200 +diff -Nur linux-4.1.10.orig/Documentation/hwlat_detector.txt linux-4.1.10/Documentation/hwlat_detector.txt +--- linux-4.1.10.orig/Documentation/hwlat_detector.txt 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/Documentation/hwlat_detector.txt 2015-10-07 18:00:07.000000000 +0200 @@ -0,0 +1,64 @@ +Introduction: +------------- @@ -5063,9 +5085,9 @@ diff -Nur linux-4.1.6.orig/Documentation/hwlat_detector.txt linux-4.1.6/Document +observe any latencies that exceed the threshold (initially 100 usecs), +then we write to a global sample ring buffer of 8K samples, which is +consumed by reading from the "sample" (pipe) debugfs file interface. -diff -Nur linux-4.1.6.orig/Documentation/sysrq.txt linux-4.1.6/Documentation/sysrq.txt ---- linux-4.1.6.orig/Documentation/sysrq.txt 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/Documentation/sysrq.txt 2015-09-08 23:49:03.494378758 +0200 +diff -Nur linux-4.1.10.orig/Documentation/sysrq.txt linux-4.1.10/Documentation/sysrq.txt +--- linux-4.1.10.orig/Documentation/sysrq.txt 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/Documentation/sysrq.txt 2015-10-07 18:00:07.000000000 +0200 @@ -59,10 +59,17 @@ On other - If you know of the key combos for other architectures, please let me know so I can add them to this section. @@ -5086,9 +5108,9 @@ diff -Nur linux-4.1.6.orig/Documentation/sysrq.txt linux-4.1.6/Documentation/sys * What are the 'command' keys? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 'b' - Will immediately reboot the system without syncing or unmounting -diff -Nur linux-4.1.6.orig/Documentation/trace/histograms.txt linux-4.1.6/Documentation/trace/histograms.txt ---- linux-4.1.6.orig/Documentation/trace/histograms.txt 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.6/Documentation/trace/histograms.txt 2015-09-08 23:49:03.498378314 +0200 +diff -Nur linux-4.1.10.orig/Documentation/trace/histograms.txt linux-4.1.10/Documentation/trace/histograms.txt +--- linux-4.1.10.orig/Documentation/trace/histograms.txt 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/Documentation/trace/histograms.txt 2015-10-07 18:00:07.000000000 +0200 @@ -0,0 +1,186 @@ + Using the Linux Kernel Latency Histograms + @@ -5276,9 +5298,9 @@ diff -Nur linux-4.1.6.orig/Documentation/trace/histograms.txt linux-4.1.6/Docume +is provided. + +These data are also reset when the wakeup histogram is reset. -diff -Nur linux-4.1.6.orig/drivers/acpi/acpica/acglobal.h linux-4.1.6/drivers/acpi/acpica/acglobal.h ---- linux-4.1.6.orig/drivers/acpi/acpica/acglobal.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/acpi/acpica/acglobal.h 2015-09-08 23:49:05.762127285 +0200 +diff -Nur linux-4.1.10.orig/drivers/acpi/acpica/acglobal.h linux-4.1.10/drivers/acpi/acpica/acglobal.h +--- linux-4.1.10.orig/drivers/acpi/acpica/acglobal.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/acpi/acpica/acglobal.h 2015-10-07 18:00:07.000000000 +0200 @@ -112,7 +112,7 @@ * interrupt level */ @@ -5288,9 +5310,9 @@ diff -Nur linux-4.1.6.orig/drivers/acpi/acpica/acglobal.h linux-4.1.6/drivers/ac ACPI_GLOBAL(acpi_spinlock, acpi_gbl_reference_count_lock); /* Mutex for _OSI support */ -diff -Nur linux-4.1.6.orig/drivers/acpi/acpica/hwregs.c linux-4.1.6/drivers/acpi/acpica/hwregs.c ---- linux-4.1.6.orig/drivers/acpi/acpica/hwregs.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/acpi/acpica/hwregs.c 2015-09-08 23:49:05.762127285 +0200 +diff -Nur linux-4.1.10.orig/drivers/acpi/acpica/hwregs.c linux-4.1.10/drivers/acpi/acpica/hwregs.c +--- linux-4.1.10.orig/drivers/acpi/acpica/hwregs.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/acpi/acpica/hwregs.c 2015-10-07 18:00:07.000000000 +0200 @@ -269,14 +269,14 @@ ACPI_BITMASK_ALL_FIXED_STATUS, ACPI_FORMAT_UINT64(acpi_gbl_xpm1a_status.address))); @@ -5308,9 +5330,9 @@ diff -Nur linux-4.1.6.orig/drivers/acpi/acpica/hwregs.c linux-4.1.6/drivers/acpi if (ACPI_FAILURE(status)) { goto exit; -diff -Nur linux-4.1.6.orig/drivers/acpi/acpica/hwxface.c linux-4.1.6/drivers/acpi/acpica/hwxface.c ---- linux-4.1.6.orig/drivers/acpi/acpica/hwxface.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/acpi/acpica/hwxface.c 2015-09-08 23:49:05.762127285 +0200 +diff -Nur linux-4.1.10.orig/drivers/acpi/acpica/hwxface.c linux-4.1.10/drivers/acpi/acpica/hwxface.c +--- linux-4.1.10.orig/drivers/acpi/acpica/hwxface.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/acpi/acpica/hwxface.c 2015-10-07 18:00:07.000000000 +0200 @@ -374,7 +374,7 @@ return_ACPI_STATUS(AE_BAD_PARAMETER); } @@ -5329,9 +5351,9 @@ diff -Nur linux-4.1.6.orig/drivers/acpi/acpica/hwxface.c linux-4.1.6/drivers/acp return_ACPI_STATUS(status); } -diff -Nur linux-4.1.6.orig/drivers/acpi/acpica/utmutex.c linux-4.1.6/drivers/acpi/acpica/utmutex.c ---- linux-4.1.6.orig/drivers/acpi/acpica/utmutex.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/acpi/acpica/utmutex.c 2015-09-08 23:49:05.762127285 +0200 +diff -Nur linux-4.1.10.orig/drivers/acpi/acpica/utmutex.c linux-4.1.10/drivers/acpi/acpica/utmutex.c +--- linux-4.1.10.orig/drivers/acpi/acpica/utmutex.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/acpi/acpica/utmutex.c 2015-10-07 18:00:07.000000000 +0200 @@ -88,7 +88,7 @@ return_ACPI_STATUS (status); } @@ -5350,9 +5372,9 @@ diff -Nur linux-4.1.6.orig/drivers/acpi/acpica/utmutex.c linux-4.1.6/drivers/acp acpi_os_delete_lock(acpi_gbl_reference_count_lock); /* Delete the reader/writer lock */ -diff -Nur linux-4.1.6.orig/drivers/ata/libata-sff.c linux-4.1.6/drivers/ata/libata-sff.c ---- linux-4.1.6.orig/drivers/ata/libata-sff.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/ata/libata-sff.c 2015-09-08 23:49:05.762127285 +0200 +diff -Nur linux-4.1.10.orig/drivers/ata/libata-sff.c linux-4.1.10/drivers/ata/libata-sff.c +--- linux-4.1.10.orig/drivers/ata/libata-sff.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/ata/libata-sff.c 2015-10-07 18:00:07.000000000 +0200 @@ -678,9 +678,9 @@ unsigned long flags; unsigned int consumed; @@ -5401,9 +5423,9 @@ diff -Nur linux-4.1.6.orig/drivers/ata/libata-sff.c linux-4.1.6/drivers/ata/liba } else { buf = page_address(page); consumed = ap->ops->sff_data_xfer(dev, buf + offset, -diff -Nur linux-4.1.6.orig/drivers/char/random.c linux-4.1.6/drivers/char/random.c ---- linux-4.1.6.orig/drivers/char/random.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/char/random.c 2015-09-08 23:49:05.766126841 +0200 +diff -Nur linux-4.1.10.orig/drivers/char/random.c linux-4.1.10/drivers/char/random.c +--- linux-4.1.10.orig/drivers/char/random.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/char/random.c 2015-10-07 18:00:07.000000000 +0200 @@ -776,8 +776,6 @@ } sample; long delta, delta2, delta3; @@ -5455,9 +5477,9 @@ diff -Nur linux-4.1.6.orig/drivers/char/random.c linux-4.1.6/drivers/char/random fast_mix(fast_pool); add_interrupt_bench(cycles); -diff -Nur linux-4.1.6.orig/drivers/clocksource/tcb_clksrc.c linux-4.1.6/drivers/clocksource/tcb_clksrc.c ---- linux-4.1.6.orig/drivers/clocksource/tcb_clksrc.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/clocksource/tcb_clksrc.c 2015-09-08 23:49:05.766126841 +0200 +diff -Nur linux-4.1.10.orig/drivers/clocksource/tcb_clksrc.c linux-4.1.10/drivers/clocksource/tcb_clksrc.c +--- linux-4.1.10.orig/drivers/clocksource/tcb_clksrc.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/clocksource/tcb_clksrc.c 2015-10-07 18:00:08.000000000 +0200 @@ -23,8 +23,7 @@ * this 32 bit free-running counter. the second channel is not used. * @@ -5572,9 +5594,9 @@ diff -Nur linux-4.1.6.orig/drivers/clocksource/tcb_clksrc.c linux-4.1.6/drivers/ if (ret) goto err_unregister_clksrc; -diff -Nur linux-4.1.6.orig/drivers/clocksource/timer-atmel-pit.c linux-4.1.6/drivers/clocksource/timer-atmel-pit.c ---- linux-4.1.6.orig/drivers/clocksource/timer-atmel-pit.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/clocksource/timer-atmel-pit.c 2015-09-08 23:49:05.766126841 +0200 +diff -Nur linux-4.1.10.orig/drivers/clocksource/timer-atmel-pit.c linux-4.1.10/drivers/clocksource/timer-atmel-pit.c +--- linux-4.1.10.orig/drivers/clocksource/timer-atmel-pit.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/clocksource/timer-atmel-pit.c 2015-10-07 18:00:08.000000000 +0200 @@ -90,6 +90,7 @@ return elapsed; } @@ -5600,9 +5622,9 @@ diff -Nur linux-4.1.6.orig/drivers/clocksource/timer-atmel-pit.c linux-4.1.6/dri break; case CLOCK_EVT_MODE_RESUME: break; -diff -Nur linux-4.1.6.orig/drivers/clocksource/timer-atmel-st.c linux-4.1.6/drivers/clocksource/timer-atmel-st.c ---- linux-4.1.6.orig/drivers/clocksource/timer-atmel-st.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/clocksource/timer-atmel-st.c 2015-09-08 23:49:05.766126841 +0200 +diff -Nur linux-4.1.10.orig/drivers/clocksource/timer-atmel-st.c linux-4.1.10/drivers/clocksource/timer-atmel-st.c +--- linux-4.1.10.orig/drivers/clocksource/timer-atmel-st.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/clocksource/timer-atmel-st.c 2015-10-07 18:00:08.000000000 +0200 @@ -131,6 +131,7 @@ break; case CLOCK_EVT_MODE_SHUTDOWN: @@ -5611,9 +5633,9 @@ diff -Nur linux-4.1.6.orig/drivers/clocksource/timer-atmel-st.c linux-4.1.6/driv case CLOCK_EVT_MODE_RESUME: irqmask = 0; break; -diff -Nur linux-4.1.6.orig/drivers/cpufreq/cpufreq.c linux-4.1.6/drivers/cpufreq/cpufreq.c ---- linux-4.1.6.orig/drivers/cpufreq/cpufreq.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/cpufreq/cpufreq.c 2015-09-08 23:49:05.818121077 +0200 +diff -Nur linux-4.1.10.orig/drivers/cpufreq/cpufreq.c linux-4.1.10/drivers/cpufreq/cpufreq.c +--- linux-4.1.10.orig/drivers/cpufreq/cpufreq.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/cpufreq/cpufreq.c 2015-10-07 18:00:08.000000000 +0200 @@ -64,12 +64,6 @@ return cpufreq_driver->target_index || cpufreq_driver->target; } @@ -5750,9 +5772,9 @@ diff -Nur linux-4.1.6.orig/drivers/cpufreq/cpufreq.c linux-4.1.6/drivers/cpufreq return 0; } -diff -Nur linux-4.1.6.orig/drivers/cpufreq/Kconfig.x86 linux-4.1.6/drivers/cpufreq/Kconfig.x86 ---- linux-4.1.6.orig/drivers/cpufreq/Kconfig.x86 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/cpufreq/Kconfig.x86 2015-09-08 23:49:05.818121077 +0200 +diff -Nur linux-4.1.10.orig/drivers/cpufreq/Kconfig.x86 linux-4.1.10/drivers/cpufreq/Kconfig.x86 +--- linux-4.1.10.orig/drivers/cpufreq/Kconfig.x86 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/cpufreq/Kconfig.x86 2015-10-07 18:00:08.000000000 +0200 @@ -123,7 +123,7 @@ config X86_POWERNOW_K8 @@ -5762,9 +5784,9 @@ diff -Nur linux-4.1.6.orig/drivers/cpufreq/Kconfig.x86 linux-4.1.6/drivers/cpufr help This adds the CPUFreq driver for K8/early Opteron/Athlon64 processors. Support for K10 and newer processors is now in acpi-cpufreq. -diff -Nur linux-4.1.6.orig/drivers/gpio/gpio-omap.c linux-4.1.6/drivers/gpio/gpio-omap.c ---- linux-4.1.6.orig/drivers/gpio/gpio-omap.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/gpio/gpio-omap.c 2015-09-08 23:49:05.818121077 +0200 +diff -Nur linux-4.1.10.orig/drivers/gpio/gpio-omap.c linux-4.1.10/drivers/gpio/gpio-omap.c +--- linux-4.1.10.orig/drivers/gpio/gpio-omap.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/gpio/gpio-omap.c 2015-10-07 18:00:08.000000000 +0200 @@ -57,7 +57,7 @@ u32 saved_datain; u32 level_mask; @@ -6045,9 +6067,9 @@ diff -Nur linux-4.1.6.orig/drivers/gpio/gpio-omap.c linux-4.1.6/drivers/gpio/gpi return 0; } -diff -Nur linux-4.1.6.orig/drivers/gpu/drm/i915/i915_gem_execbuffer.c linux-4.1.6/drivers/gpu/drm/i915/i915_gem_execbuffer.c ---- linux-4.1.6.orig/drivers/gpu/drm/i915/i915_gem_execbuffer.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/gpu/drm/i915/i915_gem_execbuffer.c 2015-09-08 23:49:05.818121077 +0200 +diff -Nur linux-4.1.10.orig/drivers/gpu/drm/i915/i915_gem_execbuffer.c linux-4.1.10/drivers/gpu/drm/i915/i915_gem_execbuffer.c +--- linux-4.1.10.orig/drivers/gpu/drm/i915/i915_gem_execbuffer.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/gpu/drm/i915/i915_gem_execbuffer.c 2015-10-07 18:00:08.000000000 +0200 @@ -32,6 +32,7 @@ #include "i915_trace.h" #include "intel_drv.h" @@ -6075,9 +6097,9 @@ diff -Nur linux-4.1.6.orig/drivers/gpu/drm/i915/i915_gem_execbuffer.c linux-4.1. i915_gem_execbuffer_move_to_active(vmas, ring); i915_gem_execbuffer_retire_commands(dev, file, ring, batch_obj); -diff -Nur linux-4.1.6.orig/drivers/gpu/drm/i915/i915_gem_shrinker.c linux-4.1.6/drivers/gpu/drm/i915/i915_gem_shrinker.c ---- linux-4.1.6.orig/drivers/gpu/drm/i915/i915_gem_shrinker.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/gpu/drm/i915/i915_gem_shrinker.c 2015-09-08 23:49:05.818121077 +0200 +diff -Nur linux-4.1.10.orig/drivers/gpu/drm/i915/i915_gem_shrinker.c linux-4.1.10/drivers/gpu/drm/i915/i915_gem_shrinker.c +--- linux-4.1.10.orig/drivers/gpu/drm/i915/i915_gem_shrinker.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/gpu/drm/i915/i915_gem_shrinker.c 2015-10-07 18:00:08.000000000 +0200 @@ -39,7 +39,7 @@ if (!mutex_is_locked(mutex)) return false; @@ -6087,9 +6109,9 @@ diff -Nur linux-4.1.6.orig/drivers/gpu/drm/i915/i915_gem_shrinker.c linux-4.1.6/ return mutex->owner == task; #else /* Since UP may be pre-empted, we cannot assume that we own the lock */ -diff -Nur linux-4.1.6.orig/drivers/gpu/drm/i915/intel_display.c linux-4.1.6/drivers/gpu/drm/i915/intel_display.c ---- linux-4.1.6.orig/drivers/gpu/drm/i915/intel_display.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/gpu/drm/i915/intel_display.c 2015-09-08 23:49:05.822120634 +0200 +diff -Nur linux-4.1.10.orig/drivers/gpu/drm/i915/intel_display.c linux-4.1.10/drivers/gpu/drm/i915/intel_display.c +--- linux-4.1.10.orig/drivers/gpu/drm/i915/intel_display.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/gpu/drm/i915/intel_display.c 2015-10-07 18:00:08.000000000 +0200 @@ -10086,7 +10086,7 @@ struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe]; struct intel_crtc *intel_crtc = to_intel_crtc(crtc); @@ -6099,9 +6121,9 @@ diff -Nur linux-4.1.6.orig/drivers/gpu/drm/i915/intel_display.c linux-4.1.6/driv if (crtc == NULL) return; -diff -Nur linux-4.1.6.orig/drivers/i2c/busses/i2c-omap.c linux-4.1.6/drivers/i2c/busses/i2c-omap.c ---- linux-4.1.6.orig/drivers/i2c/busses/i2c-omap.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/i2c/busses/i2c-omap.c 2015-09-08 23:49:06.074092692 +0200 +diff -Nur linux-4.1.10.orig/drivers/i2c/busses/i2c-omap.c linux-4.1.10/drivers/i2c/busses/i2c-omap.c +--- linux-4.1.10.orig/drivers/i2c/busses/i2c-omap.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/i2c/busses/i2c-omap.c 2015-10-07 18:00:08.000000000 +0200 @@ -996,15 +996,12 @@ u16 mask; u16 stat; @@ -6119,9 +6141,9 @@ diff -Nur linux-4.1.6.orig/drivers/i2c/busses/i2c-omap.c linux-4.1.6/drivers/i2c return ret; } -diff -Nur linux-4.1.6.orig/drivers/ide/alim15x3.c linux-4.1.6/drivers/ide/alim15x3.c ---- linux-4.1.6.orig/drivers/ide/alim15x3.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/ide/alim15x3.c 2015-09-08 23:49:06.074092692 +0200 +diff -Nur linux-4.1.10.orig/drivers/ide/alim15x3.c linux-4.1.10/drivers/ide/alim15x3.c +--- linux-4.1.10.orig/drivers/ide/alim15x3.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/ide/alim15x3.c 2015-10-07 18:00:08.000000000 +0200 @@ -234,7 +234,7 @@ isa_dev = pci_get_device(PCI_VENDOR_ID_AL, PCI_DEVICE_ID_AL_M1533, NULL); @@ -6140,9 +6162,9 @@ diff -Nur linux-4.1.6.orig/drivers/ide/alim15x3.c linux-4.1.6/drivers/ide/alim15 return 0; } -diff -Nur linux-4.1.6.orig/drivers/ide/hpt366.c linux-4.1.6/drivers/ide/hpt366.c ---- linux-4.1.6.orig/drivers/ide/hpt366.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/ide/hpt366.c 2015-09-08 23:49:06.074092692 +0200 +diff -Nur linux-4.1.10.orig/drivers/ide/hpt366.c linux-4.1.10/drivers/ide/hpt366.c +--- linux-4.1.10.orig/drivers/ide/hpt366.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/ide/hpt366.c 2015-10-07 18:00:08.000000000 +0200 @@ -1241,7 +1241,7 @@ dma_old = inb(base + 2); @@ -6161,9 +6183,9 @@ diff -Nur linux-4.1.6.orig/drivers/ide/hpt366.c linux-4.1.6/drivers/ide/hpt366.c printk(KERN_INFO " %s: BM-DMA at 0x%04lx-0x%04lx\n", hwif->name, base, base + 7); -diff -Nur linux-4.1.6.orig/drivers/ide/ide-io.c linux-4.1.6/drivers/ide/ide-io.c ---- linux-4.1.6.orig/drivers/ide/ide-io.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/ide/ide-io.c 2015-09-08 23:49:06.074092692 +0200 +diff -Nur linux-4.1.10.orig/drivers/ide/ide-io.c linux-4.1.10/drivers/ide/ide-io.c +--- linux-4.1.10.orig/drivers/ide/ide-io.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/ide/ide-io.c 2015-10-07 18:00:08.000000000 +0200 @@ -659,7 +659,7 @@ /* disable_irq_nosync ?? */ disable_irq(hwif->irq); @@ -6173,9 +6195,9 @@ diff -Nur linux-4.1.6.orig/drivers/ide/ide-io.c linux-4.1.6/drivers/ide/ide-io.c if (hwif->polling) { startstop = handler(drive); } else if (drive_is_ready(drive)) { -diff -Nur linux-4.1.6.orig/drivers/ide/ide-iops.c linux-4.1.6/drivers/ide/ide-iops.c ---- linux-4.1.6.orig/drivers/ide/ide-iops.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/ide/ide-iops.c 2015-09-08 23:49:06.074092692 +0200 +diff -Nur linux-4.1.10.orig/drivers/ide/ide-iops.c linux-4.1.10/drivers/ide/ide-iops.c +--- linux-4.1.10.orig/drivers/ide/ide-iops.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/ide/ide-iops.c 2015-10-07 18:00:08.000000000 +0200 @@ -129,12 +129,12 @@ if ((stat & ATA_BUSY) == 0) break; @@ -6191,9 +6213,9 @@ diff -Nur linux-4.1.6.orig/drivers/ide/ide-iops.c linux-4.1.6/drivers/ide/ide-io } /* * Allow status to settle, then read it again. -diff -Nur linux-4.1.6.orig/drivers/ide/ide-io-std.c linux-4.1.6/drivers/ide/ide-io-std.c ---- linux-4.1.6.orig/drivers/ide/ide-io-std.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/ide/ide-io-std.c 2015-09-08 23:49:06.074092692 +0200 +diff -Nur linux-4.1.10.orig/drivers/ide/ide-io-std.c linux-4.1.10/drivers/ide/ide-io-std.c +--- linux-4.1.10.orig/drivers/ide/ide-io-std.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/ide/ide-io-std.c 2015-10-07 18:00:08.000000000 +0200 @@ -175,7 +175,7 @@ unsigned long uninitialized_var(flags); @@ -6230,9 +6252,9 @@ diff -Nur linux-4.1.6.orig/drivers/ide/ide-io-std.c linux-4.1.6/drivers/ide/ide- if (((len + 1) & 3) < 2) return; -diff -Nur linux-4.1.6.orig/drivers/ide/ide-probe.c linux-4.1.6/drivers/ide/ide-probe.c ---- linux-4.1.6.orig/drivers/ide/ide-probe.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/ide/ide-probe.c 2015-09-08 23:49:06.074092692 +0200 +diff -Nur linux-4.1.10.orig/drivers/ide/ide-probe.c linux-4.1.10/drivers/ide/ide-probe.c +--- linux-4.1.10.orig/drivers/ide/ide-probe.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/ide/ide-probe.c 2015-10-07 18:00:08.000000000 +0200 @@ -196,10 +196,10 @@ int bswap = 1; @@ -6246,9 +6268,9 @@ diff -Nur linux-4.1.6.orig/drivers/ide/ide-probe.c linux-4.1.6/drivers/ide/ide-p drive->dev_flags |= IDE_DFLAG_ID_READ; #ifdef DEBUG -diff -Nur linux-4.1.6.orig/drivers/ide/ide-taskfile.c linux-4.1.6/drivers/ide/ide-taskfile.c ---- linux-4.1.6.orig/drivers/ide/ide-taskfile.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/ide/ide-taskfile.c 2015-09-08 23:49:06.074092692 +0200 +diff -Nur linux-4.1.10.orig/drivers/ide/ide-taskfile.c linux-4.1.10/drivers/ide/ide-taskfile.c +--- linux-4.1.10.orig/drivers/ide/ide-taskfile.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/ide/ide-taskfile.c 2015-10-07 18:00:08.000000000 +0200 @@ -250,7 +250,7 @@ page_is_high = PageHighMem(page); @@ -6276,9 +6298,9 @@ diff -Nur linux-4.1.6.orig/drivers/ide/ide-taskfile.c linux-4.1.6/drivers/ide/id ide_set_handler(drive, &task_pio_intr, WAIT_WORSTCASE); -diff -Nur linux-4.1.6.orig/drivers/infiniband/ulp/ipoib/ipoib_multicast.c linux-4.1.6/drivers/infiniband/ulp/ipoib/ipoib_multicast.c ---- linux-4.1.6.orig/drivers/infiniband/ulp/ipoib/ipoib_multicast.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/infiniband/ulp/ipoib/ipoib_multicast.c 2015-09-08 23:49:06.074092692 +0200 +diff -Nur linux-4.1.10.orig/drivers/infiniband/ulp/ipoib/ipoib_multicast.c linux-4.1.10/drivers/infiniband/ulp/ipoib/ipoib_multicast.c +--- linux-4.1.10.orig/drivers/infiniband/ulp/ipoib/ipoib_multicast.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/infiniband/ulp/ipoib/ipoib_multicast.c 2015-10-07 18:00:08.000000000 +0200 @@ -821,7 +821,7 @@ ipoib_dbg_mcast(priv, "restarting multicast task\n"); @@ -6297,9 +6319,9 @@ diff -Nur linux-4.1.6.orig/drivers/infiniband/ulp/ipoib/ipoib_multicast.c linux- /* * make sure the in-flight joins have finished before we attempt -diff -Nur linux-4.1.6.orig/drivers/input/gameport/gameport.c linux-4.1.6/drivers/input/gameport/gameport.c ---- linux-4.1.6.orig/drivers/input/gameport/gameport.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/input/gameport/gameport.c 2015-09-08 23:49:06.074092692 +0200 +diff -Nur linux-4.1.10.orig/drivers/input/gameport/gameport.c linux-4.1.10/drivers/input/gameport/gameport.c +--- linux-4.1.10.orig/drivers/input/gameport/gameport.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/input/gameport/gameport.c 2015-10-07 18:00:08.000000000 +0200 @@ -124,12 +124,12 @@ tx = 1 << 30; @@ -6329,9 +6351,9 @@ diff -Nur linux-4.1.6.orig/drivers/input/gameport/gameport.c linux-4.1.6/drivers udelay(i * 10); if (t2 - t1 < tx) tx = t2 - t1; } -diff -Nur linux-4.1.6.orig/drivers/leds/trigger/Kconfig linux-4.1.6/drivers/leds/trigger/Kconfig ---- linux-4.1.6.orig/drivers/leds/trigger/Kconfig 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/leds/trigger/Kconfig 2015-09-08 23:49:06.074092692 +0200 +diff -Nur linux-4.1.10.orig/drivers/leds/trigger/Kconfig linux-4.1.10/drivers/leds/trigger/Kconfig +--- linux-4.1.10.orig/drivers/leds/trigger/Kconfig 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/leds/trigger/Kconfig 2015-10-07 18:00:08.000000000 +0200 @@ -61,7 +61,7 @@ config LEDS_TRIGGER_CPU @@ -6341,9 +6363,9 @@ diff -Nur linux-4.1.6.orig/drivers/leds/trigger/Kconfig linux-4.1.6/drivers/leds help This allows LEDs to be controlled by active CPUs. This shows the active CPUs across an array of LEDs so you can see which -diff -Nur linux-4.1.6.orig/drivers/md/bcache/Kconfig linux-4.1.6/drivers/md/bcache/Kconfig ---- linux-4.1.6.orig/drivers/md/bcache/Kconfig 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/md/bcache/Kconfig 2015-09-08 23:49:06.074092692 +0200 +diff -Nur linux-4.1.10.orig/drivers/md/bcache/Kconfig linux-4.1.10/drivers/md/bcache/Kconfig +--- linux-4.1.10.orig/drivers/md/bcache/Kconfig 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/md/bcache/Kconfig 2015-10-07 18:00:08.000000000 +0200 @@ -1,6 +1,7 @@ config BCACHE @@ -6352,9 +6374,9 @@ diff -Nur linux-4.1.6.orig/drivers/md/bcache/Kconfig linux-4.1.6/drivers/md/bcac ---help--- Allows a block device to be used as cache for other devices; uses a btree for indexing and the layout is optimized for SSDs. -diff -Nur linux-4.1.6.orig/drivers/md/dm.c linux-4.1.6/drivers/md/dm.c ---- linux-4.1.6.orig/drivers/md/dm.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/md/dm.c 2015-09-08 23:49:06.074092692 +0200 +diff -Nur linux-4.1.10.orig/drivers/md/dm.c linux-4.1.10/drivers/md/dm.c +--- linux-4.1.10.orig/drivers/md/dm.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/md/dm.c 2015-10-07 18:00:08.000000000 +0200 @@ -2132,7 +2132,7 @@ /* Establish tio->ti before queuing work (map_tio_request) */ tio->ti = ti; @@ -6364,9 +6386,9 @@ diff -Nur linux-4.1.6.orig/drivers/md/dm.c linux-4.1.6/drivers/md/dm.c } goto out; -diff -Nur linux-4.1.6.orig/drivers/md/raid5.c linux-4.1.6/drivers/md/raid5.c ---- linux-4.1.6.orig/drivers/md/raid5.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/md/raid5.c 2015-09-08 23:49:06.078092247 +0200 +diff -Nur linux-4.1.10.orig/drivers/md/raid5.c linux-4.1.10/drivers/md/raid5.c +--- linux-4.1.10.orig/drivers/md/raid5.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/md/raid5.c 2015-10-07 18:00:08.000000000 +0200 @@ -1918,8 +1918,9 @@ struct raid5_percpu *percpu; unsigned long cpu; @@ -6388,7 +6410,7 @@ diff -Nur linux-4.1.6.orig/drivers/md/raid5.c linux-4.1.6/drivers/md/raid5.c } static struct stripe_head *alloc_stripe(struct kmem_cache *sc, gfp_t gfp) -@@ -6350,6 +6352,7 @@ +@@ -6361,6 +6363,7 @@ __func__, cpu); break; } @@ -6396,19459 +6418,55700 @@ diff -Nur linux-4.1.6.orig/drivers/md/raid5.c linux-4.1.6/drivers/md/raid5.c } put_online_cpus(); -diff -Nur linux-4.1.6.orig/drivers/md/raid5.h linux-4.1.6/drivers/md/raid5.h ---- linux-4.1.6.orig/drivers/md/raid5.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/md/raid5.h 2015-09-08 23:49:06.078092247 +0200 -@@ -494,6 +494,7 @@ - int recovery_disabled; - /* per cpu variables */ - struct raid5_percpu { -+ spinlock_t lock; /* Protection for -RT */ - struct page *spare_page; /* Used when checking P/Q in raid6 */ - struct flex_array *scribble; /* space for constructing buffer - * lists and performing address -diff -Nur linux-4.1.6.orig/drivers/misc/hwlat_detector.c linux-4.1.6/drivers/misc/hwlat_detector.c ---- linux-4.1.6.orig/drivers/misc/hwlat_detector.c 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.6/drivers/misc/hwlat_detector.c 2015-09-08 23:49:06.082091803 +0200 -@@ -0,0 +1,1240 @@ +diff -Nur linux-4.1.10.orig/drivers/md/raid5.c.orig linux-4.1.10/drivers/md/raid5.c.orig +--- linux-4.1.10.orig/drivers/md/raid5.c.orig 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/drivers/md/raid5.c.orig 2015-10-03 13:49:38.000000000 +0200 +@@ -0,0 +1,7853 @@ +/* -+ * hwlat_detector.c - A simple Hardware Latency detector. -+ * -+ * Use this module to detect large system latencies induced by the behavior of -+ * certain underlying system hardware or firmware, independent of Linux itself. -+ * The code was developed originally to detect the presence of SMIs on Intel -+ * and AMD systems, although there is no dependency upon x86 herein. -+ * -+ * The classical example usage of this module is in detecting the presence of -+ * SMIs or System Management Interrupts on Intel and AMD systems. An SMI is a -+ * somewhat special form of hardware interrupt spawned from earlier CPU debug -+ * modes in which the (BIOS/EFI/etc.) firmware arranges for the South Bridge -+ * LPC (or other device) to generate a special interrupt under certain -+ * circumstances, for example, upon expiration of a special SMI timer device, -+ * due to certain external thermal readings, on certain I/O address accesses, -+ * and other situations. An SMI hits a special CPU pin, triggers a special -+ * SMI mode (complete with special memory map), and the OS is unaware. ++ * raid5.c : Multiple Devices driver for Linux ++ * Copyright (C) 1996, 1997 Ingo Molnar, Miguel de Icaza, Gadi Oxman ++ * Copyright (C) 1999, 2000 Ingo Molnar ++ * Copyright (C) 2002, 2003 H. Peter Anvin + * -+ * Although certain hardware-inducing latencies are necessary (for example, -+ * a modern system often requires an SMI handler for correct thermal control -+ * and remote management) they can wreak havoc upon any OS-level performance -+ * guarantees toward low-latency, especially when the OS is not even made -+ * aware of the presence of these interrupts. For this reason, we need a -+ * somewhat brute force mechanism to detect these interrupts. In this case, -+ * we do it by hogging all of the CPU(s) for configurable timer intervals, -+ * sampling the built-in CPU timer, looking for discontiguous readings. ++ * RAID-4/5/6 management functions. ++ * Thanks to Penguin Computing for making the RAID-6 development possible ++ * by donating a test server! + * -+ * WARNING: This implementation necessarily introduces latencies. Therefore, -+ * you should NEVER use this module in a production environment -+ * requiring any kind of low-latency performance guarantee(s). ++ * This program is free software; you can redistribute it and/or modify ++ * it under the terms of the GNU General Public License as published by ++ * the Free Software Foundation; either version 2, or (at your option) ++ * any later version. + * -+ * Copyright (C) 2008-2009 Jon Masters, Red Hat, Inc. ++ * You should have received a copy of the GNU General Public License ++ * (for example /usr/src/linux/COPYING); if not, write to the Free ++ * Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. ++ */ ++ ++/* ++ * BITMAP UNPLUGGING: + * -+ * Includes useful feedback from Clark Williams ++ * The sequencing for updating the bitmap reliably is a little ++ * subtle (and I got it wrong the first time) so it deserves some ++ * explanation. + * -+ * This file is licensed under the terms of the GNU General Public -+ * License version 2. This program is licensed "as is" without any -+ * warranty of any kind, whether express or implied. ++ * We group bitmap updates into batches. Each batch has a number. ++ * We may write out several batches at once, but that isn't very important. ++ * conf->seq_write is the number of the last batch successfully written. ++ * conf->seq_flush is the number of the last batch that was closed to ++ * new additions. ++ * When we discover that we will need to write to any block in a stripe ++ * (in add_stripe_bio) we update the in-memory bitmap and record in sh->bm_seq ++ * the number of the batch it will be in. This is seq_flush+1. ++ * When we are ready to do a write, if that batch hasn't been written yet, ++ * we plug the array and queue the stripe for later. ++ * When an unplug happens, we increment bm_flush, thus closing the current ++ * batch. ++ * When we notice that bm_flush > bm_write, we write out all pending updates ++ * to the bitmap, and advance bm_write to where bm_flush was. ++ * This may occasionally write a bit out twice, but is sure never to ++ * miss any bits. + */ + -+#include -+#include -+#include -+#include -+#include ++#include +#include -+#include ++#include ++#include ++#include ++#include +#include -+#include -+#include -+#include ++#include +#include -+#include ++#include ++#include ++#include ++#include ++ ++#include "md.h" ++#include "raid5.h" ++#include "raid0.h" ++#include "bitmap.h" ++ ++#define cpu_to_group(cpu) cpu_to_node(cpu) ++#define ANY_GROUP NUMA_NO_NODE ++ ++static bool devices_handle_discard_safely = false; ++module_param(devices_handle_discard_safely, bool, 0644); ++MODULE_PARM_DESC(devices_handle_discard_safely, ++ "Set to Y if all devices in each array reliably return zeroes on reads from discarded regions"); ++static struct workqueue_struct *raid5_wq; ++/* ++ * Stripe cache ++ */ + -+#define BUF_SIZE_DEFAULT 262144UL /* 8K*(sizeof(entry)) */ -+#define BUF_FLAGS (RB_FL_OVERWRITE) /* no block on full */ -+#define U64STR_SIZE 22 /* 20 digits max */ ++#define NR_STRIPES 256 ++#define STRIPE_SIZE PAGE_SIZE ++#define STRIPE_SHIFT (PAGE_SHIFT - 9) ++#define STRIPE_SECTORS (STRIPE_SIZE>>9) ++#define IO_THRESHOLD 1 ++#define BYPASS_THRESHOLD 1 ++#define NR_HASH (PAGE_SIZE / sizeof(struct hlist_head)) ++#define HASH_MASK (NR_HASH - 1) ++#define MAX_STRIPE_BATCH 8 ++ ++static inline struct hlist_head *stripe_hash(struct r5conf *conf, sector_t sect) ++{ ++ int hash = (sect >> STRIPE_SHIFT) & HASH_MASK; ++ return &conf->stripe_hashtbl[hash]; ++} + -+#define VERSION "1.0.0" -+#define BANNER "hwlat_detector: " -+#define DRVNAME "hwlat_detector" -+#define DEFAULT_SAMPLE_WINDOW 1000000 /* 1s */ -+#define DEFAULT_SAMPLE_WIDTH 500000 /* 0.5s */ -+#define DEFAULT_LAT_THRESHOLD 10 /* 10us */ ++static inline int stripe_hash_locks_hash(sector_t sect) ++{ ++ return (sect >> STRIPE_SHIFT) & STRIPE_HASH_LOCKS_MASK; ++} + -+/* Module metadata */ ++static inline void lock_device_hash_lock(struct r5conf *conf, int hash) ++{ ++ spin_lock_irq(conf->hash_locks + hash); ++ spin_lock(&conf->device_lock); ++} + -+MODULE_LICENSE("GPL"); -+MODULE_AUTHOR("Jon Masters "); -+MODULE_DESCRIPTION("A simple hardware latency detector"); -+MODULE_VERSION(VERSION); ++static inline void unlock_device_hash_lock(struct r5conf *conf, int hash) ++{ ++ spin_unlock(&conf->device_lock); ++ spin_unlock_irq(conf->hash_locks + hash); ++} + -+/* Module parameters */ ++static inline void lock_all_device_hash_locks_irq(struct r5conf *conf) ++{ ++ int i; ++ local_irq_disable(); ++ spin_lock(conf->hash_locks); ++ for (i = 1; i < NR_STRIPE_HASH_LOCKS; i++) ++ spin_lock_nest_lock(conf->hash_locks + i, conf->hash_locks); ++ spin_lock(&conf->device_lock); ++} + -+static int debug; -+static int enabled; -+static int threshold; ++static inline void unlock_all_device_hash_locks_irq(struct r5conf *conf) ++{ ++ int i; ++ spin_unlock(&conf->device_lock); ++ for (i = NR_STRIPE_HASH_LOCKS; i; i--) ++ spin_unlock(conf->hash_locks + i - 1); ++ local_irq_enable(); ++} + -+module_param(debug, int, 0); /* enable debug */ -+module_param(enabled, int, 0); /* enable detector */ -+module_param(threshold, int, 0); /* latency threshold */ ++/* bio's attached to a stripe+device for I/O are linked together in bi_sector ++ * order without overlap. There may be several bio's per stripe+device, and ++ * a bio could span several devices. ++ * When walking this list for a particular stripe+device, we must never proceed ++ * beyond a bio that extends past this device, as the next bio might no longer ++ * be valid. ++ * This function is used to determine the 'next' bio in the list, given the sector ++ * of the current stripe+device ++ */ ++static inline struct bio *r5_next_bio(struct bio *bio, sector_t sector) ++{ ++ int sectors = bio_sectors(bio); ++ if (bio->bi_iter.bi_sector + sectors < sector + STRIPE_SECTORS) ++ return bio->bi_next; ++ else ++ return NULL; ++} + -+/* Buffering and sampling */ ++/* ++ * We maintain a biased count of active stripes in the bottom 16 bits of ++ * bi_phys_segments, and a count of processed stripes in the upper 16 bits ++ */ ++static inline int raid5_bi_processed_stripes(struct bio *bio) ++{ ++ atomic_t *segments = (atomic_t *)&bio->bi_phys_segments; ++ return (atomic_read(segments) >> 16) & 0xffff; ++} + -+static struct ring_buffer *ring_buffer; /* sample buffer */ -+static DEFINE_MUTEX(ring_buffer_mutex); /* lock changes */ -+static unsigned long buf_size = BUF_SIZE_DEFAULT; -+static struct task_struct *kthread; /* sampling thread */ ++static inline int raid5_dec_bi_active_stripes(struct bio *bio) ++{ ++ atomic_t *segments = (atomic_t *)&bio->bi_phys_segments; ++ return atomic_sub_return(1, segments) & 0xffff; ++} + -+/* DebugFS filesystem entries */ ++static inline void raid5_inc_bi_active_stripes(struct bio *bio) ++{ ++ atomic_t *segments = (atomic_t *)&bio->bi_phys_segments; ++ atomic_inc(segments); ++} + -+static struct dentry *debug_dir; /* debugfs directory */ -+static struct dentry *debug_max; /* maximum TSC delta */ -+static struct dentry *debug_count; /* total detect count */ -+static struct dentry *debug_sample_width; /* sample width us */ -+static struct dentry *debug_sample_window; /* sample window us */ -+static struct dentry *debug_sample; /* raw samples us */ -+static struct dentry *debug_threshold; /* threshold us */ -+static struct dentry *debug_enable; /* enable/disable */ ++static inline void raid5_set_bi_processed_stripes(struct bio *bio, ++ unsigned int cnt) ++{ ++ atomic_t *segments = (atomic_t *)&bio->bi_phys_segments; ++ int old, new; + -+/* Individual samples and global state */ ++ do { ++ old = atomic_read(segments); ++ new = (old & 0xffff) | (cnt << 16); ++ } while (atomic_cmpxchg(segments, old, new) != old); ++} + -+struct sample; /* latency sample */ -+struct data; /* Global state */ ++static inline void raid5_set_bi_stripes(struct bio *bio, unsigned int cnt) ++{ ++ atomic_t *segments = (atomic_t *)&bio->bi_phys_segments; ++ atomic_set(segments, cnt); ++} + -+/* Sampling functions */ -+static int __buffer_add_sample(struct sample *sample); -+static struct sample *buffer_get_sample(struct sample *sample); ++/* Find first data disk in a raid6 stripe */ ++static inline int raid6_d0(struct stripe_head *sh) ++{ ++ if (sh->ddf_layout) ++ /* ddf always start from first device */ ++ return 0; ++ /* md starts just after Q block */ ++ if (sh->qd_idx == sh->disks - 1) ++ return 0; ++ else ++ return sh->qd_idx + 1; ++} ++static inline int raid6_next_disk(int disk, int raid_disks) ++{ ++ disk++; ++ return (disk < raid_disks) ? disk : 0; ++} + -+/* Threading and state */ -+static int kthread_fn(void *unused); -+static int start_kthread(void); -+static int stop_kthread(void); -+static void __reset_stats(void); -+static int init_stats(void); ++/* When walking through the disks in a raid5, starting at raid6_d0, ++ * We need to map each disk to a 'slot', where the data disks are slot ++ * 0 .. raid_disks-3, the parity disk is raid_disks-2 and the Q disk ++ * is raid_disks-1. This help does that mapping. ++ */ ++static int raid6_idx_to_slot(int idx, struct stripe_head *sh, ++ int *count, int syndrome_disks) ++{ ++ int slot = *count; ++ ++ if (sh->ddf_layout) ++ (*count)++; ++ if (idx == sh->pd_idx) ++ return syndrome_disks; ++ if (idx == sh->qd_idx) ++ return syndrome_disks + 1; ++ if (!sh->ddf_layout) ++ (*count)++; ++ return slot; ++} + -+/* Debugfs interface */ -+static ssize_t simple_data_read(struct file *filp, char __user *ubuf, -+ size_t cnt, loff_t *ppos, const u64 *entry); -+static ssize_t simple_data_write(struct file *filp, const char __user *ubuf, -+ size_t cnt, loff_t *ppos, u64 *entry); -+static int debug_sample_fopen(struct inode *inode, struct file *filp); -+static ssize_t debug_sample_fread(struct file *filp, char __user *ubuf, -+ size_t cnt, loff_t *ppos); -+static int debug_sample_release(struct inode *inode, struct file *filp); -+static int debug_enable_fopen(struct inode *inode, struct file *filp); -+static ssize_t debug_enable_fread(struct file *filp, char __user *ubuf, -+ size_t cnt, loff_t *ppos); -+static ssize_t debug_enable_fwrite(struct file *file, -+ const char __user *user_buffer, -+ size_t user_size, loff_t *offset); ++static void return_io(struct bio *return_bi) ++{ ++ struct bio *bi = return_bi; ++ while (bi) { ++ ++ return_bi = bi->bi_next; ++ bi->bi_next = NULL; ++ bi->bi_iter.bi_size = 0; ++ trace_block_bio_complete(bdev_get_queue(bi->bi_bdev), ++ bi, 0); ++ bio_endio(bi, 0); ++ bi = return_bi; ++ } ++} + -+/* Initialization functions */ -+static int init_debugfs(void); -+static void free_debugfs(void); -+static int detector_init(void); -+static void detector_exit(void); ++static void print_raid5_conf (struct r5conf *conf); + -+/* Individual latency samples are stored here when detected and packed into -+ * the ring_buffer circular buffer, where they are overwritten when -+ * more than buf_size/sizeof(sample) samples are received. */ -+struct sample { -+ u64 seqnum; /* unique sequence */ -+ u64 duration; /* ktime delta */ -+ u64 outer_duration; /* ktime delta (outer loop) */ -+ struct timespec timestamp; /* wall time */ -+ unsigned long lost; -+}; ++static int stripe_operations_active(struct stripe_head *sh) ++{ ++ return sh->check_state || sh->reconstruct_state || ++ test_bit(STRIPE_BIOFILL_RUN, &sh->state) || ++ test_bit(STRIPE_COMPUTE_RUN, &sh->state); ++} + -+/* keep the global state somewhere. */ -+static struct data { ++static void raid5_wakeup_stripe_thread(struct stripe_head *sh) ++{ ++ struct r5conf *conf = sh->raid_conf; ++ struct r5worker_group *group; ++ int thread_cnt; ++ int i, cpu = sh->cpu; ++ ++ if (!cpu_online(cpu)) { ++ cpu = cpumask_any(cpu_online_mask); ++ sh->cpu = cpu; ++ } + -+ struct mutex lock; /* protect changes */ ++ if (list_empty(&sh->lru)) { ++ struct r5worker_group *group; ++ group = conf->worker_groups + cpu_to_group(cpu); ++ list_add_tail(&sh->lru, &group->handle_list); ++ group->stripes_cnt++; ++ sh->group = group; ++ } + -+ u64 count; /* total since reset */ -+ u64 max_sample; /* max hardware latency */ -+ u64 threshold; /* sample threshold level */ ++ if (conf->worker_cnt_per_group == 0) { ++ md_wakeup_thread(conf->mddev->thread); ++ return; ++ } + -+ u64 sample_window; /* total sampling window (on+off) */ -+ u64 sample_width; /* active sampling portion of window */ ++ group = conf->worker_groups + cpu_to_group(sh->cpu); + -+ atomic_t sample_open; /* whether the sample file is open */ ++ group->workers[0].working = true; ++ /* at least one worker should run to avoid race */ ++ queue_work_on(sh->cpu, raid5_wq, &group->workers[0].work); + -+ wait_queue_head_t wq; /* waitqeue for new sample values */ ++ thread_cnt = group->stripes_cnt / MAX_STRIPE_BATCH - 1; ++ /* wakeup more workers */ ++ for (i = 1; i < conf->worker_cnt_per_group && thread_cnt > 0; i++) { ++ if (group->workers[i].working == false) { ++ group->workers[i].working = true; ++ queue_work_on(sh->cpu, raid5_wq, ++ &group->workers[i].work); ++ thread_cnt--; ++ } ++ } ++} + -+} data; ++static void do_release_stripe(struct r5conf *conf, struct stripe_head *sh, ++ struct list_head *temp_inactive_list) ++{ ++ BUG_ON(!list_empty(&sh->lru)); ++ BUG_ON(atomic_read(&conf->active_stripes)==0); ++ if (test_bit(STRIPE_HANDLE, &sh->state)) { ++ if (test_bit(STRIPE_DELAYED, &sh->state) && ++ !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) ++ list_add_tail(&sh->lru, &conf->delayed_list); ++ else if (test_bit(STRIPE_BIT_DELAY, &sh->state) && ++ sh->bm_seq - conf->seq_write > 0) ++ list_add_tail(&sh->lru, &conf->bitmap_list); ++ else { ++ clear_bit(STRIPE_DELAYED, &sh->state); ++ clear_bit(STRIPE_BIT_DELAY, &sh->state); ++ if (conf->worker_cnt_per_group == 0) { ++ list_add_tail(&sh->lru, &conf->handle_list); ++ } else { ++ raid5_wakeup_stripe_thread(sh); ++ return; ++ } ++ } ++ md_wakeup_thread(conf->mddev->thread); ++ } else { ++ BUG_ON(stripe_operations_active(sh)); ++ if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) ++ if (atomic_dec_return(&conf->preread_active_stripes) ++ < IO_THRESHOLD) ++ md_wakeup_thread(conf->mddev->thread); ++ atomic_dec(&conf->active_stripes); ++ if (!test_bit(STRIPE_EXPANDING, &sh->state)) ++ list_add_tail(&sh->lru, temp_inactive_list); ++ } ++} + -+/** -+ * __buffer_add_sample - add a new latency sample recording to the ring buffer -+ * @sample: The new latency sample value -+ * -+ * This receives a new latency sample and records it in a global ring buffer. -+ * No additional locking is used in this case. -+ */ -+static int __buffer_add_sample(struct sample *sample) ++static void __release_stripe(struct r5conf *conf, struct stripe_head *sh, ++ struct list_head *temp_inactive_list) +{ -+ return ring_buffer_write(ring_buffer, -+ sizeof(struct sample), sample); ++ if (atomic_dec_and_test(&sh->count)) ++ do_release_stripe(conf, sh, temp_inactive_list); +} + -+/** -+ * buffer_get_sample - remove a hardware latency sample from the ring buffer -+ * @sample: Pre-allocated storage for the sample ++/* ++ * @hash could be NR_STRIPE_HASH_LOCKS, then we have a list of inactive_list + * -+ * This retrieves a hardware latency sample from the global circular buffer ++ * Be careful: Only one task can add/delete stripes from temp_inactive_list at ++ * given time. Adding stripes only takes device lock, while deleting stripes ++ * only takes hash lock. + */ -+static struct sample *buffer_get_sample(struct sample *sample) ++static void release_inactive_stripe_list(struct r5conf *conf, ++ struct list_head *temp_inactive_list, ++ int hash) +{ -+ struct ring_buffer_event *e = NULL; -+ struct sample *s = NULL; -+ unsigned int cpu = 0; ++ int size; ++ bool do_wakeup = false; ++ unsigned long flags; + -+ if (!sample) -+ return NULL; ++ if (hash == NR_STRIPE_HASH_LOCKS) { ++ size = NR_STRIPE_HASH_LOCKS; ++ hash = NR_STRIPE_HASH_LOCKS - 1; ++ } else ++ size = 1; ++ while (size) { ++ struct list_head *list = &temp_inactive_list[size - 1]; + -+ mutex_lock(&ring_buffer_mutex); -+ for_each_online_cpu(cpu) { -+ e = ring_buffer_consume(ring_buffer, cpu, NULL, &sample->lost); -+ if (e) -+ break; ++ /* ++ * We don't hold any lock here yet, get_active_stripe() might ++ * remove stripes from the list ++ */ ++ if (!list_empty_careful(list)) { ++ spin_lock_irqsave(conf->hash_locks + hash, flags); ++ if (list_empty(conf->inactive_list + hash) && ++ !list_empty(list)) ++ atomic_dec(&conf->empty_inactive_list_nr); ++ list_splice_tail_init(list, conf->inactive_list + hash); ++ do_wakeup = true; ++ spin_unlock_irqrestore(conf->hash_locks + hash, flags); ++ } ++ size--; ++ hash--; + } + -+ if (e) { -+ s = ring_buffer_event_data(e); -+ memcpy(sample, s, sizeof(struct sample)); -+ } else -+ sample = NULL; -+ mutex_unlock(&ring_buffer_mutex); -+ -+ return sample; ++ if (do_wakeup) { ++ wake_up(&conf->wait_for_stripe); ++ if (conf->retry_read_aligned) ++ md_wakeup_thread(conf->mddev->thread); ++ } +} + -+#ifndef CONFIG_TRACING -+#define time_type ktime_t -+#define time_get() ktime_get() -+#define time_to_us(x) ktime_to_us(x) -+#define time_sub(a, b) ktime_sub(a, b) -+#define init_time(a, b) (a).tv64 = b -+#define time_u64(a) ((a).tv64) -+#else -+#define time_type u64 -+#define time_get() trace_clock_local() -+#define time_to_us(x) div_u64(x, 1000) -+#define time_sub(a, b) ((a) - (b)) -+#define init_time(a, b) (a = b) -+#define time_u64(a) a -+#endif -+/** -+ * get_sample - sample the CPU TSC and look for likely hardware latencies -+ * -+ * Used to repeatedly capture the CPU TSC (or similar), looking for potential -+ * hardware-induced latency. Called with interrupts disabled and with -+ * data.lock held. -+ */ -+static int get_sample(void) ++/* should hold conf->device_lock already */ ++static int release_stripe_list(struct r5conf *conf, ++ struct list_head *temp_inactive_list) +{ -+ time_type start, t1, t2, last_t2; -+ s64 diff, total = 0; -+ u64 sample = 0; -+ u64 outer_sample = 0; -+ int ret = -1; -+ -+ init_time(last_t2, 0); -+ start = time_get(); /* start timestamp */ ++ struct stripe_head *sh; ++ int count = 0; ++ struct llist_node *head; ++ ++ head = llist_del_all(&conf->released_stripes); ++ head = llist_reverse_order(head); ++ while (head) { ++ int hash; ++ ++ sh = llist_entry(head, struct stripe_head, release_list); ++ head = llist_next(head); ++ /* sh could be readded after STRIPE_ON_RELEASE_LIST is cleard */ ++ smp_mb(); ++ clear_bit(STRIPE_ON_RELEASE_LIST, &sh->state); ++ /* ++ * Don't worry the bit is set here, because if the bit is set ++ * again, the count is always > 1. This is true for ++ * STRIPE_ON_UNPLUG_LIST bit too. ++ */ ++ hash = sh->hash_lock_index; ++ __release_stripe(conf, sh, &temp_inactive_list[hash]); ++ count++; ++ } + -+ do { ++ return count; ++} + -+ t1 = time_get(); /* we'll look for a discontinuity */ -+ t2 = time_get(); ++static void release_stripe(struct stripe_head *sh) ++{ ++ struct r5conf *conf = sh->raid_conf; ++ unsigned long flags; ++ struct list_head list; ++ int hash; ++ bool wakeup; + -+ if (time_u64(last_t2)) { -+ /* Check the delta from outer loop (t2 to next t1) */ -+ diff = time_to_us(time_sub(t1, last_t2)); -+ /* This shouldn't happen */ -+ if (diff < 0) { -+ pr_err(BANNER "time running backwards\n"); -+ goto out; -+ } -+ if (diff > outer_sample) -+ outer_sample = diff; -+ } -+ last_t2 = t2; ++ /* Avoid release_list until the last reference. ++ */ ++ if (atomic_add_unless(&sh->count, -1, 1)) ++ return; + -+ total = time_to_us(time_sub(t2, start)); /* sample width */ ++ if (unlikely(!conf->mddev->thread) || ++ test_and_set_bit(STRIPE_ON_RELEASE_LIST, &sh->state)) ++ goto slow_path; ++ wakeup = llist_add(&sh->release_list, &conf->released_stripes); ++ if (wakeup) ++ md_wakeup_thread(conf->mddev->thread); ++ return; ++slow_path: ++ local_irq_save(flags); ++ /* we are ok here if STRIPE_ON_RELEASE_LIST is set or not */ ++ if (atomic_dec_and_lock(&sh->count, &conf->device_lock)) { ++ INIT_LIST_HEAD(&list); ++ hash = sh->hash_lock_index; ++ do_release_stripe(conf, sh, &list); ++ spin_unlock(&conf->device_lock); ++ release_inactive_stripe_list(conf, &list, hash); ++ } ++ local_irq_restore(flags); ++} + -+ /* This checks the inner loop (t1 to t2) */ -+ diff = time_to_us(time_sub(t2, t1)); /* current diff */ ++static inline void remove_hash(struct stripe_head *sh) ++{ ++ pr_debug("remove_hash(), stripe %llu\n", ++ (unsigned long long)sh->sector); + -+ /* This shouldn't happen */ -+ if (diff < 0) { -+ pr_err(BANNER "time running backwards\n"); -+ goto out; -+ } ++ hlist_del_init(&sh->hash); ++} + -+ if (diff > sample) -+ sample = diff; /* only want highest value */ ++static inline void insert_hash(struct r5conf *conf, struct stripe_head *sh) ++{ ++ struct hlist_head *hp = stripe_hash(conf, sh->sector); + -+ } while (total <= data.sample_width); ++ pr_debug("insert_hash(), stripe %llu\n", ++ (unsigned long long)sh->sector); + -+ ret = 0; ++ hlist_add_head(&sh->hash, hp); ++} + -+ /* If we exceed the threshold value, we have found a hardware latency */ -+ if (sample > data.threshold || outer_sample > data.threshold) { -+ struct sample s; ++/* find an idle stripe, make sure it is unhashed, and return it. */ ++static struct stripe_head *get_free_stripe(struct r5conf *conf, int hash) ++{ ++ struct stripe_head *sh = NULL; ++ struct list_head *first; + -+ ret = 1; ++ if (list_empty(conf->inactive_list + hash)) ++ goto out; ++ first = (conf->inactive_list + hash)->next; ++ sh = list_entry(first, struct stripe_head, lru); ++ list_del_init(first); ++ remove_hash(sh); ++ atomic_inc(&conf->active_stripes); ++ BUG_ON(hash != sh->hash_lock_index); ++ if (list_empty(conf->inactive_list + hash)) ++ atomic_inc(&conf->empty_inactive_list_nr); ++out: ++ return sh; ++} + -+ data.count++; -+ s.seqnum = data.count; -+ s.duration = sample; -+ s.outer_duration = outer_sample; -+ s.timestamp = CURRENT_TIME; -+ __buffer_add_sample(&s); ++static void shrink_buffers(struct stripe_head *sh) ++{ ++ struct page *p; ++ int i; ++ int num = sh->raid_conf->pool_size; + -+ /* Keep a running maximum ever recorded hardware latency */ -+ if (sample > data.max_sample) -+ data.max_sample = sample; ++ for (i = 0; i < num ; i++) { ++ WARN_ON(sh->dev[i].page != sh->dev[i].orig_page); ++ p = sh->dev[i].page; ++ if (!p) ++ continue; ++ sh->dev[i].page = NULL; ++ put_page(p); + } -+ -+out: -+ return ret; +} + -+/* -+ * kthread_fn - The CPU time sampling/hardware latency detection kernel thread -+ * @unused: A required part of the kthread API. -+ * -+ * Used to periodically sample the CPU TSC via a call to get_sample. We -+ * disable interrupts, which does (intentionally) introduce latency since we -+ * need to ensure nothing else might be running (and thus pre-empting). -+ * Obviously this should never be used in production environments. -+ * -+ * Currently this runs on which ever CPU it was scheduled on, but most -+ * real-worald hardware latency situations occur across several CPUs, -+ * but we might later generalize this if we find there are any actualy -+ * systems with alternate SMI delivery or other hardware latencies. -+ */ -+static int kthread_fn(void *unused) ++static int grow_buffers(struct stripe_head *sh, gfp_t gfp) +{ -+ int ret; -+ u64 interval; -+ -+ while (!kthread_should_stop()) { ++ int i; ++ int num = sh->raid_conf->pool_size; + -+ mutex_lock(&data.lock); ++ for (i = 0; i < num; i++) { ++ struct page *page; + -+ local_irq_disable(); -+ ret = get_sample(); -+ local_irq_enable(); ++ if (!(page = alloc_page(gfp))) { ++ return 1; ++ } ++ sh->dev[i].page = page; ++ sh->dev[i].orig_page = page; ++ } ++ return 0; ++} + -+ if (ret > 0) -+ wake_up(&data.wq); /* wake up reader(s) */ ++static void raid5_build_block(struct stripe_head *sh, int i, int previous); ++static void stripe_set_idx(sector_t stripe, struct r5conf *conf, int previous, ++ struct stripe_head *sh); + -+ interval = data.sample_window - data.sample_width; -+ do_div(interval, USEC_PER_MSEC); /* modifies interval value */ ++static void init_stripe(struct stripe_head *sh, sector_t sector, int previous) ++{ ++ struct r5conf *conf = sh->raid_conf; ++ int i, seq; + -+ mutex_unlock(&data.lock); ++ BUG_ON(atomic_read(&sh->count) != 0); ++ BUG_ON(test_bit(STRIPE_HANDLE, &sh->state)); ++ BUG_ON(stripe_operations_active(sh)); ++ BUG_ON(sh->batch_head); + -+ if (msleep_interruptible(interval)) -+ break; ++ pr_debug("init_stripe called, stripe %llu\n", ++ (unsigned long long)sector); ++retry: ++ seq = read_seqcount_begin(&conf->gen_lock); ++ sh->generation = conf->generation - previous; ++ sh->disks = previous ? conf->previous_raid_disks : conf->raid_disks; ++ sh->sector = sector; ++ stripe_set_idx(sector, conf, previous, sh); ++ sh->state = 0; ++ ++ for (i = sh->disks; i--; ) { ++ struct r5dev *dev = &sh->dev[i]; ++ ++ if (dev->toread || dev->read || dev->towrite || dev->written || ++ test_bit(R5_LOCKED, &dev->flags)) { ++ printk(KERN_ERR "sector=%llx i=%d %p %p %p %p %d\n", ++ (unsigned long long)sh->sector, i, dev->toread, ++ dev->read, dev->towrite, dev->written, ++ test_bit(R5_LOCKED, &dev->flags)); ++ WARN_ON(1); ++ } ++ dev->flags = 0; ++ raid5_build_block(sh, i, previous); + } ++ if (read_seqcount_retry(&conf->gen_lock, seq)) ++ goto retry; ++ sh->overwrite_disks = 0; ++ insert_hash(conf, sh); ++ sh->cpu = smp_processor_id(); ++ set_bit(STRIPE_BATCH_READY, &sh->state); ++} + -+ return 0; ++static struct stripe_head *__find_stripe(struct r5conf *conf, sector_t sector, ++ short generation) ++{ ++ struct stripe_head *sh; ++ ++ pr_debug("__find_stripe, sector %llu\n", (unsigned long long)sector); ++ hlist_for_each_entry(sh, stripe_hash(conf, sector), hash) ++ if (sh->sector == sector && sh->generation == generation) ++ return sh; ++ pr_debug("__stripe %llu not in cache\n", (unsigned long long)sector); ++ return NULL; +} + -+/** -+ * start_kthread - Kick off the hardware latency sampling/detector kthread -+ * -+ * This starts a kernel thread that will sit and sample the CPU timestamp -+ * counter (TSC or similar) and look for potential hardware latencies. ++/* ++ * Need to check if array has failed when deciding whether to: ++ * - start an array ++ * - remove non-faulty devices ++ * - add a spare ++ * - allow a reshape ++ * This determination is simple when no reshape is happening. ++ * However if there is a reshape, we need to carefully check ++ * both the before and after sections. ++ * This is because some failed devices may only affect one ++ * of the two sections, and some non-in_sync devices may ++ * be insync in the section most affected by failed devices. + */ -+static int start_kthread(void) ++static int calc_degraded(struct r5conf *conf) +{ -+ kthread = kthread_run(kthread_fn, NULL, -+ DRVNAME); -+ if (IS_ERR(kthread)) { -+ pr_err(BANNER "could not start sampling thread\n"); -+ enabled = 0; -+ return -ENOMEM; ++ int degraded, degraded2; ++ int i; ++ ++ rcu_read_lock(); ++ degraded = 0; ++ for (i = 0; i < conf->previous_raid_disks; i++) { ++ struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev); ++ if (rdev && test_bit(Faulty, &rdev->flags)) ++ rdev = rcu_dereference(conf->disks[i].replacement); ++ if (!rdev || test_bit(Faulty, &rdev->flags)) ++ degraded++; ++ else if (test_bit(In_sync, &rdev->flags)) ++ ; ++ else ++ /* not in-sync or faulty. ++ * If the reshape increases the number of devices, ++ * this is being recovered by the reshape, so ++ * this 'previous' section is not in_sync. ++ * If the number of devices is being reduced however, ++ * the device can only be part of the array if ++ * we are reverting a reshape, so this section will ++ * be in-sync. ++ */ ++ if (conf->raid_disks >= conf->previous_raid_disks) ++ degraded++; ++ } ++ rcu_read_unlock(); ++ if (conf->raid_disks == conf->previous_raid_disks) ++ return degraded; ++ rcu_read_lock(); ++ degraded2 = 0; ++ for (i = 0; i < conf->raid_disks; i++) { ++ struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev); ++ if (rdev && test_bit(Faulty, &rdev->flags)) ++ rdev = rcu_dereference(conf->disks[i].replacement); ++ if (!rdev || test_bit(Faulty, &rdev->flags)) ++ degraded2++; ++ else if (test_bit(In_sync, &rdev->flags)) ++ ; ++ else ++ /* not in-sync or faulty. ++ * If reshape increases the number of devices, this ++ * section has already been recovered, else it ++ * almost certainly hasn't. ++ */ ++ if (conf->raid_disks <= conf->previous_raid_disks) ++ degraded2++; + } ++ rcu_read_unlock(); ++ if (degraded2 > degraded) ++ return degraded2; ++ return degraded; ++} + ++static int has_failed(struct r5conf *conf) ++{ ++ int degraded; ++ ++ if (conf->mddev->reshape_position == MaxSector) ++ return conf->mddev->degraded > conf->max_degraded; ++ ++ degraded = calc_degraded(conf); ++ if (degraded > conf->max_degraded) ++ return 1; + return 0; +} + -+/** -+ * stop_kthread - Inform the hardware latency samping/detector kthread to stop -+ * -+ * This kicks the running hardware latency sampling/detector kernel thread and -+ * tells it to stop sampling now. Use this on unload and at system shutdown. -+ */ -+static int stop_kthread(void) ++static struct stripe_head * ++get_active_stripe(struct r5conf *conf, sector_t sector, ++ int previous, int noblock, int noquiesce) +{ -+ int ret; ++ struct stripe_head *sh; ++ int hash = stripe_hash_locks_hash(sector); + -+ ret = kthread_stop(kthread); ++ pr_debug("get_stripe, sector %llu\n", (unsigned long long)sector); + -+ return ret; ++ spin_lock_irq(conf->hash_locks + hash); ++ ++ do { ++ wait_event_lock_irq(conf->wait_for_stripe, ++ conf->quiesce == 0 || noquiesce, ++ *(conf->hash_locks + hash)); ++ sh = __find_stripe(conf, sector, conf->generation - previous); ++ if (!sh) { ++ if (!test_bit(R5_INACTIVE_BLOCKED, &conf->cache_state)) { ++ sh = get_free_stripe(conf, hash); ++ if (!sh && llist_empty(&conf->released_stripes) && ++ !test_bit(R5_DID_ALLOC, &conf->cache_state)) ++ set_bit(R5_ALLOC_MORE, ++ &conf->cache_state); ++ } ++ if (noblock && sh == NULL) ++ break; ++ if (!sh) { ++ set_bit(R5_INACTIVE_BLOCKED, ++ &conf->cache_state); ++ wait_event_lock_irq( ++ conf->wait_for_stripe, ++ !list_empty(conf->inactive_list + hash) && ++ (atomic_read(&conf->active_stripes) ++ < (conf->max_nr_stripes * 3 / 4) ++ || !test_bit(R5_INACTIVE_BLOCKED, ++ &conf->cache_state)), ++ *(conf->hash_locks + hash)); ++ clear_bit(R5_INACTIVE_BLOCKED, ++ &conf->cache_state); ++ } else { ++ init_stripe(sh, sector, previous); ++ atomic_inc(&sh->count); ++ } ++ } else if (!atomic_inc_not_zero(&sh->count)) { ++ spin_lock(&conf->device_lock); ++ if (!atomic_read(&sh->count)) { ++ if (!test_bit(STRIPE_HANDLE, &sh->state)) ++ atomic_inc(&conf->active_stripes); ++ BUG_ON(list_empty(&sh->lru) && ++ !test_bit(STRIPE_EXPANDING, &sh->state)); ++ list_del_init(&sh->lru); ++ if (sh->group) { ++ sh->group->stripes_cnt--; ++ sh->group = NULL; ++ } ++ } ++ atomic_inc(&sh->count); ++ spin_unlock(&conf->device_lock); ++ } ++ } while (sh == NULL); ++ ++ spin_unlock_irq(conf->hash_locks + hash); ++ return sh; +} + -+/** -+ * __reset_stats - Reset statistics for the hardware latency detector -+ * -+ * We use data to store various statistics and global state. We call this -+ * function in order to reset those when "enable" is toggled on or off, and -+ * also at initialization. Should be called with data.lock held. -+ */ -+static void __reset_stats(void) ++static bool is_full_stripe_write(struct stripe_head *sh) +{ -+ data.count = 0; -+ data.max_sample = 0; -+ ring_buffer_reset(ring_buffer); /* flush out old sample entries */ ++ BUG_ON(sh->overwrite_disks > (sh->disks - sh->raid_conf->max_degraded)); ++ return sh->overwrite_disks == (sh->disks - sh->raid_conf->max_degraded); +} + -+/** -+ * init_stats - Setup global state statistics for the hardware latency detector -+ * -+ * We use data to store various statistics and global state. We also use -+ * a global ring buffer (ring_buffer) to keep raw samples of detected hardware -+ * induced system latencies. This function initializes these structures and -+ * allocates the global ring buffer also. -+ */ -+static int init_stats(void) ++static void lock_two_stripes(struct stripe_head *sh1, struct stripe_head *sh2) +{ -+ int ret = -ENOMEM; ++ local_irq_disable(); ++ if (sh1 > sh2) { ++ spin_lock(&sh2->stripe_lock); ++ spin_lock_nested(&sh1->stripe_lock, 1); ++ } else { ++ spin_lock(&sh1->stripe_lock); ++ spin_lock_nested(&sh2->stripe_lock, 1); ++ } ++} + -+ mutex_init(&data.lock); -+ init_waitqueue_head(&data.wq); -+ atomic_set(&data.sample_open, 0); ++static void unlock_two_stripes(struct stripe_head *sh1, struct stripe_head *sh2) ++{ ++ spin_unlock(&sh1->stripe_lock); ++ spin_unlock(&sh2->stripe_lock); ++ local_irq_enable(); ++} + -+ ring_buffer = ring_buffer_alloc(buf_size, BUF_FLAGS); ++/* Only freshly new full stripe normal write stripe can be added to a batch list */ ++static bool stripe_can_batch(struct stripe_head *sh) ++{ ++ return test_bit(STRIPE_BATCH_READY, &sh->state) && ++ !test_bit(STRIPE_BITMAP_PENDING, &sh->state) && ++ is_full_stripe_write(sh); ++} + -+ if (WARN(!ring_buffer, KERN_ERR BANNER -+ "failed to allocate ring buffer!\n")) ++/* we only do back search */ ++static void stripe_add_to_batch_list(struct r5conf *conf, struct stripe_head *sh) ++{ ++ struct stripe_head *head; ++ sector_t head_sector, tmp_sec; ++ int hash; ++ int dd_idx; ++ ++ if (!stripe_can_batch(sh)) ++ return; ++ /* Don't cross chunks, so stripe pd_idx/qd_idx is the same */ ++ tmp_sec = sh->sector; ++ if (!sector_div(tmp_sec, conf->chunk_sectors)) ++ return; ++ head_sector = sh->sector - STRIPE_SECTORS; ++ ++ hash = stripe_hash_locks_hash(head_sector); ++ spin_lock_irq(conf->hash_locks + hash); ++ head = __find_stripe(conf, head_sector, conf->generation); ++ if (head && !atomic_inc_not_zero(&head->count)) { ++ spin_lock(&conf->device_lock); ++ if (!atomic_read(&head->count)) { ++ if (!test_bit(STRIPE_HANDLE, &head->state)) ++ atomic_inc(&conf->active_stripes); ++ BUG_ON(list_empty(&head->lru) && ++ !test_bit(STRIPE_EXPANDING, &head->state)); ++ list_del_init(&head->lru); ++ if (head->group) { ++ head->group->stripes_cnt--; ++ head->group = NULL; ++ } ++ } ++ atomic_inc(&head->count); ++ spin_unlock(&conf->device_lock); ++ } ++ spin_unlock_irq(conf->hash_locks + hash); ++ ++ if (!head) ++ return; ++ if (!stripe_can_batch(head)) + goto out; + -+ __reset_stats(); -+ data.threshold = threshold ?: DEFAULT_LAT_THRESHOLD; /* threshold us */ -+ data.sample_window = DEFAULT_SAMPLE_WINDOW; /* window us */ -+ data.sample_width = DEFAULT_SAMPLE_WIDTH; /* width us */ ++ lock_two_stripes(head, sh); ++ /* clear_batch_ready clear the flag */ ++ if (!stripe_can_batch(head) || !stripe_can_batch(sh)) ++ goto unlock_out; ++ ++ if (sh->batch_head) ++ goto unlock_out; ++ ++ dd_idx = 0; ++ while (dd_idx == sh->pd_idx || dd_idx == sh->qd_idx) ++ dd_idx++; ++ if (head->dev[dd_idx].towrite->bi_rw != sh->dev[dd_idx].towrite->bi_rw) ++ goto unlock_out; ++ ++ if (head->batch_head) { ++ spin_lock(&head->batch_head->batch_lock); ++ /* This batch list is already running */ ++ if (!stripe_can_batch(head)) { ++ spin_unlock(&head->batch_head->batch_lock); ++ goto unlock_out; ++ } + -+ ret = 0; ++ /* ++ * at this point, head's BATCH_READY could be cleared, but we ++ * can still add the stripe to batch list ++ */ ++ list_add(&sh->batch_list, &head->batch_list); ++ spin_unlock(&head->batch_head->batch_lock); + -+out: -+ return ret; ++ sh->batch_head = head->batch_head; ++ } else { ++ head->batch_head = head; ++ sh->batch_head = head->batch_head; ++ spin_lock(&head->batch_lock); ++ list_add_tail(&sh->batch_list, &head->batch_list); ++ spin_unlock(&head->batch_lock); ++ } + ++ if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) ++ if (atomic_dec_return(&conf->preread_active_stripes) ++ < IO_THRESHOLD) ++ md_wakeup_thread(conf->mddev->thread); ++ ++ if (test_and_clear_bit(STRIPE_BIT_DELAY, &sh->state)) { ++ int seq = sh->bm_seq; ++ if (test_bit(STRIPE_BIT_DELAY, &sh->batch_head->state) && ++ sh->batch_head->bm_seq > seq) ++ seq = sh->batch_head->bm_seq; ++ set_bit(STRIPE_BIT_DELAY, &sh->batch_head->state); ++ sh->batch_head->bm_seq = seq; ++ } ++ ++ atomic_inc(&sh->count); ++unlock_out: ++ unlock_two_stripes(head, sh); ++out: ++ release_stripe(head); +} + -+/* -+ * simple_data_read - Wrapper read function for global state debugfs entries -+ * @filp: The active open file structure for the debugfs "file" -+ * @ubuf: The userspace provided buffer to read value into -+ * @cnt: The maximum number of bytes to read -+ * @ppos: The current "file" position -+ * @entry: The entry to read from -+ * -+ * This function provides a generic read implementation for the global state -+ * "data" structure debugfs filesystem entries. It would be nice to use -+ * simple_attr_read directly, but we need to make sure that the data.lock -+ * is held during the actual read. ++/* Determine if 'data_offset' or 'new_data_offset' should be used ++ * in this stripe_head. + */ -+static ssize_t simple_data_read(struct file *filp, char __user *ubuf, -+ size_t cnt, loff_t *ppos, const u64 *entry) ++static int use_new_offset(struct r5conf *conf, struct stripe_head *sh) +{ -+ char buf[U64STR_SIZE]; -+ u64 val = 0; -+ int len = 0; ++ sector_t progress = conf->reshape_progress; ++ /* Need a memory barrier to make sure we see the value ++ * of conf->generation, or ->data_offset that was set before ++ * reshape_progress was updated. ++ */ ++ smp_rmb(); ++ if (progress == MaxSector) ++ return 0; ++ if (sh->generation == conf->generation - 1) ++ return 0; ++ /* We are in a reshape, and this is a new-generation stripe, ++ * so use new_data_offset. ++ */ ++ return 1; ++} + -+ memset(buf, 0, sizeof(buf)); ++static void ++raid5_end_read_request(struct bio *bi, int error); ++static void ++raid5_end_write_request(struct bio *bi, int error); + -+ if (!entry) -+ return -EFAULT; ++static void ops_run_io(struct stripe_head *sh, struct stripe_head_state *s) ++{ ++ struct r5conf *conf = sh->raid_conf; ++ int i, disks = sh->disks; ++ struct stripe_head *head_sh = sh; + -+ mutex_lock(&data.lock); -+ val = *entry; -+ mutex_unlock(&data.lock); ++ might_sleep(); + -+ len = snprintf(buf, sizeof(buf), "%llu\n", (unsigned long long)val); ++ for (i = disks; i--; ) { ++ int rw; ++ int replace_only = 0; ++ struct bio *bi, *rbi; ++ struct md_rdev *rdev, *rrdev = NULL; + -+ return simple_read_from_buffer(ubuf, cnt, ppos, buf, len); ++ sh = head_sh; ++ if (test_and_clear_bit(R5_Wantwrite, &sh->dev[i].flags)) { ++ if (test_and_clear_bit(R5_WantFUA, &sh->dev[i].flags)) ++ rw = WRITE_FUA; ++ else ++ rw = WRITE; ++ if (test_bit(R5_Discard, &sh->dev[i].flags)) ++ rw |= REQ_DISCARD; ++ } else if (test_and_clear_bit(R5_Wantread, &sh->dev[i].flags)) ++ rw = READ; ++ else if (test_and_clear_bit(R5_WantReplace, ++ &sh->dev[i].flags)) { ++ rw = WRITE; ++ replace_only = 1; ++ } else ++ continue; ++ if (test_and_clear_bit(R5_SyncIO, &sh->dev[i].flags)) ++ rw |= REQ_SYNC; ++ ++again: ++ bi = &sh->dev[i].req; ++ rbi = &sh->dev[i].rreq; /* For writing to replacement */ ++ ++ rcu_read_lock(); ++ rrdev = rcu_dereference(conf->disks[i].replacement); ++ smp_mb(); /* Ensure that if rrdev is NULL, rdev won't be */ ++ rdev = rcu_dereference(conf->disks[i].rdev); ++ if (!rdev) { ++ rdev = rrdev; ++ rrdev = NULL; ++ } ++ if (rw & WRITE) { ++ if (replace_only) ++ rdev = NULL; ++ if (rdev == rrdev) ++ /* We raced and saw duplicates */ ++ rrdev = NULL; ++ } else { ++ if (test_bit(R5_ReadRepl, &head_sh->dev[i].flags) && rrdev) ++ rdev = rrdev; ++ rrdev = NULL; ++ } ++ ++ if (rdev && test_bit(Faulty, &rdev->flags)) ++ rdev = NULL; ++ if (rdev) ++ atomic_inc(&rdev->nr_pending); ++ if (rrdev && test_bit(Faulty, &rrdev->flags)) ++ rrdev = NULL; ++ if (rrdev) ++ atomic_inc(&rrdev->nr_pending); ++ rcu_read_unlock(); ++ ++ /* We have already checked bad blocks for reads. Now ++ * need to check for writes. We never accept write errors ++ * on the replacement, so we don't to check rrdev. ++ */ ++ while ((rw & WRITE) && rdev && ++ test_bit(WriteErrorSeen, &rdev->flags)) { ++ sector_t first_bad; ++ int bad_sectors; ++ int bad = is_badblock(rdev, sh->sector, STRIPE_SECTORS, ++ &first_bad, &bad_sectors); ++ if (!bad) ++ break; ++ ++ if (bad < 0) { ++ set_bit(BlockedBadBlocks, &rdev->flags); ++ if (!conf->mddev->external && ++ conf->mddev->flags) { ++ /* It is very unlikely, but we might ++ * still need to write out the ++ * bad block log - better give it ++ * a chance*/ ++ md_check_recovery(conf->mddev); ++ } ++ /* ++ * Because md_wait_for_blocked_rdev ++ * will dec nr_pending, we must ++ * increment it first. ++ */ ++ atomic_inc(&rdev->nr_pending); ++ md_wait_for_blocked_rdev(rdev, conf->mddev); ++ } else { ++ /* Acknowledged bad block - skip the write */ ++ rdev_dec_pending(rdev, conf->mddev); ++ rdev = NULL; ++ } ++ } ++ ++ if (rdev) { ++ if (s->syncing || s->expanding || s->expanded ++ || s->replacing) ++ md_sync_acct(rdev->bdev, STRIPE_SECTORS); ++ ++ set_bit(STRIPE_IO_STARTED, &sh->state); ++ ++ bio_reset(bi); ++ bi->bi_bdev = rdev->bdev; ++ bi->bi_rw = rw; ++ bi->bi_end_io = (rw & WRITE) ++ ? raid5_end_write_request ++ : raid5_end_read_request; ++ bi->bi_private = sh; ++ ++ pr_debug("%s: for %llu schedule op %ld on disc %d\n", ++ __func__, (unsigned long long)sh->sector, ++ bi->bi_rw, i); ++ atomic_inc(&sh->count); ++ if (sh != head_sh) ++ atomic_inc(&head_sh->count); ++ if (use_new_offset(conf, sh)) ++ bi->bi_iter.bi_sector = (sh->sector ++ + rdev->new_data_offset); ++ else ++ bi->bi_iter.bi_sector = (sh->sector ++ + rdev->data_offset); ++ if (test_bit(R5_ReadNoMerge, &head_sh->dev[i].flags)) ++ bi->bi_rw |= REQ_NOMERGE; ++ ++ if (test_bit(R5_SkipCopy, &sh->dev[i].flags)) ++ WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags)); ++ sh->dev[i].vec.bv_page = sh->dev[i].page; ++ bi->bi_vcnt = 1; ++ bi->bi_io_vec[0].bv_len = STRIPE_SIZE; ++ bi->bi_io_vec[0].bv_offset = 0; ++ bi->bi_iter.bi_size = STRIPE_SIZE; ++ /* ++ * If this is discard request, set bi_vcnt 0. We don't ++ * want to confuse SCSI because SCSI will replace payload ++ */ ++ if (rw & REQ_DISCARD) ++ bi->bi_vcnt = 0; ++ if (rrdev) ++ set_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags); ++ ++ if (conf->mddev->gendisk) ++ trace_block_bio_remap(bdev_get_queue(bi->bi_bdev), ++ bi, disk_devt(conf->mddev->gendisk), ++ sh->dev[i].sector); ++ generic_make_request(bi); ++ } ++ if (rrdev) { ++ if (s->syncing || s->expanding || s->expanded ++ || s->replacing) ++ md_sync_acct(rrdev->bdev, STRIPE_SECTORS); ++ ++ set_bit(STRIPE_IO_STARTED, &sh->state); ++ ++ bio_reset(rbi); ++ rbi->bi_bdev = rrdev->bdev; ++ rbi->bi_rw = rw; ++ BUG_ON(!(rw & WRITE)); ++ rbi->bi_end_io = raid5_end_write_request; ++ rbi->bi_private = sh; ++ ++ pr_debug("%s: for %llu schedule op %ld on " ++ "replacement disc %d\n", ++ __func__, (unsigned long long)sh->sector, ++ rbi->bi_rw, i); ++ atomic_inc(&sh->count); ++ if (sh != head_sh) ++ atomic_inc(&head_sh->count); ++ if (use_new_offset(conf, sh)) ++ rbi->bi_iter.bi_sector = (sh->sector ++ + rrdev->new_data_offset); ++ else ++ rbi->bi_iter.bi_sector = (sh->sector ++ + rrdev->data_offset); ++ if (test_bit(R5_SkipCopy, &sh->dev[i].flags)) ++ WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags)); ++ sh->dev[i].rvec.bv_page = sh->dev[i].page; ++ rbi->bi_vcnt = 1; ++ rbi->bi_io_vec[0].bv_len = STRIPE_SIZE; ++ rbi->bi_io_vec[0].bv_offset = 0; ++ rbi->bi_iter.bi_size = STRIPE_SIZE; ++ /* ++ * If this is discard request, set bi_vcnt 0. We don't ++ * want to confuse SCSI because SCSI will replace payload ++ */ ++ if (rw & REQ_DISCARD) ++ rbi->bi_vcnt = 0; ++ if (conf->mddev->gendisk) ++ trace_block_bio_remap(bdev_get_queue(rbi->bi_bdev), ++ rbi, disk_devt(conf->mddev->gendisk), ++ sh->dev[i].sector); ++ generic_make_request(rbi); ++ } ++ if (!rdev && !rrdev) { ++ if (rw & WRITE) ++ set_bit(STRIPE_DEGRADED, &sh->state); ++ pr_debug("skip op %ld on disc %d for sector %llu\n", ++ bi->bi_rw, i, (unsigned long long)sh->sector); ++ clear_bit(R5_LOCKED, &sh->dev[i].flags); ++ set_bit(STRIPE_HANDLE, &sh->state); ++ } + ++ if (!head_sh->batch_head) ++ continue; ++ sh = list_first_entry(&sh->batch_list, struct stripe_head, ++ batch_list); ++ if (sh != head_sh) ++ goto again; ++ } +} + -+/* -+ * simple_data_write - Wrapper write function for global state debugfs entries -+ * @filp: The active open file structure for the debugfs "file" -+ * @ubuf: The userspace provided buffer to write value from -+ * @cnt: The maximum number of bytes to write -+ * @ppos: The current "file" position -+ * @entry: The entry to write to -+ * -+ * This function provides a generic write implementation for the global state -+ * "data" structure debugfs filesystem entries. It would be nice to use -+ * simple_attr_write directly, but we need to make sure that the data.lock -+ * is held during the actual write. -+ */ -+static ssize_t simple_data_write(struct file *filp, const char __user *ubuf, -+ size_t cnt, loff_t *ppos, u64 *entry) ++static struct dma_async_tx_descriptor * ++async_copy_data(int frombio, struct bio *bio, struct page **page, ++ sector_t sector, struct dma_async_tx_descriptor *tx, ++ struct stripe_head *sh) +{ -+ char buf[U64STR_SIZE]; -+ int csize = min(cnt, sizeof(buf)); -+ u64 val = 0; -+ int err = 0; ++ struct bio_vec bvl; ++ struct bvec_iter iter; ++ struct page *bio_page; ++ int page_offset; ++ struct async_submit_ctl submit; ++ enum async_tx_flags flags = 0; ++ ++ if (bio->bi_iter.bi_sector >= sector) ++ page_offset = (signed)(bio->bi_iter.bi_sector - sector) * 512; ++ else ++ page_offset = (signed)(sector - bio->bi_iter.bi_sector) * -512; + -+ memset(buf, '\0', sizeof(buf)); -+ if (copy_from_user(buf, ubuf, csize)) -+ return -EFAULT; ++ if (frombio) ++ flags |= ASYNC_TX_FENCE; ++ init_async_submit(&submit, flags, tx, NULL, NULL, NULL); + -+ buf[U64STR_SIZE-1] = '\0'; /* just in case */ -+ err = kstrtoull(buf, 10, &val); -+ if (err) -+ return -EINVAL; ++ bio_for_each_segment(bvl, bio, iter) { ++ int len = bvl.bv_len; ++ int clen; ++ int b_offset = 0; + -+ mutex_lock(&data.lock); -+ *entry = val; -+ mutex_unlock(&data.lock); ++ if (page_offset < 0) { ++ b_offset = -page_offset; ++ page_offset += b_offset; ++ len -= b_offset; ++ } + -+ return csize; ++ if (len > 0 && page_offset + len > STRIPE_SIZE) ++ clen = STRIPE_SIZE - page_offset; ++ else ++ clen = len; ++ ++ if (clen > 0) { ++ b_offset += bvl.bv_offset; ++ bio_page = bvl.bv_page; ++ if (frombio) { ++ if (sh->raid_conf->skip_copy && ++ b_offset == 0 && page_offset == 0 && ++ clen == STRIPE_SIZE) ++ *page = bio_page; ++ else ++ tx = async_memcpy(*page, bio_page, page_offset, ++ b_offset, clen, &submit); ++ } else ++ tx = async_memcpy(bio_page, *page, b_offset, ++ page_offset, clen, &submit); ++ } ++ /* chain the operations */ ++ submit.depend_tx = tx; ++ ++ if (clen < len) /* hit end of page */ ++ break; ++ page_offset += len; ++ } ++ ++ return tx; +} + -+/** -+ * debug_count_fopen - Open function for "count" debugfs entry -+ * @inode: The in-kernel inode representation of the debugfs "file" -+ * @filp: The active open file structure for the debugfs "file" -+ * -+ * This function provides an open implementation for the "count" debugfs -+ * interface to the hardware latency detector. -+ */ -+static int debug_count_fopen(struct inode *inode, struct file *filp) ++static void ops_complete_biofill(void *stripe_head_ref) +{ -+ return 0; ++ struct stripe_head *sh = stripe_head_ref; ++ struct bio *return_bi = NULL; ++ int i; ++ ++ pr_debug("%s: stripe %llu\n", __func__, ++ (unsigned long long)sh->sector); ++ ++ /* clear completed biofills */ ++ for (i = sh->disks; i--; ) { ++ struct r5dev *dev = &sh->dev[i]; ++ ++ /* acknowledge completion of a biofill operation */ ++ /* and check if we need to reply to a read request, ++ * new R5_Wantfill requests are held off until ++ * !STRIPE_BIOFILL_RUN ++ */ ++ if (test_and_clear_bit(R5_Wantfill, &dev->flags)) { ++ struct bio *rbi, *rbi2; ++ ++ BUG_ON(!dev->read); ++ rbi = dev->read; ++ dev->read = NULL; ++ while (rbi && rbi->bi_iter.bi_sector < ++ dev->sector + STRIPE_SECTORS) { ++ rbi2 = r5_next_bio(rbi, dev->sector); ++ if (!raid5_dec_bi_active_stripes(rbi)) { ++ rbi->bi_next = return_bi; ++ return_bi = rbi; ++ } ++ rbi = rbi2; ++ } ++ } ++ } ++ clear_bit(STRIPE_BIOFILL_RUN, &sh->state); ++ ++ return_io(return_bi); ++ ++ set_bit(STRIPE_HANDLE, &sh->state); ++ release_stripe(sh); +} + -+/** -+ * debug_count_fread - Read function for "count" debugfs entry -+ * @filp: The active open file structure for the debugfs "file" -+ * @ubuf: The userspace provided buffer to read value into -+ * @cnt: The maximum number of bytes to read -+ * @ppos: The current "file" position -+ * -+ * This function provides a read implementation for the "count" debugfs -+ * interface to the hardware latency detector. Can be used to read the -+ * number of latency readings exceeding the configured threshold since -+ * the detector was last reset (e.g. by writing a zero into "count"). -+ */ -+static ssize_t debug_count_fread(struct file *filp, char __user *ubuf, -+ size_t cnt, loff_t *ppos) ++static void ops_run_biofill(struct stripe_head *sh) +{ -+ return simple_data_read(filp, ubuf, cnt, ppos, &data.count); ++ struct dma_async_tx_descriptor *tx = NULL; ++ struct async_submit_ctl submit; ++ int i; ++ ++ BUG_ON(sh->batch_head); ++ pr_debug("%s: stripe %llu\n", __func__, ++ (unsigned long long)sh->sector); ++ ++ for (i = sh->disks; i--; ) { ++ struct r5dev *dev = &sh->dev[i]; ++ if (test_bit(R5_Wantfill, &dev->flags)) { ++ struct bio *rbi; ++ spin_lock_irq(&sh->stripe_lock); ++ dev->read = rbi = dev->toread; ++ dev->toread = NULL; ++ spin_unlock_irq(&sh->stripe_lock); ++ while (rbi && rbi->bi_iter.bi_sector < ++ dev->sector + STRIPE_SECTORS) { ++ tx = async_copy_data(0, rbi, &dev->page, ++ dev->sector, tx, sh); ++ rbi = r5_next_bio(rbi, dev->sector); ++ } ++ } ++ } ++ ++ atomic_inc(&sh->count); ++ init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_biofill, sh, NULL); ++ async_trigger_callback(&submit); +} + -+/** -+ * debug_count_fwrite - Write function for "count" debugfs entry -+ * @filp: The active open file structure for the debugfs "file" -+ * @ubuf: The user buffer that contains the value to write -+ * @cnt: The maximum number of bytes to write to "file" -+ * @ppos: The current position in the debugfs "file" -+ * -+ * This function provides a write implementation for the "count" debugfs -+ * interface to the hardware latency detector. Can be used to write a -+ * desired value, especially to zero the total count. -+ */ -+static ssize_t debug_count_fwrite(struct file *filp, -+ const char __user *ubuf, -+ size_t cnt, -+ loff_t *ppos) ++static void mark_target_uptodate(struct stripe_head *sh, int target) +{ -+ return simple_data_write(filp, ubuf, cnt, ppos, &data.count); ++ struct r5dev *tgt; ++ ++ if (target < 0) ++ return; ++ ++ tgt = &sh->dev[target]; ++ set_bit(R5_UPTODATE, &tgt->flags); ++ BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags)); ++ clear_bit(R5_Wantcompute, &tgt->flags); +} + -+/** -+ * debug_enable_fopen - Dummy open function for "enable" debugfs interface -+ * @inode: The in-kernel inode representation of the debugfs "file" -+ * @filp: The active open file structure for the debugfs "file" -+ * -+ * This function provides an open implementation for the "enable" debugfs -+ * interface to the hardware latency detector. -+ */ -+static int debug_enable_fopen(struct inode *inode, struct file *filp) ++static void ops_complete_compute(void *stripe_head_ref) +{ -+ return 0; ++ struct stripe_head *sh = stripe_head_ref; ++ ++ pr_debug("%s: stripe %llu\n", __func__, ++ (unsigned long long)sh->sector); ++ ++ /* mark the computed target(s) as uptodate */ ++ mark_target_uptodate(sh, sh->ops.target); ++ mark_target_uptodate(sh, sh->ops.target2); ++ ++ clear_bit(STRIPE_COMPUTE_RUN, &sh->state); ++ if (sh->check_state == check_state_compute_run) ++ sh->check_state = check_state_compute_result; ++ set_bit(STRIPE_HANDLE, &sh->state); ++ release_stripe(sh); +} + -+/** -+ * debug_enable_fread - Read function for "enable" debugfs interface -+ * @filp: The active open file structure for the debugfs "file" -+ * @ubuf: The userspace provided buffer to read value into -+ * @cnt: The maximum number of bytes to read -+ * @ppos: The current "file" position -+ * -+ * This function provides a read implementation for the "enable" debugfs -+ * interface to the hardware latency detector. Can be used to determine -+ * whether the detector is currently enabled ("0\n" or "1\n" returned). -+ */ -+static ssize_t debug_enable_fread(struct file *filp, char __user *ubuf, -+ size_t cnt, loff_t *ppos) ++/* return a pointer to the address conversion region of the scribble buffer */ ++static addr_conv_t *to_addr_conv(struct stripe_head *sh, ++ struct raid5_percpu *percpu, int i) +{ -+ char buf[4]; ++ void *addr; + -+ if ((cnt < sizeof(buf)) || (*ppos)) -+ return 0; ++ addr = flex_array_get(percpu->scribble, i); ++ return addr + sizeof(struct page *) * (sh->disks + 2); ++} + -+ buf[0] = enabled ? '1' : '0'; -+ buf[1] = '\n'; -+ buf[2] = '\0'; -+ if (copy_to_user(ubuf, buf, strlen(buf))) -+ return -EFAULT; -+ return *ppos = strlen(buf); ++/* return a pointer to the address conversion region of the scribble buffer */ ++static struct page **to_addr_page(struct raid5_percpu *percpu, int i) ++{ ++ void *addr; ++ ++ addr = flex_array_get(percpu->scribble, i); ++ return addr; +} + -+/** -+ * debug_enable_fwrite - Write function for "enable" debugfs interface -+ * @filp: The active open file structure for the debugfs "file" -+ * @ubuf: The user buffer that contains the value to write -+ * @cnt: The maximum number of bytes to write to "file" -+ * @ppos: The current position in the debugfs "file" ++static struct dma_async_tx_descriptor * ++ops_run_compute5(struct stripe_head *sh, struct raid5_percpu *percpu) ++{ ++ int disks = sh->disks; ++ struct page **xor_srcs = to_addr_page(percpu, 0); ++ int target = sh->ops.target; ++ struct r5dev *tgt = &sh->dev[target]; ++ struct page *xor_dest = tgt->page; ++ int count = 0; ++ struct dma_async_tx_descriptor *tx; ++ struct async_submit_ctl submit; ++ int i; ++ ++ BUG_ON(sh->batch_head); ++ ++ pr_debug("%s: stripe %llu block: %d\n", ++ __func__, (unsigned long long)sh->sector, target); ++ BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags)); ++ ++ for (i = disks; i--; ) ++ if (i != target) ++ xor_srcs[count++] = sh->dev[i].page; ++ ++ atomic_inc(&sh->count); ++ ++ init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, NULL, ++ ops_complete_compute, sh, to_addr_conv(sh, percpu, 0)); ++ if (unlikely(count == 1)) ++ tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit); ++ else ++ tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit); ++ ++ return tx; ++} ++ ++/* set_syndrome_sources - populate source buffers for gen_syndrome ++ * @srcs - (struct page *) array of size sh->disks ++ * @sh - stripe_head to parse + * -+ * This function provides a write implementation for the "enable" debugfs -+ * interface to the hardware latency detector. Can be used to enable or -+ * disable the detector, which will have the side-effect of possibly -+ * also resetting the global stats and kicking off the measuring -+ * kthread (on an enable) or the converse (upon a disable). ++ * Populates srcs in proper layout order for the stripe and returns the ++ * 'count' of sources to be used in a call to async_gen_syndrome. The P ++ * destination buffer is recorded in srcs[count] and the Q destination ++ * is recorded in srcs[count+1]]. + */ -+static ssize_t debug_enable_fwrite(struct file *filp, -+ const char __user *ubuf, -+ size_t cnt, -+ loff_t *ppos) ++static int set_syndrome_sources(struct page **srcs, ++ struct stripe_head *sh, ++ int srctype) +{ -+ char buf[4]; -+ int csize = min(cnt, sizeof(buf)); -+ long val = 0; -+ int err = 0; ++ int disks = sh->disks; ++ int syndrome_disks = sh->ddf_layout ? disks : (disks - 2); ++ int d0_idx = raid6_d0(sh); ++ int count; ++ int i; + -+ memset(buf, '\0', sizeof(buf)); -+ if (copy_from_user(buf, ubuf, csize)) -+ return -EFAULT; ++ for (i = 0; i < disks; i++) ++ srcs[i] = NULL; + -+ buf[sizeof(buf)-1] = '\0'; /* just in case */ -+ err = kstrtoul(buf, 10, &val); -+ if (0 != err) -+ return -EINVAL; ++ count = 0; ++ i = d0_idx; ++ do { ++ int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks); ++ struct r5dev *dev = &sh->dev[i]; ++ ++ if (i == sh->qd_idx || i == sh->pd_idx || ++ (srctype == SYNDROME_SRC_ALL) || ++ (srctype == SYNDROME_SRC_WANT_DRAIN && ++ test_bit(R5_Wantdrain, &dev->flags)) || ++ (srctype == SYNDROME_SRC_WRITTEN && ++ dev->written)) ++ srcs[slot] = sh->dev[i].page; ++ i = raid6_next_disk(i, disks); ++ } while (i != d0_idx); ++ ++ return syndrome_disks; ++} + -+ if (val) { -+ if (enabled) -+ goto unlock; -+ enabled = 1; -+ __reset_stats(); -+ if (start_kthread()) -+ return -EFAULT; ++static struct dma_async_tx_descriptor * ++ops_run_compute6_1(struct stripe_head *sh, struct raid5_percpu *percpu) ++{ ++ int disks = sh->disks; ++ struct page **blocks = to_addr_page(percpu, 0); ++ int target; ++ int qd_idx = sh->qd_idx; ++ struct dma_async_tx_descriptor *tx; ++ struct async_submit_ctl submit; ++ struct r5dev *tgt; ++ struct page *dest; ++ int i; ++ int count; ++ ++ BUG_ON(sh->batch_head); ++ if (sh->ops.target < 0) ++ target = sh->ops.target2; ++ else if (sh->ops.target2 < 0) ++ target = sh->ops.target; ++ else ++ /* we should only have one valid target */ ++ BUG(); ++ BUG_ON(target < 0); ++ pr_debug("%s: stripe %llu block: %d\n", ++ __func__, (unsigned long long)sh->sector, target); ++ ++ tgt = &sh->dev[target]; ++ BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags)); ++ dest = tgt->page; ++ ++ atomic_inc(&sh->count); ++ ++ if (target == qd_idx) { ++ count = set_syndrome_sources(blocks, sh, SYNDROME_SRC_ALL); ++ blocks[count] = NULL; /* regenerating p is not necessary */ ++ BUG_ON(blocks[count+1] != dest); /* q should already be set */ ++ init_async_submit(&submit, ASYNC_TX_FENCE, NULL, ++ ops_complete_compute, sh, ++ to_addr_conv(sh, percpu, 0)); ++ tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit); + } else { -+ if (!enabled) -+ goto unlock; -+ enabled = 0; -+ err = stop_kthread(); -+ if (err) { -+ pr_err(BANNER "cannot stop kthread\n"); -+ return -EFAULT; ++ /* Compute any data- or p-drive using XOR */ ++ count = 0; ++ for (i = disks; i-- ; ) { ++ if (i == target || i == qd_idx) ++ continue; ++ blocks[count++] = sh->dev[i].page; + } -+ wake_up(&data.wq); /* reader(s) should return */ ++ ++ init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, ++ NULL, ops_complete_compute, sh, ++ to_addr_conv(sh, percpu, 0)); ++ tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE, &submit); + } -+unlock: -+ return csize; ++ ++ return tx; +} + -+/** -+ * debug_max_fopen - Open function for "max" debugfs entry -+ * @inode: The in-kernel inode representation of the debugfs "file" -+ * @filp: The active open file structure for the debugfs "file" -+ * -+ * This function provides an open implementation for the "max" debugfs -+ * interface to the hardware latency detector. -+ */ -+static int debug_max_fopen(struct inode *inode, struct file *filp) ++static struct dma_async_tx_descriptor * ++ops_run_compute6_2(struct stripe_head *sh, struct raid5_percpu *percpu) +{ -+ return 0; ++ int i, count, disks = sh->disks; ++ int syndrome_disks = sh->ddf_layout ? disks : disks-2; ++ int d0_idx = raid6_d0(sh); ++ int faila = -1, failb = -1; ++ int target = sh->ops.target; ++ int target2 = sh->ops.target2; ++ struct r5dev *tgt = &sh->dev[target]; ++ struct r5dev *tgt2 = &sh->dev[target2]; ++ struct dma_async_tx_descriptor *tx; ++ struct page **blocks = to_addr_page(percpu, 0); ++ struct async_submit_ctl submit; ++ ++ BUG_ON(sh->batch_head); ++ pr_debug("%s: stripe %llu block1: %d block2: %d\n", ++ __func__, (unsigned long long)sh->sector, target, target2); ++ BUG_ON(target < 0 || target2 < 0); ++ BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags)); ++ BUG_ON(!test_bit(R5_Wantcompute, &tgt2->flags)); ++ ++ /* we need to open-code set_syndrome_sources to handle the ++ * slot number conversion for 'faila' and 'failb' ++ */ ++ for (i = 0; i < disks ; i++) ++ blocks[i] = NULL; ++ count = 0; ++ i = d0_idx; ++ do { ++ int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks); ++ ++ blocks[slot] = sh->dev[i].page; ++ ++ if (i == target) ++ faila = slot; ++ if (i == target2) ++ failb = slot; ++ i = raid6_next_disk(i, disks); ++ } while (i != d0_idx); ++ ++ BUG_ON(faila == failb); ++ if (failb < faila) ++ swap(faila, failb); ++ pr_debug("%s: stripe: %llu faila: %d failb: %d\n", ++ __func__, (unsigned long long)sh->sector, faila, failb); ++ ++ atomic_inc(&sh->count); ++ ++ if (failb == syndrome_disks+1) { ++ /* Q disk is one of the missing disks */ ++ if (faila == syndrome_disks) { ++ /* Missing P+Q, just recompute */ ++ init_async_submit(&submit, ASYNC_TX_FENCE, NULL, ++ ops_complete_compute, sh, ++ to_addr_conv(sh, percpu, 0)); ++ return async_gen_syndrome(blocks, 0, syndrome_disks+2, ++ STRIPE_SIZE, &submit); ++ } else { ++ struct page *dest; ++ int data_target; ++ int qd_idx = sh->qd_idx; ++ ++ /* Missing D+Q: recompute D from P, then recompute Q */ ++ if (target == qd_idx) ++ data_target = target2; ++ else ++ data_target = target; ++ ++ count = 0; ++ for (i = disks; i-- ; ) { ++ if (i == data_target || i == qd_idx) ++ continue; ++ blocks[count++] = sh->dev[i].page; ++ } ++ dest = sh->dev[data_target].page; ++ init_async_submit(&submit, ++ ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, ++ NULL, NULL, NULL, ++ to_addr_conv(sh, percpu, 0)); ++ tx = async_xor(dest, blocks, 0, count, STRIPE_SIZE, ++ &submit); ++ ++ count = set_syndrome_sources(blocks, sh, SYNDROME_SRC_ALL); ++ init_async_submit(&submit, ASYNC_TX_FENCE, tx, ++ ops_complete_compute, sh, ++ to_addr_conv(sh, percpu, 0)); ++ return async_gen_syndrome(blocks, 0, count+2, ++ STRIPE_SIZE, &submit); ++ } ++ } else { ++ init_async_submit(&submit, ASYNC_TX_FENCE, NULL, ++ ops_complete_compute, sh, ++ to_addr_conv(sh, percpu, 0)); ++ if (failb == syndrome_disks) { ++ /* We're missing D+P. */ ++ return async_raid6_datap_recov(syndrome_disks+2, ++ STRIPE_SIZE, faila, ++ blocks, &submit); ++ } else { ++ /* We're missing D+D. */ ++ return async_raid6_2data_recov(syndrome_disks+2, ++ STRIPE_SIZE, faila, failb, ++ blocks, &submit); ++ } ++ } +} + -+/** -+ * debug_max_fread - Read function for "max" debugfs entry -+ * @filp: The active open file structure for the debugfs "file" -+ * @ubuf: The userspace provided buffer to read value into -+ * @cnt: The maximum number of bytes to read -+ * @ppos: The current "file" position -+ * -+ * This function provides a read implementation for the "max" debugfs -+ * interface to the hardware latency detector. Can be used to determine -+ * the maximum latency value observed since it was last reset. -+ */ -+static ssize_t debug_max_fread(struct file *filp, char __user *ubuf, -+ size_t cnt, loff_t *ppos) ++static void ops_complete_prexor(void *stripe_head_ref) +{ -+ return simple_data_read(filp, ubuf, cnt, ppos, &data.max_sample); ++ struct stripe_head *sh = stripe_head_ref; ++ ++ pr_debug("%s: stripe %llu\n", __func__, ++ (unsigned long long)sh->sector); +} + -+/** -+ * debug_max_fwrite - Write function for "max" debugfs entry -+ * @filp: The active open file structure for the debugfs "file" -+ * @ubuf: The user buffer that contains the value to write -+ * @cnt: The maximum number of bytes to write to "file" -+ * @ppos: The current position in the debugfs "file" -+ * -+ * This function provides a write implementation for the "max" debugfs -+ * interface to the hardware latency detector. Can be used to reset the -+ * maximum or set it to some other desired value - if, then, subsequent -+ * measurements exceed this value, the maximum will be updated. -+ */ -+static ssize_t debug_max_fwrite(struct file *filp, -+ const char __user *ubuf, -+ size_t cnt, -+ loff_t *ppos) ++static struct dma_async_tx_descriptor * ++ops_run_prexor5(struct stripe_head *sh, struct raid5_percpu *percpu, ++ struct dma_async_tx_descriptor *tx) +{ -+ return simple_data_write(filp, ubuf, cnt, ppos, &data.max_sample); -+} ++ int disks = sh->disks; ++ struct page **xor_srcs = to_addr_page(percpu, 0); ++ int count = 0, pd_idx = sh->pd_idx, i; ++ struct async_submit_ctl submit; ++ ++ /* existing parity data subtracted */ ++ struct page *xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page; ++ ++ BUG_ON(sh->batch_head); ++ pr_debug("%s: stripe %llu\n", __func__, ++ (unsigned long long)sh->sector); ++ ++ for (i = disks; i--; ) { ++ struct r5dev *dev = &sh->dev[i]; ++ /* Only process blocks that are known to be uptodate */ ++ if (test_bit(R5_Wantdrain, &dev->flags)) ++ xor_srcs[count++] = dev->page; ++ } + ++ init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx, ++ ops_complete_prexor, sh, to_addr_conv(sh, percpu, 0)); ++ tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit); + -+/** -+ * debug_sample_fopen - An open function for "sample" debugfs interface -+ * @inode: The in-kernel inode representation of this debugfs "file" -+ * @filp: The active open file structure for the debugfs "file" -+ * -+ * This function handles opening the "sample" file within the hardware -+ * latency detector debugfs directory interface. This file is used to read -+ * raw samples from the global ring_buffer and allows the user to see a -+ * running latency history. Can be opened blocking or non-blocking, -+ * affecting whether it behaves as a buffer read pipe, or does not. -+ * Implements simple locking to prevent multiple simultaneous use. -+ */ -+static int debug_sample_fopen(struct inode *inode, struct file *filp) -+{ -+ if (!atomic_add_unless(&data.sample_open, 1, 1)) -+ return -EBUSY; -+ else -+ return 0; ++ return tx; +} + -+/** -+ * debug_sample_fread - A read function for "sample" debugfs interface -+ * @filp: The active open file structure for the debugfs "file" -+ * @ubuf: The user buffer that will contain the samples read -+ * @cnt: The maximum bytes to read from the debugfs "file" -+ * @ppos: The current position in the debugfs "file" -+ * -+ * This function handles reading from the "sample" file within the hardware -+ * latency detector debugfs directory interface. This file is used to read -+ * raw samples from the global ring_buffer and allows the user to see a -+ * running latency history. By default this will block pending a new -+ * value written into the sample buffer, unless there are already a -+ * number of value(s) waiting in the buffer, or the sample file was -+ * previously opened in a non-blocking mode of operation. -+ */ -+static ssize_t debug_sample_fread(struct file *filp, char __user *ubuf, -+ size_t cnt, loff_t *ppos) ++static struct dma_async_tx_descriptor * ++ops_run_prexor6(struct stripe_head *sh, struct raid5_percpu *percpu, ++ struct dma_async_tx_descriptor *tx) +{ -+ int len = 0; -+ char buf[64]; -+ struct sample *sample = NULL; ++ struct page **blocks = to_addr_page(percpu, 0); ++ int count; ++ struct async_submit_ctl submit; + -+ if (!enabled) -+ return 0; ++ pr_debug("%s: stripe %llu\n", __func__, ++ (unsigned long long)sh->sector); + -+ sample = kzalloc(sizeof(struct sample), GFP_KERNEL); -+ if (!sample) -+ return -ENOMEM; ++ count = set_syndrome_sources(blocks, sh, SYNDROME_SRC_WANT_DRAIN); + -+ while (!buffer_get_sample(sample)) { ++ init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_PQ_XOR_DST, tx, ++ ops_complete_prexor, sh, to_addr_conv(sh, percpu, 0)); ++ tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit); + -+ DEFINE_WAIT(wait); ++ return tx; ++} + -+ if (filp->f_flags & O_NONBLOCK) { -+ len = -EAGAIN; -+ goto out; -+ } ++static struct dma_async_tx_descriptor * ++ops_run_biodrain(struct stripe_head *sh, struct dma_async_tx_descriptor *tx) ++{ ++ int disks = sh->disks; ++ int i; ++ struct stripe_head *head_sh = sh; + -+ prepare_to_wait(&data.wq, &wait, TASK_INTERRUPTIBLE); -+ schedule(); -+ finish_wait(&data.wq, &wait); ++ pr_debug("%s: stripe %llu\n", __func__, ++ (unsigned long long)sh->sector); + -+ if (signal_pending(current)) { -+ len = -EINTR; -+ goto out; -+ } ++ for (i = disks; i--; ) { ++ struct r5dev *dev; ++ struct bio *chosen; + -+ if (!enabled) { /* enable was toggled */ -+ len = 0; -+ goto out; ++ sh = head_sh; ++ if (test_and_clear_bit(R5_Wantdrain, &head_sh->dev[i].flags)) { ++ struct bio *wbi; ++ ++again: ++ dev = &sh->dev[i]; ++ spin_lock_irq(&sh->stripe_lock); ++ chosen = dev->towrite; ++ dev->towrite = NULL; ++ sh->overwrite_disks = 0; ++ BUG_ON(dev->written); ++ wbi = dev->written = chosen; ++ spin_unlock_irq(&sh->stripe_lock); ++ WARN_ON(dev->page != dev->orig_page); ++ ++ while (wbi && wbi->bi_iter.bi_sector < ++ dev->sector + STRIPE_SECTORS) { ++ if (wbi->bi_rw & REQ_FUA) ++ set_bit(R5_WantFUA, &dev->flags); ++ if (wbi->bi_rw & REQ_SYNC) ++ set_bit(R5_SyncIO, &dev->flags); ++ if (wbi->bi_rw & REQ_DISCARD) ++ set_bit(R5_Discard, &dev->flags); ++ else { ++ tx = async_copy_data(1, wbi, &dev->page, ++ dev->sector, tx, sh); ++ if (dev->page != dev->orig_page) { ++ set_bit(R5_SkipCopy, &dev->flags); ++ clear_bit(R5_UPTODATE, &dev->flags); ++ clear_bit(R5_OVERWRITE, &dev->flags); ++ } ++ } ++ wbi = r5_next_bio(wbi, dev->sector); ++ } ++ ++ if (head_sh->batch_head) { ++ sh = list_first_entry(&sh->batch_list, ++ struct stripe_head, ++ batch_list); ++ if (sh == head_sh) ++ continue; ++ goto again; ++ } + } + } + -+ len = snprintf(buf, sizeof(buf), "%010lu.%010lu\t%llu\t%llu\n", -+ sample->timestamp.tv_sec, -+ sample->timestamp.tv_nsec, -+ sample->duration, -+ sample->outer_duration); ++ return tx; ++} + ++static void ops_complete_reconstruct(void *stripe_head_ref) ++{ ++ struct stripe_head *sh = stripe_head_ref; ++ int disks = sh->disks; ++ int pd_idx = sh->pd_idx; ++ int qd_idx = sh->qd_idx; ++ int i; ++ bool fua = false, sync = false, discard = false; + -+ /* handling partial reads is more trouble than it's worth */ -+ if (len > cnt) -+ goto out; ++ pr_debug("%s: stripe %llu\n", __func__, ++ (unsigned long long)sh->sector); + -+ if (copy_to_user(ubuf, buf, len)) -+ len = -EFAULT; ++ for (i = disks; i--; ) { ++ fua |= test_bit(R5_WantFUA, &sh->dev[i].flags); ++ sync |= test_bit(R5_SyncIO, &sh->dev[i].flags); ++ discard |= test_bit(R5_Discard, &sh->dev[i].flags); ++ } + -+out: -+ kfree(sample); -+ return len; -+} ++ for (i = disks; i--; ) { ++ struct r5dev *dev = &sh->dev[i]; + -+/** -+ * debug_sample_release - Release function for "sample" debugfs interface -+ * @inode: The in-kernel inode represenation of the debugfs "file" -+ * @filp: The active open file structure for the debugfs "file" -+ * -+ * This function completes the close of the debugfs interface "sample" file. -+ * Frees the sample_open "lock" so that other users may open the interface. -+ */ -+static int debug_sample_release(struct inode *inode, struct file *filp) -+{ -+ atomic_dec(&data.sample_open); ++ if (dev->written || i == pd_idx || i == qd_idx) { ++ if (!discard && !test_bit(R5_SkipCopy, &dev->flags)) ++ set_bit(R5_UPTODATE, &dev->flags); ++ if (fua) ++ set_bit(R5_WantFUA, &dev->flags); ++ if (sync) ++ set_bit(R5_SyncIO, &dev->flags); ++ } ++ } + -+ return 0; ++ if (sh->reconstruct_state == reconstruct_state_drain_run) ++ sh->reconstruct_state = reconstruct_state_drain_result; ++ else if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) ++ sh->reconstruct_state = reconstruct_state_prexor_drain_result; ++ else { ++ BUG_ON(sh->reconstruct_state != reconstruct_state_run); ++ sh->reconstruct_state = reconstruct_state_result; ++ } ++ ++ set_bit(STRIPE_HANDLE, &sh->state); ++ release_stripe(sh); +} + -+/** -+ * debug_threshold_fopen - Open function for "threshold" debugfs entry -+ * @inode: The in-kernel inode representation of the debugfs "file" -+ * @filp: The active open file structure for the debugfs "file" -+ * -+ * This function provides an open implementation for the "threshold" debugfs -+ * interface to the hardware latency detector. -+ */ -+static int debug_threshold_fopen(struct inode *inode, struct file *filp) ++static void ++ops_run_reconstruct5(struct stripe_head *sh, struct raid5_percpu *percpu, ++ struct dma_async_tx_descriptor *tx) +{ -+ return 0; ++ int disks = sh->disks; ++ struct page **xor_srcs; ++ struct async_submit_ctl submit; ++ int count, pd_idx = sh->pd_idx, i; ++ struct page *xor_dest; ++ int prexor = 0; ++ unsigned long flags; ++ int j = 0; ++ struct stripe_head *head_sh = sh; ++ int last_stripe; ++ ++ pr_debug("%s: stripe %llu\n", __func__, ++ (unsigned long long)sh->sector); ++ ++ for (i = 0; i < sh->disks; i++) { ++ if (pd_idx == i) ++ continue; ++ if (!test_bit(R5_Discard, &sh->dev[i].flags)) ++ break; ++ } ++ if (i >= sh->disks) { ++ atomic_inc(&sh->count); ++ set_bit(R5_Discard, &sh->dev[pd_idx].flags); ++ ops_complete_reconstruct(sh); ++ return; ++ } ++again: ++ count = 0; ++ xor_srcs = to_addr_page(percpu, j); ++ /* check if prexor is active which means only process blocks ++ * that are part of a read-modify-write (written) ++ */ ++ if (head_sh->reconstruct_state == reconstruct_state_prexor_drain_run) { ++ prexor = 1; ++ xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page; ++ for (i = disks; i--; ) { ++ struct r5dev *dev = &sh->dev[i]; ++ if (head_sh->dev[i].written) ++ xor_srcs[count++] = dev->page; ++ } ++ } else { ++ xor_dest = sh->dev[pd_idx].page; ++ for (i = disks; i--; ) { ++ struct r5dev *dev = &sh->dev[i]; ++ if (i != pd_idx) ++ xor_srcs[count++] = dev->page; ++ } ++ } ++ ++ /* 1/ if we prexor'd then the dest is reused as a source ++ * 2/ if we did not prexor then we are redoing the parity ++ * set ASYNC_TX_XOR_DROP_DST and ASYNC_TX_XOR_ZERO_DST ++ * for the synchronous xor case ++ */ ++ last_stripe = !head_sh->batch_head || ++ list_first_entry(&sh->batch_list, ++ struct stripe_head, batch_list) == head_sh; ++ if (last_stripe) { ++ flags = ASYNC_TX_ACK | ++ (prexor ? ASYNC_TX_XOR_DROP_DST : ASYNC_TX_XOR_ZERO_DST); ++ ++ atomic_inc(&head_sh->count); ++ init_async_submit(&submit, flags, tx, ops_complete_reconstruct, head_sh, ++ to_addr_conv(sh, percpu, j)); ++ } else { ++ flags = prexor ? ASYNC_TX_XOR_DROP_DST : ASYNC_TX_XOR_ZERO_DST; ++ init_async_submit(&submit, flags, tx, NULL, NULL, ++ to_addr_conv(sh, percpu, j)); ++ } ++ ++ if (unlikely(count == 1)) ++ tx = async_memcpy(xor_dest, xor_srcs[0], 0, 0, STRIPE_SIZE, &submit); ++ else ++ tx = async_xor(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, &submit); ++ if (!last_stripe) { ++ j++; ++ sh = list_first_entry(&sh->batch_list, struct stripe_head, ++ batch_list); ++ goto again; ++ } +} + -+/** -+ * debug_threshold_fread - Read function for "threshold" debugfs entry -+ * @filp: The active open file structure for the debugfs "file" -+ * @ubuf: The userspace provided buffer to read value into -+ * @cnt: The maximum number of bytes to read -+ * @ppos: The current "file" position -+ * -+ * This function provides a read implementation for the "threshold" debugfs -+ * interface to the hardware latency detector. It can be used to determine -+ * the current threshold level at which a latency will be recorded in the -+ * global ring buffer, typically on the order of 10us. -+ */ -+static ssize_t debug_threshold_fread(struct file *filp, char __user *ubuf, -+ size_t cnt, loff_t *ppos) ++static void ++ops_run_reconstruct6(struct stripe_head *sh, struct raid5_percpu *percpu, ++ struct dma_async_tx_descriptor *tx) +{ -+ return simple_data_read(filp, ubuf, cnt, ppos, &data.threshold); ++ struct async_submit_ctl submit; ++ struct page **blocks; ++ int count, i, j = 0; ++ struct stripe_head *head_sh = sh; ++ int last_stripe; ++ int synflags; ++ unsigned long txflags; ++ ++ pr_debug("%s: stripe %llu\n", __func__, (unsigned long long)sh->sector); ++ ++ for (i = 0; i < sh->disks; i++) { ++ if (sh->pd_idx == i || sh->qd_idx == i) ++ continue; ++ if (!test_bit(R5_Discard, &sh->dev[i].flags)) ++ break; ++ } ++ if (i >= sh->disks) { ++ atomic_inc(&sh->count); ++ set_bit(R5_Discard, &sh->dev[sh->pd_idx].flags); ++ set_bit(R5_Discard, &sh->dev[sh->qd_idx].flags); ++ ops_complete_reconstruct(sh); ++ return; ++ } ++ ++again: ++ blocks = to_addr_page(percpu, j); ++ ++ if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) { ++ synflags = SYNDROME_SRC_WRITTEN; ++ txflags = ASYNC_TX_ACK | ASYNC_TX_PQ_XOR_DST; ++ } else { ++ synflags = SYNDROME_SRC_ALL; ++ txflags = ASYNC_TX_ACK; ++ } ++ ++ count = set_syndrome_sources(blocks, sh, synflags); ++ last_stripe = !head_sh->batch_head || ++ list_first_entry(&sh->batch_list, ++ struct stripe_head, batch_list) == head_sh; ++ ++ if (last_stripe) { ++ atomic_inc(&head_sh->count); ++ init_async_submit(&submit, txflags, tx, ops_complete_reconstruct, ++ head_sh, to_addr_conv(sh, percpu, j)); ++ } else ++ init_async_submit(&submit, 0, tx, NULL, NULL, ++ to_addr_conv(sh, percpu, j)); ++ tx = async_gen_syndrome(blocks, 0, count+2, STRIPE_SIZE, &submit); ++ if (!last_stripe) { ++ j++; ++ sh = list_first_entry(&sh->batch_list, struct stripe_head, ++ batch_list); ++ goto again; ++ } +} + -+/** -+ * debug_threshold_fwrite - Write function for "threshold" debugfs entry -+ * @filp: The active open file structure for the debugfs "file" -+ * @ubuf: The user buffer that contains the value to write -+ * @cnt: The maximum number of bytes to write to "file" -+ * @ppos: The current position in the debugfs "file" -+ * -+ * This function provides a write implementation for the "threshold" debugfs -+ * interface to the hardware latency detector. It can be used to configure -+ * the threshold level at which any subsequently detected latencies will -+ * be recorded into the global ring buffer. -+ */ -+static ssize_t debug_threshold_fwrite(struct file *filp, -+ const char __user *ubuf, -+ size_t cnt, -+ loff_t *ppos) ++static void ops_complete_check(void *stripe_head_ref) +{ -+ int ret; -+ -+ ret = simple_data_write(filp, ubuf, cnt, ppos, &data.threshold); ++ struct stripe_head *sh = stripe_head_ref; + -+ if (enabled) -+ wake_up_process(kthread); ++ pr_debug("%s: stripe %llu\n", __func__, ++ (unsigned long long)sh->sector); + -+ return ret; ++ sh->check_state = check_state_check_result; ++ set_bit(STRIPE_HANDLE, &sh->state); ++ release_stripe(sh); +} + -+/** -+ * debug_width_fopen - Open function for "width" debugfs entry -+ * @inode: The in-kernel inode representation of the debugfs "file" -+ * @filp: The active open file structure for the debugfs "file" -+ * -+ * This function provides an open implementation for the "width" debugfs -+ * interface to the hardware latency detector. -+ */ -+static int debug_width_fopen(struct inode *inode, struct file *filp) ++static void ops_run_check_p(struct stripe_head *sh, struct raid5_percpu *percpu) +{ -+ return 0; ++ int disks = sh->disks; ++ int pd_idx = sh->pd_idx; ++ int qd_idx = sh->qd_idx; ++ struct page *xor_dest; ++ struct page **xor_srcs = to_addr_page(percpu, 0); ++ struct dma_async_tx_descriptor *tx; ++ struct async_submit_ctl submit; ++ int count; ++ int i; ++ ++ pr_debug("%s: stripe %llu\n", __func__, ++ (unsigned long long)sh->sector); ++ ++ BUG_ON(sh->batch_head); ++ count = 0; ++ xor_dest = sh->dev[pd_idx].page; ++ xor_srcs[count++] = xor_dest; ++ for (i = disks; i--; ) { ++ if (i == pd_idx || i == qd_idx) ++ continue; ++ xor_srcs[count++] = sh->dev[i].page; ++ } ++ ++ init_async_submit(&submit, 0, NULL, NULL, NULL, ++ to_addr_conv(sh, percpu, 0)); ++ tx = async_xor_val(xor_dest, xor_srcs, 0, count, STRIPE_SIZE, ++ &sh->ops.zero_sum_result, &submit); ++ ++ atomic_inc(&sh->count); ++ init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_check, sh, NULL); ++ tx = async_trigger_callback(&submit); +} + -+/** -+ * debug_width_fread - Read function for "width" debugfs entry -+ * @filp: The active open file structure for the debugfs "file" -+ * @ubuf: The userspace provided buffer to read value into -+ * @cnt: The maximum number of bytes to read -+ * @ppos: The current "file" position -+ * -+ * This function provides a read implementation for the "width" debugfs -+ * interface to the hardware latency detector. It can be used to determine -+ * for how many us of the total window us we will actively sample for any -+ * hardware-induced latecy periods. Obviously, it is not possible to -+ * sample constantly and have the system respond to a sample reader, or, -+ * worse, without having the system appear to have gone out to lunch. -+ */ -+static ssize_t debug_width_fread(struct file *filp, char __user *ubuf, -+ size_t cnt, loff_t *ppos) ++static void ops_run_check_pq(struct stripe_head *sh, struct raid5_percpu *percpu, int checkp) +{ -+ return simple_data_read(filp, ubuf, cnt, ppos, &data.sample_width); ++ struct page **srcs = to_addr_page(percpu, 0); ++ struct async_submit_ctl submit; ++ int count; ++ ++ pr_debug("%s: stripe %llu checkp: %d\n", __func__, ++ (unsigned long long)sh->sector, checkp); ++ ++ BUG_ON(sh->batch_head); ++ count = set_syndrome_sources(srcs, sh, SYNDROME_SRC_ALL); ++ if (!checkp) ++ srcs[count] = NULL; ++ ++ atomic_inc(&sh->count); ++ init_async_submit(&submit, ASYNC_TX_ACK, NULL, ops_complete_check, ++ sh, to_addr_conv(sh, percpu, 0)); ++ async_syndrome_val(srcs, 0, count+2, STRIPE_SIZE, ++ &sh->ops.zero_sum_result, percpu->spare_page, &submit); +} + -+/** -+ * debug_width_fwrite - Write function for "width" debugfs entry -+ * @filp: The active open file structure for the debugfs "file" -+ * @ubuf: The user buffer that contains the value to write -+ * @cnt: The maximum number of bytes to write to "file" -+ * @ppos: The current position in the debugfs "file" -+ * -+ * This function provides a write implementation for the "width" debugfs -+ * interface to the hardware latency detector. It can be used to configure -+ * for how many us of the total window us we will actively sample for any -+ * hardware-induced latency periods. Obviously, it is not possible to -+ * sample constantly and have the system respond to a sample reader, or, -+ * worse, without having the system appear to have gone out to lunch. It -+ * is enforced that width is less that the total window size. -+ */ -+static ssize_t debug_width_fwrite(struct file *filp, -+ const char __user *ubuf, -+ size_t cnt, -+ loff_t *ppos) ++static void raid_run_ops(struct stripe_head *sh, unsigned long ops_request) +{ -+ char buf[U64STR_SIZE]; -+ int csize = min(cnt, sizeof(buf)); -+ u64 val = 0; -+ int err = 0; ++ int overlap_clear = 0, i, disks = sh->disks; ++ struct dma_async_tx_descriptor *tx = NULL; ++ struct r5conf *conf = sh->raid_conf; ++ int level = conf->level; ++ struct raid5_percpu *percpu; ++ unsigned long cpu; + -+ memset(buf, '\0', sizeof(buf)); -+ if (copy_from_user(buf, ubuf, csize)) -+ return -EFAULT; ++ cpu = get_cpu(); ++ percpu = per_cpu_ptr(conf->percpu, cpu); ++ if (test_bit(STRIPE_OP_BIOFILL, &ops_request)) { ++ ops_run_biofill(sh); ++ overlap_clear++; ++ } + -+ buf[U64STR_SIZE-1] = '\0'; /* just in case */ -+ err = kstrtoull(buf, 10, &val); -+ if (0 != err) -+ return -EINVAL; ++ if (test_bit(STRIPE_OP_COMPUTE_BLK, &ops_request)) { ++ if (level < 6) ++ tx = ops_run_compute5(sh, percpu); ++ else { ++ if (sh->ops.target2 < 0 || sh->ops.target < 0) ++ tx = ops_run_compute6_1(sh, percpu); ++ else ++ tx = ops_run_compute6_2(sh, percpu); ++ } ++ /* terminate the chain if reconstruct is not set to be run */ ++ if (tx && !test_bit(STRIPE_OP_RECONSTRUCT, &ops_request)) ++ async_tx_ack(tx); ++ } + -+ mutex_lock(&data.lock); -+ if (val < data.sample_window) -+ data.sample_width = val; -+ else { -+ mutex_unlock(&data.lock); -+ return -EINVAL; ++ if (test_bit(STRIPE_OP_PREXOR, &ops_request)) { ++ if (level < 6) ++ tx = ops_run_prexor5(sh, percpu, tx); ++ else ++ tx = ops_run_prexor6(sh, percpu, tx); + } -+ mutex_unlock(&data.lock); + -+ if (enabled) -+ wake_up_process(kthread); ++ if (test_bit(STRIPE_OP_BIODRAIN, &ops_request)) { ++ tx = ops_run_biodrain(sh, tx); ++ overlap_clear++; ++ } + -+ return csize; ++ if (test_bit(STRIPE_OP_RECONSTRUCT, &ops_request)) { ++ if (level < 6) ++ ops_run_reconstruct5(sh, percpu, tx); ++ else ++ ops_run_reconstruct6(sh, percpu, tx); ++ } ++ ++ if (test_bit(STRIPE_OP_CHECK, &ops_request)) { ++ if (sh->check_state == check_state_run) ++ ops_run_check_p(sh, percpu); ++ else if (sh->check_state == check_state_run_q) ++ ops_run_check_pq(sh, percpu, 0); ++ else if (sh->check_state == check_state_run_pq) ++ ops_run_check_pq(sh, percpu, 1); ++ else ++ BUG(); ++ } ++ ++ if (overlap_clear && !sh->batch_head) ++ for (i = disks; i--; ) { ++ struct r5dev *dev = &sh->dev[i]; ++ if (test_and_clear_bit(R5_Overlap, &dev->flags)) ++ wake_up(&sh->raid_conf->wait_for_overlap); ++ } ++ put_cpu(); +} + -+/** -+ * debug_window_fopen - Open function for "window" debugfs entry -+ * @inode: The in-kernel inode representation of the debugfs "file" -+ * @filp: The active open file structure for the debugfs "file" -+ * -+ * This function provides an open implementation for the "window" debugfs -+ * interface to the hardware latency detector. The window is the total time -+ * in us that will be considered one sample period. Conceptually, windows -+ * occur back-to-back and contain a sample width period during which -+ * actual sampling occurs. -+ */ -+static int debug_window_fopen(struct inode *inode, struct file *filp) ++static struct stripe_head *alloc_stripe(struct kmem_cache *sc, gfp_t gfp) ++{ ++ struct stripe_head *sh; ++ ++ sh = kmem_cache_zalloc(sc, gfp); ++ if (sh) { ++ spin_lock_init(&sh->stripe_lock); ++ spin_lock_init(&sh->batch_lock); ++ INIT_LIST_HEAD(&sh->batch_list); ++ INIT_LIST_HEAD(&sh->lru); ++ atomic_set(&sh->count, 1); ++ } ++ return sh; ++} ++static int grow_one_stripe(struct r5conf *conf, gfp_t gfp) ++{ ++ struct stripe_head *sh; ++ ++ sh = alloc_stripe(conf->slab_cache, gfp); ++ if (!sh) ++ return 0; ++ ++ sh->raid_conf = conf; ++ ++ if (grow_buffers(sh, gfp)) { ++ shrink_buffers(sh); ++ kmem_cache_free(conf->slab_cache, sh); ++ return 0; ++ } ++ sh->hash_lock_index = ++ conf->max_nr_stripes % NR_STRIPE_HASH_LOCKS; ++ /* we just created an active stripe so... */ ++ atomic_inc(&conf->active_stripes); ++ ++ release_stripe(sh); ++ conf->max_nr_stripes++; ++ return 1; ++} ++ ++static int grow_stripes(struct r5conf *conf, int num) +{ ++ struct kmem_cache *sc; ++ int devs = max(conf->raid_disks, conf->previous_raid_disks); ++ ++ if (conf->mddev->gendisk) ++ sprintf(conf->cache_name[0], ++ "raid%d-%s", conf->level, mdname(conf->mddev)); ++ else ++ sprintf(conf->cache_name[0], ++ "raid%d-%p", conf->level, conf->mddev); ++ sprintf(conf->cache_name[1], "%s-alt", conf->cache_name[0]); ++ ++ conf->active_name = 0; ++ sc = kmem_cache_create(conf->cache_name[conf->active_name], ++ sizeof(struct stripe_head)+(devs-1)*sizeof(struct r5dev), ++ 0, 0, NULL); ++ if (!sc) ++ return 1; ++ conf->slab_cache = sc; ++ conf->pool_size = devs; ++ while (num--) ++ if (!grow_one_stripe(conf, GFP_KERNEL)) ++ return 1; ++ + return 0; +} + +/** -+ * debug_window_fread - Read function for "window" debugfs entry -+ * @filp: The active open file structure for the debugfs "file" -+ * @ubuf: The userspace provided buffer to read value into -+ * @cnt: The maximum number of bytes to read -+ * @ppos: The current "file" position ++ * scribble_len - return the required size of the scribble region ++ * @num - total number of disks in the array + * -+ * This function provides a read implementation for the "window" debugfs -+ * interface to the hardware latency detector. The window is the total time -+ * in us that will be considered one sample period. Conceptually, windows -+ * occur back-to-back and contain a sample width period during which -+ * actual sampling occurs. Can be used to read the total window size. ++ * The size must be enough to contain: ++ * 1/ a struct page pointer for each device in the array +2 ++ * 2/ room to convert each entry in (1) to its corresponding dma ++ * (dma_map_page()) or page (page_address()) address. ++ * ++ * Note: the +2 is for the destination buffers of the ddf/raid6 case where we ++ * calculate over all devices (not just the data blocks), using zeros in place ++ * of the P and Q blocks. + */ -+static ssize_t debug_window_fread(struct file *filp, char __user *ubuf, -+ size_t cnt, loff_t *ppos) ++static struct flex_array *scribble_alloc(int num, int cnt, gfp_t flags) +{ -+ return simple_data_read(filp, ubuf, cnt, ppos, &data.sample_window); ++ struct flex_array *ret; ++ size_t len; ++ ++ len = sizeof(struct page *) * (num+2) + sizeof(addr_conv_t) * (num+2); ++ ret = flex_array_alloc(len, cnt, flags); ++ if (!ret) ++ return NULL; ++ /* always prealloc all elements, so no locking is required */ ++ if (flex_array_prealloc(ret, 0, cnt, flags)) { ++ flex_array_free(ret); ++ return NULL; ++ } ++ return ret; +} + -+/** -+ * debug_window_fwrite - Write function for "window" debugfs entry -+ * @filp: The active open file structure for the debugfs "file" -+ * @ubuf: The user buffer that contains the value to write -+ * @cnt: The maximum number of bytes to write to "file" -+ * @ppos: The current position in the debugfs "file" -+ * -+ * This function provides a write implementation for the "window" debufds -+ * interface to the hardware latency detetector. The window is the total time -+ * in us that will be considered one sample period. Conceptually, windows -+ * occur back-to-back and contain a sample width period during which -+ * actual sampling occurs. Can be used to write a new total window size. It -+ * is enfoced that any value written must be greater than the sample width -+ * size, or an error results. -+ */ -+static ssize_t debug_window_fwrite(struct file *filp, -+ const char __user *ubuf, -+ size_t cnt, -+ loff_t *ppos) ++static int resize_chunks(struct r5conf *conf, int new_disks, int new_sectors) +{ -+ char buf[U64STR_SIZE]; -+ int csize = min(cnt, sizeof(buf)); -+ u64 val = 0; ++ unsigned long cpu; + int err = 0; + -+ memset(buf, '\0', sizeof(buf)); -+ if (copy_from_user(buf, ubuf, csize)) -+ return -EFAULT; ++ mddev_suspend(conf->mddev); ++ get_online_cpus(); ++ for_each_present_cpu(cpu) { ++ struct raid5_percpu *percpu; ++ struct flex_array *scribble; ++ ++ percpu = per_cpu_ptr(conf->percpu, cpu); ++ scribble = scribble_alloc(new_disks, ++ new_sectors / STRIPE_SECTORS, ++ GFP_NOIO); ++ ++ if (scribble) { ++ flex_array_free(percpu->scribble); ++ percpu->scribble = scribble; ++ } else { ++ err = -ENOMEM; ++ break; ++ } ++ } ++ put_online_cpus(); ++ mddev_resume(conf->mddev); ++ return err; ++} + -+ buf[U64STR_SIZE-1] = '\0'; /* just in case */ -+ err = kstrtoull(buf, 10, &val); -+ if (0 != err) -+ return -EINVAL; ++static int resize_stripes(struct r5conf *conf, int newsize) ++{ ++ /* Make all the stripes able to hold 'newsize' devices. ++ * New slots in each stripe get 'page' set to a new page. ++ * ++ * This happens in stages: ++ * 1/ create a new kmem_cache and allocate the required number of ++ * stripe_heads. ++ * 2/ gather all the old stripe_heads and transfer the pages across ++ * to the new stripe_heads. This will have the side effect of ++ * freezing the array as once all stripe_heads have been collected, ++ * no IO will be possible. Old stripe heads are freed once their ++ * pages have been transferred over, and the old kmem_cache is ++ * freed when all stripes are done. ++ * 3/ reallocate conf->disks to be suitable bigger. If this fails, ++ * we simple return a failre status - no need to clean anything up. ++ * 4/ allocate new pages for the new slots in the new stripe_heads. ++ * If this fails, we don't bother trying the shrink the ++ * stripe_heads down again, we just leave them as they are. ++ * As each stripe_head is processed the new one is released into ++ * active service. ++ * ++ * Once step2 is started, we cannot afford to wait for a write, ++ * so we use GFP_NOIO allocations. ++ */ ++ struct stripe_head *osh, *nsh; ++ LIST_HEAD(newstripes); ++ struct disk_info *ndisks; ++ int err; ++ struct kmem_cache *sc; ++ int i; ++ int hash, cnt; + -+ mutex_lock(&data.lock); -+ if (data.sample_width < val) -+ data.sample_window = val; -+ else { -+ mutex_unlock(&data.lock); -+ return -EINVAL; ++ if (newsize <= conf->pool_size) ++ return 0; /* never bother to shrink */ ++ ++ err = md_allow_write(conf->mddev); ++ if (err) ++ return err; ++ ++ /* Step 1 */ ++ sc = kmem_cache_create(conf->cache_name[1-conf->active_name], ++ sizeof(struct stripe_head)+(newsize-1)*sizeof(struct r5dev), ++ 0, 0, NULL); ++ if (!sc) ++ return -ENOMEM; ++ ++ /* Need to ensure auto-resizing doesn't interfere */ ++ mutex_lock(&conf->cache_size_mutex); ++ ++ for (i = conf->max_nr_stripes; i; i--) { ++ nsh = alloc_stripe(sc, GFP_KERNEL); ++ if (!nsh) ++ break; ++ ++ nsh->raid_conf = conf; ++ list_add(&nsh->lru, &newstripes); + } -+ mutex_unlock(&data.lock); ++ if (i) { ++ /* didn't get enough, give up */ ++ while (!list_empty(&newstripes)) { ++ nsh = list_entry(newstripes.next, struct stripe_head, lru); ++ list_del(&nsh->lru); ++ kmem_cache_free(sc, nsh); ++ } ++ kmem_cache_destroy(sc); ++ mutex_unlock(&conf->cache_size_mutex); ++ return -ENOMEM; ++ } ++ /* Step 2 - Must use GFP_NOIO now. ++ * OK, we have enough stripes, start collecting inactive ++ * stripes and copying them over ++ */ ++ hash = 0; ++ cnt = 0; ++ list_for_each_entry(nsh, &newstripes, lru) { ++ lock_device_hash_lock(conf, hash); ++ wait_event_cmd(conf->wait_for_stripe, ++ !list_empty(conf->inactive_list + hash), ++ unlock_device_hash_lock(conf, hash), ++ lock_device_hash_lock(conf, hash)); ++ osh = get_free_stripe(conf, hash); ++ unlock_device_hash_lock(conf, hash); ++ ++ for(i=0; ipool_size; i++) { ++ nsh->dev[i].page = osh->dev[i].page; ++ nsh->dev[i].orig_page = osh->dev[i].page; ++ } ++ nsh->hash_lock_index = hash; ++ kmem_cache_free(conf->slab_cache, osh); ++ cnt++; ++ if (cnt >= conf->max_nr_stripes / NR_STRIPE_HASH_LOCKS + ++ !!((conf->max_nr_stripes % NR_STRIPE_HASH_LOCKS) > hash)) { ++ hash++; ++ cnt = 0; ++ } ++ } ++ kmem_cache_destroy(conf->slab_cache); + -+ return csize; ++ /* Step 3. ++ * At this point, we are holding all the stripes so the array ++ * is completely stalled, so now is a good time to resize ++ * conf->disks and the scribble region ++ */ ++ ndisks = kzalloc(newsize * sizeof(struct disk_info), GFP_NOIO); ++ if (ndisks) { ++ for (i=0; iraid_disks; i++) ++ ndisks[i] = conf->disks[i]; ++ kfree(conf->disks); ++ conf->disks = ndisks; ++ } else ++ err = -ENOMEM; ++ ++ mutex_unlock(&conf->cache_size_mutex); ++ /* Step 4, return new stripes to service */ ++ while(!list_empty(&newstripes)) { ++ nsh = list_entry(newstripes.next, struct stripe_head, lru); ++ list_del_init(&nsh->lru); ++ ++ for (i=conf->raid_disks; i < newsize; i++) ++ if (nsh->dev[i].page == NULL) { ++ struct page *p = alloc_page(GFP_NOIO); ++ nsh->dev[i].page = p; ++ nsh->dev[i].orig_page = p; ++ if (!p) ++ err = -ENOMEM; ++ } ++ release_stripe(nsh); ++ } ++ /* critical section pass, GFP_NOIO no longer needed */ ++ ++ conf->slab_cache = sc; ++ conf->active_name = 1-conf->active_name; ++ if (!err) ++ conf->pool_size = newsize; ++ return err; +} + -+/* -+ * Function pointers for the "count" debugfs file operations -+ */ -+static const struct file_operations count_fops = { -+ .open = debug_count_fopen, -+ .read = debug_count_fread, -+ .write = debug_count_fwrite, -+ .owner = THIS_MODULE, -+}; ++static int drop_one_stripe(struct r5conf *conf) ++{ ++ struct stripe_head *sh; ++ int hash = (conf->max_nr_stripes - 1) & STRIPE_HASH_LOCKS_MASK; + -+/* -+ * Function pointers for the "enable" debugfs file operations -+ */ -+static const struct file_operations enable_fops = { -+ .open = debug_enable_fopen, -+ .read = debug_enable_fread, -+ .write = debug_enable_fwrite, -+ .owner = THIS_MODULE, -+}; ++ spin_lock_irq(conf->hash_locks + hash); ++ sh = get_free_stripe(conf, hash); ++ spin_unlock_irq(conf->hash_locks + hash); ++ if (!sh) ++ return 0; ++ BUG_ON(atomic_read(&sh->count)); ++ shrink_buffers(sh); ++ kmem_cache_free(conf->slab_cache, sh); ++ atomic_dec(&conf->active_stripes); ++ conf->max_nr_stripes--; ++ return 1; ++} + -+/* -+ * Function pointers for the "max" debugfs file operations -+ */ -+static const struct file_operations max_fops = { -+ .open = debug_max_fopen, -+ .read = debug_max_fread, -+ .write = debug_max_fwrite, -+ .owner = THIS_MODULE, -+}; ++static void shrink_stripes(struct r5conf *conf) ++{ ++ while (conf->max_nr_stripes && ++ drop_one_stripe(conf)) ++ ; + -+/* -+ * Function pointers for the "sample" debugfs file operations -+ */ -+static const struct file_operations sample_fops = { -+ .open = debug_sample_fopen, -+ .read = debug_sample_fread, -+ .release = debug_sample_release, -+ .owner = THIS_MODULE, -+}; ++ if (conf->slab_cache) ++ kmem_cache_destroy(conf->slab_cache); ++ conf->slab_cache = NULL; ++} + -+/* -+ * Function pointers for the "threshold" debugfs file operations -+ */ -+static const struct file_operations threshold_fops = { -+ .open = debug_threshold_fopen, -+ .read = debug_threshold_fread, -+ .write = debug_threshold_fwrite, -+ .owner = THIS_MODULE, -+}; ++static void raid5_end_read_request(struct bio * bi, int error) ++{ ++ struct stripe_head *sh = bi->bi_private; ++ struct r5conf *conf = sh->raid_conf; ++ int disks = sh->disks, i; ++ int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags); ++ char b[BDEVNAME_SIZE]; ++ struct md_rdev *rdev = NULL; ++ sector_t s; ++ ++ for (i=0 ; idev[i].req) ++ break; + -+/* -+ * Function pointers for the "width" debugfs file operations -+ */ -+static const struct file_operations width_fops = { -+ .open = debug_width_fopen, -+ .read = debug_width_fread, -+ .write = debug_width_fwrite, -+ .owner = THIS_MODULE, -+}; ++ pr_debug("end_read_request %llu/%d, count: %d, uptodate %d.\n", ++ (unsigned long long)sh->sector, i, atomic_read(&sh->count), ++ uptodate); ++ if (i == disks) { ++ BUG(); ++ return; ++ } ++ if (test_bit(R5_ReadRepl, &sh->dev[i].flags)) ++ /* If replacement finished while this request was outstanding, ++ * 'replacement' might be NULL already. ++ * In that case it moved down to 'rdev'. ++ * rdev is not removed until all requests are finished. ++ */ ++ rdev = conf->disks[i].replacement; ++ if (!rdev) ++ rdev = conf->disks[i].rdev; + -+/* -+ * Function pointers for the "window" debugfs file operations -+ */ -+static const struct file_operations window_fops = { -+ .open = debug_window_fopen, -+ .read = debug_window_fread, -+ .write = debug_window_fwrite, -+ .owner = THIS_MODULE, -+}; ++ if (use_new_offset(conf, sh)) ++ s = sh->sector + rdev->new_data_offset; ++ else ++ s = sh->sector + rdev->data_offset; ++ if (uptodate) { ++ set_bit(R5_UPTODATE, &sh->dev[i].flags); ++ if (test_bit(R5_ReadError, &sh->dev[i].flags)) { ++ /* Note that this cannot happen on a ++ * replacement device. We just fail those on ++ * any error ++ */ ++ printk_ratelimited( ++ KERN_INFO ++ "md/raid:%s: read error corrected" ++ " (%lu sectors at %llu on %s)\n", ++ mdname(conf->mddev), STRIPE_SECTORS, ++ (unsigned long long)s, ++ bdevname(rdev->bdev, b)); ++ atomic_add(STRIPE_SECTORS, &rdev->corrected_errors); ++ clear_bit(R5_ReadError, &sh->dev[i].flags); ++ clear_bit(R5_ReWrite, &sh->dev[i].flags); ++ } else if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags)) ++ clear_bit(R5_ReadNoMerge, &sh->dev[i].flags); ++ ++ if (atomic_read(&rdev->read_errors)) ++ atomic_set(&rdev->read_errors, 0); ++ } else { ++ const char *bdn = bdevname(rdev->bdev, b); ++ int retry = 0; ++ int set_bad = 0; ++ ++ clear_bit(R5_UPTODATE, &sh->dev[i].flags); ++ atomic_inc(&rdev->read_errors); ++ if (test_bit(R5_ReadRepl, &sh->dev[i].flags)) ++ printk_ratelimited( ++ KERN_WARNING ++ "md/raid:%s: read error on replacement device " ++ "(sector %llu on %s).\n", ++ mdname(conf->mddev), ++ (unsigned long long)s, ++ bdn); ++ else if (conf->mddev->degraded >= conf->max_degraded) { ++ set_bad = 1; ++ printk_ratelimited( ++ KERN_WARNING ++ "md/raid:%s: read error not correctable " ++ "(sector %llu on %s).\n", ++ mdname(conf->mddev), ++ (unsigned long long)s, ++ bdn); ++ } else if (test_bit(R5_ReWrite, &sh->dev[i].flags)) { ++ /* Oh, no!!! */ ++ set_bad = 1; ++ printk_ratelimited( ++ KERN_WARNING ++ "md/raid:%s: read error NOT corrected!! " ++ "(sector %llu on %s).\n", ++ mdname(conf->mddev), ++ (unsigned long long)s, ++ bdn); ++ } else if (atomic_read(&rdev->read_errors) ++ > conf->max_nr_stripes) ++ printk(KERN_WARNING ++ "md/raid:%s: Too many read errors, failing device %s.\n", ++ mdname(conf->mddev), bdn); ++ else ++ retry = 1; ++ if (set_bad && test_bit(In_sync, &rdev->flags) ++ && !test_bit(R5_ReadNoMerge, &sh->dev[i].flags)) ++ retry = 1; ++ if (retry) ++ if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags)) { ++ set_bit(R5_ReadError, &sh->dev[i].flags); ++ clear_bit(R5_ReadNoMerge, &sh->dev[i].flags); ++ } else ++ set_bit(R5_ReadNoMerge, &sh->dev[i].flags); ++ else { ++ clear_bit(R5_ReadError, &sh->dev[i].flags); ++ clear_bit(R5_ReWrite, &sh->dev[i].flags); ++ if (!(set_bad ++ && test_bit(In_sync, &rdev->flags) ++ && rdev_set_badblocks( ++ rdev, sh->sector, STRIPE_SECTORS, 0))) ++ md_error(conf->mddev, rdev); ++ } ++ } ++ rdev_dec_pending(rdev, conf->mddev); ++ clear_bit(R5_LOCKED, &sh->dev[i].flags); ++ set_bit(STRIPE_HANDLE, &sh->state); ++ release_stripe(sh); ++} + -+/** -+ * init_debugfs - A function to initialize the debugfs interface files -+ * -+ * This function creates entries in debugfs for "hwlat_detector", including -+ * files to read values from the detector, current samples, and the -+ * maximum sample that has been captured since the hardware latency -+ * dectector was started. -+ */ -+static int init_debugfs(void) ++static void raid5_end_write_request(struct bio *bi, int error) +{ -+ int ret = -ENOMEM; -+ -+ debug_dir = debugfs_create_dir(DRVNAME, NULL); -+ if (!debug_dir) -+ goto err_debug_dir; ++ struct stripe_head *sh = bi->bi_private; ++ struct r5conf *conf = sh->raid_conf; ++ int disks = sh->disks, i; ++ struct md_rdev *uninitialized_var(rdev); ++ int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags); ++ sector_t first_bad; ++ int bad_sectors; ++ int replacement = 0; ++ ++ for (i = 0 ; i < disks; i++) { ++ if (bi == &sh->dev[i].req) { ++ rdev = conf->disks[i].rdev; ++ break; ++ } ++ if (bi == &sh->dev[i].rreq) { ++ rdev = conf->disks[i].replacement; ++ if (rdev) ++ replacement = 1; ++ else ++ /* rdev was removed and 'replacement' ++ * replaced it. rdev is not removed ++ * until all requests are finished. ++ */ ++ rdev = conf->disks[i].rdev; ++ break; ++ } ++ } ++ pr_debug("end_write_request %llu/%d, count %d, uptodate: %d.\n", ++ (unsigned long long)sh->sector, i, atomic_read(&sh->count), ++ uptodate); ++ if (i == disks) { ++ BUG(); ++ return; ++ } + -+ debug_sample = debugfs_create_file("sample", 0444, -+ debug_dir, NULL, -+ &sample_fops); -+ if (!debug_sample) -+ goto err_sample; ++ if (replacement) { ++ if (!uptodate) ++ md_error(conf->mddev, rdev); ++ else if (is_badblock(rdev, sh->sector, ++ STRIPE_SECTORS, ++ &first_bad, &bad_sectors)) ++ set_bit(R5_MadeGoodRepl, &sh->dev[i].flags); ++ } else { ++ if (!uptodate) { ++ set_bit(STRIPE_DEGRADED, &sh->state); ++ set_bit(WriteErrorSeen, &rdev->flags); ++ set_bit(R5_WriteError, &sh->dev[i].flags); ++ if (!test_and_set_bit(WantReplacement, &rdev->flags)) ++ set_bit(MD_RECOVERY_NEEDED, ++ &rdev->mddev->recovery); ++ } else if (is_badblock(rdev, sh->sector, ++ STRIPE_SECTORS, ++ &first_bad, &bad_sectors)) { ++ set_bit(R5_MadeGood, &sh->dev[i].flags); ++ if (test_bit(R5_ReadError, &sh->dev[i].flags)) ++ /* That was a successful write so make ++ * sure it looks like we already did ++ * a re-write. ++ */ ++ set_bit(R5_ReWrite, &sh->dev[i].flags); ++ } ++ } ++ rdev_dec_pending(rdev, conf->mddev); + -+ debug_count = debugfs_create_file("count", 0444, -+ debug_dir, NULL, -+ &count_fops); -+ if (!debug_count) -+ goto err_count; ++ if (sh->batch_head && !uptodate && !replacement) ++ set_bit(STRIPE_BATCH_ERR, &sh->batch_head->state); + -+ debug_max = debugfs_create_file("max", 0444, -+ debug_dir, NULL, -+ &max_fops); -+ if (!debug_max) -+ goto err_max; ++ if (!test_and_clear_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags)) ++ clear_bit(R5_LOCKED, &sh->dev[i].flags); ++ set_bit(STRIPE_HANDLE, &sh->state); ++ release_stripe(sh); + -+ debug_sample_window = debugfs_create_file("window", 0644, -+ debug_dir, NULL, -+ &window_fops); -+ if (!debug_sample_window) -+ goto err_window; ++ if (sh->batch_head && sh != sh->batch_head) ++ release_stripe(sh->batch_head); ++} + -+ debug_sample_width = debugfs_create_file("width", 0644, -+ debug_dir, NULL, -+ &width_fops); -+ if (!debug_sample_width) -+ goto err_width; ++static sector_t compute_blocknr(struct stripe_head *sh, int i, int previous); + -+ debug_threshold = debugfs_create_file("threshold", 0644, -+ debug_dir, NULL, -+ &threshold_fops); -+ if (!debug_threshold) -+ goto err_threshold; ++static void raid5_build_block(struct stripe_head *sh, int i, int previous) ++{ ++ struct r5dev *dev = &sh->dev[i]; + -+ debug_enable = debugfs_create_file("enable", 0644, -+ debug_dir, &enabled, -+ &enable_fops); -+ if (!debug_enable) -+ goto err_enable; ++ bio_init(&dev->req); ++ dev->req.bi_io_vec = &dev->vec; ++ dev->req.bi_max_vecs = 1; ++ dev->req.bi_private = sh; + -+ else { -+ ret = 0; -+ goto out; -+ } ++ bio_init(&dev->rreq); ++ dev->rreq.bi_io_vec = &dev->rvec; ++ dev->rreq.bi_max_vecs = 1; ++ dev->rreq.bi_private = sh; + -+err_enable: -+ debugfs_remove(debug_threshold); -+err_threshold: -+ debugfs_remove(debug_sample_width); -+err_width: -+ debugfs_remove(debug_sample_window); -+err_window: -+ debugfs_remove(debug_max); -+err_max: -+ debugfs_remove(debug_count); -+err_count: -+ debugfs_remove(debug_sample); -+err_sample: -+ debugfs_remove(debug_dir); -+err_debug_dir: -+out: -+ return ret; ++ dev->flags = 0; ++ dev->sector = compute_blocknr(sh, i, previous); +} + -+/** -+ * free_debugfs - A function to cleanup the debugfs file interface -+ */ -+static void free_debugfs(void) ++static void error(struct mddev *mddev, struct md_rdev *rdev) +{ -+ /* could also use a debugfs_remove_recursive */ -+ debugfs_remove(debug_enable); -+ debugfs_remove(debug_threshold); -+ debugfs_remove(debug_sample_width); -+ debugfs_remove(debug_sample_window); -+ debugfs_remove(debug_max); -+ debugfs_remove(debug_count); -+ debugfs_remove(debug_sample); -+ debugfs_remove(debug_dir); ++ char b[BDEVNAME_SIZE]; ++ struct r5conf *conf = mddev->private; ++ unsigned long flags; ++ pr_debug("raid456: error called\n"); ++ ++ spin_lock_irqsave(&conf->device_lock, flags); ++ clear_bit(In_sync, &rdev->flags); ++ mddev->degraded = calc_degraded(conf); ++ spin_unlock_irqrestore(&conf->device_lock, flags); ++ set_bit(MD_RECOVERY_INTR, &mddev->recovery); ++ ++ set_bit(Blocked, &rdev->flags); ++ set_bit(Faulty, &rdev->flags); ++ set_bit(MD_CHANGE_DEVS, &mddev->flags); ++ printk(KERN_ALERT ++ "md/raid:%s: Disk failure on %s, disabling device.\n" ++ "md/raid:%s: Operation continuing on %d devices.\n", ++ mdname(mddev), ++ bdevname(rdev->bdev, b), ++ mdname(mddev), ++ conf->raid_disks - mddev->degraded); +} + -+/** -+ * detector_init - Standard module initialization code ++/* ++ * Input: a 'big' sector number, ++ * Output: index of the data and parity disk, and the sector # in them. + */ -+static int detector_init(void) ++static sector_t raid5_compute_sector(struct r5conf *conf, sector_t r_sector, ++ int previous, int *dd_idx, ++ struct stripe_head *sh) +{ -+ int ret = -ENOMEM; ++ sector_t stripe, stripe2; ++ sector_t chunk_number; ++ unsigned int chunk_offset; ++ int pd_idx, qd_idx; ++ int ddf_layout = 0; ++ sector_t new_sector; ++ int algorithm = previous ? conf->prev_algo ++ : conf->algorithm; ++ int sectors_per_chunk = previous ? conf->prev_chunk_sectors ++ : conf->chunk_sectors; ++ int raid_disks = previous ? conf->previous_raid_disks ++ : conf->raid_disks; ++ int data_disks = raid_disks - conf->max_degraded; ++ ++ /* First compute the information on this sector */ + -+ pr_info(BANNER "version %s\n", VERSION); ++ /* ++ * Compute the chunk number and the sector offset inside the chunk ++ */ ++ chunk_offset = sector_div(r_sector, sectors_per_chunk); ++ chunk_number = r_sector; + -+ ret = init_stats(); -+ if (0 != ret) -+ goto out; ++ /* ++ * Compute the stripe number ++ */ ++ stripe = chunk_number; ++ *dd_idx = sector_div(stripe, data_disks); ++ stripe2 = stripe; ++ /* ++ * Select the parity disk based on the user selected algorithm. ++ */ ++ pd_idx = qd_idx = -1; ++ switch(conf->level) { ++ case 4: ++ pd_idx = data_disks; ++ break; ++ case 5: ++ switch (algorithm) { ++ case ALGORITHM_LEFT_ASYMMETRIC: ++ pd_idx = data_disks - sector_div(stripe2, raid_disks); ++ if (*dd_idx >= pd_idx) ++ (*dd_idx)++; ++ break; ++ case ALGORITHM_RIGHT_ASYMMETRIC: ++ pd_idx = sector_div(stripe2, raid_disks); ++ if (*dd_idx >= pd_idx) ++ (*dd_idx)++; ++ break; ++ case ALGORITHM_LEFT_SYMMETRIC: ++ pd_idx = data_disks - sector_div(stripe2, raid_disks); ++ *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks; ++ break; ++ case ALGORITHM_RIGHT_SYMMETRIC: ++ pd_idx = sector_div(stripe2, raid_disks); ++ *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks; ++ break; ++ case ALGORITHM_PARITY_0: ++ pd_idx = 0; ++ (*dd_idx)++; ++ break; ++ case ALGORITHM_PARITY_N: ++ pd_idx = data_disks; ++ break; ++ default: ++ BUG(); ++ } ++ break; ++ case 6: ++ ++ switch (algorithm) { ++ case ALGORITHM_LEFT_ASYMMETRIC: ++ pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks); ++ qd_idx = pd_idx + 1; ++ if (pd_idx == raid_disks-1) { ++ (*dd_idx)++; /* Q D D D P */ ++ qd_idx = 0; ++ } else if (*dd_idx >= pd_idx) ++ (*dd_idx) += 2; /* D D P Q D */ ++ break; ++ case ALGORITHM_RIGHT_ASYMMETRIC: ++ pd_idx = sector_div(stripe2, raid_disks); ++ qd_idx = pd_idx + 1; ++ if (pd_idx == raid_disks-1) { ++ (*dd_idx)++; /* Q D D D P */ ++ qd_idx = 0; ++ } else if (*dd_idx >= pd_idx) ++ (*dd_idx) += 2; /* D D P Q D */ ++ break; ++ case ALGORITHM_LEFT_SYMMETRIC: ++ pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks); ++ qd_idx = (pd_idx + 1) % raid_disks; ++ *dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks; ++ break; ++ case ALGORITHM_RIGHT_SYMMETRIC: ++ pd_idx = sector_div(stripe2, raid_disks); ++ qd_idx = (pd_idx + 1) % raid_disks; ++ *dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks; ++ break; + -+ ret = init_debugfs(); -+ if (0 != ret) -+ goto err_stats; ++ case ALGORITHM_PARITY_0: ++ pd_idx = 0; ++ qd_idx = 1; ++ (*dd_idx) += 2; ++ break; ++ case ALGORITHM_PARITY_N: ++ pd_idx = data_disks; ++ qd_idx = data_disks + 1; ++ break; + -+ if (enabled) -+ ret = start_kthread(); ++ case ALGORITHM_ROTATING_ZERO_RESTART: ++ /* Exactly the same as RIGHT_ASYMMETRIC, but or ++ * of blocks for computing Q is different. ++ */ ++ pd_idx = sector_div(stripe2, raid_disks); ++ qd_idx = pd_idx + 1; ++ if (pd_idx == raid_disks-1) { ++ (*dd_idx)++; /* Q D D D P */ ++ qd_idx = 0; ++ } else if (*dd_idx >= pd_idx) ++ (*dd_idx) += 2; /* D D P Q D */ ++ ddf_layout = 1; ++ break; + -+ goto out; ++ case ALGORITHM_ROTATING_N_RESTART: ++ /* Same a left_asymmetric, by first stripe is ++ * D D D P Q rather than ++ * Q D D D P ++ */ ++ stripe2 += 1; ++ pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks); ++ qd_idx = pd_idx + 1; ++ if (pd_idx == raid_disks-1) { ++ (*dd_idx)++; /* Q D D D P */ ++ qd_idx = 0; ++ } else if (*dd_idx >= pd_idx) ++ (*dd_idx) += 2; /* D D P Q D */ ++ ddf_layout = 1; ++ break; + -+err_stats: -+ ring_buffer_free(ring_buffer); -+out: -+ return ret; ++ case ALGORITHM_ROTATING_N_CONTINUE: ++ /* Same as left_symmetric but Q is before P */ ++ pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks); ++ qd_idx = (pd_idx + raid_disks - 1) % raid_disks; ++ *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks; ++ ddf_layout = 1; ++ break; ++ ++ case ALGORITHM_LEFT_ASYMMETRIC_6: ++ /* RAID5 left_asymmetric, with Q on last device */ ++ pd_idx = data_disks - sector_div(stripe2, raid_disks-1); ++ if (*dd_idx >= pd_idx) ++ (*dd_idx)++; ++ qd_idx = raid_disks - 1; ++ break; ++ ++ case ALGORITHM_RIGHT_ASYMMETRIC_6: ++ pd_idx = sector_div(stripe2, raid_disks-1); ++ if (*dd_idx >= pd_idx) ++ (*dd_idx)++; ++ qd_idx = raid_disks - 1; ++ break; + ++ case ALGORITHM_LEFT_SYMMETRIC_6: ++ pd_idx = data_disks - sector_div(stripe2, raid_disks-1); ++ *dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1); ++ qd_idx = raid_disks - 1; ++ break; ++ ++ case ALGORITHM_RIGHT_SYMMETRIC_6: ++ pd_idx = sector_div(stripe2, raid_disks-1); ++ *dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1); ++ qd_idx = raid_disks - 1; ++ break; ++ ++ case ALGORITHM_PARITY_0_6: ++ pd_idx = 0; ++ (*dd_idx)++; ++ qd_idx = raid_disks - 1; ++ break; ++ ++ default: ++ BUG(); ++ } ++ break; ++ } ++ ++ if (sh) { ++ sh->pd_idx = pd_idx; ++ sh->qd_idx = qd_idx; ++ sh->ddf_layout = ddf_layout; ++ } ++ /* ++ * Finally, compute the new sector number ++ */ ++ new_sector = (sector_t)stripe * sectors_per_chunk + chunk_offset; ++ return new_sector; +} + -+/** -+ * detector_exit - Standard module cleanup code -+ */ -+static void detector_exit(void) ++static sector_t compute_blocknr(struct stripe_head *sh, int i, int previous) +{ -+ int err; -+ -+ if (enabled) { -+ enabled = 0; -+ err = stop_kthread(); -+ if (err) -+ pr_err(BANNER "cannot stop kthread\n"); ++ struct r5conf *conf = sh->raid_conf; ++ int raid_disks = sh->disks; ++ int data_disks = raid_disks - conf->max_degraded; ++ sector_t new_sector = sh->sector, check; ++ int sectors_per_chunk = previous ? conf->prev_chunk_sectors ++ : conf->chunk_sectors; ++ int algorithm = previous ? conf->prev_algo ++ : conf->algorithm; ++ sector_t stripe; ++ int chunk_offset; ++ sector_t chunk_number; ++ int dummy1, dd_idx = i; ++ sector_t r_sector; ++ struct stripe_head sh2; ++ ++ chunk_offset = sector_div(new_sector, sectors_per_chunk); ++ stripe = new_sector; ++ ++ if (i == sh->pd_idx) ++ return 0; ++ switch(conf->level) { ++ case 4: break; ++ case 5: ++ switch (algorithm) { ++ case ALGORITHM_LEFT_ASYMMETRIC: ++ case ALGORITHM_RIGHT_ASYMMETRIC: ++ if (i > sh->pd_idx) ++ i--; ++ break; ++ case ALGORITHM_LEFT_SYMMETRIC: ++ case ALGORITHM_RIGHT_SYMMETRIC: ++ if (i < sh->pd_idx) ++ i += raid_disks; ++ i -= (sh->pd_idx + 1); ++ break; ++ case ALGORITHM_PARITY_0: ++ i -= 1; ++ break; ++ case ALGORITHM_PARITY_N: ++ break; ++ default: ++ BUG(); ++ } ++ break; ++ case 6: ++ if (i == sh->qd_idx) ++ return 0; /* It is the Q disk */ ++ switch (algorithm) { ++ case ALGORITHM_LEFT_ASYMMETRIC: ++ case ALGORITHM_RIGHT_ASYMMETRIC: ++ case ALGORITHM_ROTATING_ZERO_RESTART: ++ case ALGORITHM_ROTATING_N_RESTART: ++ if (sh->pd_idx == raid_disks-1) ++ i--; /* Q D D D P */ ++ else if (i > sh->pd_idx) ++ i -= 2; /* D D P Q D */ ++ break; ++ case ALGORITHM_LEFT_SYMMETRIC: ++ case ALGORITHM_RIGHT_SYMMETRIC: ++ if (sh->pd_idx == raid_disks-1) ++ i--; /* Q D D D P */ ++ else { ++ /* D D P Q D */ ++ if (i < sh->pd_idx) ++ i += raid_disks; ++ i -= (sh->pd_idx + 2); ++ } ++ break; ++ case ALGORITHM_PARITY_0: ++ i -= 2; ++ break; ++ case ALGORITHM_PARITY_N: ++ break; ++ case ALGORITHM_ROTATING_N_CONTINUE: ++ /* Like left_symmetric, but P is before Q */ ++ if (sh->pd_idx == 0) ++ i--; /* P D D D Q */ ++ else { ++ /* D D Q P D */ ++ if (i < sh->pd_idx) ++ i += raid_disks; ++ i -= (sh->pd_idx + 1); ++ } ++ break; ++ case ALGORITHM_LEFT_ASYMMETRIC_6: ++ case ALGORITHM_RIGHT_ASYMMETRIC_6: ++ if (i > sh->pd_idx) ++ i--; ++ break; ++ case ALGORITHM_LEFT_SYMMETRIC_6: ++ case ALGORITHM_RIGHT_SYMMETRIC_6: ++ if (i < sh->pd_idx) ++ i += data_disks + 1; ++ i -= (sh->pd_idx + 1); ++ break; ++ case ALGORITHM_PARITY_0_6: ++ i -= 1; ++ break; ++ default: ++ BUG(); ++ } ++ break; + } + -+ free_debugfs(); -+ ring_buffer_free(ring_buffer); /* free up the ring buffer */ ++ chunk_number = stripe * data_disks + i; ++ r_sector = chunk_number * sectors_per_chunk + chunk_offset; + ++ check = raid5_compute_sector(conf, r_sector, ++ previous, &dummy1, &sh2); ++ if (check != sh->sector || dummy1 != dd_idx || sh2.pd_idx != sh->pd_idx ++ || sh2.qd_idx != sh->qd_idx) { ++ printk(KERN_ERR "md/raid:%s: compute_blocknr: map not correct\n", ++ mdname(conf->mddev)); ++ return 0; ++ } ++ return r_sector; +} + -+module_init(detector_init); -+module_exit(detector_exit); -diff -Nur linux-4.1.6.orig/drivers/misc/Kconfig linux-4.1.6/drivers/misc/Kconfig ---- linux-4.1.6.orig/drivers/misc/Kconfig 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/misc/Kconfig 2015-09-08 23:49:06.082091803 +0200 -@@ -54,6 +54,7 @@ - config ATMEL_TCLIB - bool "Atmel AT32/AT91 Timer/Counter Library" - depends on (AVR32 || ARCH_AT91) -+ default y if PREEMPT_RT_FULL - help - Select this if you want a library to allocate the Timer/Counter - blocks found on many Atmel processors. This facilitates using -@@ -69,8 +70,7 @@ - are combined to make a single 32-bit timer. - - When GENERIC_CLOCKEVENTS is defined, the third timer channel -- may be used as a clock event device supporting oneshot mode -- (delays of up to two seconds) based on the 32 KiHz clock. -+ may be used as a clock event device supporting oneshot mode. - - config ATMEL_TCB_CLKSRC_BLOCK - int -@@ -84,6 +84,15 @@ - TC can be used for other purposes, such as PWM generation and - interval timing. - -+config ATMEL_TCB_CLKSRC_USE_SLOW_CLOCK -+ bool "TC Block use 32 KiHz clock" -+ depends on ATMEL_TCB_CLKSRC -+ default y if !PREEMPT_RT_FULL -+ help -+ Select this to use 32 KiHz base clock rate as TC block clock -+ source for clock events. -+ ++static void ++schedule_reconstruction(struct stripe_head *sh, struct stripe_head_state *s, ++ int rcw, int expand) ++{ ++ int i, pd_idx = sh->pd_idx, qd_idx = sh->qd_idx, disks = sh->disks; ++ struct r5conf *conf = sh->raid_conf; ++ int level = conf->level; + - config DUMMY_IRQ - tristate "Dummy IRQ handler" - default n -@@ -113,6 +122,35 @@ - for information on the specific driver level and support statement - for your IBM server. - -+config HWLAT_DETECTOR -+ tristate "Testing module to detect hardware-induced latencies" -+ depends on DEBUG_FS -+ depends on RING_BUFFER -+ default m -+ ---help--- -+ A simple hardware latency detector. Use this module to detect -+ large latencies introduced by the behavior of the underlying -+ system firmware external to Linux. We do this using periodic -+ use of stop_machine to grab all available CPUs and measure -+ for unexplainable gaps in the CPU timestamp counter(s). By -+ default, the module is not enabled until the "enable" file -+ within the "hwlat_detector" debugfs directory is toggled. ++ if (rcw) { + -+ This module is often used to detect SMI (System Management -+ Interrupts) on x86 systems, though is not x86 specific. To -+ this end, we default to using a sample window of 1 second, -+ during which we will sample for 0.5 seconds. If an SMI or -+ similar event occurs during that time, it is recorded -+ into an 8K samples global ring buffer until retreived. ++ for (i = disks; i--; ) { ++ struct r5dev *dev = &sh->dev[i]; + -+ WARNING: This software should never be enabled (it can be built -+ but should not be turned on after it is loaded) in a production -+ environment where high latencies are a concern since the -+ sampling mechanism actually introduces latencies for -+ regular tasks while the CPU(s) are being held. ++ if (dev->towrite) { ++ set_bit(R5_LOCKED, &dev->flags); ++ set_bit(R5_Wantdrain, &dev->flags); ++ if (!expand) ++ clear_bit(R5_UPTODATE, &dev->flags); ++ s->locked++; ++ } ++ } ++ /* if we are not expanding this is a proper write request, and ++ * there will be bios with new data to be drained into the ++ * stripe cache ++ */ ++ if (!expand) { ++ if (!s->locked) ++ /* False alarm, nothing to do */ ++ return; ++ sh->reconstruct_state = reconstruct_state_drain_run; ++ set_bit(STRIPE_OP_BIODRAIN, &s->ops_request); ++ } else ++ sh->reconstruct_state = reconstruct_state_run; + -+ If unsure, say N ++ set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request); + - config PHANTOM - tristate "Sensable PHANToM (PCI)" - depends on PCI -diff -Nur linux-4.1.6.orig/drivers/misc/Makefile linux-4.1.6/drivers/misc/Makefile ---- linux-4.1.6.orig/drivers/misc/Makefile 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/misc/Makefile 2015-09-08 23:49:06.082091803 +0200 -@@ -38,6 +38,7 @@ - obj-$(CONFIG_HMC6352) += hmc6352.o - obj-y += eeprom/ - obj-y += cb710/ -+obj-$(CONFIG_HWLAT_DETECTOR) += hwlat_detector.o - obj-$(CONFIG_SPEAR13XX_PCIE_GADGET) += spear13xx_pcie_gadget.o - obj-$(CONFIG_VMWARE_BALLOON) += vmw_balloon.o - obj-$(CONFIG_ARM_CHARLCD) += arm-charlcd.o -diff -Nur linux-4.1.6.orig/drivers/mmc/host/mmci.c linux-4.1.6/drivers/mmc/host/mmci.c ---- linux-4.1.6.orig/drivers/mmc/host/mmci.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/mmc/host/mmci.c 2015-09-08 23:49:06.082091803 +0200 -@@ -1155,15 +1155,12 @@ - struct sg_mapping_iter *sg_miter = &host->sg_miter; - struct variant_data *variant = host->variant; - void __iomem *base = host->base; -- unsigned long flags; - u32 status; - - status = readl(base + MMCISTATUS); - - dev_dbg(mmc_dev(host->mmc), "irq1 (pio) %08x\n", status); - -- local_irq_save(flags); -- - do { - unsigned int remain, len; - char *buffer; -@@ -1203,8 +1200,6 @@ - - sg_miter_stop(sg_miter); - -- local_irq_restore(flags); -- - /* - * If we have less than the fifo 'half-full' threshold to transfer, - * trigger a PIO interrupt as soon as any data is available. -diff -Nur linux-4.1.6.orig/drivers/mmc/host/sdhci.c linux-4.1.6/drivers/mmc/host/sdhci.c ---- linux-4.1.6.orig/drivers/mmc/host/sdhci.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/mmc/host/sdhci.c 2015-09-08 23:49:06.082091803 +0200 -@@ -2691,6 +2691,31 @@ - return isr ? IRQ_HANDLED : IRQ_NONE; - } - -+#ifdef CONFIG_PREEMPT_RT_BASE -+static irqreturn_t sdhci_rt_irq(int irq, void *dev_id) -+{ -+ irqreturn_t ret; ++ if (s->locked + conf->max_degraded == disks) ++ if (!test_and_set_bit(STRIPE_FULL_WRITE, &sh->state)) ++ atomic_inc(&conf->pending_full_writes); ++ } else { ++ BUG_ON(!(test_bit(R5_UPTODATE, &sh->dev[pd_idx].flags) || ++ test_bit(R5_Wantcompute, &sh->dev[pd_idx].flags))); ++ BUG_ON(level == 6 && ++ (!(test_bit(R5_UPTODATE, &sh->dev[qd_idx].flags) || ++ test_bit(R5_Wantcompute, &sh->dev[qd_idx].flags)))); ++ ++ for (i = disks; i--; ) { ++ struct r5dev *dev = &sh->dev[i]; ++ if (i == pd_idx || i == qd_idx) ++ continue; + -+ local_bh_disable(); -+ ret = sdhci_irq(irq, dev_id); -+ local_bh_enable(); -+ if (ret == IRQ_WAKE_THREAD) -+ ret = sdhci_thread_irq(irq, dev_id); -+ return ret; -+} -+#endif ++ if (dev->towrite && ++ (test_bit(R5_UPTODATE, &dev->flags) || ++ test_bit(R5_Wantcompute, &dev->flags))) { ++ set_bit(R5_Wantdrain, &dev->flags); ++ set_bit(R5_LOCKED, &dev->flags); ++ clear_bit(R5_UPTODATE, &dev->flags); ++ s->locked++; ++ } ++ } ++ if (!s->locked) ++ /* False alarm - nothing to do */ ++ return; ++ sh->reconstruct_state = reconstruct_state_prexor_drain_run; ++ set_bit(STRIPE_OP_PREXOR, &s->ops_request); ++ set_bit(STRIPE_OP_BIODRAIN, &s->ops_request); ++ set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request); ++ } + -+static int sdhci_req_irq(struct sdhci_host *host) -+{ -+#ifdef CONFIG_PREEMPT_RT_BASE -+ return request_threaded_irq(host->irq, NULL, sdhci_rt_irq, -+ IRQF_SHARED, mmc_hostname(host->mmc), host); -+#else -+ return request_threaded_irq(host->irq, sdhci_irq, sdhci_thread_irq, -+ IRQF_SHARED, mmc_hostname(host->mmc), host); -+#endif -+} ++ /* keep the parity disk(s) locked while asynchronous operations ++ * are in flight ++ */ ++ set_bit(R5_LOCKED, &sh->dev[pd_idx].flags); ++ clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags); ++ s->locked++; + - /*****************************************************************************\ - * * - * Suspend/resume * -@@ -2758,9 +2783,7 @@ - } - - if (!device_may_wakeup(mmc_dev(host->mmc))) { -- ret = request_threaded_irq(host->irq, sdhci_irq, -- sdhci_thread_irq, IRQF_SHARED, -- mmc_hostname(host->mmc), host); -+ ret = sdhci_req_irq(host); - if (ret) - return ret; - } else { -@@ -3421,8 +3444,7 @@ - - sdhci_init(host, 0); - -- ret = request_threaded_irq(host->irq, sdhci_irq, sdhci_thread_irq, -- IRQF_SHARED, mmc_hostname(mmc), host); -+ ret = sdhci_req_irq(host); - if (ret) { - pr_err("%s: Failed to request IRQ %d: %d\n", - mmc_hostname(mmc), host->irq, ret); -diff -Nur linux-4.1.6.orig/drivers/net/ethernet/3com/3c59x.c linux-4.1.6/drivers/net/ethernet/3com/3c59x.c ---- linux-4.1.6.orig/drivers/net/ethernet/3com/3c59x.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/net/ethernet/3com/3c59x.c 2015-09-08 23:49:06.086091361 +0200 -@@ -842,9 +842,9 @@ - { - struct vortex_private *vp = netdev_priv(dev); - unsigned long flags; -- local_irq_save(flags); -+ local_irq_save_nort(flags); - (vp->full_bus_master_rx ? boomerang_interrupt:vortex_interrupt)(dev->irq,dev); -- local_irq_restore(flags); -+ local_irq_restore_nort(flags); - } - #endif - -@@ -1916,12 +1916,12 @@ - * Block interrupts because vortex_interrupt does a bare spin_lock() - */ - unsigned long flags; -- local_irq_save(flags); -+ local_irq_save_nort(flags); - if (vp->full_bus_master_tx) - boomerang_interrupt(dev->irq, dev); - else - vortex_interrupt(dev->irq, dev); -- local_irq_restore(flags); -+ local_irq_restore_nort(flags); - } - } - -diff -Nur linux-4.1.6.orig/drivers/net/ethernet/atheros/atl1c/atl1c_main.c linux-4.1.6/drivers/net/ethernet/atheros/atl1c/atl1c_main.c ---- linux-4.1.6.orig/drivers/net/ethernet/atheros/atl1c/atl1c_main.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/net/ethernet/atheros/atl1c/atl1c_main.c 2015-09-08 23:49:06.086091361 +0200 -@@ -2213,11 +2213,7 @@ - } - - tpd_req = atl1c_cal_tpd_req(skb); -- if (!spin_trylock_irqsave(&adapter->tx_lock, flags)) { -- if (netif_msg_pktdata(adapter)) -- dev_info(&adapter->pdev->dev, "tx locked\n"); -- return NETDEV_TX_LOCKED; -- } -+ spin_lock_irqsave(&adapter->tx_lock, flags); - - if (atl1c_tpd_avail(adapter, type) < tpd_req) { - /* no enough descriptor, just stop queue */ -diff -Nur linux-4.1.6.orig/drivers/net/ethernet/atheros/atl1e/atl1e_main.c linux-4.1.6/drivers/net/ethernet/atheros/atl1e/atl1e_main.c ---- linux-4.1.6.orig/drivers/net/ethernet/atheros/atl1e/atl1e_main.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/net/ethernet/atheros/atl1e/atl1e_main.c 2015-09-08 23:49:06.086091361 +0200 -@@ -1880,8 +1880,7 @@ - return NETDEV_TX_OK; - } - tpd_req = atl1e_cal_tdp_req(skb); -- if (!spin_trylock_irqsave(&adapter->tx_lock, flags)) -- return NETDEV_TX_LOCKED; -+ spin_lock_irqsave(&adapter->tx_lock, flags); - - if (atl1e_tpd_avail(adapter) < tpd_req) { - /* no enough descriptor, just stop queue */ -diff -Nur linux-4.1.6.orig/drivers/net/ethernet/chelsio/cxgb/sge.c linux-4.1.6/drivers/net/ethernet/chelsio/cxgb/sge.c ---- linux-4.1.6.orig/drivers/net/ethernet/chelsio/cxgb/sge.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/net/ethernet/chelsio/cxgb/sge.c 2015-09-08 23:49:06.086091361 +0200 -@@ -1664,8 +1664,7 @@ - struct cmdQ *q = &sge->cmdQ[qid]; - unsigned int credits, pidx, genbit, count, use_sched_skb = 0; - -- if (!spin_trylock(&q->lock)) -- return NETDEV_TX_LOCKED; -+ spin_lock(&q->lock); - - reclaim_completed_tx(sge, q); - -diff -Nur linux-4.1.6.orig/drivers/net/ethernet/freescale/gianfar.c linux-4.1.6/drivers/net/ethernet/freescale/gianfar.c ---- linux-4.1.6.orig/drivers/net/ethernet/freescale/gianfar.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/net/ethernet/freescale/gianfar.c 2015-09-08 23:49:06.086091361 +0200 -@@ -1540,7 +1540,7 @@ - - if (netif_running(ndev)) { - -- local_irq_save(flags); -+ local_irq_save_nort(flags); - lock_tx_qs(priv); - - gfar_halt_nodisable(priv); -@@ -1556,7 +1556,7 @@ - gfar_write(®s->maccfg1, tempval); - - unlock_tx_qs(priv); -- local_irq_restore(flags); -+ local_irq_restore_nort(flags); - - disable_napi(priv); - -@@ -1598,7 +1598,7 @@ - /* Disable Magic Packet mode, in case something - * else woke us up. - */ -- local_irq_save(flags); -+ local_irq_save_nort(flags); - lock_tx_qs(priv); - - tempval = gfar_read(®s->maccfg2); -@@ -1608,7 +1608,7 @@ - gfar_start(priv); - - unlock_tx_qs(priv); -- local_irq_restore(flags); -+ local_irq_restore_nort(flags); - - netif_device_attach(ndev); - -@@ -3418,14 +3418,14 @@ - dev->stats.tx_dropped++; - atomic64_inc(&priv->extra_stats.tx_underrun); - -- local_irq_save(flags); -+ local_irq_save_nort(flags); - lock_tx_qs(priv); - - /* Reactivate the Tx Queues */ - gfar_write(®s->tstat, gfargrp->tstat); - - unlock_tx_qs(priv); -- local_irq_restore(flags); -+ local_irq_restore_nort(flags); - } - netif_dbg(priv, tx_err, dev, "Transmit Error\n"); - } -diff -Nur linux-4.1.6.orig/drivers/net/ethernet/neterion/s2io.c linux-4.1.6/drivers/net/ethernet/neterion/s2io.c ---- linux-4.1.6.orig/drivers/net/ethernet/neterion/s2io.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/net/ethernet/neterion/s2io.c 2015-09-08 23:49:06.098090029 +0200 -@@ -4084,12 +4084,7 @@ - [skb->priority & (MAX_TX_FIFOS - 1)]; - fifo = &mac_control->fifos[queue]; - -- if (do_spin_lock) -- spin_lock_irqsave(&fifo->tx_lock, flags); -- else { -- if (unlikely(!spin_trylock_irqsave(&fifo->tx_lock, flags))) -- return NETDEV_TX_LOCKED; -- } -+ spin_lock_irqsave(&fifo->tx_lock, flags); - - if (sp->config.multiq) { - if (__netif_subqueue_stopped(dev, fifo->fifo_no)) { -diff -Nur linux-4.1.6.orig/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c linux-4.1.6/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c ---- linux-4.1.6.orig/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c 2015-09-08 23:49:06.302067412 +0200 -@@ -2137,10 +2137,8 @@ - struct pch_gbe_tx_ring *tx_ring = adapter->tx_ring; - unsigned long flags; - -- if (!spin_trylock_irqsave(&tx_ring->tx_lock, flags)) { -- /* Collision - tell upper layer to requeue */ -- return NETDEV_TX_LOCKED; -- } -+ spin_lock_irqsave(&tx_ring->tx_lock, flags); ++ if (level == 6) { ++ int qd_idx = sh->qd_idx; ++ struct r5dev *dev = &sh->dev[qd_idx]; + - if (unlikely(!PCH_GBE_DESC_UNUSED(tx_ring))) { - netif_stop_queue(netdev); - spin_unlock_irqrestore(&tx_ring->tx_lock, flags); -diff -Nur linux-4.1.6.orig/drivers/net/ethernet/realtek/8139too.c linux-4.1.6/drivers/net/ethernet/realtek/8139too.c ---- linux-4.1.6.orig/drivers/net/ethernet/realtek/8139too.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/net/ethernet/realtek/8139too.c 2015-09-08 23:49:06.306066968 +0200 -@@ -2229,7 +2229,7 @@ - struct rtl8139_private *tp = netdev_priv(dev); - const int irq = tp->pci_dev->irq; - -- disable_irq(irq); -+ disable_irq_nosync(irq); - rtl8139_interrupt(irq, dev); - enable_irq(irq); - } -diff -Nur linux-4.1.6.orig/drivers/net/ethernet/tehuti/tehuti.c linux-4.1.6/drivers/net/ethernet/tehuti/tehuti.c ---- linux-4.1.6.orig/drivers/net/ethernet/tehuti/tehuti.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/net/ethernet/tehuti/tehuti.c 2015-09-08 23:49:06.306066968 +0200 -@@ -1629,13 +1629,8 @@ - unsigned long flags; - - ENTER; -- local_irq_save(flags); -- if (!spin_trylock(&priv->tx_lock)) { -- local_irq_restore(flags); -- DBG("%s[%s]: TX locked, returning NETDEV_TX_LOCKED\n", -- BDX_DRV_NAME, ndev->name); -- return NETDEV_TX_LOCKED; -- } ++ set_bit(R5_LOCKED, &dev->flags); ++ clear_bit(R5_UPTODATE, &dev->flags); ++ s->locked++; ++ } + -+ spin_lock_irqsave(&priv->tx_lock, flags); - - /* build tx descriptor */ - BDX_ASSERT(f->m.wptr >= f->m.memsz); /* started with valid wptr */ -diff -Nur linux-4.1.6.orig/drivers/net/rionet.c linux-4.1.6/drivers/net/rionet.c ---- linux-4.1.6.orig/drivers/net/rionet.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/net/rionet.c 2015-09-08 23:49:06.306066968 +0200 -@@ -174,11 +174,7 @@ - unsigned long flags; - int add_num = 1; - -- local_irq_save(flags); -- if (!spin_trylock(&rnet->tx_lock)) { -- local_irq_restore(flags); -- return NETDEV_TX_LOCKED; -- } -+ spin_lock_irqsave(&rnet->tx_lock, flags); - - if (is_multicast_ether_addr(eth->h_dest)) - add_num = nets[rnet->mport->id].nact; -diff -Nur linux-4.1.6.orig/drivers/net/wireless/orinoco/orinoco_usb.c linux-4.1.6/drivers/net/wireless/orinoco/orinoco_usb.c ---- linux-4.1.6.orig/drivers/net/wireless/orinoco/orinoco_usb.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/net/wireless/orinoco/orinoco_usb.c 2015-09-08 23:49:06.306066968 +0200 -@@ -697,7 +697,7 @@ - while (!ctx->done.done && msecs--) - udelay(1000); - } else { -- wait_event_interruptible(ctx->done.wait, -+ swait_event_interruptible(ctx->done.wait, - ctx->done.done); - } - break; -diff -Nur linux-4.1.6.orig/drivers/pci/access.c linux-4.1.6/drivers/pci/access.c ---- linux-4.1.6.orig/drivers/pci/access.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/pci/access.c 2015-09-08 23:49:06.310066523 +0200 -@@ -521,7 +521,7 @@ - WARN_ON(!dev->block_cfg_access); - - dev->block_cfg_access = 0; -- wake_up_all(&pci_cfg_wait); -+ wake_up_all_locked(&pci_cfg_wait); - raw_spin_unlock_irqrestore(&pci_lock, flags); - } - EXPORT_SYMBOL_GPL(pci_cfg_access_unlock); -diff -Nur linux-4.1.6.orig/drivers/scsi/fcoe/fcoe.c linux-4.1.6/drivers/scsi/fcoe/fcoe.c ---- linux-4.1.6.orig/drivers/scsi/fcoe/fcoe.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/scsi/fcoe/fcoe.c 2015-09-08 23:49:06.314066079 +0200 -@@ -1287,7 +1287,7 @@ - struct sk_buff *skb; - #ifdef CONFIG_SMP - struct fcoe_percpu_s *p0; -- unsigned targ_cpu = get_cpu(); -+ unsigned targ_cpu = get_cpu_light(); - #endif /* CONFIG_SMP */ - - FCOE_DBG("Destroying receive thread for CPU %d\n", cpu); -@@ -1343,7 +1343,7 @@ - kfree_skb(skb); - spin_unlock_bh(&p->fcoe_rx_list.lock); - } -- put_cpu(); -+ put_cpu_light(); - #else - /* - * This a non-SMP scenario where the singular Rx thread is -@@ -1567,11 +1567,11 @@ - static int fcoe_alloc_paged_crc_eof(struct sk_buff *skb, int tlen) - { - struct fcoe_percpu_s *fps; -- int rc; -+ int rc, cpu = get_cpu_light(); - -- fps = &get_cpu_var(fcoe_percpu); -+ fps = &per_cpu(fcoe_percpu, cpu); - rc = fcoe_get_paged_crc_eof(skb, tlen, fps); -- put_cpu_var(fcoe_percpu); -+ put_cpu_light(); - - return rc; - } -@@ -1767,11 +1767,11 @@ - return 0; - } - -- stats = per_cpu_ptr(lport->stats, get_cpu()); -+ stats = per_cpu_ptr(lport->stats, get_cpu_light()); - stats->InvalidCRCCount++; - if (stats->InvalidCRCCount < 5) - printk(KERN_WARNING "fcoe: dropping frame with CRC error\n"); -- put_cpu(); -+ put_cpu_light(); - return -EINVAL; - } - -@@ -1847,13 +1847,13 @@ - goto drop; - - if (!fcoe_filter_frames(lport, fp)) { -- put_cpu(); -+ put_cpu_light(); - fc_exch_recv(lport, fp); - return; - } - drop: - stats->ErrorFrames++; -- put_cpu(); -+ put_cpu_light(); - kfree_skb(skb); - } - -diff -Nur linux-4.1.6.orig/drivers/scsi/fcoe/fcoe_ctlr.c linux-4.1.6/drivers/scsi/fcoe/fcoe_ctlr.c ---- linux-4.1.6.orig/drivers/scsi/fcoe/fcoe_ctlr.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/scsi/fcoe/fcoe_ctlr.c 2015-09-08 23:49:06.314066079 +0200 -@@ -831,7 +831,7 @@ - - INIT_LIST_HEAD(&del_list); - -- stats = per_cpu_ptr(fip->lp->stats, get_cpu()); -+ stats = per_cpu_ptr(fip->lp->stats, get_cpu_light()); - - list_for_each_entry_safe(fcf, next, &fip->fcfs, list) { - deadline = fcf->time + fcf->fka_period + fcf->fka_period / 2; -@@ -867,7 +867,7 @@ - sel_time = fcf->time; - } - } -- put_cpu(); -+ put_cpu_light(); - - list_for_each_entry_safe(fcf, next, &del_list, list) { - /* Removes fcf from current list */ -diff -Nur linux-4.1.6.orig/drivers/scsi/libfc/fc_exch.c linux-4.1.6/drivers/scsi/libfc/fc_exch.c ---- linux-4.1.6.orig/drivers/scsi/libfc/fc_exch.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/scsi/libfc/fc_exch.c 2015-09-08 23:49:06.314066079 +0200 -@@ -816,10 +816,10 @@ - } - memset(ep, 0, sizeof(*ep)); - -- cpu = get_cpu(); -+ cpu = get_cpu_light(); - pool = per_cpu_ptr(mp->pool, cpu); - spin_lock_bh(&pool->lock); -- put_cpu(); -+ put_cpu_light(); - - /* peek cache of free slot */ - if (pool->left != FC_XID_UNKNOWN) { -diff -Nur linux-4.1.6.orig/drivers/scsi/libsas/sas_ata.c linux-4.1.6/drivers/scsi/libsas/sas_ata.c ---- linux-4.1.6.orig/drivers/scsi/libsas/sas_ata.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/scsi/libsas/sas_ata.c 2015-09-08 23:49:06.314066079 +0200 -@@ -190,7 +190,7 @@ - /* TODO: audit callers to ensure they are ready for qc_issue to - * unconditionally re-enable interrupts - */ -- local_irq_save(flags); -+ local_irq_save_nort(flags); - spin_unlock(ap->lock); - - /* If the device fell off, no sense in issuing commands */ -@@ -255,7 +255,7 @@ - - out: - spin_lock(ap->lock); -- local_irq_restore(flags); -+ local_irq_restore_nort(flags); - return ret; - } - -diff -Nur linux-4.1.6.orig/drivers/scsi/qla2xxx/qla_inline.h linux-4.1.6/drivers/scsi/qla2xxx/qla_inline.h ---- linux-4.1.6.orig/drivers/scsi/qla2xxx/qla_inline.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/scsi/qla2xxx/qla_inline.h 2015-09-08 23:49:06.314066079 +0200 -@@ -59,12 +59,12 @@ - { - unsigned long flags; - struct qla_hw_data *ha = rsp->hw; -- local_irq_save(flags); -+ local_irq_save_nort(flags); - if (IS_P3P_TYPE(ha)) - qla82xx_poll(0, rsp); - else - ha->isp_ops->intr_handler(0, rsp); -- local_irq_restore(flags); -+ local_irq_restore_nort(flags); - } - - static inline uint8_t * -diff -Nur linux-4.1.6.orig/drivers/thermal/x86_pkg_temp_thermal.c linux-4.1.6/drivers/thermal/x86_pkg_temp_thermal.c ---- linux-4.1.6.orig/drivers/thermal/x86_pkg_temp_thermal.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/thermal/x86_pkg_temp_thermal.c 2015-09-08 23:49:06.314066079 +0200 -@@ -29,6 +29,7 @@ - #include - #include - #include -+#include - #include - #include - -@@ -352,7 +353,7 @@ - } - } - --static int pkg_temp_thermal_platform_thermal_notify(__u64 msr_val) -+static void platform_thermal_notify_work(struct swork_event *event) - { - unsigned long flags; - int cpu = smp_processor_id(); -@@ -369,7 +370,7 @@ - pkg_work_scheduled[phy_id]) { - disable_pkg_thres_interrupt(); - spin_unlock_irqrestore(&pkg_work_lock, flags); -- return -EINVAL; -+ return; - } - pkg_work_scheduled[phy_id] = 1; - spin_unlock_irqrestore(&pkg_work_lock, flags); -@@ -378,9 +379,48 @@ - schedule_delayed_work_on(cpu, - &per_cpu(pkg_temp_thermal_threshold_work, cpu), - msecs_to_jiffies(notify_delay_ms)); ++ pr_debug("%s: stripe %llu locked: %d ops_request: %lx\n", ++ __func__, (unsigned long long)sh->sector, ++ s->locked, s->ops_request); +} + -+#ifdef CONFIG_PREEMPT_RT_FULL -+static struct swork_event notify_work; -+ -+static int thermal_notify_work_init(void) ++/* ++ * Each stripe/dev can have one or more bion attached. ++ * toread/towrite point to the first in a chain. ++ * The bi_next chain must be in order. ++ */ ++static int add_stripe_bio(struct stripe_head *sh, struct bio *bi, int dd_idx, ++ int forwrite, int previous) +{ -+ int err; ++ struct bio **bip; ++ struct r5conf *conf = sh->raid_conf; ++ int firstwrite=0; + -+ err = swork_get(); -+ if (err) -+ return err; ++ pr_debug("adding bi b#%llu to stripe s#%llu\n", ++ (unsigned long long)bi->bi_iter.bi_sector, ++ (unsigned long long)sh->sector); + -+ INIT_SWORK(¬ify_work, platform_thermal_notify_work); - return 0; - } - -+static void thermal_notify_work_cleanup(void) -+{ -+ swork_put(); -+} ++ /* ++ * If several bio share a stripe. The bio bi_phys_segments acts as a ++ * reference count to avoid race. The reference count should already be ++ * increased before this function is called (for example, in ++ * make_request()), so other bio sharing this stripe will not free the ++ * stripe. If a stripe is owned by one stripe, the stripe lock will ++ * protect it. ++ */ ++ spin_lock_irq(&sh->stripe_lock); ++ /* Don't allow new IO added to stripes in batch list */ ++ if (sh->batch_head) ++ goto overlap; ++ if (forwrite) { ++ bip = &sh->dev[dd_idx].towrite; ++ if (*bip == NULL) ++ firstwrite = 1; ++ } else ++ bip = &sh->dev[dd_idx].toread; ++ while (*bip && (*bip)->bi_iter.bi_sector < bi->bi_iter.bi_sector) { ++ if (bio_end_sector(*bip) > bi->bi_iter.bi_sector) ++ goto overlap; ++ bip = & (*bip)->bi_next; ++ } ++ if (*bip && (*bip)->bi_iter.bi_sector < bio_end_sector(bi)) ++ goto overlap; ++ ++ if (!forwrite || previous) ++ clear_bit(STRIPE_BATCH_READY, &sh->state); ++ ++ BUG_ON(*bip && bi->bi_next && (*bip) != bi->bi_next); ++ if (*bip) ++ bi->bi_next = *bip; ++ *bip = bi; ++ raid5_inc_bi_active_stripes(bi); ++ ++ if (forwrite) { ++ /* check if page is covered */ ++ sector_t sector = sh->dev[dd_idx].sector; ++ for (bi=sh->dev[dd_idx].towrite; ++ sector < sh->dev[dd_idx].sector + STRIPE_SECTORS && ++ bi && bi->bi_iter.bi_sector <= sector; ++ bi = r5_next_bio(bi, sh->dev[dd_idx].sector)) { ++ if (bio_end_sector(bi) >= sector) ++ sector = bio_end_sector(bi); ++ } ++ if (sector >= sh->dev[dd_idx].sector + STRIPE_SECTORS) ++ if (!test_and_set_bit(R5_OVERWRITE, &sh->dev[dd_idx].flags)) ++ sh->overwrite_disks++; ++ } + -+static int pkg_temp_thermal_platform_thermal_notify(__u64 msr_val) -+{ -+ swork_queue(¬ify_work); -+ return 0; -+} ++ pr_debug("added bi b#%llu to stripe s#%llu, disk %d.\n", ++ (unsigned long long)(*bip)->bi_iter.bi_sector, ++ (unsigned long long)sh->sector, dd_idx); ++ ++ if (conf->mddev->bitmap && firstwrite) { ++ /* Cannot hold spinlock over bitmap_startwrite, ++ * but must ensure this isn't added to a batch until ++ * we have added to the bitmap and set bm_seq. ++ * So set STRIPE_BITMAP_PENDING to prevent ++ * batching. ++ * If multiple add_stripe_bio() calls race here they ++ * much all set STRIPE_BITMAP_PENDING. So only the first one ++ * to complete "bitmap_startwrite" gets to set ++ * STRIPE_BIT_DELAY. This is important as once a stripe ++ * is added to a batch, STRIPE_BIT_DELAY cannot be changed ++ * any more. ++ */ ++ set_bit(STRIPE_BITMAP_PENDING, &sh->state); ++ spin_unlock_irq(&sh->stripe_lock); ++ bitmap_startwrite(conf->mddev->bitmap, sh->sector, ++ STRIPE_SECTORS, 0); ++ spin_lock_irq(&sh->stripe_lock); ++ clear_bit(STRIPE_BITMAP_PENDING, &sh->state); ++ if (!sh->batch_head) { ++ sh->bm_seq = conf->seq_flush+1; ++ set_bit(STRIPE_BIT_DELAY, &sh->state); ++ } ++ } ++ spin_unlock_irq(&sh->stripe_lock); + -+#else /* !CONFIG_PREEMPT_RT_FULL */ ++ if (stripe_can_batch(sh)) ++ stripe_add_to_batch_list(conf, sh); ++ return 1; + -+static int thermal_notify_work_init(void) { return 0; } ++ overlap: ++ set_bit(R5_Overlap, &sh->dev[dd_idx].flags); ++ spin_unlock_irq(&sh->stripe_lock); ++ return 0; ++} + -+static void thermal_notify_work_cleanup(void) { } ++static void end_reshape(struct r5conf *conf); + -+static int pkg_temp_thermal_platform_thermal_notify(__u64 msr_val) ++static void stripe_set_idx(sector_t stripe, struct r5conf *conf, int previous, ++ struct stripe_head *sh) +{ -+ platform_thermal_notify_work(NULL); -+ -+ return 0; ++ int sectors_per_chunk = ++ previous ? conf->prev_chunk_sectors : conf->chunk_sectors; ++ int dd_idx; ++ int chunk_offset = sector_div(stripe, sectors_per_chunk); ++ int disks = previous ? conf->previous_raid_disks : conf->raid_disks; ++ ++ raid5_compute_sector(conf, ++ stripe * (disks - conf->max_degraded) ++ *sectors_per_chunk + chunk_offset, ++ previous, ++ &dd_idx, sh); +} -+#endif /* CONFIG_PREEMPT_RT_FULL */ + - static int find_siblings_cpu(int cpu) - { - int i; -@@ -584,6 +624,9 @@ - if (!x86_match_cpu(pkg_temp_thermal_ids)) - return -ENODEV; - -+ if (!thermal_notify_work_init()) -+ return -ENODEV; ++static void ++handle_failed_stripe(struct r5conf *conf, struct stripe_head *sh, ++ struct stripe_head_state *s, int disks, ++ struct bio **return_bi) ++{ ++ int i; ++ BUG_ON(sh->batch_head); ++ for (i = disks; i--; ) { ++ struct bio *bi; ++ int bitmap_end = 0; ++ ++ if (test_bit(R5_ReadError, &sh->dev[i].flags)) { ++ struct md_rdev *rdev; ++ rcu_read_lock(); ++ rdev = rcu_dereference(conf->disks[i].rdev); ++ if (rdev && test_bit(In_sync, &rdev->flags)) ++ atomic_inc(&rdev->nr_pending); ++ else ++ rdev = NULL; ++ rcu_read_unlock(); ++ if (rdev) { ++ if (!rdev_set_badblocks( ++ rdev, ++ sh->sector, ++ STRIPE_SECTORS, 0)) ++ md_error(conf->mddev, rdev); ++ rdev_dec_pending(rdev, conf->mddev); ++ } ++ } ++ spin_lock_irq(&sh->stripe_lock); ++ /* fail all writes first */ ++ bi = sh->dev[i].towrite; ++ sh->dev[i].towrite = NULL; ++ sh->overwrite_disks = 0; ++ spin_unlock_irq(&sh->stripe_lock); ++ if (bi) ++ bitmap_end = 1; ++ ++ if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags)) ++ wake_up(&conf->wait_for_overlap); ++ ++ while (bi && bi->bi_iter.bi_sector < ++ sh->dev[i].sector + STRIPE_SECTORS) { ++ struct bio *nextbi = r5_next_bio(bi, sh->dev[i].sector); ++ clear_bit(BIO_UPTODATE, &bi->bi_flags); ++ if (!raid5_dec_bi_active_stripes(bi)) { ++ md_write_end(conf->mddev); ++ bi->bi_next = *return_bi; ++ *return_bi = bi; ++ } ++ bi = nextbi; ++ } ++ if (bitmap_end) ++ bitmap_endwrite(conf->mddev->bitmap, sh->sector, ++ STRIPE_SECTORS, 0, 0); ++ bitmap_end = 0; ++ /* and fail all 'written' */ ++ bi = sh->dev[i].written; ++ sh->dev[i].written = NULL; ++ if (test_and_clear_bit(R5_SkipCopy, &sh->dev[i].flags)) { ++ WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags)); ++ sh->dev[i].page = sh->dev[i].orig_page; ++ } + - spin_lock_init(&pkg_work_lock); - platform_thermal_package_notify = - pkg_temp_thermal_platform_thermal_notify; -@@ -608,7 +651,7 @@ - kfree(pkg_work_scheduled); - platform_thermal_package_notify = NULL; - platform_thermal_package_rate_control = NULL; -- -+ thermal_notify_work_cleanup(); - return -ENODEV; - } - -@@ -633,6 +676,7 @@ - mutex_unlock(&phy_dev_list_mutex); - platform_thermal_package_notify = NULL; - platform_thermal_package_rate_control = NULL; -+ thermal_notify_work_cleanup(); - for_each_online_cpu(i) - cancel_delayed_work_sync( - &per_cpu(pkg_temp_thermal_threshold_work, i)); -diff -Nur linux-4.1.6.orig/drivers/tty/serial/8250/8250_core.c linux-4.1.6/drivers/tty/serial/8250/8250_core.c ---- linux-4.1.6.orig/drivers/tty/serial/8250/8250_core.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/tty/serial/8250/8250_core.c 2015-09-08 23:49:06.314066079 +0200 -@@ -36,6 +36,7 @@ - #include - #include - #include -+#include - #include - #include - #ifdef CONFIG_SPARC -@@ -80,7 +81,16 @@ - #define DEBUG_INTR(fmt...) do { } while (0) - #endif - --#define PASS_LIMIT 512 -+/* -+ * On -rt we can have a more delays, and legitimately -+ * so - so don't drop work spuriously and spam the -+ * syslog: ++ if (bi) bitmap_end = 1; ++ while (bi && bi->bi_iter.bi_sector < ++ sh->dev[i].sector + STRIPE_SECTORS) { ++ struct bio *bi2 = r5_next_bio(bi, sh->dev[i].sector); ++ clear_bit(BIO_UPTODATE, &bi->bi_flags); ++ if (!raid5_dec_bi_active_stripes(bi)) { ++ md_write_end(conf->mddev); ++ bi->bi_next = *return_bi; ++ *return_bi = bi; ++ } ++ bi = bi2; ++ } ++ ++ /* fail any reads if this device is non-operational and ++ * the data has not reached the cache yet. ++ */ ++ if (!test_bit(R5_Wantfill, &sh->dev[i].flags) && ++ (!test_bit(R5_Insync, &sh->dev[i].flags) || ++ test_bit(R5_ReadError, &sh->dev[i].flags))) { ++ spin_lock_irq(&sh->stripe_lock); ++ bi = sh->dev[i].toread; ++ sh->dev[i].toread = NULL; ++ spin_unlock_irq(&sh->stripe_lock); ++ if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags)) ++ wake_up(&conf->wait_for_overlap); ++ while (bi && bi->bi_iter.bi_sector < ++ sh->dev[i].sector + STRIPE_SECTORS) { ++ struct bio *nextbi = ++ r5_next_bio(bi, sh->dev[i].sector); ++ clear_bit(BIO_UPTODATE, &bi->bi_flags); ++ if (!raid5_dec_bi_active_stripes(bi)) { ++ bi->bi_next = *return_bi; ++ *return_bi = bi; ++ } ++ bi = nextbi; ++ } ++ } ++ if (bitmap_end) ++ bitmap_endwrite(conf->mddev->bitmap, sh->sector, ++ STRIPE_SECTORS, 0, 0); ++ /* If we were in the middle of a write the parity block might ++ * still be locked - so just clear all R5_LOCKED flags ++ */ ++ clear_bit(R5_LOCKED, &sh->dev[i].flags); ++ } ++ ++ if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state)) ++ if (atomic_dec_and_test(&conf->pending_full_writes)) ++ md_wakeup_thread(conf->mddev->thread); ++} ++ ++static void ++handle_failed_sync(struct r5conf *conf, struct stripe_head *sh, ++ struct stripe_head_state *s) ++{ ++ int abort = 0; ++ int i; ++ ++ BUG_ON(sh->batch_head); ++ clear_bit(STRIPE_SYNCING, &sh->state); ++ if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags)) ++ wake_up(&conf->wait_for_overlap); ++ s->syncing = 0; ++ s->replacing = 0; ++ /* There is nothing more to do for sync/check/repair. ++ * Don't even need to abort as that is handled elsewhere ++ * if needed, and not always wanted e.g. if there is a known ++ * bad block here. ++ * For recover/replace we need to record a bad block on all ++ * non-sync devices, or abort the recovery ++ */ ++ if (test_bit(MD_RECOVERY_RECOVER, &conf->mddev->recovery)) { ++ /* During recovery devices cannot be removed, so ++ * locking and refcounting of rdevs is not needed ++ */ ++ for (i = 0; i < conf->raid_disks; i++) { ++ struct md_rdev *rdev = conf->disks[i].rdev; ++ if (rdev ++ && !test_bit(Faulty, &rdev->flags) ++ && !test_bit(In_sync, &rdev->flags) ++ && !rdev_set_badblocks(rdev, sh->sector, ++ STRIPE_SECTORS, 0)) ++ abort = 1; ++ rdev = conf->disks[i].replacement; ++ if (rdev ++ && !test_bit(Faulty, &rdev->flags) ++ && !test_bit(In_sync, &rdev->flags) ++ && !rdev_set_badblocks(rdev, sh->sector, ++ STRIPE_SECTORS, 0)) ++ abort = 1; ++ } ++ if (abort) ++ conf->recovery_disabled = ++ conf->mddev->recovery_disabled; ++ } ++ md_done_sync(conf->mddev, STRIPE_SECTORS, !abort); ++} ++ ++static int want_replace(struct stripe_head *sh, int disk_idx) ++{ ++ struct md_rdev *rdev; ++ int rv = 0; ++ /* Doing recovery so rcu locking not required */ ++ rdev = sh->raid_conf->disks[disk_idx].replacement; ++ if (rdev ++ && !test_bit(Faulty, &rdev->flags) ++ && !test_bit(In_sync, &rdev->flags) ++ && (rdev->recovery_offset <= sh->sector ++ || rdev->mddev->recovery_cp <= sh->sector)) ++ rv = 1; ++ ++ return rv; ++} ++ ++/* fetch_block - checks the given member device to see if its data needs ++ * to be read or computed to satisfy a request. ++ * ++ * Returns 1 when no more member devices need to be checked, otherwise returns ++ * 0 to tell the loop in handle_stripe_fill to continue + */ -+#ifdef CONFIG_PREEMPT_RT_FULL -+# define PASS_LIMIT 1000000 -+#else -+# define PASS_LIMIT 512 -+#endif - - #define BOTH_EMPTY (UART_LSR_TEMT | UART_LSR_THRE) - -@@ -3364,7 +3374,7 @@ - - if (port->sysrq) - locked = 0; -- else if (oops_in_progress) -+ else if (oops_in_progress || in_kdb_printk()) - locked = spin_trylock_irqsave(&port->lock, flags); - else - spin_lock_irqsave(&port->lock, flags); -diff -Nur linux-4.1.6.orig/drivers/tty/serial/amba-pl011.c linux-4.1.6/drivers/tty/serial/amba-pl011.c ---- linux-4.1.6.orig/drivers/tty/serial/amba-pl011.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/tty/serial/amba-pl011.c 2015-09-08 23:49:06.314066079 +0200 -@@ -2000,13 +2000,19 @@ - - clk_enable(uap->clk); - -- local_irq_save(flags); -+ /* -+ * local_irq_save(flags); -+ * -+ * This local_irq_save() is nonsense. If we come in via sysrq -+ * handling then interrupts are already disabled. Aside of -+ * that the port.sysrq check is racy on SMP regardless. -+ */ - if (uap->port.sysrq) - locked = 0; - else if (oops_in_progress) -- locked = spin_trylock(&uap->port.lock); -+ locked = spin_trylock_irqsave(&uap->port.lock, flags); - else -- spin_lock(&uap->port.lock); -+ spin_lock_irqsave(&uap->port.lock, flags); - - /* - * First save the CR then disable the interrupts -@@ -2028,8 +2034,7 @@ - writew(old_cr, uap->port.membase + UART011_CR); - - if (locked) -- spin_unlock(&uap->port.lock); -- local_irq_restore(flags); -+ spin_unlock_irqrestore(&uap->port.lock, flags); - - clk_disable(uap->clk); - } -diff -Nur linux-4.1.6.orig/drivers/tty/serial/omap-serial.c linux-4.1.6/drivers/tty/serial/omap-serial.c ---- linux-4.1.6.orig/drivers/tty/serial/omap-serial.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/tty/serial/omap-serial.c 2015-09-08 23:49:06.314066079 +0200 -@@ -1282,13 +1282,10 @@ - - pm_runtime_get_sync(up->dev); - -- local_irq_save(flags); -- if (up->port.sysrq) -- locked = 0; -- else if (oops_in_progress) -- locked = spin_trylock(&up->port.lock); -+ if (up->port.sysrq || oops_in_progress) -+ locked = spin_trylock_irqsave(&up->port.lock, flags); - else -- spin_lock(&up->port.lock); -+ spin_lock_irqsave(&up->port.lock, flags); - - /* - * First save the IER then disable the interrupts -@@ -1317,8 +1314,7 @@ - pm_runtime_mark_last_busy(up->dev); - pm_runtime_put_autosuspend(up->dev); - if (locked) -- spin_unlock(&up->port.lock); -- local_irq_restore(flags); -+ spin_unlock_irqrestore(&up->port.lock, flags); - } - - static int __init -diff -Nur linux-4.1.6.orig/drivers/usb/core/hcd.c linux-4.1.6/drivers/usb/core/hcd.c ---- linux-4.1.6.orig/drivers/usb/core/hcd.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/usb/core/hcd.c 2015-09-08 23:49:06.314066079 +0200 -@@ -1684,9 +1684,9 @@ - * and no one may trigger the above deadlock situation when - * running complete() in tasklet. - */ -- local_irq_save(flags); -+ local_irq_save_nort(flags); - urb->complete(urb); -- local_irq_restore(flags); -+ local_irq_restore_nort(flags); - - usb_anchor_resume_wakeups(anchor); - atomic_dec(&urb->use_count); -diff -Nur linux-4.1.6.orig/drivers/usb/gadget/function/f_fs.c linux-4.1.6/drivers/usb/gadget/function/f_fs.c ---- linux-4.1.6.orig/drivers/usb/gadget/function/f_fs.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/usb/gadget/function/f_fs.c 2015-09-08 23:49:06.318065636 +0200 -@@ -1405,7 +1405,7 @@ - pr_info("%s(): freeing\n", __func__); - ffs_data_clear(ffs); - BUG_ON(waitqueue_active(&ffs->ev.waitq) || -- waitqueue_active(&ffs->ep0req_completion.wait)); -+ swaitqueue_active(&ffs->ep0req_completion.wait)); - kfree(ffs->dev_name); - kfree(ffs); - } -diff -Nur linux-4.1.6.orig/drivers/usb/gadget/legacy/inode.c linux-4.1.6/drivers/usb/gadget/legacy/inode.c ---- linux-4.1.6.orig/drivers/usb/gadget/legacy/inode.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/drivers/usb/gadget/legacy/inode.c 2015-09-08 23:49:06.318065636 +0200 -@@ -345,7 +345,7 @@ - spin_unlock_irq (&epdata->dev->lock); - - if (likely (value == 0)) { -- value = wait_event_interruptible (done.wait, done.done); -+ value = swait_event_interruptible (done.wait, done.done); - if (value != 0) { - spin_lock_irq (&epdata->dev->lock); - if (likely (epdata->ep != NULL)) { -@@ -354,7 +354,7 @@ - usb_ep_dequeue (epdata->ep, epdata->req); - spin_unlock_irq (&epdata->dev->lock); - -- wait_event (done.wait, done.done); -+ swait_event (done.wait, done.done); - if (epdata->status == -ECONNRESET) - epdata->status = -EINTR; - } else { -diff -Nur linux-4.1.6.orig/fs/aio.c linux-4.1.6/fs/aio.c ---- linux-4.1.6.orig/fs/aio.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/fs/aio.c 2015-09-08 23:49:06.318065636 +0200 -@@ -40,6 +40,7 @@ - #include - #include - #include -+#include - - #include - #include -@@ -115,7 +116,7 @@ - struct page **ring_pages; - long nr_pages; - -- struct work_struct free_work; -+ struct swork_event free_work; - - /* - * signals when all in-flight requests are done -@@ -253,6 +254,7 @@ - .mount = aio_mount, - .kill_sb = kill_anon_super, - }; -+ BUG_ON(swork_get()); - aio_mnt = kern_mount(&aio_fs); - if (IS_ERR(aio_mnt)) - panic("Failed to create aio fs mount."); -@@ -559,9 +561,9 @@ - return cancel(&kiocb->common); - } - --static void free_ioctx(struct work_struct *work) -+static void free_ioctx(struct swork_event *sev) - { -- struct kioctx *ctx = container_of(work, struct kioctx, free_work); -+ struct kioctx *ctx = container_of(sev, struct kioctx, free_work); - - pr_debug("freeing %p\n", ctx); - -@@ -580,8 +582,8 @@ - if (ctx->rq_wait && atomic_dec_and_test(&ctx->rq_wait->count)) - complete(&ctx->rq_wait->comp); - -- INIT_WORK(&ctx->free_work, free_ioctx); -- schedule_work(&ctx->free_work); -+ INIT_SWORK(&ctx->free_work, free_ioctx); -+ swork_queue(&ctx->free_work); - } - - /* -@@ -589,9 +591,9 @@ - * and ctx->users has dropped to 0, so we know no more kiocbs can be submitted - - * now it's safe to cancel any that need to be. - */ --static void free_ioctx_users(struct percpu_ref *ref) -+static void free_ioctx_users_work(struct swork_event *sev) - { -- struct kioctx *ctx = container_of(ref, struct kioctx, users); -+ struct kioctx *ctx = container_of(sev, struct kioctx, free_work); - struct aio_kiocb *req; - - spin_lock_irq(&ctx->ctx_lock); -@@ -610,6 +612,14 @@ - percpu_ref_put(&ctx->reqs); - } - -+static void free_ioctx_users(struct percpu_ref *ref) ++ ++static int need_this_block(struct stripe_head *sh, struct stripe_head_state *s, ++ int disk_idx, int disks) +{ -+ struct kioctx *ctx = container_of(ref, struct kioctx, users); ++ struct r5dev *dev = &sh->dev[disk_idx]; ++ struct r5dev *fdev[2] = { &sh->dev[s->failed_num[0]], ++ &sh->dev[s->failed_num[1]] }; ++ int i; + -+ INIT_SWORK(&ctx->free_work, free_ioctx_users_work); -+ swork_queue(&ctx->free_work); ++ ++ if (test_bit(R5_LOCKED, &dev->flags) || ++ test_bit(R5_UPTODATE, &dev->flags)) ++ /* No point reading this as we already have it or have ++ * decided to get it. ++ */ ++ return 0; ++ ++ if (dev->toread || ++ (dev->towrite && !test_bit(R5_OVERWRITE, &dev->flags))) ++ /* We need this block to directly satisfy a request */ ++ return 1; ++ ++ if (s->syncing || s->expanding || ++ (s->replacing && want_replace(sh, disk_idx))) ++ /* When syncing, or expanding we read everything. ++ * When replacing, we need the replaced block. ++ */ ++ return 1; ++ ++ if ((s->failed >= 1 && fdev[0]->toread) || ++ (s->failed >= 2 && fdev[1]->toread)) ++ /* If we want to read from a failed device, then ++ * we need to actually read every other device. ++ */ ++ return 1; ++ ++ /* Sometimes neither read-modify-write nor reconstruct-write ++ * cycles can work. In those cases we read every block we ++ * can. Then the parity-update is certain to have enough to ++ * work with. ++ * This can only be a problem when we need to write something, ++ * and some device has failed. If either of those tests ++ * fail we need look no further. ++ */ ++ if (!s->failed || !s->to_write) ++ return 0; ++ ++ if (test_bit(R5_Insync, &dev->flags) && ++ !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) ++ /* Pre-reads at not permitted until after short delay ++ * to gather multiple requests. However if this ++ * device is no Insync, the block could only be be computed ++ * and there is no need to delay that. ++ */ ++ return 0; ++ ++ for (i = 0; i < s->failed; i++) { ++ if (fdev[i]->towrite && ++ !test_bit(R5_UPTODATE, &fdev[i]->flags) && ++ !test_bit(R5_OVERWRITE, &fdev[i]->flags)) ++ /* If we have a partial write to a failed ++ * device, then we will need to reconstruct ++ * the content of that device, so all other ++ * devices must be read. ++ */ ++ return 1; ++ } ++ ++ /* If we are forced to do a reconstruct-write, either because ++ * the current RAID6 implementation only supports that, or ++ * or because parity cannot be trusted and we are currently ++ * recovering it, there is extra need to be careful. ++ * If one of the devices that we would need to read, because ++ * it is not being overwritten (and maybe not written at all) ++ * is missing/faulty, then we need to read everything we can. ++ */ ++ if (sh->raid_conf->level != 6 && ++ sh->sector < sh->raid_conf->mddev->recovery_cp) ++ /* reconstruct-write isn't being forced */ ++ return 0; ++ for (i = 0; i < s->failed; i++) { ++ if (s->failed_num[i] != sh->pd_idx && ++ s->failed_num[i] != sh->qd_idx && ++ !test_bit(R5_UPTODATE, &fdev[i]->flags) && ++ !test_bit(R5_OVERWRITE, &fdev[i]->flags)) ++ return 1; ++ } ++ ++ return 0; +} + - static int ioctx_add_table(struct kioctx *ctx, struct mm_struct *mm) - { - unsigned i, new_nr; -diff -Nur linux-4.1.6.orig/fs/autofs4/autofs_i.h linux-4.1.6/fs/autofs4/autofs_i.h ---- linux-4.1.6.orig/fs/autofs4/autofs_i.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/fs/autofs4/autofs_i.h 2015-09-08 23:49:06.318065636 +0200 -@@ -34,6 +34,7 @@ - #include - #include - #include -+#include - #include - #include - -diff -Nur linux-4.1.6.orig/fs/autofs4/expire.c linux-4.1.6/fs/autofs4/expire.c ---- linux-4.1.6.orig/fs/autofs4/expire.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/fs/autofs4/expire.c 2015-09-08 23:49:06.318065636 +0200 -@@ -150,7 +150,7 @@ - parent = p->d_parent; - if (!spin_trylock(&parent->d_lock)) { - spin_unlock(&p->d_lock); -- cpu_relax(); -+ cpu_chill(); - goto relock; - } - spin_unlock(&p->d_lock); -diff -Nur linux-4.1.6.orig/fs/buffer.c linux-4.1.6/fs/buffer.c ---- linux-4.1.6.orig/fs/buffer.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/fs/buffer.c 2015-09-08 23:49:06.318065636 +0200 -@@ -301,8 +301,7 @@ - * decide that the page is now completely done. - */ - first = page_buffers(page); -- local_irq_save(flags); -- bit_spin_lock(BH_Uptodate_Lock, &first->b_state); -+ flags = bh_uptodate_lock_irqsave(first); - clear_buffer_async_read(bh); - unlock_buffer(bh); - tmp = bh; -@@ -315,8 +314,7 @@ - } - tmp = tmp->b_this_page; - } while (tmp != bh); -- bit_spin_unlock(BH_Uptodate_Lock, &first->b_state); -- local_irq_restore(flags); -+ bh_uptodate_unlock_irqrestore(first, flags); - - /* - * If none of the buffers had errors and they are all -@@ -328,9 +326,7 @@ - return; - - still_busy: -- bit_spin_unlock(BH_Uptodate_Lock, &first->b_state); -- local_irq_restore(flags); -- return; -+ bh_uptodate_unlock_irqrestore(first, flags); - } - - /* -@@ -358,8 +354,7 @@ - } - - first = page_buffers(page); -- local_irq_save(flags); -- bit_spin_lock(BH_Uptodate_Lock, &first->b_state); -+ flags = bh_uptodate_lock_irqsave(first); - - clear_buffer_async_write(bh); - unlock_buffer(bh); -@@ -371,15 +366,12 @@ - } - tmp = tmp->b_this_page; - } -- bit_spin_unlock(BH_Uptodate_Lock, &first->b_state); -- local_irq_restore(flags); -+ bh_uptodate_unlock_irqrestore(first, flags); - end_page_writeback(page); - return; - - still_busy: -- bit_spin_unlock(BH_Uptodate_Lock, &first->b_state); -- local_irq_restore(flags); -- return; -+ bh_uptodate_unlock_irqrestore(first, flags); - } - EXPORT_SYMBOL(end_buffer_async_write); - -@@ -3325,6 +3317,7 @@ - struct buffer_head *ret = kmem_cache_zalloc(bh_cachep, gfp_flags); - if (ret) { - INIT_LIST_HEAD(&ret->b_assoc_buffers); -+ buffer_head_init_locks(ret); - preempt_disable(); - __this_cpu_inc(bh_accounting.nr); - recalc_bh_state(); -diff -Nur linux-4.1.6.orig/fs/dcache.c linux-4.1.6/fs/dcache.c ---- linux-4.1.6.orig/fs/dcache.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/fs/dcache.c 2015-09-08 23:49:06.318065636 +0200 -@@ -19,6 +19,7 @@ - #include - #include - #include -+#include - #include - #include - #include -@@ -589,7 +590,7 @@ - - failed: - spin_unlock(&dentry->d_lock); -- cpu_relax(); -+ cpu_chill(); - return dentry; /* try again with same dentry */ - } - -@@ -2395,7 +2396,7 @@ - if (dentry->d_lockref.count == 1) { - if (!spin_trylock(&inode->i_lock)) { - spin_unlock(&dentry->d_lock); -- cpu_relax(); -+ cpu_chill(); - goto again; - } - dentry->d_flags &= ~DCACHE_CANT_MOUNT; -diff -Nur linux-4.1.6.orig/fs/eventpoll.c linux-4.1.6/fs/eventpoll.c ---- linux-4.1.6.orig/fs/eventpoll.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/fs/eventpoll.c 2015-09-08 23:49:06.318065636 +0200 -@@ -505,12 +505,12 @@ - */ - static void ep_poll_safewake(wait_queue_head_t *wq) - { -- int this_cpu = get_cpu(); -+ int this_cpu = get_cpu_light(); - - ep_call_nested(&poll_safewake_ncalls, EP_MAX_NESTS, - ep_poll_wakeup_proc, NULL, wq, (void *) (long) this_cpu); - -- put_cpu(); -+ put_cpu_light(); - } - - static void ep_remove_wait_queue(struct eppoll_entry *pwq) -diff -Nur linux-4.1.6.orig/fs/exec.c linux-4.1.6/fs/exec.c ---- linux-4.1.6.orig/fs/exec.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/fs/exec.c 2015-09-08 23:49:06.318065636 +0200 -@@ -859,12 +859,14 @@ - } - } - task_lock(tsk); -+ preempt_disable_rt(); - active_mm = tsk->active_mm; - tsk->mm = mm; - tsk->active_mm = mm; - activate_mm(active_mm, mm); - tsk->mm->vmacache_seqnum = 0; - vmacache_flush(tsk); -+ preempt_enable_rt(); - task_unlock(tsk); - if (old_mm) { - up_read(&old_mm->mmap_sem); -diff -Nur linux-4.1.6.orig/fs/jbd/checkpoint.c linux-4.1.6/fs/jbd/checkpoint.c ---- linux-4.1.6.orig/fs/jbd/checkpoint.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/fs/jbd/checkpoint.c 2015-09-08 23:49:06.318065636 +0200 -@@ -129,6 +129,8 @@ - if (journal->j_flags & JFS_ABORT) - return; - spin_unlock(&journal->j_state_lock); -+ if (current->plug) -+ io_schedule(); - mutex_lock(&journal->j_checkpoint_mutex); - - /* -diff -Nur linux-4.1.6.orig/fs/jbd2/checkpoint.c linux-4.1.6/fs/jbd2/checkpoint.c ---- linux-4.1.6.orig/fs/jbd2/checkpoint.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/fs/jbd2/checkpoint.c 2015-09-08 23:49:06.322065194 +0200 -@@ -116,6 +116,8 @@ - nblocks = jbd2_space_needed(journal); - while (jbd2_log_space_left(journal) < nblocks) { - write_unlock(&journal->j_state_lock); -+ if (current->plug) -+ io_schedule(); - mutex_lock(&journal->j_checkpoint_mutex); - - /* -diff -Nur linux-4.1.6.orig/fs/namespace.c linux-4.1.6/fs/namespace.c ---- linux-4.1.6.orig/fs/namespace.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/fs/namespace.c 2015-09-08 23:49:06.322065194 +0200 -@@ -14,6 +14,7 @@ - #include - #include - #include -+#include - #include - #include - #include /* init_rootfs */ -@@ -353,8 +354,11 @@ - * incremented count after it has set MNT_WRITE_HOLD. - */ - smp_mb(); -- while (ACCESS_ONCE(mnt->mnt.mnt_flags) & MNT_WRITE_HOLD) -- cpu_relax(); -+ while (ACCESS_ONCE(mnt->mnt.mnt_flags) & MNT_WRITE_HOLD) { -+ preempt_enable(); -+ cpu_chill(); -+ preempt_disable(); ++static int fetch_block(struct stripe_head *sh, struct stripe_head_state *s, ++ int disk_idx, int disks) ++{ ++ struct r5dev *dev = &sh->dev[disk_idx]; ++ ++ /* is the data in this block needed, and can we get it? */ ++ if (need_this_block(sh, s, disk_idx, disks)) { ++ /* we would like to get this block, possibly by computing it, ++ * otherwise read it if the backing disk is insync ++ */ ++ BUG_ON(test_bit(R5_Wantcompute, &dev->flags)); ++ BUG_ON(test_bit(R5_Wantread, &dev->flags)); ++ BUG_ON(sh->batch_head); ++ if ((s->uptodate == disks - 1) && ++ (s->failed && (disk_idx == s->failed_num[0] || ++ disk_idx == s->failed_num[1]))) { ++ /* have disk failed, and we're requested to fetch it; ++ * do compute it ++ */ ++ pr_debug("Computing stripe %llu block %d\n", ++ (unsigned long long)sh->sector, disk_idx); ++ set_bit(STRIPE_COMPUTE_RUN, &sh->state); ++ set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request); ++ set_bit(R5_Wantcompute, &dev->flags); ++ sh->ops.target = disk_idx; ++ sh->ops.target2 = -1; /* no 2nd target */ ++ s->req_compute = 1; ++ /* Careful: from this point on 'uptodate' is in the eye ++ * of raid_run_ops which services 'compute' operations ++ * before writes. R5_Wantcompute flags a block that will ++ * be R5_UPTODATE by the time it is needed for a ++ * subsequent operation. ++ */ ++ s->uptodate++; ++ return 1; ++ } else if (s->uptodate == disks-2 && s->failed >= 2) { ++ /* Computing 2-failure is *very* expensive; only ++ * do it if failed >= 2 ++ */ ++ int other; ++ for (other = disks; other--; ) { ++ if (other == disk_idx) ++ continue; ++ if (!test_bit(R5_UPTODATE, ++ &sh->dev[other].flags)) ++ break; ++ } ++ BUG_ON(other < 0); ++ pr_debug("Computing stripe %llu blocks %d,%d\n", ++ (unsigned long long)sh->sector, ++ disk_idx, other); ++ set_bit(STRIPE_COMPUTE_RUN, &sh->state); ++ set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request); ++ set_bit(R5_Wantcompute, &sh->dev[disk_idx].flags); ++ set_bit(R5_Wantcompute, &sh->dev[other].flags); ++ sh->ops.target = disk_idx; ++ sh->ops.target2 = other; ++ s->uptodate += 2; ++ s->req_compute = 1; ++ return 1; ++ } else if (test_bit(R5_Insync, &dev->flags)) { ++ set_bit(R5_LOCKED, &dev->flags); ++ set_bit(R5_Wantread, &dev->flags); ++ s->locked++; ++ pr_debug("Reading block %d (sync=%d)\n", ++ disk_idx, s->syncing); ++ } + } - /* - * After the slowpath clears MNT_WRITE_HOLD, mnt_is_readonly will - * be set to match its requirements. So we must not load that until -diff -Nur linux-4.1.6.orig/fs/ntfs/aops.c linux-4.1.6/fs/ntfs/aops.c ---- linux-4.1.6.orig/fs/ntfs/aops.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/fs/ntfs/aops.c 2015-09-08 23:49:06.322065194 +0200 -@@ -107,8 +107,7 @@ - "0x%llx.", (unsigned long long)bh->b_blocknr); - } - first = page_buffers(page); -- local_irq_save(flags); -- bit_spin_lock(BH_Uptodate_Lock, &first->b_state); -+ flags = bh_uptodate_lock_irqsave(first); - clear_buffer_async_read(bh); - unlock_buffer(bh); - tmp = bh; -@@ -123,8 +122,7 @@ - } - tmp = tmp->b_this_page; - } while (tmp != bh); -- bit_spin_unlock(BH_Uptodate_Lock, &first->b_state); -- local_irq_restore(flags); -+ bh_uptodate_unlock_irqrestore(first, flags); - /* - * If none of the buffers had errors then we can set the page uptodate, - * but we first have to perform the post read mst fixups, if the -@@ -145,13 +143,13 @@ - recs = PAGE_CACHE_SIZE / rec_size; - /* Should have been verified before we got here... */ - BUG_ON(!recs); -- local_irq_save(flags); -+ local_irq_save_nort(flags); - kaddr = kmap_atomic(page); - for (i = 0; i < recs; i++) - post_read_mst_fixup((NTFS_RECORD*)(kaddr + - i * rec_size), rec_size); - kunmap_atomic(kaddr); -- local_irq_restore(flags); -+ local_irq_restore_nort(flags); - flush_dcache_page(page); - if (likely(page_uptodate && !PageError(page))) - SetPageUptodate(page); -@@ -159,9 +157,7 @@ - unlock_page(page); - return; - still_busy: -- bit_spin_unlock(BH_Uptodate_Lock, &first->b_state); -- local_irq_restore(flags); -- return; -+ bh_uptodate_unlock_irqrestore(first, flags); - } - - /** -diff -Nur linux-4.1.6.orig/fs/timerfd.c linux-4.1.6/fs/timerfd.c ---- linux-4.1.6.orig/fs/timerfd.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/fs/timerfd.c 2015-09-08 23:49:06.322065194 +0200 -@@ -450,7 +450,10 @@ - break; - } - spin_unlock_irq(&ctx->wqh.lock); -- cpu_relax(); -+ if (isalarm(ctx)) -+ hrtimer_wait_for_timer(&ctx->t.alarm.timer); -+ else -+ hrtimer_wait_for_timer(&ctx->t.tmr); - } - - /* -diff -Nur linux-4.1.6.orig/include/acpi/platform/aclinux.h linux-4.1.6/include/acpi/platform/aclinux.h ---- linux-4.1.6.orig/include/acpi/platform/aclinux.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/acpi/platform/aclinux.h 2015-09-08 23:49:06.322065194 +0200 -@@ -123,6 +123,7 @@ - - #define acpi_cache_t struct kmem_cache - #define acpi_spinlock spinlock_t * -+#define acpi_raw_spinlock raw_spinlock_t * - #define acpi_cpu_flags unsigned long - - /* Use native linux version of acpi_os_allocate_zeroed */ -@@ -141,6 +142,20 @@ - #define ACPI_USE_ALTERNATE_PROTOTYPE_acpi_os_get_thread_id - #define ACPI_USE_ALTERNATE_PROTOTYPE_acpi_os_create_lock - -+#define acpi_os_create_raw_lock(__handle) \ -+({ \ -+ raw_spinlock_t *lock = ACPI_ALLOCATE(sizeof(*lock)); \ -+ \ -+ if (lock) { \ -+ *(__handle) = lock; \ -+ raw_spin_lock_init(*(__handle)); \ -+ } \ -+ lock ? AE_OK : AE_NO_MEMORY; \ -+ }) + -+#define acpi_os_delete_raw_lock(__handle) kfree(__handle) ++ return 0; ++} + ++/** ++ * handle_stripe_fill - read or compute data to satisfy pending requests. ++ */ ++static void handle_stripe_fill(struct stripe_head *sh, ++ struct stripe_head_state *s, ++ int disks) ++{ ++ int i; + - /* - * OSL interfaces used by debugger/disassembler - */ -diff -Nur linux-4.1.6.orig/include/asm-generic/bug.h linux-4.1.6/include/asm-generic/bug.h ---- linux-4.1.6.orig/include/asm-generic/bug.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/asm-generic/bug.h 2015-09-08 23:49:06.322065194 +0200 -@@ -206,6 +206,20 @@ - # define WARN_ON_SMP(x) ({0;}) - #endif - -+#ifdef CONFIG_PREEMPT_RT_BASE -+# define BUG_ON_RT(c) BUG_ON(c) -+# define BUG_ON_NONRT(c) do { } while (0) -+# define WARN_ON_RT(condition) WARN_ON(condition) -+# define WARN_ON_NONRT(condition) do { } while (0) -+# define WARN_ON_ONCE_NONRT(condition) do { } while (0) -+#else -+# define BUG_ON_RT(c) do { } while (0) -+# define BUG_ON_NONRT(c) BUG_ON(c) -+# define WARN_ON_RT(condition) do { } while (0) -+# define WARN_ON_NONRT(condition) WARN_ON(condition) -+# define WARN_ON_ONCE_NONRT(condition) WARN_ON_ONCE(condition) -+#endif ++ /* look for blocks to read/compute, skip this if a compute ++ * is already in flight, or if the stripe contents are in the ++ * midst of changing due to a write ++ */ ++ if (!test_bit(STRIPE_COMPUTE_RUN, &sh->state) && !sh->check_state && ++ !sh->reconstruct_state) ++ for (i = disks; i--; ) ++ if (fetch_block(sh, s, i, disks)) ++ break; ++ set_bit(STRIPE_HANDLE, &sh->state); ++} + - #endif /* __ASSEMBLY__ */ - - #endif -diff -Nur linux-4.1.6.orig/include/asm-generic/futex.h linux-4.1.6/include/asm-generic/futex.h ---- linux-4.1.6.orig/include/asm-generic/futex.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/asm-generic/futex.h 2015-09-08 23:49:06.322065194 +0200 -@@ -8,8 +8,7 @@ - #ifndef CONFIG_SMP - /* - * The following implementation only for uniprocessor machines. -- * For UP, it's relies on the fact that pagefault_disable() also disables -- * preemption to ensure mutual exclusion. -+ * It relies on preempt_disable() ensuring mutual exclusion. - * - */ - -@@ -38,6 +37,7 @@ - if (encoded_op & (FUTEX_OP_OPARG_SHIFT << 28)) - oparg = 1 << oparg; - -+ preempt_disable(); - pagefault_disable(); - - ret = -EFAULT; -@@ -72,6 +72,7 @@ - - out_pagefault_enable: - pagefault_enable(); -+ preempt_enable(); - - if (ret == 0) { - switch (cmp) { -@@ -106,6 +107,7 @@ - { - u32 val; - -+ preempt_disable(); - if (unlikely(get_user(val, uaddr) != 0)) - return -EFAULT; - -@@ -113,6 +115,7 @@ - return -EFAULT; - - *uval = val; -+ preempt_enable(); - - return 0; - } -diff -Nur linux-4.1.6.orig/include/linux/blkdev.h linux-4.1.6/include/linux/blkdev.h ---- linux-4.1.6.orig/include/linux/blkdev.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/blkdev.h 2015-09-08 23:49:06.530042129 +0200 -@@ -101,6 +101,7 @@ - struct list_head queuelist; - union { - struct call_single_data csd; -+ struct work_struct work; - unsigned long fifo_time; - }; - -@@ -482,7 +483,7 @@ - struct throtl_data *td; - #endif - struct rcu_head rcu_head; -- wait_queue_head_t mq_freeze_wq; -+ struct swait_head mq_freeze_wq; - struct percpu_ref mq_usage_counter; - struct list_head all_q_node; - -diff -Nur linux-4.1.6.orig/include/linux/blk-mq.h linux-4.1.6/include/linux/blk-mq.h ---- linux-4.1.6.orig/include/linux/blk-mq.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/blk-mq.h 2015-09-08 23:49:06.322065194 +0200 -@@ -202,6 +202,7 @@ - - struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *, const int ctx_index); - struct blk_mq_hw_ctx *blk_mq_alloc_single_hw_queue(struct blk_mq_tag_set *, unsigned int, int); -+void __blk_mq_complete_request_remote_work(struct work_struct *work); - - int blk_mq_request_started(struct request *rq); - void blk_mq_start_request(struct request *rq); -diff -Nur linux-4.1.6.orig/include/linux/bottom_half.h linux-4.1.6/include/linux/bottom_half.h ---- linux-4.1.6.orig/include/linux/bottom_half.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/bottom_half.h 2015-09-08 23:49:06.530042129 +0200 -@@ -4,6 +4,17 @@ - #include - #include - -+#ifdef CONFIG_PREEMPT_RT_FULL ++static void break_stripe_batch_list(struct stripe_head *head_sh, ++ unsigned long handle_flags); ++/* handle_stripe_clean_event ++ * any written block on an uptodate or failed drive can be returned. ++ * Note that if we 'wrote' to a failed drive, it will be UPTODATE, but ++ * never LOCKED, so we don't need to test 'failed' directly. ++ */ ++static void handle_stripe_clean_event(struct r5conf *conf, ++ struct stripe_head *sh, int disks, struct bio **return_bi) ++{ ++ int i; ++ struct r5dev *dev; ++ int discard_pending = 0; ++ struct stripe_head *head_sh = sh; ++ bool do_endio = false; ++ ++ for (i = disks; i--; ) ++ if (sh->dev[i].written) { ++ dev = &sh->dev[i]; ++ if (!test_bit(R5_LOCKED, &dev->flags) && ++ (test_bit(R5_UPTODATE, &dev->flags) || ++ test_bit(R5_Discard, &dev->flags) || ++ test_bit(R5_SkipCopy, &dev->flags))) { ++ /* We can return any write requests */ ++ struct bio *wbi, *wbi2; ++ pr_debug("Return write for disc %d\n", i); ++ if (test_and_clear_bit(R5_Discard, &dev->flags)) ++ clear_bit(R5_UPTODATE, &dev->flags); ++ if (test_and_clear_bit(R5_SkipCopy, &dev->flags)) { ++ WARN_ON(test_bit(R5_UPTODATE, &dev->flags)); ++ } ++ do_endio = true; ++ ++returnbi: ++ dev->page = dev->orig_page; ++ wbi = dev->written; ++ dev->written = NULL; ++ while (wbi && wbi->bi_iter.bi_sector < ++ dev->sector + STRIPE_SECTORS) { ++ wbi2 = r5_next_bio(wbi, dev->sector); ++ if (!raid5_dec_bi_active_stripes(wbi)) { ++ md_write_end(conf->mddev); ++ wbi->bi_next = *return_bi; ++ *return_bi = wbi; ++ } ++ wbi = wbi2; ++ } ++ bitmap_endwrite(conf->mddev->bitmap, sh->sector, ++ STRIPE_SECTORS, ++ !test_bit(STRIPE_DEGRADED, &sh->state), ++ 0); ++ if (head_sh->batch_head) { ++ sh = list_first_entry(&sh->batch_list, ++ struct stripe_head, ++ batch_list); ++ if (sh != head_sh) { ++ dev = &sh->dev[i]; ++ goto returnbi; ++ } ++ } ++ sh = head_sh; ++ dev = &sh->dev[i]; ++ } else if (test_bit(R5_Discard, &dev->flags)) ++ discard_pending = 1; ++ WARN_ON(test_bit(R5_SkipCopy, &dev->flags)); ++ WARN_ON(dev->page != dev->orig_page); ++ } ++ if (!discard_pending && ++ test_bit(R5_Discard, &sh->dev[sh->pd_idx].flags)) { ++ clear_bit(R5_Discard, &sh->dev[sh->pd_idx].flags); ++ clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags); ++ if (sh->qd_idx >= 0) { ++ clear_bit(R5_Discard, &sh->dev[sh->qd_idx].flags); ++ clear_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags); ++ } ++ /* now that discard is done we can proceed with any sync */ ++ clear_bit(STRIPE_DISCARD, &sh->state); ++ /* ++ * SCSI discard will change some bio fields and the stripe has ++ * no updated data, so remove it from hash list and the stripe ++ * will be reinitialized ++ */ ++ spin_lock_irq(&conf->device_lock); ++unhash: ++ remove_hash(sh); ++ if (head_sh->batch_head) { ++ sh = list_first_entry(&sh->batch_list, ++ struct stripe_head, batch_list); ++ if (sh != head_sh) ++ goto unhash; ++ } ++ spin_unlock_irq(&conf->device_lock); ++ sh = head_sh; + -+extern void local_bh_disable(void); -+extern void _local_bh_enable(void); -+extern void local_bh_enable(void); -+extern void local_bh_enable_ip(unsigned long ip); -+extern void __local_bh_disable_ip(unsigned long ip, unsigned int cnt); -+extern void __local_bh_enable_ip(unsigned long ip, unsigned int cnt); ++ if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state)) ++ set_bit(STRIPE_HANDLE, &sh->state); + -+#else ++ } + - #ifdef CONFIG_TRACE_IRQFLAGS - extern void __local_bh_disable_ip(unsigned long ip, unsigned int cnt); - #else -@@ -31,5 +42,6 @@ - { - __local_bh_enable_ip(_THIS_IP_, SOFTIRQ_DISABLE_OFFSET); - } -+#endif - - #endif /* _LINUX_BH_H */ -diff -Nur linux-4.1.6.orig/include/linux/buffer_head.h linux-4.1.6/include/linux/buffer_head.h ---- linux-4.1.6.orig/include/linux/buffer_head.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/buffer_head.h 2015-09-08 23:49:06.530042129 +0200 -@@ -75,8 +75,52 @@ - struct address_space *b_assoc_map; /* mapping this buffer is - associated with */ - atomic_t b_count; /* users using this buffer_head */ -+#ifdef CONFIG_PREEMPT_RT_BASE -+ spinlock_t b_uptodate_lock; -+#if defined(CONFIG_JBD) || defined(CONFIG_JBD_MODULE) || \ -+ defined(CONFIG_JBD2) || defined(CONFIG_JBD2_MODULE) -+ spinlock_t b_state_lock; -+ spinlock_t b_journal_head_lock; -+#endif -+#endif - }; - -+static inline unsigned long bh_uptodate_lock_irqsave(struct buffer_head *bh) -+{ -+ unsigned long flags; ++ if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state)) ++ if (atomic_dec_and_test(&conf->pending_full_writes)) ++ md_wakeup_thread(conf->mddev->thread); + -+#ifndef CONFIG_PREEMPT_RT_BASE -+ local_irq_save(flags); -+ bit_spin_lock(BH_Uptodate_Lock, &bh->b_state); -+#else -+ spin_lock_irqsave(&bh->b_uptodate_lock, flags); -+#endif -+ return flags; ++ if (head_sh->batch_head && do_endio) ++ break_stripe_batch_list(head_sh, STRIPE_EXPAND_SYNC_FLAGS); +} + -+static inline void -+bh_uptodate_unlock_irqrestore(struct buffer_head *bh, unsigned long flags) ++static void handle_stripe_dirtying(struct r5conf *conf, ++ struct stripe_head *sh, ++ struct stripe_head_state *s, ++ int disks) +{ -+#ifndef CONFIG_PREEMPT_RT_BASE -+ bit_spin_unlock(BH_Uptodate_Lock, &bh->b_state); -+ local_irq_restore(flags); -+#else -+ spin_unlock_irqrestore(&bh->b_uptodate_lock, flags); -+#endif ++ int rmw = 0, rcw = 0, i; ++ sector_t recovery_cp = conf->mddev->recovery_cp; ++ ++ /* Check whether resync is now happening or should start. ++ * If yes, then the array is dirty (after unclean shutdown or ++ * initial creation), so parity in some stripes might be inconsistent. ++ * In this case, we need to always do reconstruct-write, to ensure ++ * that in case of drive failure or read-error correction, we ++ * generate correct data from the parity. ++ */ ++ if (conf->rmw_level == PARITY_DISABLE_RMW || ++ (recovery_cp < MaxSector && sh->sector >= recovery_cp && ++ s->failed == 0)) { ++ /* Calculate the real rcw later - for now make it ++ * look like rcw is cheaper ++ */ ++ rcw = 1; rmw = 2; ++ pr_debug("force RCW rmw_level=%u, recovery_cp=%llu sh->sector=%llu\n", ++ conf->rmw_level, (unsigned long long)recovery_cp, ++ (unsigned long long)sh->sector); ++ } else for (i = disks; i--; ) { ++ /* would I have to read this buffer for read_modify_write */ ++ struct r5dev *dev = &sh->dev[i]; ++ if ((dev->towrite || i == sh->pd_idx || i == sh->qd_idx) && ++ !test_bit(R5_LOCKED, &dev->flags) && ++ !(test_bit(R5_UPTODATE, &dev->flags) || ++ test_bit(R5_Wantcompute, &dev->flags))) { ++ if (test_bit(R5_Insync, &dev->flags)) ++ rmw++; ++ else ++ rmw += 2*disks; /* cannot read it */ ++ } ++ /* Would I have to read this buffer for reconstruct_write */ ++ if (!test_bit(R5_OVERWRITE, &dev->flags) && ++ i != sh->pd_idx && i != sh->qd_idx && ++ !test_bit(R5_LOCKED, &dev->flags) && ++ !(test_bit(R5_UPTODATE, &dev->flags) || ++ test_bit(R5_Wantcompute, &dev->flags))) { ++ if (test_bit(R5_Insync, &dev->flags)) ++ rcw++; ++ else ++ rcw += 2*disks; ++ } ++ } ++ pr_debug("for sector %llu, rmw=%d rcw=%d\n", ++ (unsigned long long)sh->sector, rmw, rcw); ++ set_bit(STRIPE_HANDLE, &sh->state); ++ if ((rmw < rcw || (rmw == rcw && conf->rmw_level == PARITY_ENABLE_RMW)) && rmw > 0) { ++ /* prefer read-modify-write, but need to get some data */ ++ if (conf->mddev->queue) ++ blk_add_trace_msg(conf->mddev->queue, ++ "raid5 rmw %llu %d", ++ (unsigned long long)sh->sector, rmw); ++ for (i = disks; i--; ) { ++ struct r5dev *dev = &sh->dev[i]; ++ if ((dev->towrite || i == sh->pd_idx || i == sh->qd_idx) && ++ !test_bit(R5_LOCKED, &dev->flags) && ++ !(test_bit(R5_UPTODATE, &dev->flags) || ++ test_bit(R5_Wantcompute, &dev->flags)) && ++ test_bit(R5_Insync, &dev->flags)) { ++ if (test_bit(STRIPE_PREREAD_ACTIVE, ++ &sh->state)) { ++ pr_debug("Read_old block %d for r-m-w\n", ++ i); ++ set_bit(R5_LOCKED, &dev->flags); ++ set_bit(R5_Wantread, &dev->flags); ++ s->locked++; ++ } else { ++ set_bit(STRIPE_DELAYED, &sh->state); ++ set_bit(STRIPE_HANDLE, &sh->state); ++ } ++ } ++ } ++ } ++ if ((rcw < rmw || (rcw == rmw && conf->rmw_level != PARITY_ENABLE_RMW)) && rcw > 0) { ++ /* want reconstruct write, but need to get some data */ ++ int qread =0; ++ rcw = 0; ++ for (i = disks; i--; ) { ++ struct r5dev *dev = &sh->dev[i]; ++ if (!test_bit(R5_OVERWRITE, &dev->flags) && ++ i != sh->pd_idx && i != sh->qd_idx && ++ !test_bit(R5_LOCKED, &dev->flags) && ++ !(test_bit(R5_UPTODATE, &dev->flags) || ++ test_bit(R5_Wantcompute, &dev->flags))) { ++ rcw++; ++ if (test_bit(R5_Insync, &dev->flags) && ++ test_bit(STRIPE_PREREAD_ACTIVE, ++ &sh->state)) { ++ pr_debug("Read_old block " ++ "%d for Reconstruct\n", i); ++ set_bit(R5_LOCKED, &dev->flags); ++ set_bit(R5_Wantread, &dev->flags); ++ s->locked++; ++ qread++; ++ } else { ++ set_bit(STRIPE_DELAYED, &sh->state); ++ set_bit(STRIPE_HANDLE, &sh->state); ++ } ++ } ++ } ++ if (rcw && conf->mddev->queue) ++ blk_add_trace_msg(conf->mddev->queue, "raid5 rcw %llu %d %d %d", ++ (unsigned long long)sh->sector, ++ rcw, qread, test_bit(STRIPE_DELAYED, &sh->state)); ++ } ++ ++ if (rcw > disks && rmw > disks && ++ !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) ++ set_bit(STRIPE_DELAYED, &sh->state); ++ ++ /* now if nothing is locked, and if we have enough data, ++ * we can start a write request ++ */ ++ /* since handle_stripe can be called at any time we need to handle the ++ * case where a compute block operation has been submitted and then a ++ * subsequent call wants to start a write request. raid_run_ops only ++ * handles the case where compute block and reconstruct are requested ++ * simultaneously. If this is not the case then new writes need to be ++ * held off until the compute completes. ++ */ ++ if ((s->req_compute || !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) && ++ (s->locked == 0 && (rcw == 0 || rmw == 0) && ++ !test_bit(STRIPE_BIT_DELAY, &sh->state))) ++ schedule_reconstruction(sh, s, rcw == 0, 0); +} + -+static inline void buffer_head_init_locks(struct buffer_head *bh) ++static void handle_parity_checks5(struct r5conf *conf, struct stripe_head *sh, ++ struct stripe_head_state *s, int disks) +{ -+#ifdef CONFIG_PREEMPT_RT_BASE -+ spin_lock_init(&bh->b_uptodate_lock); -+#if defined(CONFIG_JBD) || defined(CONFIG_JBD_MODULE) || \ -+ defined(CONFIG_JBD2) || defined(CONFIG_JBD2_MODULE) -+ spin_lock_init(&bh->b_state_lock); -+ spin_lock_init(&bh->b_journal_head_lock); -+#endif -+#endif ++ struct r5dev *dev = NULL; ++ ++ BUG_ON(sh->batch_head); ++ set_bit(STRIPE_HANDLE, &sh->state); ++ ++ switch (sh->check_state) { ++ case check_state_idle: ++ /* start a new check operation if there are no failures */ ++ if (s->failed == 0) { ++ BUG_ON(s->uptodate != disks); ++ sh->check_state = check_state_run; ++ set_bit(STRIPE_OP_CHECK, &s->ops_request); ++ clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags); ++ s->uptodate--; ++ break; ++ } ++ dev = &sh->dev[s->failed_num[0]]; ++ /* fall through */ ++ case check_state_compute_result: ++ sh->check_state = check_state_idle; ++ if (!dev) ++ dev = &sh->dev[sh->pd_idx]; ++ ++ /* check that a write has not made the stripe insync */ ++ if (test_bit(STRIPE_INSYNC, &sh->state)) ++ break; ++ ++ /* either failed parity check, or recovery is happening */ ++ BUG_ON(!test_bit(R5_UPTODATE, &dev->flags)); ++ BUG_ON(s->uptodate != disks); ++ ++ set_bit(R5_LOCKED, &dev->flags); ++ s->locked++; ++ set_bit(R5_Wantwrite, &dev->flags); ++ ++ clear_bit(STRIPE_DEGRADED, &sh->state); ++ set_bit(STRIPE_INSYNC, &sh->state); ++ break; ++ case check_state_run: ++ break; /* we will be called again upon completion */ ++ case check_state_check_result: ++ sh->check_state = check_state_idle; ++ ++ /* if a failure occurred during the check operation, leave ++ * STRIPE_INSYNC not set and let the stripe be handled again ++ */ ++ if (s->failed) ++ break; ++ ++ /* handle a successful check operation, if parity is correct ++ * we are done. Otherwise update the mismatch count and repair ++ * parity if !MD_RECOVERY_CHECK ++ */ ++ if ((sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) == 0) ++ /* parity is correct (on disc, ++ * not in buffer any more) ++ */ ++ set_bit(STRIPE_INSYNC, &sh->state); ++ else { ++ atomic64_add(STRIPE_SECTORS, &conf->mddev->resync_mismatches); ++ if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery)) ++ /* don't try to repair!! */ ++ set_bit(STRIPE_INSYNC, &sh->state); ++ else { ++ sh->check_state = check_state_compute_run; ++ set_bit(STRIPE_COMPUTE_RUN, &sh->state); ++ set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request); ++ set_bit(R5_Wantcompute, ++ &sh->dev[sh->pd_idx].flags); ++ sh->ops.target = sh->pd_idx; ++ sh->ops.target2 = -1; ++ s->uptodate++; ++ } ++ } ++ break; ++ case check_state_compute_run: ++ break; ++ default: ++ printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n", ++ __func__, sh->check_state, ++ (unsigned long long) sh->sector); ++ BUG(); ++ } +} + - /* - * macro tricks to expand the set_buffer_foo(), clear_buffer_foo() - * and buffer_foo() functions. -diff -Nur linux-4.1.6.orig/include/linux/cgroup.h linux-4.1.6/include/linux/cgroup.h ---- linux-4.1.6.orig/include/linux/cgroup.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/cgroup.h 2015-09-08 23:49:06.534041686 +0200 -@@ -22,6 +22,7 @@ - #include - #include - #include -+#include - - #ifdef CONFIG_CGROUPS - -@@ -91,6 +92,7 @@ - /* percpu_ref killing and RCU release */ - struct rcu_head rcu_head; - struct work_struct destroy_work; -+ struct swork_event destroy_swork; - }; - - /* bits in struct cgroup_subsys_state flags field */ -diff -Nur linux-4.1.6.orig/include/linux/completion.h linux-4.1.6/include/linux/completion.h ---- linux-4.1.6.orig/include/linux/completion.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/completion.h 2015-09-08 23:49:06.534041686 +0200 -@@ -7,8 +7,7 @@ - * Atomic wait-for-completion handler data structures. - * See kernel/sched/completion.c for details. - */ -- --#include -+#include - - /* - * struct completion - structure used to maintain state for a "completion" -@@ -24,11 +23,11 @@ - */ - struct completion { - unsigned int done; -- wait_queue_head_t wait; -+ struct swait_head wait; - }; - - #define COMPLETION_INITIALIZER(work) \ -- { 0, __WAIT_QUEUE_HEAD_INITIALIZER((work).wait) } -+ { 0, SWAIT_HEAD_INITIALIZER((work).wait) } - - #define COMPLETION_INITIALIZER_ONSTACK(work) \ - ({ init_completion(&work); work; }) -@@ -73,7 +72,7 @@ - static inline void init_completion(struct completion *x) - { - x->done = 0; -- init_waitqueue_head(&x->wait); -+ init_swait_head(&x->wait); - } - - /** -diff -Nur linux-4.1.6.orig/include/linux/cpu.h linux-4.1.6/include/linux/cpu.h ---- linux-4.1.6.orig/include/linux/cpu.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/cpu.h 2015-09-08 23:49:06.534041686 +0200 -@@ -231,6 +231,8 @@ - extern void put_online_cpus(void); - extern void cpu_hotplug_disable(void); - extern void cpu_hotplug_enable(void); -+extern void pin_current_cpu(void); -+extern void unpin_current_cpu(void); - #define hotcpu_notifier(fn, pri) cpu_notifier(fn, pri) - #define __hotcpu_notifier(fn, pri) __cpu_notifier(fn, pri) - #define register_hotcpu_notifier(nb) register_cpu_notifier(nb) -@@ -249,6 +251,8 @@ - #define put_online_cpus() do { } while (0) - #define cpu_hotplug_disable() do { } while (0) - #define cpu_hotplug_enable() do { } while (0) -+static inline void pin_current_cpu(void) { } -+static inline void unpin_current_cpu(void) { } - #define hotcpu_notifier(fn, pri) do { (void)(fn); } while (0) - #define __hotcpu_notifier(fn, pri) do { (void)(fn); } while (0) - /* These aren't inline functions due to a GCC bug. */ -diff -Nur linux-4.1.6.orig/include/linux/delay.h linux-4.1.6/include/linux/delay.h ---- linux-4.1.6.orig/include/linux/delay.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/delay.h 2015-09-08 23:49:06.534041686 +0200 -@@ -52,4 +52,10 @@ - msleep(seconds * 1000); - } - -+#ifdef CONFIG_PREEMPT_RT_FULL -+extern void cpu_chill(void); -+#else -+# define cpu_chill() cpu_relax() -+#endif ++static void handle_parity_checks6(struct r5conf *conf, struct stripe_head *sh, ++ struct stripe_head_state *s, ++ int disks) ++{ ++ int pd_idx = sh->pd_idx; ++ int qd_idx = sh->qd_idx; ++ struct r5dev *dev; + - #endif /* defined(_LINUX_DELAY_H) */ -diff -Nur linux-4.1.6.orig/include/linux/ftrace_event.h linux-4.1.6/include/linux/ftrace_event.h ---- linux-4.1.6.orig/include/linux/ftrace_event.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/ftrace_event.h 2015-09-08 23:49:06.534041686 +0200 -@@ -66,6 +66,9 @@ - unsigned char flags; - unsigned char preempt_count; - int pid; -+ unsigned short migrate_disable; -+ unsigned short padding; -+ unsigned char preempt_lazy_count; - }; - - #define FTRACE_MAX_EVENT \ -diff -Nur linux-4.1.6.orig/include/linux/highmem.h linux-4.1.6/include/linux/highmem.h ---- linux-4.1.6.orig/include/linux/highmem.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/highmem.h 2015-09-08 23:49:06.534041686 +0200 -@@ -7,6 +7,7 @@ - #include - #include - #include -+#include - - #include - -@@ -65,6 +66,7 @@ - - static inline void *kmap_atomic(struct page *page) - { -+ preempt_disable(); - pagefault_disable(); - return page_address(page); - } -@@ -73,6 +75,7 @@ - static inline void __kunmap_atomic(void *addr) - { - pagefault_enable(); -+ preempt_enable(); - } - - #define kmap_atomic_pfn(pfn) kmap_atomic(pfn_to_page(pfn)) -@@ -85,32 +88,51 @@ - - #if defined(CONFIG_HIGHMEM) || defined(CONFIG_X86_32) - -+#ifndef CONFIG_PREEMPT_RT_FULL - DECLARE_PER_CPU(int, __kmap_atomic_idx); -+#endif - - static inline int kmap_atomic_idx_push(void) - { -+#ifndef CONFIG_PREEMPT_RT_FULL - int idx = __this_cpu_inc_return(__kmap_atomic_idx) - 1; - --#ifdef CONFIG_DEBUG_HIGHMEM -+# ifdef CONFIG_DEBUG_HIGHMEM - WARN_ON_ONCE(in_irq() && !irqs_disabled()); - BUG_ON(idx >= KM_TYPE_NR); --#endif -+# endif - return idx; -+#else -+ current->kmap_idx++; -+ BUG_ON(current->kmap_idx > KM_TYPE_NR); -+ return current->kmap_idx - 1; -+#endif - } - - static inline int kmap_atomic_idx(void) - { -+#ifndef CONFIG_PREEMPT_RT_FULL - return __this_cpu_read(__kmap_atomic_idx) - 1; -+#else -+ return current->kmap_idx - 1; -+#endif - } - - static inline void kmap_atomic_idx_pop(void) - { --#ifdef CONFIG_DEBUG_HIGHMEM -+#ifndef CONFIG_PREEMPT_RT_FULL -+# ifdef CONFIG_DEBUG_HIGHMEM - int idx = __this_cpu_dec_return(__kmap_atomic_idx); - - BUG_ON(idx < 0); --#else -+# else - __this_cpu_dec(__kmap_atomic_idx); -+# endif -+#else -+ current->kmap_idx--; -+# ifdef CONFIG_DEBUG_HIGHMEM -+ BUG_ON(current->kmap_idx < 0); -+# endif - #endif - } - -diff -Nur linux-4.1.6.orig/include/linux/hrtimer.h linux-4.1.6/include/linux/hrtimer.h ---- linux-4.1.6.orig/include/linux/hrtimer.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/hrtimer.h 2015-09-08 23:49:06.534041686 +0200 -@@ -111,6 +111,11 @@ - enum hrtimer_restart (*function)(struct hrtimer *); - struct hrtimer_clock_base *base; - unsigned long state; -+ struct list_head cb_entry; -+ int irqsafe; -+#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST -+ ktime_t praecox; -+#endif - #ifdef CONFIG_TIMER_STATS - int start_pid; - void *start_site; -@@ -147,6 +152,7 @@ - int index; - clockid_t clockid; - struct timerqueue_head active; -+ struct list_head expired; - ktime_t resolution; - ktime_t (*get_time)(void); - ktime_t softirq_time; -@@ -194,6 +200,9 @@ - unsigned long nr_hangs; - ktime_t max_hang_time; - #endif -+#ifdef CONFIG_PREEMPT_RT_BASE -+ wait_queue_head_t wait; -+#endif - struct hrtimer_clock_base clock_base[HRTIMER_MAX_CLOCK_BASES]; - }; - -@@ -381,6 +390,13 @@ - return hrtimer_start_expires(timer, HRTIMER_MODE_ABS); - } - -+/* Softirq preemption could deadlock timer removal */ -+#ifdef CONFIG_PREEMPT_RT_BASE -+ extern void hrtimer_wait_for_timer(const struct hrtimer *timer); -+#else -+# define hrtimer_wait_for_timer(timer) do { cpu_relax(); } while (0) -+#endif ++ BUG_ON(sh->batch_head); ++ set_bit(STRIPE_HANDLE, &sh->state); + - /* Query timers: */ - extern ktime_t hrtimer_get_remaining(const struct hrtimer *timer); - extern int hrtimer_get_res(const clockid_t which_clock, struct timespec *tp); -diff -Nur linux-4.1.6.orig/include/linux/idr.h linux-4.1.6/include/linux/idr.h ---- linux-4.1.6.orig/include/linux/idr.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/idr.h 2015-09-08 23:49:06.534041686 +0200 -@@ -95,10 +95,14 @@ - * Each idr_preload() should be matched with an invocation of this - * function. See idr_preload() for details. - */ -+#ifdef CONFIG_PREEMPT_RT_FULL -+void idr_preload_end(void); -+#else - static inline void idr_preload_end(void) - { - preempt_enable(); - } -+#endif - - /** - * idr_find - return pointer for given id -diff -Nur linux-4.1.6.orig/include/linux/init_task.h linux-4.1.6/include/linux/init_task.h ---- linux-4.1.6.orig/include/linux/init_task.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/init_task.h 2015-09-08 23:49:06.542040801 +0200 -@@ -147,9 +147,16 @@ - # define INIT_PERF_EVENTS(tsk) - #endif - -+#ifdef CONFIG_PREEMPT_RT_BASE -+# define INIT_TIMER_LIST .posix_timer_list = NULL, -+#else -+# define INIT_TIMER_LIST -+#endif ++ BUG_ON(s->failed > 2); + - #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN - # define INIT_VTIME(tsk) \ -- .vtime_seqlock = __SEQLOCK_UNLOCKED(tsk.vtime_seqlock), \ -+ .vtime_lock = __RAW_SPIN_LOCK_UNLOCKED(tsk.vtime_lock), \ -+ .vtime_seq = SEQCNT_ZERO(tsk.vtime_seq), \ - .vtime_snap = 0, \ - .vtime_snap_whence = VTIME_SYS, - #else -@@ -238,6 +245,7 @@ - .cpu_timers = INIT_CPU_TIMERS(tsk.cpu_timers), \ - .pi_lock = __RAW_SPIN_LOCK_UNLOCKED(tsk.pi_lock), \ - .timer_slack_ns = 50000, /* 50 usec default slack */ \ -+ INIT_TIMER_LIST \ - .pids = { \ - [PIDTYPE_PID] = INIT_PID_LINK(PIDTYPE_PID), \ - [PIDTYPE_PGID] = INIT_PID_LINK(PIDTYPE_PGID), \ -diff -Nur linux-4.1.6.orig/include/linux/interrupt.h linux-4.1.6/include/linux/interrupt.h ---- linux-4.1.6.orig/include/linux/interrupt.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/interrupt.h 2015-09-08 23:49:06.542040801 +0200 -@@ -61,6 +61,7 @@ - * interrupt handler after suspending interrupts. For system - * wakeup devices users need to implement wakeup detection in - * their interrupt handlers. -+ * IRQF_NO_SOFTIRQ_CALL - Do not process softirqs in the irq thread context (RT) - */ - #define IRQF_SHARED 0x00000080 - #define IRQF_PROBE_SHARED 0x00000100 -@@ -74,6 +75,7 @@ - #define IRQF_NO_THREAD 0x00010000 - #define IRQF_EARLY_RESUME 0x00020000 - #define IRQF_COND_SUSPEND 0x00040000 -+#define IRQF_NO_SOFTIRQ_CALL 0x00080000 - - #define IRQF_TIMER (__IRQF_TIMER | IRQF_NO_SUSPEND | IRQF_NO_THREAD) - -@@ -184,7 +186,7 @@ - #ifdef CONFIG_LOCKDEP - # define local_irq_enable_in_hardirq() do { } while (0) - #else --# define local_irq_enable_in_hardirq() local_irq_enable() -+# define local_irq_enable_in_hardirq() local_irq_enable_nort() - #endif - - extern void disable_irq_nosync(unsigned int irq); -@@ -215,6 +217,7 @@ - unsigned int irq; - struct kref kref; - struct work_struct work; -+ struct list_head list; - void (*notify)(struct irq_affinity_notify *, const cpumask_t *mask); - void (*release)(struct kref *ref); - }; -@@ -377,9 +380,13 @@ - bool state); - - #ifdef CONFIG_IRQ_FORCED_THREADING -+# ifndef CONFIG_PREEMPT_RT_BASE - extern bool force_irqthreads; -+# else -+# define force_irqthreads (true) -+# endif - #else --#define force_irqthreads (0) -+#define force_irqthreads (false) - #endif - - #ifndef __ARCH_SET_SOFTIRQ_PENDING -@@ -435,9 +442,10 @@ - void (*action)(struct softirq_action *); - }; - -+#ifndef CONFIG_PREEMPT_RT_FULL - asmlinkage void do_softirq(void); - asmlinkage void __do_softirq(void); -- -+static inline void thread_do_softirq(void) { do_softirq(); } - #ifdef __ARCH_HAS_DO_SOFTIRQ - void do_softirq_own_stack(void); - #else -@@ -446,6 +454,9 @@ - __do_softirq(); - } - #endif -+#else -+extern void thread_do_softirq(void); -+#endif - - extern void open_softirq(int nr, void (*action)(struct softirq_action *)); - extern void softirq_init(void); -@@ -453,6 +464,7 @@ - - extern void raise_softirq_irqoff(unsigned int nr); - extern void raise_softirq(unsigned int nr); -+extern void softirq_check_pending_idle(void); - - DECLARE_PER_CPU(struct task_struct *, ksoftirqd); - -@@ -474,8 +486,9 @@ - to be executed on some cpu at least once after this. - * If the tasklet is already scheduled, but its execution is still not - started, it will be executed only once. -- * If this tasklet is already running on another CPU (or schedule is called -- from tasklet itself), it is rescheduled for later. -+ * If this tasklet is already running on another CPU, it is rescheduled -+ for later. -+ * Schedule must not be called from the tasklet itself (a lockup occurs) - * Tasklet is strictly serialized wrt itself, but not - wrt another tasklets. If client needs some intertask synchronization, - he makes it with spinlocks. -@@ -500,27 +513,36 @@ - enum - { - TASKLET_STATE_SCHED, /* Tasklet is scheduled for execution */ -- TASKLET_STATE_RUN /* Tasklet is running (SMP only) */ -+ TASKLET_STATE_RUN, /* Tasklet is running (SMP only) */ -+ TASKLET_STATE_PENDING /* Tasklet is pending */ - }; - --#ifdef CONFIG_SMP -+#define TASKLET_STATEF_SCHED (1 << TASKLET_STATE_SCHED) -+#define TASKLET_STATEF_RUN (1 << TASKLET_STATE_RUN) -+#define TASKLET_STATEF_PENDING (1 << TASKLET_STATE_PENDING) ++ /* Want to check and possibly repair P and Q. ++ * However there could be one 'failed' device, in which ++ * case we can only check one of them, possibly using the ++ * other to generate missing data ++ */ + -+#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT_FULL) - static inline int tasklet_trylock(struct tasklet_struct *t) - { - return !test_and_set_bit(TASKLET_STATE_RUN, &(t)->state); - } - -+static inline int tasklet_tryunlock(struct tasklet_struct *t) -+{ -+ return cmpxchg(&t->state, TASKLET_STATEF_RUN, 0) == TASKLET_STATEF_RUN; ++ switch (sh->check_state) { ++ case check_state_idle: ++ /* start a new check operation if there are < 2 failures */ ++ if (s->failed == s->q_failed) { ++ /* The only possible failed device holds Q, so it ++ * makes sense to check P (If anything else were failed, ++ * we would have used P to recreate it). ++ */ ++ sh->check_state = check_state_run; ++ } ++ if (!s->q_failed && s->failed < 2) { ++ /* Q is not failed, and we didn't use it to generate ++ * anything, so it makes sense to check it ++ */ ++ if (sh->check_state == check_state_run) ++ sh->check_state = check_state_run_pq; ++ else ++ sh->check_state = check_state_run_q; ++ } ++ ++ /* discard potentially stale zero_sum_result */ ++ sh->ops.zero_sum_result = 0; ++ ++ if (sh->check_state == check_state_run) { ++ /* async_xor_zero_sum destroys the contents of P */ ++ clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags); ++ s->uptodate--; ++ } ++ if (sh->check_state >= check_state_run && ++ sh->check_state <= check_state_run_pq) { ++ /* async_syndrome_zero_sum preserves P and Q, so ++ * no need to mark them !uptodate here ++ */ ++ set_bit(STRIPE_OP_CHECK, &s->ops_request); ++ break; ++ } ++ ++ /* we have 2-disk failure */ ++ BUG_ON(s->failed != 2); ++ /* fall through */ ++ case check_state_compute_result: ++ sh->check_state = check_state_idle; ++ ++ /* check that a write has not made the stripe insync */ ++ if (test_bit(STRIPE_INSYNC, &sh->state)) ++ break; ++ ++ /* now write out any block on a failed drive, ++ * or P or Q if they were recomputed ++ */ ++ BUG_ON(s->uptodate < disks - 1); /* We don't need Q to recover */ ++ if (s->failed == 2) { ++ dev = &sh->dev[s->failed_num[1]]; ++ s->locked++; ++ set_bit(R5_LOCKED, &dev->flags); ++ set_bit(R5_Wantwrite, &dev->flags); ++ } ++ if (s->failed >= 1) { ++ dev = &sh->dev[s->failed_num[0]]; ++ s->locked++; ++ set_bit(R5_LOCKED, &dev->flags); ++ set_bit(R5_Wantwrite, &dev->flags); ++ } ++ if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) { ++ dev = &sh->dev[pd_idx]; ++ s->locked++; ++ set_bit(R5_LOCKED, &dev->flags); ++ set_bit(R5_Wantwrite, &dev->flags); ++ } ++ if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) { ++ dev = &sh->dev[qd_idx]; ++ s->locked++; ++ set_bit(R5_LOCKED, &dev->flags); ++ set_bit(R5_Wantwrite, &dev->flags); ++ } ++ clear_bit(STRIPE_DEGRADED, &sh->state); ++ ++ set_bit(STRIPE_INSYNC, &sh->state); ++ break; ++ case check_state_run: ++ case check_state_run_q: ++ case check_state_run_pq: ++ break; /* we will be called again upon completion */ ++ case check_state_check_result: ++ sh->check_state = check_state_idle; ++ ++ /* handle a successful check operation, if parity is correct ++ * we are done. Otherwise update the mismatch count and repair ++ * parity if !MD_RECOVERY_CHECK ++ */ ++ if (sh->ops.zero_sum_result == 0) { ++ /* both parities are correct */ ++ if (!s->failed) ++ set_bit(STRIPE_INSYNC, &sh->state); ++ else { ++ /* in contrast to the raid5 case we can validate ++ * parity, but still have a failure to write ++ * back ++ */ ++ sh->check_state = check_state_compute_result; ++ /* Returning at this point means that we may go ++ * off and bring p and/or q uptodate again so ++ * we make sure to check zero_sum_result again ++ * to verify if p or q need writeback ++ */ ++ } ++ } else { ++ atomic64_add(STRIPE_SECTORS, &conf->mddev->resync_mismatches); ++ if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery)) ++ /* don't try to repair!! */ ++ set_bit(STRIPE_INSYNC, &sh->state); ++ else { ++ int *target = &sh->ops.target; ++ ++ sh->ops.target = -1; ++ sh->ops.target2 = -1; ++ sh->check_state = check_state_compute_run; ++ set_bit(STRIPE_COMPUTE_RUN, &sh->state); ++ set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request); ++ if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) { ++ set_bit(R5_Wantcompute, ++ &sh->dev[pd_idx].flags); ++ *target = pd_idx; ++ target = &sh->ops.target2; ++ s->uptodate++; ++ } ++ if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) { ++ set_bit(R5_Wantcompute, ++ &sh->dev[qd_idx].flags); ++ *target = qd_idx; ++ s->uptodate++; ++ } ++ } ++ } ++ break; ++ case check_state_compute_run: ++ break; ++ default: ++ printk(KERN_ERR "%s: unknown check_state: %d sector: %llu\n", ++ __func__, sh->check_state, ++ (unsigned long long) sh->sector); ++ BUG(); ++ } +} + - static inline void tasklet_unlock(struct tasklet_struct *t) - { - smp_mb__before_atomic(); - clear_bit(TASKLET_STATE_RUN, &(t)->state); - } - --static inline void tasklet_unlock_wait(struct tasklet_struct *t) --{ -- while (test_bit(TASKLET_STATE_RUN, &(t)->state)) { barrier(); } --} -+extern void tasklet_unlock_wait(struct tasklet_struct *t); ++static void handle_stripe_expansion(struct r5conf *conf, struct stripe_head *sh) ++{ ++ int i; + - #else - #define tasklet_trylock(t) 1 -+#define tasklet_tryunlock(t) 1 - #define tasklet_unlock_wait(t) do { } while (0) - #define tasklet_unlock(t) do { } while (0) - #endif -@@ -569,12 +591,7 @@ - smp_mb(); - } - --static inline void tasklet_enable(struct tasklet_struct *t) --{ -- smp_mb__before_atomic(); -- atomic_dec(&t->count); --} -- -+extern void tasklet_enable(struct tasklet_struct *t); - extern void tasklet_kill(struct tasklet_struct *t); - extern void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu); - extern void tasklet_init(struct tasklet_struct *t, -@@ -605,6 +622,12 @@ - tasklet_kill(&ttimer->tasklet); - } - -+#ifdef CONFIG_PREEMPT_RT_FULL -+extern void softirq_early_init(void); -+#else -+static inline void softirq_early_init(void) { } -+#endif ++ /* We have read all the blocks in this stripe and now we need to ++ * copy some of them into a target stripe for expand. ++ */ ++ struct dma_async_tx_descriptor *tx = NULL; ++ BUG_ON(sh->batch_head); ++ clear_bit(STRIPE_EXPAND_SOURCE, &sh->state); ++ for (i = 0; i < sh->disks; i++) ++ if (i != sh->pd_idx && i != sh->qd_idx) { ++ int dd_idx, j; ++ struct stripe_head *sh2; ++ struct async_submit_ctl submit; ++ ++ sector_t bn = compute_blocknr(sh, i, 1); ++ sector_t s = raid5_compute_sector(conf, bn, 0, ++ &dd_idx, NULL); ++ sh2 = get_active_stripe(conf, s, 0, 1, 1); ++ if (sh2 == NULL) ++ /* so far only the early blocks of this stripe ++ * have been requested. When later blocks ++ * get requested, we will try again ++ */ ++ continue; ++ if (!test_bit(STRIPE_EXPANDING, &sh2->state) || ++ test_bit(R5_Expanded, &sh2->dev[dd_idx].flags)) { ++ /* must have already done this block */ ++ release_stripe(sh2); ++ continue; ++ } + - /* - * Autoprobing for irqs: - * -diff -Nur linux-4.1.6.orig/include/linux/io-mapping.h linux-4.1.6/include/linux/io-mapping.h ---- linux-4.1.6.orig/include/linux/io-mapping.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/io-mapping.h 2015-09-08 23:49:06.542040801 +0200 -@@ -141,6 +141,7 @@ - io_mapping_map_atomic_wc(struct io_mapping *mapping, - unsigned long offset) - { -+ preempt_disable(); - pagefault_disable(); - return ((char __force __iomem *) mapping) + offset; - } -@@ -149,6 +150,7 @@ - io_mapping_unmap_atomic(void __iomem *vaddr) - { - pagefault_enable(); -+ preempt_enable(); - } - - /* Non-atomic map/unmap */ -diff -Nur linux-4.1.6.orig/include/linux/irqdesc.h linux-4.1.6/include/linux/irqdesc.h ---- linux-4.1.6.orig/include/linux/irqdesc.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/irqdesc.h 2015-09-08 23:49:06.546040357 +0200 -@@ -63,6 +63,7 @@ - unsigned int irqs_unhandled; - atomic_t threads_handled; - int threads_handled_last; -+ u64 random_ip; - raw_spinlock_t lock; - struct cpumask *percpu_enabled; - #ifdef CONFIG_SMP -diff -Nur linux-4.1.6.orig/include/linux/irqflags.h linux-4.1.6/include/linux/irqflags.h ---- linux-4.1.6.orig/include/linux/irqflags.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/irqflags.h 2015-09-08 23:49:06.546040357 +0200 -@@ -25,8 +25,6 @@ - # define trace_softirqs_enabled(p) ((p)->softirqs_enabled) - # define trace_hardirq_enter() do { current->hardirq_context++; } while (0) - # define trace_hardirq_exit() do { current->hardirq_context--; } while (0) --# define lockdep_softirq_enter() do { current->softirq_context++; } while (0) --# define lockdep_softirq_exit() do { current->softirq_context--; } while (0) - # define INIT_TRACE_IRQFLAGS .softirqs_enabled = 1, - #else - # define trace_hardirqs_on() do { } while (0) -@@ -39,9 +37,15 @@ - # define trace_softirqs_enabled(p) 0 - # define trace_hardirq_enter() do { } while (0) - # define trace_hardirq_exit() do { } while (0) -+# define INIT_TRACE_IRQFLAGS -+#endif ++ /* place all the copies on one channel */ ++ init_async_submit(&submit, 0, tx, NULL, NULL, NULL); ++ tx = async_memcpy(sh2->dev[dd_idx].page, ++ sh->dev[i].page, 0, 0, STRIPE_SIZE, ++ &submit); ++ ++ set_bit(R5_Expanded, &sh2->dev[dd_idx].flags); ++ set_bit(R5_UPTODATE, &sh2->dev[dd_idx].flags); ++ for (j = 0; j < conf->raid_disks; j++) ++ if (j != sh2->pd_idx && ++ j != sh2->qd_idx && ++ !test_bit(R5_Expanded, &sh2->dev[j].flags)) ++ break; ++ if (j == conf->raid_disks) { ++ set_bit(STRIPE_EXPAND_READY, &sh2->state); ++ set_bit(STRIPE_HANDLE, &sh2->state); ++ } ++ release_stripe(sh2); ++ ++ } ++ /* done submitting copies, wait for them to complete */ ++ async_tx_quiesce(&tx); ++} + -+#if defined(CONFIG_TRACE_IRQFLAGS) && !defined(CONFIG_PREEMPT_RT_FULL) -+# define lockdep_softirq_enter() do { current->softirq_context++; } while (0) -+# define lockdep_softirq_exit() do { current->softirq_context--; } while (0) -+#else - # define lockdep_softirq_enter() do { } while (0) - # define lockdep_softirq_exit() do { } while (0) --# define INIT_TRACE_IRQFLAGS - #endif - - #if defined(CONFIG_IRQSOFF_TRACER) || \ -@@ -148,4 +152,23 @@ - - #define irqs_disabled_flags(flags) raw_irqs_disabled_flags(flags) - +/* -+ * local_irq* variants depending on RT/!RT ++ * handle_stripe - do things to a stripe. ++ * ++ * We lock the stripe by setting STRIPE_ACTIVE and then examine the ++ * state of various bits to see what needs to be done. ++ * Possible results: ++ * return some read requests which now have data ++ * return some write requests which are safely on storage ++ * schedule a read on some buffers ++ * schedule a write of some buffers ++ * return confirmation of parity correctness ++ * + */ -+#ifdef CONFIG_PREEMPT_RT_FULL -+# define local_irq_disable_nort() do { } while (0) -+# define local_irq_enable_nort() do { } while (0) -+# define local_irq_save_nort(flags) local_save_flags(flags) -+# define local_irq_restore_nort(flags) (void)(flags) -+# define local_irq_disable_rt() local_irq_disable() -+# define local_irq_enable_rt() local_irq_enable() -+#else -+# define local_irq_disable_nort() local_irq_disable() -+# define local_irq_enable_nort() local_irq_enable() -+# define local_irq_save_nort(flags) local_irq_save(flags) -+# define local_irq_restore_nort(flags) local_irq_restore(flags) -+# define local_irq_disable_rt() do { } while (0) -+# define local_irq_enable_rt() do { } while (0) -+#endif + - #endif -diff -Nur linux-4.1.6.orig/include/linux/irq.h linux-4.1.6/include/linux/irq.h ---- linux-4.1.6.orig/include/linux/irq.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/irq.h 2015-09-08 23:49:06.542040801 +0200 -@@ -72,6 +72,7 @@ - * IRQ_IS_POLLED - Always polled by another interrupt. Exclude - * it from the spurious interrupt detection - * mechanism and from core side polling. -+ * IRQ_NO_SOFTIRQ_CALL - No softirq processing in the irq thread context (RT) - */ - enum { - IRQ_TYPE_NONE = 0x00000000, -@@ -97,13 +98,14 @@ - IRQ_NOTHREAD = (1 << 16), - IRQ_PER_CPU_DEVID = (1 << 17), - IRQ_IS_POLLED = (1 << 18), -+ IRQ_NO_SOFTIRQ_CALL = (1 << 19), - }; - - #define IRQF_MODIFY_MASK \ - (IRQ_TYPE_SENSE_MASK | IRQ_NOPROBE | IRQ_NOREQUEST | \ - IRQ_NOAUTOEN | IRQ_MOVE_PCNTXT | IRQ_LEVEL | IRQ_NO_BALANCING | \ - IRQ_PER_CPU | IRQ_NESTED_THREAD | IRQ_NOTHREAD | IRQ_PER_CPU_DEVID | \ -- IRQ_IS_POLLED) -+ IRQ_IS_POLLED | IRQ_NO_SOFTIRQ_CALL) - - #define IRQ_NO_BALANCING_MASK (IRQ_PER_CPU | IRQ_NO_BALANCING) - -diff -Nur linux-4.1.6.orig/include/linux/irq_work.h linux-4.1.6/include/linux/irq_work.h ---- linux-4.1.6.orig/include/linux/irq_work.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/irq_work.h 2015-09-08 23:49:06.542040801 +0200 -@@ -16,6 +16,7 @@ - #define IRQ_WORK_BUSY 2UL - #define IRQ_WORK_FLAGS 3UL - #define IRQ_WORK_LAZY 4UL /* Doesn't want IPI, wait for tick */ -+#define IRQ_WORK_HARD_IRQ 8UL /* Run hard IRQ context, even on RT */ - - struct irq_work { - unsigned long flags; -diff -Nur linux-4.1.6.orig/include/linux/jbd_common.h linux-4.1.6/include/linux/jbd_common.h ---- linux-4.1.6.orig/include/linux/jbd_common.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/jbd_common.h 2015-09-08 23:49:06.546040357 +0200 -@@ -15,32 +15,56 @@ - - static inline void jbd_lock_bh_state(struct buffer_head *bh) - { -+#ifndef CONFIG_PREEMPT_RT_BASE - bit_spin_lock(BH_State, &bh->b_state); -+#else -+ spin_lock(&bh->b_state_lock); -+#endif - } - - static inline int jbd_trylock_bh_state(struct buffer_head *bh) - { -+#ifndef CONFIG_PREEMPT_RT_BASE - return bit_spin_trylock(BH_State, &bh->b_state); -+#else -+ return spin_trylock(&bh->b_state_lock); -+#endif - } - - static inline int jbd_is_locked_bh_state(struct buffer_head *bh) - { -+#ifndef CONFIG_PREEMPT_RT_BASE - return bit_spin_is_locked(BH_State, &bh->b_state); -+#else -+ return spin_is_locked(&bh->b_state_lock); -+#endif - } - - static inline void jbd_unlock_bh_state(struct buffer_head *bh) - { -+#ifndef CONFIG_PREEMPT_RT_BASE - bit_spin_unlock(BH_State, &bh->b_state); -+#else -+ spin_unlock(&bh->b_state_lock); -+#endif - } - - static inline void jbd_lock_bh_journal_head(struct buffer_head *bh) - { -+#ifndef CONFIG_PREEMPT_RT_BASE - bit_spin_lock(BH_JournalHead, &bh->b_state); -+#else -+ spin_lock(&bh->b_journal_head_lock); -+#endif - } - - static inline void jbd_unlock_bh_journal_head(struct buffer_head *bh) - { -+#ifndef CONFIG_PREEMPT_RT_BASE - bit_spin_unlock(BH_JournalHead, &bh->b_state); -+#else -+ spin_unlock(&bh->b_journal_head_lock); -+#endif - } - - #endif -diff -Nur linux-4.1.6.orig/include/linux/kdb.h linux-4.1.6/include/linux/kdb.h ---- linux-4.1.6.orig/include/linux/kdb.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/kdb.h 2015-09-08 23:49:06.546040357 +0200 -@@ -167,6 +167,7 @@ - extern __printf(1, 2) int kdb_printf(const char *, ...); - typedef __printf(1, 2) int (*kdb_printf_t)(const char *, ...); - -+#define in_kdb_printk() (kdb_trap_printk) - extern void kdb_init(int level); - - /* Access to kdb specific polling devices */ -@@ -201,6 +202,7 @@ - extern int kdb_unregister(char *); - #else /* ! CONFIG_KGDB_KDB */ - static inline __printf(1, 2) int kdb_printf(const char *fmt, ...) { return 0; } -+#define in_kdb_printk() (0) - static inline void kdb_init(int level) {} - static inline int kdb_register(char *cmd, kdb_func_t func, char *usage, - char *help, short minlen) { return 0; } -diff -Nur linux-4.1.6.orig/include/linux/kernel.h linux-4.1.6/include/linux/kernel.h ---- linux-4.1.6.orig/include/linux/kernel.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/kernel.h 2015-09-08 23:49:06.546040357 +0200 -@@ -188,6 +188,9 @@ - */ - # define might_sleep() \ - do { __might_sleep(__FILE__, __LINE__, 0); might_resched(); } while (0) ++static void analyse_stripe(struct stripe_head *sh, struct stripe_head_state *s) ++{ ++ struct r5conf *conf = sh->raid_conf; ++ int disks = sh->disks; ++ struct r5dev *dev; ++ int i; ++ int do_recovery = 0; + -+# define might_sleep_no_state_check() \ -+ do { ___might_sleep(__FILE__, __LINE__, 0); might_resched(); } while (0) - # define sched_annotate_sleep() (current->task_state_change = 0) - #else - static inline void ___might_sleep(const char *file, int line, -@@ -195,6 +198,7 @@ - static inline void __might_sleep(const char *file, int line, - int preempt_offset) { } - # define might_sleep() do { might_resched(); } while (0) -+# define might_sleep_no_state_check() do { might_resched(); } while (0) - # define sched_annotate_sleep() do { } while (0) - #endif - -@@ -244,7 +248,8 @@ - - #if defined(CONFIG_MMU) && \ - (defined(CONFIG_PROVE_LOCKING) || defined(CONFIG_DEBUG_ATOMIC_SLEEP)) --void might_fault(void); -+#define might_fault() __might_fault(__FILE__, __LINE__) -+void __might_fault(const char *file, int line); - #else - static inline void might_fault(void) { } - #endif -@@ -466,6 +471,7 @@ - SYSTEM_HALT, - SYSTEM_POWER_OFF, - SYSTEM_RESTART, -+ SYSTEM_SUSPEND, - } system_state; - - #define TAINT_PROPRIETARY_MODULE 0 -diff -Nur linux-4.1.6.orig/include/linux/kvm_host.h linux-4.1.6/include/linux/kvm_host.h ---- linux-4.1.6.orig/include/linux/kvm_host.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/kvm_host.h 2015-09-08 23:49:06.546040357 +0200 -@@ -230,7 +230,7 @@ - - int fpu_active; - int guest_fpu_loaded, guest_xcr0_loaded; -- wait_queue_head_t wq; -+ struct swait_head wq; - struct pid *pid; - int sigset_active; - sigset_t sigset; -@@ -690,7 +690,7 @@ - } - #endif - --static inline wait_queue_head_t *kvm_arch_vcpu_wq(struct kvm_vcpu *vcpu) -+static inline struct swait_head *kvm_arch_vcpu_wq(struct kvm_vcpu *vcpu) - { - #ifdef __KVM_HAVE_ARCH_WQP - return vcpu->arch.wqp; -diff -Nur linux-4.1.6.orig/include/linux/lglock.h linux-4.1.6/include/linux/lglock.h ---- linux-4.1.6.orig/include/linux/lglock.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/lglock.h 2015-09-08 23:49:06.546040357 +0200 -@@ -34,22 +34,39 @@ - #endif - - struct lglock { -+#ifndef CONFIG_PREEMPT_RT_FULL - arch_spinlock_t __percpu *lock; -+#else -+ struct rt_mutex __percpu *lock; -+#endif - #ifdef CONFIG_DEBUG_LOCK_ALLOC - struct lock_class_key lock_key; - struct lockdep_map lock_dep_map; - #endif - }; - --#define DEFINE_LGLOCK(name) \ -+#ifndef CONFIG_PREEMPT_RT_FULL -+# define DEFINE_LGLOCK(name) \ - static DEFINE_PER_CPU(arch_spinlock_t, name ## _lock) \ - = __ARCH_SPIN_LOCK_UNLOCKED; \ - struct lglock name = { .lock = &name ## _lock } - --#define DEFINE_STATIC_LGLOCK(name) \ -+# define DEFINE_STATIC_LGLOCK(name) \ - static DEFINE_PER_CPU(arch_spinlock_t, name ## _lock) \ - = __ARCH_SPIN_LOCK_UNLOCKED; \ - static struct lglock name = { .lock = &name ## _lock } -+#else ++ memset(s, 0, sizeof(*s)); + -+# define DEFINE_LGLOCK(name) \ -+ static DEFINE_PER_CPU(struct rt_mutex, name ## _lock) \ -+ = __RT_MUTEX_INITIALIZER( name ## _lock); \ -+ struct lglock name = { .lock = &name ## _lock } ++ s->expanding = test_bit(STRIPE_EXPAND_SOURCE, &sh->state) && !sh->batch_head; ++ s->expanded = test_bit(STRIPE_EXPAND_READY, &sh->state) && !sh->batch_head; ++ s->failed_num[0] = -1; ++ s->failed_num[1] = -1; + -+# define DEFINE_STATIC_LGLOCK(name) \ -+ static DEFINE_PER_CPU(struct rt_mutex, name ## _lock) \ -+ = __RT_MUTEX_INITIALIZER( name ## _lock); \ -+ static struct lglock name = { .lock = &name ## _lock } -+#endif - - void lg_lock_init(struct lglock *lg, char *name); - void lg_local_lock(struct lglock *lg); -@@ -59,6 +76,12 @@ - void lg_global_lock(struct lglock *lg); - void lg_global_unlock(struct lglock *lg); - -+#ifndef CONFIG_PREEMPT_RT_FULL -+#define lg_global_trylock_relax(name) lg_global_lock(name) -+#else -+void lg_global_trylock_relax(struct lglock *lg); -+#endif ++ /* Now to look around and see what can be done */ ++ rcu_read_lock(); ++ for (i=disks; i--; ) { ++ struct md_rdev *rdev; ++ sector_t first_bad; ++ int bad_sectors; ++ int is_bad = 0; ++ ++ dev = &sh->dev[i]; ++ ++ pr_debug("check %d: state 0x%lx read %p write %p written %p\n", ++ i, dev->flags, ++ dev->toread, dev->towrite, dev->written); ++ /* maybe we can reply to a read ++ * ++ * new wantfill requests are only permitted while ++ * ops_complete_biofill is guaranteed to be inactive ++ */ ++ if (test_bit(R5_UPTODATE, &dev->flags) && dev->toread && ++ !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) ++ set_bit(R5_Wantfill, &dev->flags); ++ ++ /* now count some things */ ++ if (test_bit(R5_LOCKED, &dev->flags)) ++ s->locked++; ++ if (test_bit(R5_UPTODATE, &dev->flags)) ++ s->uptodate++; ++ if (test_bit(R5_Wantcompute, &dev->flags)) { ++ s->compute++; ++ BUG_ON(s->compute > 2); ++ } + - #else - /* When !CONFIG_SMP, map lglock to spinlock */ - #define lglock spinlock -diff -Nur linux-4.1.6.orig/include/linux/list_bl.h linux-4.1.6/include/linux/list_bl.h ---- linux-4.1.6.orig/include/linux/list_bl.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/list_bl.h 2015-09-08 23:49:06.546040357 +0200 -@@ -2,6 +2,7 @@ - #define _LINUX_LIST_BL_H - - #include -+#include - #include - - /* -@@ -32,13 +33,22 @@ - - struct hlist_bl_head { - struct hlist_bl_node *first; -+#ifdef CONFIG_PREEMPT_RT_BASE -+ raw_spinlock_t lock; -+#endif - }; - - struct hlist_bl_node { - struct hlist_bl_node *next, **pprev; - }; --#define INIT_HLIST_BL_HEAD(ptr) \ -- ((ptr)->first = NULL) ++ if (test_bit(R5_Wantfill, &dev->flags)) ++ s->to_fill++; ++ else if (dev->toread) ++ s->to_read++; ++ if (dev->towrite) { ++ s->to_write++; ++ if (!test_bit(R5_OVERWRITE, &dev->flags)) ++ s->non_overwrite++; ++ } ++ if (dev->written) ++ s->written++; ++ /* Prefer to use the replacement for reads, but only ++ * if it is recovered enough and has no bad blocks. ++ */ ++ rdev = rcu_dereference(conf->disks[i].replacement); ++ if (rdev && !test_bit(Faulty, &rdev->flags) && ++ rdev->recovery_offset >= sh->sector + STRIPE_SECTORS && ++ !is_badblock(rdev, sh->sector, STRIPE_SECTORS, ++ &first_bad, &bad_sectors)) ++ set_bit(R5_ReadRepl, &dev->flags); ++ else { ++ if (rdev) ++ set_bit(R5_NeedReplace, &dev->flags); ++ rdev = rcu_dereference(conf->disks[i].rdev); ++ clear_bit(R5_ReadRepl, &dev->flags); ++ } ++ if (rdev && test_bit(Faulty, &rdev->flags)) ++ rdev = NULL; ++ if (rdev) { ++ is_bad = is_badblock(rdev, sh->sector, STRIPE_SECTORS, ++ &first_bad, &bad_sectors); ++ if (s->blocked_rdev == NULL ++ && (test_bit(Blocked, &rdev->flags) ++ || is_bad < 0)) { ++ if (is_bad < 0) ++ set_bit(BlockedBadBlocks, ++ &rdev->flags); ++ s->blocked_rdev = rdev; ++ atomic_inc(&rdev->nr_pending); ++ } ++ } ++ clear_bit(R5_Insync, &dev->flags); ++ if (!rdev) ++ /* Not in-sync */; ++ else if (is_bad) { ++ /* also not in-sync */ ++ if (!test_bit(WriteErrorSeen, &rdev->flags) && ++ test_bit(R5_UPTODATE, &dev->flags)) { ++ /* treat as in-sync, but with a read error ++ * which we can now try to correct ++ */ ++ set_bit(R5_Insync, &dev->flags); ++ set_bit(R5_ReadError, &dev->flags); ++ } ++ } else if (test_bit(In_sync, &rdev->flags)) ++ set_bit(R5_Insync, &dev->flags); ++ else if (sh->sector + STRIPE_SECTORS <= rdev->recovery_offset) ++ /* in sync if before recovery_offset */ ++ set_bit(R5_Insync, &dev->flags); ++ else if (test_bit(R5_UPTODATE, &dev->flags) && ++ test_bit(R5_Expanded, &dev->flags)) ++ /* If we've reshaped into here, we assume it is Insync. ++ * We will shortly update recovery_offset to make ++ * it official. ++ */ ++ set_bit(R5_Insync, &dev->flags); ++ ++ if (test_bit(R5_WriteError, &dev->flags)) { ++ /* This flag does not apply to '.replacement' ++ * only to .rdev, so make sure to check that*/ ++ struct md_rdev *rdev2 = rcu_dereference( ++ conf->disks[i].rdev); ++ if (rdev2 == rdev) ++ clear_bit(R5_Insync, &dev->flags); ++ if (rdev2 && !test_bit(Faulty, &rdev2->flags)) { ++ s->handle_bad_blocks = 1; ++ atomic_inc(&rdev2->nr_pending); ++ } else ++ clear_bit(R5_WriteError, &dev->flags); ++ } ++ if (test_bit(R5_MadeGood, &dev->flags)) { ++ /* This flag does not apply to '.replacement' ++ * only to .rdev, so make sure to check that*/ ++ struct md_rdev *rdev2 = rcu_dereference( ++ conf->disks[i].rdev); ++ if (rdev2 && !test_bit(Faulty, &rdev2->flags)) { ++ s->handle_bad_blocks = 1; ++ atomic_inc(&rdev2->nr_pending); ++ } else ++ clear_bit(R5_MadeGood, &dev->flags); ++ } ++ if (test_bit(R5_MadeGoodRepl, &dev->flags)) { ++ struct md_rdev *rdev2 = rcu_dereference( ++ conf->disks[i].replacement); ++ if (rdev2 && !test_bit(Faulty, &rdev2->flags)) { ++ s->handle_bad_blocks = 1; ++ atomic_inc(&rdev2->nr_pending); ++ } else ++ clear_bit(R5_MadeGoodRepl, &dev->flags); ++ } ++ if (!test_bit(R5_Insync, &dev->flags)) { ++ /* The ReadError flag will just be confusing now */ ++ clear_bit(R5_ReadError, &dev->flags); ++ clear_bit(R5_ReWrite, &dev->flags); ++ } ++ if (test_bit(R5_ReadError, &dev->flags)) ++ clear_bit(R5_Insync, &dev->flags); ++ if (!test_bit(R5_Insync, &dev->flags)) { ++ if (s->failed < 2) ++ s->failed_num[s->failed] = i; ++ s->failed++; ++ if (rdev && !test_bit(Faulty, &rdev->flags)) ++ do_recovery = 1; ++ } ++ } ++ if (test_bit(STRIPE_SYNCING, &sh->state)) { ++ /* If there is a failed device being replaced, ++ * we must be recovering. ++ * else if we are after recovery_cp, we must be syncing ++ * else if MD_RECOVERY_REQUESTED is set, we also are syncing. ++ * else we can only be replacing ++ * sync and recovery both need to read all devices, and so ++ * use the same flag. ++ */ ++ if (do_recovery || ++ sh->sector >= conf->mddev->recovery_cp || ++ test_bit(MD_RECOVERY_REQUESTED, &(conf->mddev->recovery))) ++ s->syncing = 1; ++ else ++ s->replacing = 1; ++ } ++ rcu_read_unlock(); ++} + -+static inline void INIT_HLIST_BL_HEAD(struct hlist_bl_head *h) ++static int clear_batch_ready(struct stripe_head *sh) +{ -+ h->first = NULL; -+#ifdef CONFIG_PREEMPT_RT_BASE -+ raw_spin_lock_init(&h->lock); -+#endif ++ /* Return '1' if this is a member of batch, or ++ * '0' if it is a lone stripe or a head which can now be ++ * handled. ++ */ ++ struct stripe_head *tmp; ++ if (!test_and_clear_bit(STRIPE_BATCH_READY, &sh->state)) ++ return (sh->batch_head && sh->batch_head != sh); ++ spin_lock(&sh->stripe_lock); ++ if (!sh->batch_head) { ++ spin_unlock(&sh->stripe_lock); ++ return 0; ++ } ++ ++ /* ++ * this stripe could be added to a batch list before we check ++ * BATCH_READY, skips it ++ */ ++ if (sh->batch_head != sh) { ++ spin_unlock(&sh->stripe_lock); ++ return 1; ++ } ++ spin_lock(&sh->batch_lock); ++ list_for_each_entry(tmp, &sh->batch_list, batch_list) ++ clear_bit(STRIPE_BATCH_READY, &tmp->state); ++ spin_unlock(&sh->batch_lock); ++ spin_unlock(&sh->stripe_lock); ++ ++ /* ++ * BATCH_READY is cleared, no new stripes can be added. ++ * batch_list can be accessed without lock ++ */ ++ return 0; +} - - static inline void INIT_HLIST_BL_NODE(struct hlist_bl_node *h) - { -@@ -117,12 +127,26 @@ - - static inline void hlist_bl_lock(struct hlist_bl_head *b) - { -+#ifndef CONFIG_PREEMPT_RT_BASE - bit_spin_lock(0, (unsigned long *)b); -+#else -+ raw_spin_lock(&b->lock); -+#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK) -+ __set_bit(0, (unsigned long *)b); -+#endif -+#endif - } - - static inline void hlist_bl_unlock(struct hlist_bl_head *b) - { -+#ifndef CONFIG_PREEMPT_RT_BASE - __bit_spin_unlock(0, (unsigned long *)b); -+#else -+#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK) -+ __clear_bit(0, (unsigned long *)b); -+#endif -+ raw_spin_unlock(&b->lock); -+#endif - } - - static inline bool hlist_bl_is_locked(struct hlist_bl_head *b) -diff -Nur linux-4.1.6.orig/include/linux/locallock.h linux-4.1.6/include/linux/locallock.h ---- linux-4.1.6.orig/include/linux/locallock.h 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.6/include/linux/locallock.h 2015-09-08 23:49:06.550039912 +0200 -@@ -0,0 +1,270 @@ -+#ifndef _LINUX_LOCALLOCK_H -+#define _LINUX_LOCALLOCK_H + -+#include -+#include ++static void break_stripe_batch_list(struct stripe_head *head_sh, ++ unsigned long handle_flags) ++{ ++ struct stripe_head *sh, *next; ++ int i; ++ int do_wakeup = 0; ++ ++ list_for_each_entry_safe(sh, next, &head_sh->batch_list, batch_list) { ++ ++ list_del_init(&sh->batch_list); ++ ++ WARN_ON_ONCE(sh->state & ((1 << STRIPE_ACTIVE) | ++ (1 << STRIPE_SYNCING) | ++ (1 << STRIPE_REPLACED) | ++ (1 << STRIPE_PREREAD_ACTIVE) | ++ (1 << STRIPE_DELAYED) | ++ (1 << STRIPE_BIT_DELAY) | ++ (1 << STRIPE_FULL_WRITE) | ++ (1 << STRIPE_BIOFILL_RUN) | ++ (1 << STRIPE_COMPUTE_RUN) | ++ (1 << STRIPE_OPS_REQ_PENDING) | ++ (1 << STRIPE_DISCARD) | ++ (1 << STRIPE_BATCH_READY) | ++ (1 << STRIPE_BATCH_ERR) | ++ (1 << STRIPE_BITMAP_PENDING))); ++ WARN_ON_ONCE(head_sh->state & ((1 << STRIPE_DISCARD) | ++ (1 << STRIPE_REPLACED))); ++ ++ set_mask_bits(&sh->state, ~(STRIPE_EXPAND_SYNC_FLAGS | ++ (1 << STRIPE_DEGRADED)), ++ head_sh->state & (1 << STRIPE_INSYNC)); ++ ++ sh->check_state = head_sh->check_state; ++ sh->reconstruct_state = head_sh->reconstruct_state; ++ for (i = 0; i < sh->disks; i++) { ++ if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags)) ++ do_wakeup = 1; ++ sh->dev[i].flags = head_sh->dev[i].flags & ++ (~((1 << R5_WriteError) | (1 << R5_Overlap))); ++ } ++ spin_lock_irq(&sh->stripe_lock); ++ sh->batch_head = NULL; ++ spin_unlock_irq(&sh->stripe_lock); ++ if (handle_flags == 0 || ++ sh->state & handle_flags) ++ set_bit(STRIPE_HANDLE, &sh->state); ++ release_stripe(sh); ++ } ++ spin_lock_irq(&head_sh->stripe_lock); ++ head_sh->batch_head = NULL; ++ spin_unlock_irq(&head_sh->stripe_lock); ++ for (i = 0; i < head_sh->disks; i++) ++ if (test_and_clear_bit(R5_Overlap, &head_sh->dev[i].flags)) ++ do_wakeup = 1; ++ if (head_sh->state & handle_flags) ++ set_bit(STRIPE_HANDLE, &head_sh->state); ++ ++ if (do_wakeup) ++ wake_up(&head_sh->raid_conf->wait_for_overlap); ++} + -+#ifdef CONFIG_PREEMPT_RT_BASE ++static void handle_stripe(struct stripe_head *sh) ++{ ++ struct stripe_head_state s; ++ struct r5conf *conf = sh->raid_conf; ++ int i; ++ int prexor; ++ int disks = sh->disks; ++ struct r5dev *pdev, *qdev; ++ ++ clear_bit(STRIPE_HANDLE, &sh->state); ++ if (test_and_set_bit_lock(STRIPE_ACTIVE, &sh->state)) { ++ /* already being handled, ensure it gets handled ++ * again when current action finishes */ ++ set_bit(STRIPE_HANDLE, &sh->state); ++ return; ++ } + -+#ifdef CONFIG_DEBUG_SPINLOCK -+# define LL_WARN(cond) WARN_ON(cond) -+#else -+# define LL_WARN(cond) do { } while (0) -+#endif ++ if (clear_batch_ready(sh) ) { ++ clear_bit_unlock(STRIPE_ACTIVE, &sh->state); ++ return; ++ } + -+/* -+ * per cpu lock based substitute for local_irq_*() -+ */ -+struct local_irq_lock { -+ spinlock_t lock; -+ struct task_struct *owner; -+ int nestcnt; -+ unsigned long flags; -+}; ++ if (test_and_clear_bit(STRIPE_BATCH_ERR, &sh->state)) ++ break_stripe_batch_list(sh, 0); ++ ++ if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state) && !sh->batch_head) { ++ spin_lock(&sh->stripe_lock); ++ /* Cannot process 'sync' concurrently with 'discard' */ ++ if (!test_bit(STRIPE_DISCARD, &sh->state) && ++ test_and_clear_bit(STRIPE_SYNC_REQUESTED, &sh->state)) { ++ set_bit(STRIPE_SYNCING, &sh->state); ++ clear_bit(STRIPE_INSYNC, &sh->state); ++ clear_bit(STRIPE_REPLACED, &sh->state); ++ } ++ spin_unlock(&sh->stripe_lock); ++ } ++ clear_bit(STRIPE_DELAYED, &sh->state); + -+#define DEFINE_LOCAL_IRQ_LOCK(lvar) \ -+ DEFINE_PER_CPU(struct local_irq_lock, lvar) = { \ -+ .lock = __SPIN_LOCK_UNLOCKED((lvar).lock) } ++ pr_debug("handling stripe %llu, state=%#lx cnt=%d, " ++ "pd_idx=%d, qd_idx=%d\n, check:%d, reconstruct:%d\n", ++ (unsigned long long)sh->sector, sh->state, ++ atomic_read(&sh->count), sh->pd_idx, sh->qd_idx, ++ sh->check_state, sh->reconstruct_state); + -+#define DECLARE_LOCAL_IRQ_LOCK(lvar) \ -+ DECLARE_PER_CPU(struct local_irq_lock, lvar) ++ analyse_stripe(sh, &s); + -+#define local_irq_lock_init(lvar) \ -+ do { \ -+ int __cpu; \ -+ for_each_possible_cpu(__cpu) \ -+ spin_lock_init(&per_cpu(lvar, __cpu).lock); \ -+ } while (0) ++ if (s.handle_bad_blocks) { ++ set_bit(STRIPE_HANDLE, &sh->state); ++ goto finish; ++ } + -+/* -+ * spin_lock|trylock|unlock_local flavour that does not migrate disable -+ * used for __local_lock|trylock|unlock where get_local_var/put_local_var -+ * already takes care of the migrate_disable/enable -+ * for CONFIG_PREEMPT_BASE map to the normal spin_* calls. -+ */ -+#ifdef CONFIG_PREEMPT_RT_FULL -+# define spin_lock_local(lock) rt_spin_lock(lock) -+# define spin_trylock_local(lock) rt_spin_trylock(lock) -+# define spin_unlock_local(lock) rt_spin_unlock(lock) -+#else -+# define spin_lock_local(lock) spin_lock(lock) -+# define spin_trylock_local(lock) spin_trylock(lock) -+# define spin_unlock_local(lock) spin_unlock(lock) -+#endif ++ if (unlikely(s.blocked_rdev)) { ++ if (s.syncing || s.expanding || s.expanded || ++ s.replacing || s.to_write || s.written) { ++ set_bit(STRIPE_HANDLE, &sh->state); ++ goto finish; ++ } ++ /* There is nothing for the blocked_rdev to block */ ++ rdev_dec_pending(s.blocked_rdev, conf->mddev); ++ s.blocked_rdev = NULL; ++ } + -+static inline void __local_lock(struct local_irq_lock *lv) ++ if (s.to_fill && !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) { ++ set_bit(STRIPE_OP_BIOFILL, &s.ops_request); ++ set_bit(STRIPE_BIOFILL_RUN, &sh->state); ++ } ++ ++ pr_debug("locked=%d uptodate=%d to_read=%d" ++ " to_write=%d failed=%d failed_num=%d,%d\n", ++ s.locked, s.uptodate, s.to_read, s.to_write, s.failed, ++ s.failed_num[0], s.failed_num[1]); ++ /* check if the array has lost more than max_degraded devices and, ++ * if so, some requests might need to be failed. ++ */ ++ if (s.failed > conf->max_degraded) { ++ sh->check_state = 0; ++ sh->reconstruct_state = 0; ++ break_stripe_batch_list(sh, 0); ++ if (s.to_read+s.to_write+s.written) ++ handle_failed_stripe(conf, sh, &s, disks, &s.return_bi); ++ if (s.syncing + s.replacing) ++ handle_failed_sync(conf, sh, &s); ++ } ++ ++ /* Now we check to see if any write operations have recently ++ * completed ++ */ ++ prexor = 0; ++ if (sh->reconstruct_state == reconstruct_state_prexor_drain_result) ++ prexor = 1; ++ if (sh->reconstruct_state == reconstruct_state_drain_result || ++ sh->reconstruct_state == reconstruct_state_prexor_drain_result) { ++ sh->reconstruct_state = reconstruct_state_idle; ++ ++ /* All the 'written' buffers and the parity block are ready to ++ * be written back to disk ++ */ ++ BUG_ON(!test_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags) && ++ !test_bit(R5_Discard, &sh->dev[sh->pd_idx].flags)); ++ BUG_ON(sh->qd_idx >= 0 && ++ !test_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags) && ++ !test_bit(R5_Discard, &sh->dev[sh->qd_idx].flags)); ++ for (i = disks; i--; ) { ++ struct r5dev *dev = &sh->dev[i]; ++ if (test_bit(R5_LOCKED, &dev->flags) && ++ (i == sh->pd_idx || i == sh->qd_idx || ++ dev->written)) { ++ pr_debug("Writing block %d\n", i); ++ set_bit(R5_Wantwrite, &dev->flags); ++ if (prexor) ++ continue; ++ if (s.failed > 1) ++ continue; ++ if (!test_bit(R5_Insync, &dev->flags) || ++ ((i == sh->pd_idx || i == sh->qd_idx) && ++ s.failed == 0)) ++ set_bit(STRIPE_INSYNC, &sh->state); ++ } ++ } ++ if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) ++ s.dec_preread_active = 1; ++ } ++ ++ /* ++ * might be able to return some write requests if the parity blocks ++ * are safe, or on a failed drive ++ */ ++ pdev = &sh->dev[sh->pd_idx]; ++ s.p_failed = (s.failed >= 1 && s.failed_num[0] == sh->pd_idx) ++ || (s.failed >= 2 && s.failed_num[1] == sh->pd_idx); ++ qdev = &sh->dev[sh->qd_idx]; ++ s.q_failed = (s.failed >= 1 && s.failed_num[0] == sh->qd_idx) ++ || (s.failed >= 2 && s.failed_num[1] == sh->qd_idx) ++ || conf->level < 6; ++ ++ if (s.written && ++ (s.p_failed || ((test_bit(R5_Insync, &pdev->flags) ++ && !test_bit(R5_LOCKED, &pdev->flags) ++ && (test_bit(R5_UPTODATE, &pdev->flags) || ++ test_bit(R5_Discard, &pdev->flags))))) && ++ (s.q_failed || ((test_bit(R5_Insync, &qdev->flags) ++ && !test_bit(R5_LOCKED, &qdev->flags) ++ && (test_bit(R5_UPTODATE, &qdev->flags) || ++ test_bit(R5_Discard, &qdev->flags)))))) ++ handle_stripe_clean_event(conf, sh, disks, &s.return_bi); ++ ++ /* Now we might consider reading some blocks, either to check/generate ++ * parity, or to satisfy requests ++ * or to load a block that is being partially written. ++ */ ++ if (s.to_read || s.non_overwrite ++ || (conf->level == 6 && s.to_write && s.failed) ++ || (s.syncing && (s.uptodate + s.compute < disks)) ++ || s.replacing ++ || s.expanding) ++ handle_stripe_fill(sh, &s, disks); ++ ++ /* Now to consider new write requests and what else, if anything ++ * should be read. We do not handle new writes when: ++ * 1/ A 'write' operation (copy+xor) is already in flight. ++ * 2/ A 'check' operation is in flight, as it may clobber the parity ++ * block. ++ */ ++ if (s.to_write && !sh->reconstruct_state && !sh->check_state) ++ handle_stripe_dirtying(conf, sh, &s, disks); ++ ++ /* maybe we need to check and possibly fix the parity for this stripe ++ * Any reads will already have been scheduled, so we just see if enough ++ * data is available. The parity check is held off while parity ++ * dependent operations are in flight. ++ */ ++ if (sh->check_state || ++ (s.syncing && s.locked == 0 && ++ !test_bit(STRIPE_COMPUTE_RUN, &sh->state) && ++ !test_bit(STRIPE_INSYNC, &sh->state))) { ++ if (conf->level == 6) ++ handle_parity_checks6(conf, sh, &s, disks); ++ else ++ handle_parity_checks5(conf, sh, &s, disks); ++ } ++ ++ if ((s.replacing || s.syncing) && s.locked == 0 ++ && !test_bit(STRIPE_COMPUTE_RUN, &sh->state) ++ && !test_bit(STRIPE_REPLACED, &sh->state)) { ++ /* Write out to replacement devices where possible */ ++ for (i = 0; i < conf->raid_disks; i++) ++ if (test_bit(R5_NeedReplace, &sh->dev[i].flags)) { ++ WARN_ON(!test_bit(R5_UPTODATE, &sh->dev[i].flags)); ++ set_bit(R5_WantReplace, &sh->dev[i].flags); ++ set_bit(R5_LOCKED, &sh->dev[i].flags); ++ s.locked++; ++ } ++ if (s.replacing) ++ set_bit(STRIPE_INSYNC, &sh->state); ++ set_bit(STRIPE_REPLACED, &sh->state); ++ } ++ if ((s.syncing || s.replacing) && s.locked == 0 && ++ !test_bit(STRIPE_COMPUTE_RUN, &sh->state) && ++ test_bit(STRIPE_INSYNC, &sh->state)) { ++ md_done_sync(conf->mddev, STRIPE_SECTORS, 1); ++ clear_bit(STRIPE_SYNCING, &sh->state); ++ if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags)) ++ wake_up(&conf->wait_for_overlap); ++ } ++ ++ /* If the failed drives are just a ReadError, then we might need ++ * to progress the repair/check process ++ */ ++ if (s.failed <= conf->max_degraded && !conf->mddev->ro) ++ for (i = 0; i < s.failed; i++) { ++ struct r5dev *dev = &sh->dev[s.failed_num[i]]; ++ if (test_bit(R5_ReadError, &dev->flags) ++ && !test_bit(R5_LOCKED, &dev->flags) ++ && test_bit(R5_UPTODATE, &dev->flags) ++ ) { ++ if (!test_bit(R5_ReWrite, &dev->flags)) { ++ set_bit(R5_Wantwrite, &dev->flags); ++ set_bit(R5_ReWrite, &dev->flags); ++ set_bit(R5_LOCKED, &dev->flags); ++ s.locked++; ++ } else { ++ /* let's read it back */ ++ set_bit(R5_Wantread, &dev->flags); ++ set_bit(R5_LOCKED, &dev->flags); ++ s.locked++; ++ } ++ } ++ } ++ ++ /* Finish reconstruct operations initiated by the expansion process */ ++ if (sh->reconstruct_state == reconstruct_state_result) { ++ struct stripe_head *sh_src ++ = get_active_stripe(conf, sh->sector, 1, 1, 1); ++ if (sh_src && test_bit(STRIPE_EXPAND_SOURCE, &sh_src->state)) { ++ /* sh cannot be written until sh_src has been read. ++ * so arrange for sh to be delayed a little ++ */ ++ set_bit(STRIPE_DELAYED, &sh->state); ++ set_bit(STRIPE_HANDLE, &sh->state); ++ if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, ++ &sh_src->state)) ++ atomic_inc(&conf->preread_active_stripes); ++ release_stripe(sh_src); ++ goto finish; ++ } ++ if (sh_src) ++ release_stripe(sh_src); ++ ++ sh->reconstruct_state = reconstruct_state_idle; ++ clear_bit(STRIPE_EXPANDING, &sh->state); ++ for (i = conf->raid_disks; i--; ) { ++ set_bit(R5_Wantwrite, &sh->dev[i].flags); ++ set_bit(R5_LOCKED, &sh->dev[i].flags); ++ s.locked++; ++ } ++ } ++ ++ if (s.expanded && test_bit(STRIPE_EXPANDING, &sh->state) && ++ !sh->reconstruct_state) { ++ /* Need to write out all blocks after computing parity */ ++ sh->disks = conf->raid_disks; ++ stripe_set_idx(sh->sector, conf, 0, sh); ++ schedule_reconstruction(sh, &s, 1, 1); ++ } else if (s.expanded && !sh->reconstruct_state && s.locked == 0) { ++ clear_bit(STRIPE_EXPAND_READY, &sh->state); ++ atomic_dec(&conf->reshape_stripes); ++ wake_up(&conf->wait_for_overlap); ++ md_done_sync(conf->mddev, STRIPE_SECTORS, 1); ++ } ++ ++ if (s.expanding && s.locked == 0 && ++ !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) ++ handle_stripe_expansion(conf, sh); ++ ++finish: ++ /* wait for this device to become unblocked */ ++ if (unlikely(s.blocked_rdev)) { ++ if (conf->mddev->external) ++ md_wait_for_blocked_rdev(s.blocked_rdev, ++ conf->mddev); ++ else ++ /* Internal metadata will immediately ++ * be written by raid5d, so we don't ++ * need to wait here. ++ */ ++ rdev_dec_pending(s.blocked_rdev, ++ conf->mddev); ++ } ++ ++ if (s.handle_bad_blocks) ++ for (i = disks; i--; ) { ++ struct md_rdev *rdev; ++ struct r5dev *dev = &sh->dev[i]; ++ if (test_and_clear_bit(R5_WriteError, &dev->flags)) { ++ /* We own a safe reference to the rdev */ ++ rdev = conf->disks[i].rdev; ++ if (!rdev_set_badblocks(rdev, sh->sector, ++ STRIPE_SECTORS, 0)) ++ md_error(conf->mddev, rdev); ++ rdev_dec_pending(rdev, conf->mddev); ++ } ++ if (test_and_clear_bit(R5_MadeGood, &dev->flags)) { ++ rdev = conf->disks[i].rdev; ++ rdev_clear_badblocks(rdev, sh->sector, ++ STRIPE_SECTORS, 0); ++ rdev_dec_pending(rdev, conf->mddev); ++ } ++ if (test_and_clear_bit(R5_MadeGoodRepl, &dev->flags)) { ++ rdev = conf->disks[i].replacement; ++ if (!rdev) ++ /* rdev have been moved down */ ++ rdev = conf->disks[i].rdev; ++ rdev_clear_badblocks(rdev, sh->sector, ++ STRIPE_SECTORS, 0); ++ rdev_dec_pending(rdev, conf->mddev); ++ } ++ } ++ ++ if (s.ops_request) ++ raid_run_ops(sh, s.ops_request); ++ ++ ops_run_io(sh, &s); ++ ++ if (s.dec_preread_active) { ++ /* We delay this until after ops_run_io so that if make_request ++ * is waiting on a flush, it won't continue until the writes ++ * have actually been submitted. ++ */ ++ atomic_dec(&conf->preread_active_stripes); ++ if (atomic_read(&conf->preread_active_stripes) < ++ IO_THRESHOLD) ++ md_wakeup_thread(conf->mddev->thread); ++ } ++ ++ return_io(s.return_bi); ++ ++ clear_bit_unlock(STRIPE_ACTIVE, &sh->state); ++} ++ ++static void raid5_activate_delayed(struct r5conf *conf) +{ -+ if (lv->owner != current) { -+ spin_lock_local(&lv->lock); -+ LL_WARN(lv->owner); -+ LL_WARN(lv->nestcnt); -+ lv->owner = current; ++ if (atomic_read(&conf->preread_active_stripes) < IO_THRESHOLD) { ++ while (!list_empty(&conf->delayed_list)) { ++ struct list_head *l = conf->delayed_list.next; ++ struct stripe_head *sh; ++ sh = list_entry(l, struct stripe_head, lru); ++ list_del_init(l); ++ clear_bit(STRIPE_DELAYED, &sh->state); ++ if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) ++ atomic_inc(&conf->preread_active_stripes); ++ list_add_tail(&sh->lru, &conf->hold_list); ++ raid5_wakeup_stripe_thread(sh); ++ } + } -+ lv->nestcnt++; +} + -+#define local_lock(lvar) \ -+ do { __local_lock(&get_local_var(lvar)); } while (0) ++static void activate_bit_delay(struct r5conf *conf, ++ struct list_head *temp_inactive_list) ++{ ++ /* device_lock is held */ ++ struct list_head head; ++ list_add(&head, &conf->bitmap_list); ++ list_del_init(&conf->bitmap_list); ++ while (!list_empty(&head)) { ++ struct stripe_head *sh = list_entry(head.next, struct stripe_head, lru); ++ int hash; ++ list_del_init(&sh->lru); ++ atomic_inc(&sh->count); ++ hash = sh->hash_lock_index; ++ __release_stripe(conf, sh, &temp_inactive_list[hash]); ++ } ++} + -+static inline int __local_trylock(struct local_irq_lock *lv) ++static int raid5_congested(struct mddev *mddev, int bits) +{ -+ if (lv->owner != current && spin_trylock_local(&lv->lock)) { -+ LL_WARN(lv->owner); -+ LL_WARN(lv->nestcnt); -+ lv->owner = current; -+ lv->nestcnt = 1; ++ struct r5conf *conf = mddev->private; ++ ++ /* No difference between reads and writes. Just check ++ * how busy the stripe_cache is ++ */ ++ ++ if (test_bit(R5_INACTIVE_BLOCKED, &conf->cache_state)) + return 1; -+ } ++ if (conf->quiesce) ++ return 1; ++ if (atomic_read(&conf->empty_inactive_list_nr)) ++ return 1; ++ + return 0; +} + -+#define local_trylock(lvar) \ -+ ({ \ -+ int __locked; \ -+ __locked = __local_trylock(&get_local_var(lvar)); \ -+ if (!__locked) \ -+ put_local_var(lvar); \ -+ __locked; \ -+ }) -+ -+static inline void __local_unlock(struct local_irq_lock *lv) ++/* We want read requests to align with chunks where possible, ++ * but write requests don't need to. ++ */ ++static int raid5_mergeable_bvec(struct mddev *mddev, ++ struct bvec_merge_data *bvm, ++ struct bio_vec *biovec) +{ -+ LL_WARN(lv->nestcnt == 0); -+ LL_WARN(lv->owner != current); -+ if (--lv->nestcnt) -+ return; ++ sector_t sector = bvm->bi_sector + get_start_sect(bvm->bi_bdev); ++ int max; ++ unsigned int chunk_sectors = mddev->chunk_sectors; ++ unsigned int bio_sectors = bvm->bi_size >> 9; + -+ lv->owner = NULL; -+ spin_unlock_local(&lv->lock); ++ /* ++ * always allow writes to be mergeable, read as well if array ++ * is degraded as we'll go through stripe cache anyway. ++ */ ++ if ((bvm->bi_rw & 1) == WRITE || mddev->degraded) ++ return biovec->bv_len; ++ ++ if (mddev->new_chunk_sectors < mddev->chunk_sectors) ++ chunk_sectors = mddev->new_chunk_sectors; ++ max = (chunk_sectors - ((sector & (chunk_sectors - 1)) + bio_sectors)) << 9; ++ if (max < 0) max = 0; ++ if (max <= biovec->bv_len && bio_sectors == 0) ++ return biovec->bv_len; ++ else ++ return max; +} + -+#define local_unlock(lvar) \ -+ do { \ -+ __local_unlock(this_cpu_ptr(&lvar)); \ -+ put_local_var(lvar); \ -+ } while (0) -+ -+static inline void __local_lock_irq(struct local_irq_lock *lv) ++static int in_chunk_boundary(struct mddev *mddev, struct bio *bio) +{ -+ spin_lock_irqsave(&lv->lock, lv->flags); -+ LL_WARN(lv->owner); -+ LL_WARN(lv->nestcnt); -+ lv->owner = current; -+ lv->nestcnt = 1; ++ sector_t sector = bio->bi_iter.bi_sector + get_start_sect(bio->bi_bdev); ++ unsigned int chunk_sectors = mddev->chunk_sectors; ++ unsigned int bio_sectors = bio_sectors(bio); ++ ++ if (mddev->new_chunk_sectors < mddev->chunk_sectors) ++ chunk_sectors = mddev->new_chunk_sectors; ++ return chunk_sectors >= ++ ((sector & (chunk_sectors - 1)) + bio_sectors); +} + -+#define local_lock_irq(lvar) \ -+ do { __local_lock_irq(&get_local_var(lvar)); } while (0) ++/* ++ * add bio to the retry LIFO ( in O(1) ... we are in interrupt ) ++ * later sampled by raid5d. ++ */ ++static void add_bio_to_retry(struct bio *bi,struct r5conf *conf) ++{ ++ unsigned long flags; + -+#define local_lock_irq_on(lvar, cpu) \ -+ do { __local_lock_irq(&per_cpu(lvar, cpu)); } while (0) ++ spin_lock_irqsave(&conf->device_lock, flags); + -+static inline void __local_unlock_irq(struct local_irq_lock *lv) -+{ -+ LL_WARN(!lv->nestcnt); -+ LL_WARN(lv->owner != current); -+ lv->owner = NULL; -+ lv->nestcnt = 0; -+ spin_unlock_irq(&lv->lock); ++ bi->bi_next = conf->retry_read_aligned_list; ++ conf->retry_read_aligned_list = bi; ++ ++ spin_unlock_irqrestore(&conf->device_lock, flags); ++ md_wakeup_thread(conf->mddev->thread); +} + -+#define local_unlock_irq(lvar) \ -+ do { \ -+ __local_unlock_irq(this_cpu_ptr(&lvar)); \ -+ put_local_var(lvar); \ -+ } while (0) ++static struct bio *remove_bio_from_retry(struct r5conf *conf) ++{ ++ struct bio *bi; + -+#define local_unlock_irq_on(lvar, cpu) \ -+ do { \ -+ __local_unlock_irq(&per_cpu(lvar, cpu)); \ -+ } while (0) ++ bi = conf->retry_read_aligned; ++ if (bi) { ++ conf->retry_read_aligned = NULL; ++ return bi; ++ } ++ bi = conf->retry_read_aligned_list; ++ if(bi) { ++ conf->retry_read_aligned_list = bi->bi_next; ++ bi->bi_next = NULL; ++ /* ++ * this sets the active strip count to 1 and the processed ++ * strip count to zero (upper 8 bits) ++ */ ++ raid5_set_bi_stripes(bi, 1); /* biased count of active stripes */ ++ } + -+static inline int __local_lock_irqsave(struct local_irq_lock *lv) ++ return bi; ++} ++ ++/* ++ * The "raid5_align_endio" should check if the read succeeded and if it ++ * did, call bio_endio on the original bio (having bio_put the new bio ++ * first). ++ * If the read failed.. ++ */ ++static void raid5_align_endio(struct bio *bi, int error) +{ -+ if (lv->owner != current) { -+ __local_lock_irq(lv); -+ return 0; -+ } else { -+ lv->nestcnt++; -+ return 1; ++ struct bio* raid_bi = bi->bi_private; ++ struct mddev *mddev; ++ struct r5conf *conf; ++ int uptodate = test_bit(BIO_UPTODATE, &bi->bi_flags); ++ struct md_rdev *rdev; ++ ++ bio_put(bi); ++ ++ rdev = (void*)raid_bi->bi_next; ++ raid_bi->bi_next = NULL; ++ mddev = rdev->mddev; ++ conf = mddev->private; ++ ++ rdev_dec_pending(rdev, conf->mddev); ++ ++ if (!error && uptodate) { ++ trace_block_bio_complete(bdev_get_queue(raid_bi->bi_bdev), ++ raid_bi, 0); ++ bio_endio(raid_bi, 0); ++ if (atomic_dec_and_test(&conf->active_aligned_reads)) ++ wake_up(&conf->wait_for_stripe); ++ return; + } -+} + -+#define local_lock_irqsave(lvar, _flags) \ -+ do { \ -+ if (__local_lock_irqsave(&get_local_var(lvar))) \ -+ put_local_var(lvar); \ -+ _flags = __this_cpu_read(lvar.flags); \ -+ } while (0) ++ pr_debug("raid5_align_endio : io error...handing IO for a retry\n"); + -+#define local_lock_irqsave_on(lvar, _flags, cpu) \ -+ do { \ -+ __local_lock_irqsave(&per_cpu(lvar, cpu)); \ -+ _flags = per_cpu(lvar, cpu).flags; \ -+ } while (0) ++ add_bio_to_retry(raid_bi, conf); ++} + -+static inline int __local_unlock_irqrestore(struct local_irq_lock *lv, -+ unsigned long flags) ++static int bio_fits_rdev(struct bio *bi) +{ -+ LL_WARN(!lv->nestcnt); -+ LL_WARN(lv->owner != current); -+ if (--lv->nestcnt) ++ struct request_queue *q = bdev_get_queue(bi->bi_bdev); ++ ++ if (bio_sectors(bi) > queue_max_sectors(q)) ++ return 0; ++ blk_recount_segments(q, bi); ++ if (bi->bi_phys_segments > queue_max_segments(q)) ++ return 0; ++ ++ if (q->merge_bvec_fn) ++ /* it's too hard to apply the merge_bvec_fn at this stage, ++ * just just give up ++ */ + return 0; + -+ lv->owner = NULL; -+ spin_unlock_irqrestore(&lv->lock, lv->flags); + return 1; +} + -+#define local_unlock_irqrestore(lvar, flags) \ -+ do { \ -+ if (__local_unlock_irqrestore(this_cpu_ptr(&lvar), flags)) \ -+ put_local_var(lvar); \ -+ } while (0) ++static int chunk_aligned_read(struct mddev *mddev, struct bio * raid_bio) ++{ ++ struct r5conf *conf = mddev->private; ++ int dd_idx; ++ struct bio* align_bi; ++ struct md_rdev *rdev; ++ sector_t end_sector; ++ ++ if (!in_chunk_boundary(mddev, raid_bio)) { ++ pr_debug("chunk_aligned_read : non aligned\n"); ++ return 0; ++ } ++ /* ++ * use bio_clone_mddev to make a copy of the bio ++ */ ++ align_bi = bio_clone_mddev(raid_bio, GFP_NOIO, mddev); ++ if (!align_bi) ++ return 0; ++ /* ++ * set bi_end_io to a new function, and set bi_private to the ++ * original bio. ++ */ ++ align_bi->bi_end_io = raid5_align_endio; ++ align_bi->bi_private = raid_bio; ++ /* ++ * compute position ++ */ ++ align_bi->bi_iter.bi_sector = ++ raid5_compute_sector(conf, raid_bio->bi_iter.bi_sector, ++ 0, &dd_idx, NULL); + -+#define local_unlock_irqrestore_on(lvar, flags, cpu) \ -+ do { \ -+ __local_unlock_irqrestore(&per_cpu(lvar, cpu), flags); \ -+ } while (0) ++ end_sector = bio_end_sector(align_bi); ++ rcu_read_lock(); ++ rdev = rcu_dereference(conf->disks[dd_idx].replacement); ++ if (!rdev || test_bit(Faulty, &rdev->flags) || ++ rdev->recovery_offset < end_sector) { ++ rdev = rcu_dereference(conf->disks[dd_idx].rdev); ++ if (rdev && ++ (test_bit(Faulty, &rdev->flags) || ++ !(test_bit(In_sync, &rdev->flags) || ++ rdev->recovery_offset >= end_sector))) ++ rdev = NULL; ++ } ++ if (rdev) { ++ sector_t first_bad; ++ int bad_sectors; + -+#define local_spin_trylock_irq(lvar, lock) \ -+ ({ \ -+ int __locked; \ -+ local_lock_irq(lvar); \ -+ __locked = spin_trylock(lock); \ -+ if (!__locked) \ -+ local_unlock_irq(lvar); \ -+ __locked; \ -+ }) ++ atomic_inc(&rdev->nr_pending); ++ rcu_read_unlock(); ++ raid_bio->bi_next = (void*)rdev; ++ align_bi->bi_bdev = rdev->bdev; ++ __clear_bit(BIO_SEG_VALID, &align_bi->bi_flags); ++ ++ if (!bio_fits_rdev(align_bi) || ++ is_badblock(rdev, align_bi->bi_iter.bi_sector, ++ bio_sectors(align_bi), ++ &first_bad, &bad_sectors)) { ++ /* too big in some way, or has a known bad block */ ++ bio_put(align_bi); ++ rdev_dec_pending(rdev, mddev); ++ return 0; ++ } + -+#define local_spin_lock_irq(lvar, lock) \ -+ do { \ -+ local_lock_irq(lvar); \ -+ spin_lock(lock); \ -+ } while (0) ++ /* No reshape active, so we can trust rdev->data_offset */ ++ align_bi->bi_iter.bi_sector += rdev->data_offset; ++ ++ spin_lock_irq(&conf->device_lock); ++ wait_event_lock_irq(conf->wait_for_stripe, ++ conf->quiesce == 0, ++ conf->device_lock); ++ atomic_inc(&conf->active_aligned_reads); ++ spin_unlock_irq(&conf->device_lock); ++ ++ if (mddev->gendisk) ++ trace_block_bio_remap(bdev_get_queue(align_bi->bi_bdev), ++ align_bi, disk_devt(mddev->gendisk), ++ raid_bio->bi_iter.bi_sector); ++ generic_make_request(align_bi); ++ return 1; ++ } else { ++ rcu_read_unlock(); ++ bio_put(align_bi); ++ return 0; ++ } ++} + -+#define local_spin_unlock_irq(lvar, lock) \ -+ do { \ -+ spin_unlock(lock); \ -+ local_unlock_irq(lvar); \ -+ } while (0) ++/* __get_priority_stripe - get the next stripe to process ++ * ++ * Full stripe writes are allowed to pass preread active stripes up until ++ * the bypass_threshold is exceeded. In general the bypass_count ++ * increments when the handle_list is handled before the hold_list; however, it ++ * will not be incremented when STRIPE_IO_STARTED is sampled set signifying a ++ * stripe with in flight i/o. The bypass_count will be reset when the ++ * head of the hold_list has changed, i.e. the head was promoted to the ++ * handle_list. ++ */ ++static struct stripe_head *__get_priority_stripe(struct r5conf *conf, int group) ++{ ++ struct stripe_head *sh = NULL, *tmp; ++ struct list_head *handle_list = NULL; ++ struct r5worker_group *wg = NULL; ++ ++ if (conf->worker_cnt_per_group == 0) { ++ handle_list = &conf->handle_list; ++ } else if (group != ANY_GROUP) { ++ handle_list = &conf->worker_groups[group].handle_list; ++ wg = &conf->worker_groups[group]; ++ } else { ++ int i; ++ for (i = 0; i < conf->group_cnt; i++) { ++ handle_list = &conf->worker_groups[i].handle_list; ++ wg = &conf->worker_groups[i]; ++ if (!list_empty(handle_list)) ++ break; ++ } ++ } + -+#define local_spin_lock_irqsave(lvar, lock, flags) \ -+ do { \ -+ local_lock_irqsave(lvar, flags); \ -+ spin_lock(lock); \ -+ } while (0) ++ pr_debug("%s: handle: %s hold: %s full_writes: %d bypass_count: %d\n", ++ __func__, ++ list_empty(handle_list) ? "empty" : "busy", ++ list_empty(&conf->hold_list) ? "empty" : "busy", ++ atomic_read(&conf->pending_full_writes), conf->bypass_count); ++ ++ if (!list_empty(handle_list)) { ++ sh = list_entry(handle_list->next, typeof(*sh), lru); ++ ++ if (list_empty(&conf->hold_list)) ++ conf->bypass_count = 0; ++ else if (!test_bit(STRIPE_IO_STARTED, &sh->state)) { ++ if (conf->hold_list.next == conf->last_hold) ++ conf->bypass_count++; ++ else { ++ conf->last_hold = conf->hold_list.next; ++ conf->bypass_count -= conf->bypass_threshold; ++ if (conf->bypass_count < 0) ++ conf->bypass_count = 0; ++ } ++ } ++ } else if (!list_empty(&conf->hold_list) && ++ ((conf->bypass_threshold && ++ conf->bypass_count > conf->bypass_threshold) || ++ atomic_read(&conf->pending_full_writes) == 0)) { ++ ++ list_for_each_entry(tmp, &conf->hold_list, lru) { ++ if (conf->worker_cnt_per_group == 0 || ++ group == ANY_GROUP || ++ !cpu_online(tmp->cpu) || ++ cpu_to_group(tmp->cpu) == group) { ++ sh = tmp; ++ break; ++ } ++ } + -+#define local_spin_unlock_irqrestore(lvar, lock, flags) \ -+ do { \ -+ spin_unlock(lock); \ -+ local_unlock_irqrestore(lvar, flags); \ -+ } while (0) ++ if (sh) { ++ conf->bypass_count -= conf->bypass_threshold; ++ if (conf->bypass_count < 0) ++ conf->bypass_count = 0; ++ } ++ wg = NULL; ++ } + -+#define get_locked_var(lvar, var) \ -+ (*({ \ -+ local_lock(lvar); \ -+ this_cpu_ptr(&var); \ -+ })) ++ if (!sh) ++ return NULL; + -+#define put_locked_var(lvar, var) local_unlock(lvar); ++ if (wg) { ++ wg->stripes_cnt--; ++ sh->group = NULL; ++ } ++ list_del_init(&sh->lru); ++ BUG_ON(atomic_inc_return(&sh->count) != 1); ++ return sh; ++} + -+#define local_lock_cpu(lvar) \ -+ ({ \ -+ local_lock(lvar); \ -+ smp_processor_id(); \ -+ }) ++struct raid5_plug_cb { ++ struct blk_plug_cb cb; ++ struct list_head list; ++ struct list_head temp_inactive_list[NR_STRIPE_HASH_LOCKS]; ++}; + -+#define local_unlock_cpu(lvar) local_unlock(lvar) ++static void raid5_unplug(struct blk_plug_cb *blk_cb, bool from_schedule) ++{ ++ struct raid5_plug_cb *cb = container_of( ++ blk_cb, struct raid5_plug_cb, cb); ++ struct stripe_head *sh; ++ struct mddev *mddev = cb->cb.data; ++ struct r5conf *conf = mddev->private; ++ int cnt = 0; ++ int hash; ++ ++ if (cb->list.next && !list_empty(&cb->list)) { ++ spin_lock_irq(&conf->device_lock); ++ while (!list_empty(&cb->list)) { ++ sh = list_first_entry(&cb->list, struct stripe_head, lru); ++ list_del_init(&sh->lru); ++ /* ++ * avoid race release_stripe_plug() sees ++ * STRIPE_ON_UNPLUG_LIST clear but the stripe ++ * is still in our list ++ */ ++ smp_mb__before_atomic(); ++ clear_bit(STRIPE_ON_UNPLUG_LIST, &sh->state); ++ /* ++ * STRIPE_ON_RELEASE_LIST could be set here. In that ++ * case, the count is always > 1 here ++ */ ++ hash = sh->hash_lock_index; ++ __release_stripe(conf, sh, &cb->temp_inactive_list[hash]); ++ cnt++; ++ } ++ spin_unlock_irq(&conf->device_lock); ++ } ++ release_inactive_stripe_list(conf, cb->temp_inactive_list, ++ NR_STRIPE_HASH_LOCKS); ++ if (mddev->queue) ++ trace_block_unplug(mddev->queue, cnt, !from_schedule); ++ kfree(cb); ++} + -+#else /* PREEMPT_RT_BASE */ ++static void release_stripe_plug(struct mddev *mddev, ++ struct stripe_head *sh) ++{ ++ struct blk_plug_cb *blk_cb = blk_check_plugged( ++ raid5_unplug, mddev, ++ sizeof(struct raid5_plug_cb)); ++ struct raid5_plug_cb *cb; + -+#define DEFINE_LOCAL_IRQ_LOCK(lvar) __typeof__(const int) lvar -+#define DECLARE_LOCAL_IRQ_LOCK(lvar) extern __typeof__(const int) lvar ++ if (!blk_cb) { ++ release_stripe(sh); ++ return; ++ } + -+static inline void local_irq_lock_init(int lvar) { } ++ cb = container_of(blk_cb, struct raid5_plug_cb, cb); + -+#define local_lock(lvar) preempt_disable() -+#define local_unlock(lvar) preempt_enable() -+#define local_lock_irq(lvar) local_irq_disable() -+#define local_unlock_irq(lvar) local_irq_enable() -+#define local_lock_irqsave(lvar, flags) local_irq_save(flags) -+#define local_unlock_irqrestore(lvar, flags) local_irq_restore(flags) ++ if (cb->list.next == NULL) { ++ int i; ++ INIT_LIST_HEAD(&cb->list); ++ for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++) ++ INIT_LIST_HEAD(cb->temp_inactive_list + i); ++ } + -+#define local_spin_trylock_irq(lvar, lock) spin_trylock_irq(lock) -+#define local_spin_lock_irq(lvar, lock) spin_lock_irq(lock) -+#define local_spin_unlock_irq(lvar, lock) spin_unlock_irq(lock) -+#define local_spin_lock_irqsave(lvar, lock, flags) \ -+ spin_lock_irqsave(lock, flags) -+#define local_spin_unlock_irqrestore(lvar, lock, flags) \ -+ spin_unlock_irqrestore(lock, flags) ++ if (!test_and_set_bit(STRIPE_ON_UNPLUG_LIST, &sh->state)) ++ list_add_tail(&sh->lru, &cb->list); ++ else ++ release_stripe(sh); ++} + -+#define get_locked_var(lvar, var) get_cpu_var(var) -+#define put_locked_var(lvar, var) put_cpu_var(var) ++static void make_discard_request(struct mddev *mddev, struct bio *bi) ++{ ++ struct r5conf *conf = mddev->private; ++ sector_t logical_sector, last_sector; ++ struct stripe_head *sh; ++ int remaining; ++ int stripe_sectors; ++ ++ if (mddev->reshape_position != MaxSector) ++ /* Skip discard while reshape is happening */ ++ return; + -+#define local_lock_cpu(lvar) get_cpu() -+#define local_unlock_cpu(lvar) put_cpu() ++ logical_sector = bi->bi_iter.bi_sector & ~((sector_t)STRIPE_SECTORS-1); ++ last_sector = bi->bi_iter.bi_sector + (bi->bi_iter.bi_size>>9); ++ ++ bi->bi_next = NULL; ++ bi->bi_phys_segments = 1; /* over-loaded to count active stripes */ ++ ++ stripe_sectors = conf->chunk_sectors * ++ (conf->raid_disks - conf->max_degraded); ++ logical_sector = DIV_ROUND_UP_SECTOR_T(logical_sector, ++ stripe_sectors); ++ sector_div(last_sector, stripe_sectors); ++ ++ logical_sector *= conf->chunk_sectors; ++ last_sector *= conf->chunk_sectors; ++ ++ for (; logical_sector < last_sector; ++ logical_sector += STRIPE_SECTORS) { ++ DEFINE_WAIT(w); ++ int d; ++ again: ++ sh = get_active_stripe(conf, logical_sector, 0, 0, 0); ++ prepare_to_wait(&conf->wait_for_overlap, &w, ++ TASK_UNINTERRUPTIBLE); ++ set_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags); ++ if (test_bit(STRIPE_SYNCING, &sh->state)) { ++ release_stripe(sh); ++ schedule(); ++ goto again; ++ } ++ clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags); ++ spin_lock_irq(&sh->stripe_lock); ++ for (d = 0; d < conf->raid_disks; d++) { ++ if (d == sh->pd_idx || d == sh->qd_idx) ++ continue; ++ if (sh->dev[d].towrite || sh->dev[d].toread) { ++ set_bit(R5_Overlap, &sh->dev[d].flags); ++ spin_unlock_irq(&sh->stripe_lock); ++ release_stripe(sh); ++ schedule(); ++ goto again; ++ } ++ } ++ set_bit(STRIPE_DISCARD, &sh->state); ++ finish_wait(&conf->wait_for_overlap, &w); ++ sh->overwrite_disks = 0; ++ for (d = 0; d < conf->raid_disks; d++) { ++ if (d == sh->pd_idx || d == sh->qd_idx) ++ continue; ++ sh->dev[d].towrite = bi; ++ set_bit(R5_OVERWRITE, &sh->dev[d].flags); ++ raid5_inc_bi_active_stripes(bi); ++ sh->overwrite_disks++; ++ } ++ spin_unlock_irq(&sh->stripe_lock); ++ if (conf->mddev->bitmap) { ++ for (d = 0; ++ d < conf->raid_disks - conf->max_degraded; ++ d++) ++ bitmap_startwrite(mddev->bitmap, ++ sh->sector, ++ STRIPE_SECTORS, ++ 0); ++ sh->bm_seq = conf->seq_flush + 1; ++ set_bit(STRIPE_BIT_DELAY, &sh->state); ++ } + -+#endif ++ set_bit(STRIPE_HANDLE, &sh->state); ++ clear_bit(STRIPE_DELAYED, &sh->state); ++ if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) ++ atomic_inc(&conf->preread_active_stripes); ++ release_stripe_plug(mddev, sh); ++ } + -+#endif -diff -Nur linux-4.1.6.orig/include/linux/mm_types.h linux-4.1.6/include/linux/mm_types.h ---- linux-4.1.6.orig/include/linux/mm_types.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/mm_types.h 2015-09-08 23:49:06.550039912 +0200 -@@ -11,6 +11,7 @@ - #include - #include - #include -+#include - #include - #include - #include -@@ -462,6 +463,9 @@ - bool tlb_flush_pending; - #endif - struct uprobes_state uprobes_state; -+#ifdef CONFIG_PREEMPT_RT_BASE -+ struct rcu_head delayed_drop; -+#endif - #ifdef CONFIG_X86_INTEL_MPX - /* address of the bounds directory */ - void __user *bd_addr; -diff -Nur linux-4.1.6.orig/include/linux/mutex.h linux-4.1.6/include/linux/mutex.h ---- linux-4.1.6.orig/include/linux/mutex.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/mutex.h 2015-09-08 23:49:06.774015077 +0200 -@@ -19,6 +19,17 @@ - #include - #include - -+#ifdef CONFIG_DEBUG_LOCK_ALLOC -+# define __DEP_MAP_MUTEX_INITIALIZER(lockname) \ -+ , .dep_map = { .name = #lockname } -+#else -+# define __DEP_MAP_MUTEX_INITIALIZER(lockname) -+#endif ++ remaining = raid5_dec_bi_active_stripes(bi); ++ if (remaining == 0) { ++ md_write_end(mddev); ++ bio_endio(bi, 0); ++ } ++} + -+#ifdef CONFIG_PREEMPT_RT_FULL -+# include -+#else ++static void make_request(struct mddev *mddev, struct bio * bi) ++{ ++ struct r5conf *conf = mddev->private; ++ int dd_idx; ++ sector_t new_sector; ++ sector_t logical_sector, last_sector; ++ struct stripe_head *sh; ++ const int rw = bio_data_dir(bi); ++ int remaining; ++ DEFINE_WAIT(w); ++ bool do_prepare; ++ ++ if (unlikely(bi->bi_rw & REQ_FLUSH)) { ++ md_flush_request(mddev, bi); ++ return; ++ } + - /* - * Simple, straightforward mutexes with strict semantics: - * -@@ -99,13 +110,6 @@ - static inline void mutex_destroy(struct mutex *lock) {} - #endif - --#ifdef CONFIG_DEBUG_LOCK_ALLOC --# define __DEP_MAP_MUTEX_INITIALIZER(lockname) \ -- , .dep_map = { .name = #lockname } --#else --# define __DEP_MAP_MUTEX_INITIALIZER(lockname) --#endif -- - #define __MUTEX_INITIALIZER(lockname) \ - { .count = ATOMIC_INIT(1) \ - , .wait_lock = __SPIN_LOCK_UNLOCKED(lockname.wait_lock) \ -@@ -173,6 +177,8 @@ - extern int mutex_trylock(struct mutex *lock); - extern void mutex_unlock(struct mutex *lock); - -+#endif /* !PREEMPT_RT_FULL */ ++ md_write_start(mddev, bi); + - extern int atomic_dec_and_mutex_lock(atomic_t *cnt, struct mutex *lock); - - #endif /* __LINUX_MUTEX_H */ -diff -Nur linux-4.1.6.orig/include/linux/mutex_rt.h linux-4.1.6/include/linux/mutex_rt.h ---- linux-4.1.6.orig/include/linux/mutex_rt.h 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.6/include/linux/mutex_rt.h 2015-09-08 23:49:06.774015077 +0200 -@@ -0,0 +1,84 @@ -+#ifndef __LINUX_MUTEX_RT_H -+#define __LINUX_MUTEX_RT_H ++ /* ++ * If array is degraded, better not do chunk aligned read because ++ * later we might have to read it again in order to reconstruct ++ * data on failed drives. ++ */ ++ if (rw == READ && mddev->degraded == 0 && ++ mddev->reshape_position == MaxSector && ++ chunk_aligned_read(mddev,bi)) ++ return; + -+#ifndef __LINUX_MUTEX_H -+#error "Please include mutex.h" -+#endif ++ if (unlikely(bi->bi_rw & REQ_DISCARD)) { ++ make_discard_request(mddev, bi); ++ return; ++ } + -+#include ++ logical_sector = bi->bi_iter.bi_sector & ~((sector_t)STRIPE_SECTORS-1); ++ last_sector = bio_end_sector(bi); ++ bi->bi_next = NULL; ++ bi->bi_phys_segments = 1; /* over-loaded to count active stripes */ ++ ++ prepare_to_wait(&conf->wait_for_overlap, &w, TASK_UNINTERRUPTIBLE); ++ for (;logical_sector < last_sector; logical_sector += STRIPE_SECTORS) { ++ int previous; ++ int seq; ++ ++ do_prepare = false; ++ retry: ++ seq = read_seqcount_begin(&conf->gen_lock); ++ previous = 0; ++ if (do_prepare) ++ prepare_to_wait(&conf->wait_for_overlap, &w, ++ TASK_UNINTERRUPTIBLE); ++ if (unlikely(conf->reshape_progress != MaxSector)) { ++ /* spinlock is needed as reshape_progress may be ++ * 64bit on a 32bit platform, and so it might be ++ * possible to see a half-updated value ++ * Of course reshape_progress could change after ++ * the lock is dropped, so once we get a reference ++ * to the stripe that we think it is, we will have ++ * to check again. ++ */ ++ spin_lock_irq(&conf->device_lock); ++ if (mddev->reshape_backwards ++ ? logical_sector < conf->reshape_progress ++ : logical_sector >= conf->reshape_progress) { ++ previous = 1; ++ } else { ++ if (mddev->reshape_backwards ++ ? logical_sector < conf->reshape_safe ++ : logical_sector >= conf->reshape_safe) { ++ spin_unlock_irq(&conf->device_lock); ++ schedule(); ++ do_prepare = true; ++ goto retry; ++ } ++ } ++ spin_unlock_irq(&conf->device_lock); ++ } + -+/* FIXME: Just for __lockfunc */ -+#include ++ new_sector = raid5_compute_sector(conf, logical_sector, ++ previous, ++ &dd_idx, NULL); ++ pr_debug("raid456: make_request, sector %llu logical %llu\n", ++ (unsigned long long)new_sector, ++ (unsigned long long)logical_sector); ++ ++ sh = get_active_stripe(conf, new_sector, previous, ++ (bi->bi_rw&RWA_MASK), 0); ++ if (sh) { ++ if (unlikely(previous)) { ++ /* expansion might have moved on while waiting for a ++ * stripe, so we must do the range check again. ++ * Expansion could still move past after this ++ * test, but as we are holding a reference to ++ * 'sh', we know that if that happens, ++ * STRIPE_EXPANDING will get set and the expansion ++ * won't proceed until we finish with the stripe. ++ */ ++ int must_retry = 0; ++ spin_lock_irq(&conf->device_lock); ++ if (mddev->reshape_backwards ++ ? logical_sector >= conf->reshape_progress ++ : logical_sector < conf->reshape_progress) ++ /* mismatch, need to try again */ ++ must_retry = 1; ++ spin_unlock_irq(&conf->device_lock); ++ if (must_retry) { ++ release_stripe(sh); ++ schedule(); ++ do_prepare = true; ++ goto retry; ++ } ++ } ++ if (read_seqcount_retry(&conf->gen_lock, seq)) { ++ /* Might have got the wrong stripe_head ++ * by accident ++ */ ++ release_stripe(sh); ++ goto retry; ++ } + -+struct mutex { -+ struct rt_mutex lock; -+#ifdef CONFIG_DEBUG_LOCK_ALLOC -+ struct lockdep_map dep_map; -+#endif -+}; ++ if (rw == WRITE && ++ logical_sector >= mddev->suspend_lo && ++ logical_sector < mddev->suspend_hi) { ++ release_stripe(sh); ++ /* As the suspend_* range is controlled by ++ * userspace, we want an interruptible ++ * wait. ++ */ ++ flush_signals(current); ++ prepare_to_wait(&conf->wait_for_overlap, ++ &w, TASK_INTERRUPTIBLE); ++ if (logical_sector >= mddev->suspend_lo && ++ logical_sector < mddev->suspend_hi) { ++ schedule(); ++ do_prepare = true; ++ } ++ goto retry; ++ } + -+#define __MUTEX_INITIALIZER(mutexname) \ -+ { \ -+ .lock = __RT_MUTEX_INITIALIZER(mutexname.lock) \ -+ __DEP_MAP_MUTEX_INITIALIZER(mutexname) \ ++ if (test_bit(STRIPE_EXPANDING, &sh->state) || ++ !add_stripe_bio(sh, bi, dd_idx, rw, previous)) { ++ /* Stripe is busy expanding or ++ * add failed due to overlap. Flush everything ++ * and wait a while ++ */ ++ md_wakeup_thread(mddev->thread); ++ release_stripe(sh); ++ schedule(); ++ do_prepare = true; ++ goto retry; ++ } ++ set_bit(STRIPE_HANDLE, &sh->state); ++ clear_bit(STRIPE_DELAYED, &sh->state); ++ if ((!sh->batch_head || sh == sh->batch_head) && ++ (bi->bi_rw & REQ_SYNC) && ++ !test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state)) ++ atomic_inc(&conf->preread_active_stripes); ++ release_stripe_plug(mddev, sh); ++ } else { ++ /* cannot get stripe for read-ahead, just give-up */ ++ clear_bit(BIO_UPTODATE, &bi->bi_flags); ++ break; ++ } + } ++ finish_wait(&conf->wait_for_overlap, &w); + -+#define DEFINE_MUTEX(mutexname) \ -+ struct mutex mutexname = __MUTEX_INITIALIZER(mutexname) ++ remaining = raid5_dec_bi_active_stripes(bi); ++ if (remaining == 0) { + -+extern void __mutex_do_init(struct mutex *lock, const char *name, struct lock_class_key *key); -+extern void __lockfunc _mutex_lock(struct mutex *lock); -+extern int __lockfunc _mutex_lock_interruptible(struct mutex *lock); -+extern int __lockfunc _mutex_lock_killable(struct mutex *lock); -+extern void __lockfunc _mutex_lock_nested(struct mutex *lock, int subclass); -+extern void __lockfunc _mutex_lock_nest_lock(struct mutex *lock, struct lockdep_map *nest_lock); -+extern int __lockfunc _mutex_lock_interruptible_nested(struct mutex *lock, int subclass); -+extern int __lockfunc _mutex_lock_killable_nested(struct mutex *lock, int subclass); -+extern int __lockfunc _mutex_trylock(struct mutex *lock); -+extern void __lockfunc _mutex_unlock(struct mutex *lock); ++ if ( rw == WRITE ) ++ md_write_end(mddev); + -+#define mutex_is_locked(l) rt_mutex_is_locked(&(l)->lock) -+#define mutex_lock(l) _mutex_lock(l) -+#define mutex_lock_interruptible(l) _mutex_lock_interruptible(l) -+#define mutex_lock_killable(l) _mutex_lock_killable(l) -+#define mutex_trylock(l) _mutex_trylock(l) -+#define mutex_unlock(l) _mutex_unlock(l) -+#define mutex_destroy(l) rt_mutex_destroy(&(l)->lock) ++ trace_block_bio_complete(bdev_get_queue(bi->bi_bdev), ++ bi, 0); ++ bio_endio(bi, 0); ++ } ++} + -+#ifdef CONFIG_DEBUG_LOCK_ALLOC -+# define mutex_lock_nested(l, s) _mutex_lock_nested(l, s) -+# define mutex_lock_interruptible_nested(l, s) \ -+ _mutex_lock_interruptible_nested(l, s) -+# define mutex_lock_killable_nested(l, s) \ -+ _mutex_lock_killable_nested(l, s) ++static sector_t raid5_size(struct mddev *mddev, sector_t sectors, int raid_disks); + -+# define mutex_lock_nest_lock(lock, nest_lock) \ -+do { \ -+ typecheck(struct lockdep_map *, &(nest_lock)->dep_map); \ -+ _mutex_lock_nest_lock(lock, &(nest_lock)->dep_map); \ -+} while (0) ++static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, int *skipped) ++{ ++ /* reshaping is quite different to recovery/resync so it is ++ * handled quite separately ... here. ++ * ++ * On each call to sync_request, we gather one chunk worth of ++ * destination stripes and flag them as expanding. ++ * Then we find all the source stripes and request reads. ++ * As the reads complete, handle_stripe will copy the data ++ * into the destination stripe and release that stripe. ++ */ ++ struct r5conf *conf = mddev->private; ++ struct stripe_head *sh; ++ sector_t first_sector, last_sector; ++ int raid_disks = conf->previous_raid_disks; ++ int data_disks = raid_disks - conf->max_degraded; ++ int new_data_disks = conf->raid_disks - conf->max_degraded; ++ int i; ++ int dd_idx; ++ sector_t writepos, readpos, safepos; ++ sector_t stripe_addr; ++ int reshape_sectors; ++ struct list_head stripes; ++ ++ if (sector_nr == 0) { ++ /* If restarting in the middle, skip the initial sectors */ ++ if (mddev->reshape_backwards && ++ conf->reshape_progress < raid5_size(mddev, 0, 0)) { ++ sector_nr = raid5_size(mddev, 0, 0) ++ - conf->reshape_progress; ++ } else if (!mddev->reshape_backwards && ++ conf->reshape_progress > 0) ++ sector_nr = conf->reshape_progress; ++ sector_div(sector_nr, new_data_disks); ++ if (sector_nr) { ++ mddev->curr_resync_completed = sector_nr; ++ sysfs_notify(&mddev->kobj, NULL, "sync_completed"); ++ *skipped = 1; ++ return sector_nr; ++ } ++ } + -+#else -+# define mutex_lock_nested(l, s) _mutex_lock(l) -+# define mutex_lock_interruptible_nested(l, s) \ -+ _mutex_lock_interruptible(l) -+# define mutex_lock_killable_nested(l, s) \ -+ _mutex_lock_killable(l) -+# define mutex_lock_nest_lock(lock, nest_lock) mutex_lock(lock) -+#endif ++ /* We need to process a full chunk at a time. ++ * If old and new chunk sizes differ, we need to process the ++ * largest of these ++ */ ++ if (mddev->new_chunk_sectors > mddev->chunk_sectors) ++ reshape_sectors = mddev->new_chunk_sectors; ++ else ++ reshape_sectors = mddev->chunk_sectors; + -+# define mutex_init(mutex) \ -+do { \ -+ static struct lock_class_key __key; \ -+ \ -+ rt_mutex_init(&(mutex)->lock); \ -+ __mutex_do_init((mutex), #mutex, &__key); \ -+} while (0) ++ /* We update the metadata at least every 10 seconds, or when ++ * the data about to be copied would over-write the source of ++ * the data at the front of the range. i.e. one new_stripe ++ * along from reshape_progress new_maps to after where ++ * reshape_safe old_maps to ++ */ ++ writepos = conf->reshape_progress; ++ sector_div(writepos, new_data_disks); ++ readpos = conf->reshape_progress; ++ sector_div(readpos, data_disks); ++ safepos = conf->reshape_safe; ++ sector_div(safepos, data_disks); ++ if (mddev->reshape_backwards) { ++ writepos -= min_t(sector_t, reshape_sectors, writepos); ++ readpos += reshape_sectors; ++ safepos += reshape_sectors; ++ } else { ++ writepos += reshape_sectors; ++ readpos -= min_t(sector_t, reshape_sectors, readpos); ++ safepos -= min_t(sector_t, reshape_sectors, safepos); ++ } + -+# define __mutex_init(mutex, name, key) \ -+do { \ -+ rt_mutex_init(&(mutex)->lock); \ -+ __mutex_do_init((mutex), name, key); \ -+} while (0) ++ /* Having calculated the 'writepos' possibly use it ++ * to set 'stripe_addr' which is where we will write to. ++ */ ++ if (mddev->reshape_backwards) { ++ BUG_ON(conf->reshape_progress == 0); ++ stripe_addr = writepos; ++ BUG_ON((mddev->dev_sectors & ++ ~((sector_t)reshape_sectors - 1)) ++ - reshape_sectors - stripe_addr ++ != sector_nr); ++ } else { ++ BUG_ON(writepos != sector_nr + reshape_sectors); ++ stripe_addr = sector_nr; ++ } + -+#endif -diff -Nur linux-4.1.6.orig/include/linux/netdevice.h linux-4.1.6/include/linux/netdevice.h ---- linux-4.1.6.orig/include/linux/netdevice.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/netdevice.h 2015-09-08 23:49:06.778014632 +0200 -@@ -2469,6 +2469,7 @@ - unsigned int dropped; - struct sk_buff_head input_pkt_queue; - struct napi_struct backlog; -+ struct sk_buff_head tofree_queue; - - }; - -diff -Nur linux-4.1.6.orig/include/linux/netfilter/x_tables.h linux-4.1.6/include/linux/netfilter/x_tables.h ---- linux-4.1.6.orig/include/linux/netfilter/x_tables.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/netfilter/x_tables.h 2015-09-08 23:49:06.778014632 +0200 -@@ -3,6 +3,7 @@ - - - #include -+#include - #include - - /** -@@ -282,6 +283,8 @@ - */ - DECLARE_PER_CPU(seqcount_t, xt_recseq); - -+DECLARE_LOCAL_IRQ_LOCK(xt_write_lock); ++ /* 'writepos' is the most advanced device address we might write. ++ * 'readpos' is the least advanced device address we might read. ++ * 'safepos' is the least address recorded in the metadata as having ++ * been reshaped. ++ * If there is a min_offset_diff, these are adjusted either by ++ * increasing the safepos/readpos if diff is negative, or ++ * increasing writepos if diff is positive. ++ * If 'readpos' is then behind 'writepos', there is no way that we can ++ * ensure safety in the face of a crash - that must be done by userspace ++ * making a backup of the data. So in that case there is no particular ++ * rush to update metadata. ++ * Otherwise if 'safepos' is behind 'writepos', then we really need to ++ * update the metadata to advance 'safepos' to match 'readpos' so that ++ * we can be safe in the event of a crash. ++ * So we insist on updating metadata if safepos is behind writepos and ++ * readpos is beyond writepos. ++ * In any case, update the metadata every 10 seconds. ++ * Maybe that number should be configurable, but I'm not sure it is ++ * worth it.... maybe it could be a multiple of safemode_delay??? ++ */ ++ if (conf->min_offset_diff < 0) { ++ safepos += -conf->min_offset_diff; ++ readpos += -conf->min_offset_diff; ++ } else ++ writepos += conf->min_offset_diff; ++ ++ if ((mddev->reshape_backwards ++ ? (safepos > writepos && readpos < writepos) ++ : (safepos < writepos && readpos > writepos)) || ++ time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) { ++ /* Cannot proceed until we've updated the superblock... */ ++ wait_event(conf->wait_for_overlap, ++ atomic_read(&conf->reshape_stripes)==0 ++ || test_bit(MD_RECOVERY_INTR, &mddev->recovery)); ++ if (atomic_read(&conf->reshape_stripes) != 0) ++ return 0; ++ mddev->reshape_position = conf->reshape_progress; ++ mddev->curr_resync_completed = sector_nr; ++ conf->reshape_checkpoint = jiffies; ++ set_bit(MD_CHANGE_DEVS, &mddev->flags); ++ md_wakeup_thread(mddev->thread); ++ wait_event(mddev->sb_wait, mddev->flags == 0 || ++ test_bit(MD_RECOVERY_INTR, &mddev->recovery)); ++ if (test_bit(MD_RECOVERY_INTR, &mddev->recovery)) ++ return 0; ++ spin_lock_irq(&conf->device_lock); ++ conf->reshape_safe = mddev->reshape_position; ++ spin_unlock_irq(&conf->device_lock); ++ wake_up(&conf->wait_for_overlap); ++ sysfs_notify(&mddev->kobj, NULL, "sync_completed"); ++ } + - /** - * xt_write_recseq_begin - start of a write section - * -@@ -296,6 +299,9 @@ - { - unsigned int addend; - -+ /* RT protection */ -+ local_lock(xt_write_lock); ++ INIT_LIST_HEAD(&stripes); ++ for (i = 0; i < reshape_sectors; i += STRIPE_SECTORS) { ++ int j; ++ int skipped_disk = 0; ++ sh = get_active_stripe(conf, stripe_addr+i, 0, 0, 1); ++ set_bit(STRIPE_EXPANDING, &sh->state); ++ atomic_inc(&conf->reshape_stripes); ++ /* If any of this stripe is beyond the end of the old ++ * array, then we need to zero those blocks ++ */ ++ for (j=sh->disks; j--;) { ++ sector_t s; ++ if (j == sh->pd_idx) ++ continue; ++ if (conf->level == 6 && ++ j == sh->qd_idx) ++ continue; ++ s = compute_blocknr(sh, j, 0); ++ if (s < raid5_size(mddev, 0, 0)) { ++ skipped_disk = 1; ++ continue; ++ } ++ memset(page_address(sh->dev[j].page), 0, STRIPE_SIZE); ++ set_bit(R5_Expanded, &sh->dev[j].flags); ++ set_bit(R5_UPTODATE, &sh->dev[j].flags); ++ } ++ if (!skipped_disk) { ++ set_bit(STRIPE_EXPAND_READY, &sh->state); ++ set_bit(STRIPE_HANDLE, &sh->state); ++ } ++ list_add(&sh->lru, &stripes); ++ } ++ spin_lock_irq(&conf->device_lock); ++ if (mddev->reshape_backwards) ++ conf->reshape_progress -= reshape_sectors * new_data_disks; ++ else ++ conf->reshape_progress += reshape_sectors * new_data_disks; ++ spin_unlock_irq(&conf->device_lock); ++ /* Ok, those stripe are ready. We can start scheduling ++ * reads on the source stripes. ++ * The source stripes are determined by mapping the first and last ++ * block on the destination stripes. ++ */ ++ first_sector = ++ raid5_compute_sector(conf, stripe_addr*(new_data_disks), ++ 1, &dd_idx, NULL); ++ last_sector = ++ raid5_compute_sector(conf, ((stripe_addr+reshape_sectors) ++ * new_data_disks - 1), ++ 1, &dd_idx, NULL); ++ if (last_sector >= mddev->dev_sectors) ++ last_sector = mddev->dev_sectors - 1; ++ while (first_sector <= last_sector) { ++ sh = get_active_stripe(conf, first_sector, 1, 0, 1); ++ set_bit(STRIPE_EXPAND_SOURCE, &sh->state); ++ set_bit(STRIPE_HANDLE, &sh->state); ++ release_stripe(sh); ++ first_sector += STRIPE_SECTORS; ++ } ++ /* Now that the sources are clearly marked, we can release ++ * the destination stripes ++ */ ++ while (!list_empty(&stripes)) { ++ sh = list_entry(stripes.next, struct stripe_head, lru); ++ list_del_init(&sh->lru); ++ release_stripe(sh); ++ } ++ /* If this takes us to the resync_max point where we have to pause, ++ * then we need to write out the superblock. ++ */ ++ sector_nr += reshape_sectors; ++ if ((sector_nr - mddev->curr_resync_completed) * 2 ++ >= mddev->resync_max - mddev->curr_resync_completed) { ++ /* Cannot proceed until we've updated the superblock... */ ++ wait_event(conf->wait_for_overlap, ++ atomic_read(&conf->reshape_stripes) == 0 ++ || test_bit(MD_RECOVERY_INTR, &mddev->recovery)); ++ if (atomic_read(&conf->reshape_stripes) != 0) ++ goto ret; ++ mddev->reshape_position = conf->reshape_progress; ++ mddev->curr_resync_completed = sector_nr; ++ conf->reshape_checkpoint = jiffies; ++ set_bit(MD_CHANGE_DEVS, &mddev->flags); ++ md_wakeup_thread(mddev->thread); ++ wait_event(mddev->sb_wait, ++ !test_bit(MD_CHANGE_DEVS, &mddev->flags) ++ || test_bit(MD_RECOVERY_INTR, &mddev->recovery)); ++ if (test_bit(MD_RECOVERY_INTR, &mddev->recovery)) ++ goto ret; ++ spin_lock_irq(&conf->device_lock); ++ conf->reshape_safe = mddev->reshape_position; ++ spin_unlock_irq(&conf->device_lock); ++ wake_up(&conf->wait_for_overlap); ++ sysfs_notify(&mddev->kobj, NULL, "sync_completed"); ++ } ++ret: ++ return reshape_sectors; ++} + - /* - * Low order bit of sequence is set if we already - * called xt_write_recseq_begin(). -@@ -326,6 +332,7 @@ - /* this is kind of a write_seqcount_end(), but addend is 0 or 1 */ - smp_wmb(); - __this_cpu_add(xt_recseq.sequence, addend); -+ local_unlock(xt_write_lock); - } - - /* -diff -Nur linux-4.1.6.orig/include/linux/notifier.h linux-4.1.6/include/linux/notifier.h ---- linux-4.1.6.orig/include/linux/notifier.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/notifier.h 2015-09-08 23:49:06.778014632 +0200 -@@ -6,7 +6,7 @@ - * - * Alan Cox - */ -- ++static inline sector_t sync_request(struct mddev *mddev, sector_t sector_nr, int *skipped) ++{ ++ struct r5conf *conf = mddev->private; ++ struct stripe_head *sh; ++ sector_t max_sector = mddev->dev_sectors; ++ sector_t sync_blocks; ++ int still_degraded = 0; ++ int i; + - #ifndef _LINUX_NOTIFIER_H - #define _LINUX_NOTIFIER_H - #include -@@ -42,9 +42,7 @@ - * in srcu_notifier_call_chain(): no cache bounces and no memory barriers. - * As compensation, srcu_notifier_chain_unregister() is rather expensive. - * SRCU notifier chains should be used when the chain will be called very -- * often but notifier_blocks will seldom be removed. Also, SRCU notifier -- * chains are slightly more difficult to use because they require special -- * runtime initialization. -+ * often but notifier_blocks will seldom be removed. - */ - - typedef int (*notifier_fn_t)(struct notifier_block *nb, -@@ -88,7 +86,7 @@ - (name)->head = NULL; \ - } while (0) - --/* srcu_notifier_heads must be initialized and cleaned up dynamically */ -+/* srcu_notifier_heads must be cleaned up dynamically */ - extern void srcu_init_notifier_head(struct srcu_notifier_head *nh); - #define srcu_cleanup_notifier_head(name) \ - cleanup_srcu_struct(&(name)->srcu); -@@ -101,7 +99,13 @@ - .head = NULL } - #define RAW_NOTIFIER_INIT(name) { \ - .head = NULL } --/* srcu_notifier_heads cannot be initialized statically */ ++ if (sector_nr >= max_sector) { ++ /* just being told to finish up .. nothing much to do */ + -+#define SRCU_NOTIFIER_INIT(name, pcpu) \ -+ { \ -+ .mutex = __MUTEX_INITIALIZER(name.mutex), \ -+ .head = NULL, \ -+ .srcu = __SRCU_STRUCT_INIT(name.srcu, pcpu), \ ++ if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) { ++ end_reshape(conf); ++ return 0; ++ } ++ ++ if (mddev->curr_resync < max_sector) /* aborted */ ++ bitmap_end_sync(mddev->bitmap, mddev->curr_resync, ++ &sync_blocks, 1); ++ else /* completed sync */ ++ conf->fullsync = 0; ++ bitmap_close_sync(mddev->bitmap); ++ ++ return 0; + } - - #define ATOMIC_NOTIFIER_HEAD(name) \ - struct atomic_notifier_head name = \ -@@ -113,6 +117,18 @@ - struct raw_notifier_head name = \ - RAW_NOTIFIER_INIT(name) - -+#define _SRCU_NOTIFIER_HEAD(name, mod) \ -+ static DEFINE_PER_CPU(struct srcu_struct_array, \ -+ name##_head_srcu_array); \ -+ mod struct srcu_notifier_head name = \ -+ SRCU_NOTIFIER_INIT(name, name##_head_srcu_array) + -+#define SRCU_NOTIFIER_HEAD(name) \ -+ _SRCU_NOTIFIER_HEAD(name, ) ++ /* Allow raid5_quiesce to complete */ ++ wait_event(conf->wait_for_overlap, conf->quiesce != 2); + -+#define SRCU_NOTIFIER_HEAD_STATIC(name) \ -+ _SRCU_NOTIFIER_HEAD(name, static) ++ if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) ++ return reshape_request(mddev, sector_nr, skipped); + - #ifdef __KERNEL__ - - extern int atomic_notifier_chain_register(struct atomic_notifier_head *nh, -@@ -182,12 +198,12 @@ - - /* - * Declared notifiers so far. I can imagine quite a few more chains -- * over time (eg laptop power reset chains, reboot chain (to clean -+ * over time (eg laptop power reset chains, reboot chain (to clean - * device units up), device [un]mount chain, module load/unload chain, -- * low memory chain, screenblank chain (for plug in modular screenblankers) -+ * low memory chain, screenblank chain (for plug in modular screenblankers) - * VC switch chains (for loadable kernel svgalib VC switch helpers) etc... - */ -- ++ /* No need to check resync_max as we never do more than one ++ * stripe, and as resync_max will always be on a chunk boundary, ++ * if the check in md_do_sync didn't fire, there is no chance ++ * of overstepping resync_max here ++ */ + - /* CPU notfiers are defined in include/linux/cpu.h. */ - - /* netdevice notifiers are defined in include/linux/netdevice.h */ -diff -Nur linux-4.1.6.orig/include/linux/percpu.h linux-4.1.6/include/linux/percpu.h ---- linux-4.1.6.orig/include/linux/percpu.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/percpu.h 2015-09-08 23:49:06.778014632 +0200 -@@ -24,6 +24,35 @@ - PERCPU_MODULE_RESERVE) - #endif - -+#ifdef CONFIG_PREEMPT_RT_FULL ++ /* if there is too many failed drives and we are trying ++ * to resync, then assert that we are finished, because there is ++ * nothing we can do. ++ */ ++ if (mddev->degraded >= conf->max_degraded && ++ test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) { ++ sector_t rv = mddev->dev_sectors - sector_nr; ++ *skipped = 1; ++ return rv; ++ } ++ if (!test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) && ++ !conf->fullsync && ++ !bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, 1) && ++ sync_blocks >= STRIPE_SECTORS) { ++ /* we can skip this block, and probably more */ ++ sync_blocks /= STRIPE_SECTORS; ++ *skipped = 1; ++ return sync_blocks * STRIPE_SECTORS; /* keep things rounded to whole stripes */ ++ } + -+#define get_local_var(var) (*({ \ -+ migrate_disable(); \ -+ this_cpu_ptr(&var); })) ++ bitmap_cond_end_sync(mddev->bitmap, sector_nr); + -+#define put_local_var(var) do { \ -+ (void)&(var); \ -+ migrate_enable(); \ -+} while (0) ++ sh = get_active_stripe(conf, sector_nr, 0, 1, 0); ++ if (sh == NULL) { ++ sh = get_active_stripe(conf, sector_nr, 0, 0, 0); ++ /* make sure we don't swamp the stripe cache if someone else ++ * is trying to get access ++ */ ++ schedule_timeout_uninterruptible(1); ++ } ++ /* Need to check if array will still be degraded after recovery/resync ++ * Note in case of > 1 drive failures it's possible we're rebuilding ++ * one drive while leaving another faulty drive in array. ++ */ ++ rcu_read_lock(); ++ for (i = 0; i < conf->raid_disks; i++) { ++ struct md_rdev *rdev = ACCESS_ONCE(conf->disks[i].rdev); + -+# define get_local_ptr(var) ({ \ -+ migrate_disable(); \ -+ this_cpu_ptr(var); }) ++ if (rdev == NULL || test_bit(Faulty, &rdev->flags)) ++ still_degraded = 1; ++ } ++ rcu_read_unlock(); + -+# define put_local_ptr(var) do { \ -+ (void)(var); \ -+ migrate_enable(); \ -+} while (0) ++ bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, still_degraded); + -+#else ++ set_bit(STRIPE_SYNC_REQUESTED, &sh->state); ++ set_bit(STRIPE_HANDLE, &sh->state); + -+#define get_local_var(var) get_cpu_var(var) -+#define put_local_var(var) put_cpu_var(var) -+#define get_local_ptr(var) get_cpu_ptr(var) -+#define put_local_ptr(var) put_cpu_ptr(var) ++ release_stripe(sh); + -+#endif ++ return STRIPE_SECTORS; ++} + - /* minimum unit size, also is the maximum supported allocation size */ - #define PCPU_MIN_UNIT_SIZE PFN_ALIGN(32 << 10) - -diff -Nur linux-4.1.6.orig/include/linux/pid.h linux-4.1.6/include/linux/pid.h ---- linux-4.1.6.orig/include/linux/pid.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/pid.h 2015-09-08 23:49:06.778014632 +0200 -@@ -2,6 +2,7 @@ - #define _LINUX_PID_H - - #include -+#include - - enum pid_type - { -diff -Nur linux-4.1.6.orig/include/linux/preempt.h linux-4.1.6/include/linux/preempt.h ---- linux-4.1.6.orig/include/linux/preempt.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/preempt.h 2015-09-08 23:49:06.778014632 +0200 -@@ -33,6 +33,20 @@ - #define preempt_count_inc() preempt_count_add(1) - #define preempt_count_dec() preempt_count_sub(1) - -+#ifdef CONFIG_PREEMPT_LAZY -+#define add_preempt_lazy_count(val) do { preempt_lazy_count() += (val); } while (0) -+#define sub_preempt_lazy_count(val) do { preempt_lazy_count() -= (val); } while (0) -+#define inc_preempt_lazy_count() add_preempt_lazy_count(1) -+#define dec_preempt_lazy_count() sub_preempt_lazy_count(1) -+#define preempt_lazy_count() (current_thread_info()->preempt_lazy_count) -+#else -+#define add_preempt_lazy_count(val) do { } while (0) -+#define sub_preempt_lazy_count(val) do { } while (0) -+#define inc_preempt_lazy_count() do { } while (0) -+#define dec_preempt_lazy_count() do { } while (0) -+#define preempt_lazy_count() (0) -+#endif ++static int retry_aligned_read(struct r5conf *conf, struct bio *raid_bio) ++{ ++ /* We may not be able to submit a whole bio at once as there ++ * may not be enough stripe_heads available. ++ * We cannot pre-allocate enough stripe_heads as we may need ++ * more than exist in the cache (if we allow ever large chunks). ++ * So we do one stripe head at a time and record in ++ * ->bi_hw_segments how many have been done. ++ * ++ * We *know* that this entire raid_bio is in one chunk, so ++ * it will be only one 'dd_idx' and only need one call to raid5_compute_sector. ++ */ ++ struct stripe_head *sh; ++ int dd_idx; ++ sector_t sector, logical_sector, last_sector; ++ int scnt = 0; ++ int remaining; ++ int handled = 0; ++ ++ logical_sector = raid_bio->bi_iter.bi_sector & ++ ~((sector_t)STRIPE_SECTORS-1); ++ sector = raid5_compute_sector(conf, logical_sector, ++ 0, &dd_idx, NULL); ++ last_sector = bio_end_sector(raid_bio); ++ ++ for (; logical_sector < last_sector; ++ logical_sector += STRIPE_SECTORS, ++ sector += STRIPE_SECTORS, ++ scnt++) { ++ ++ if (scnt < raid5_bi_processed_stripes(raid_bio)) ++ /* already done this stripe */ ++ continue; + - #ifdef CONFIG_PREEMPT_COUNT - - #define preempt_disable() \ -@@ -41,13 +55,25 @@ - barrier(); \ - } while (0) - -+#define preempt_lazy_disable() \ -+do { \ -+ inc_preempt_lazy_count(); \ -+ barrier(); \ -+} while (0) ++ sh = get_active_stripe(conf, sector, 0, 1, 1); + - #define sched_preempt_enable_no_resched() \ - do { \ - barrier(); \ - preempt_count_dec(); \ - } while (0) - --#define preempt_enable_no_resched() sched_preempt_enable_no_resched() -+#ifdef CONFIG_PREEMPT_RT_BASE -+# define preempt_enable_no_resched() sched_preempt_enable_no_resched() -+# define preempt_check_resched_rt() preempt_check_resched() -+#else -+# define preempt_enable_no_resched() preempt_enable() -+# define preempt_check_resched_rt() barrier(); -+#endif - - #ifdef CONFIG_PREEMPT - #define preempt_enable() \ -@@ -63,6 +89,13 @@ - __preempt_schedule(); \ - } while (0) - -+#define preempt_lazy_enable() \ -+do { \ -+ dec_preempt_lazy_count(); \ -+ barrier(); \ -+ preempt_check_resched(); \ -+} while (0) ++ if (!sh) { ++ /* failed to get a stripe - must wait */ ++ raid5_set_bi_processed_stripes(raid_bio, scnt); ++ conf->retry_read_aligned = raid_bio; ++ return handled; ++ } + - #else - #define preempt_enable() \ - do { \ -@@ -121,6 +154,7 @@ - #define preempt_disable_notrace() barrier() - #define preempt_enable_no_resched_notrace() barrier() - #define preempt_enable_notrace() barrier() -+#define preempt_check_resched_rt() barrier() - - #endif /* CONFIG_PREEMPT_COUNT */ - -@@ -140,10 +174,31 @@ - } while (0) - #define preempt_fold_need_resched() \ - do { \ -- if (tif_need_resched()) \ -+ if (tif_need_resched_now()) \ - set_preempt_need_resched(); \ - } while (0) - -+#ifdef CONFIG_PREEMPT_RT_FULL -+# define preempt_disable_rt() preempt_disable() -+# define preempt_enable_rt() preempt_enable() -+# define preempt_disable_nort() barrier() -+# define preempt_enable_nort() barrier() -+# ifdef CONFIG_SMP -+ extern void migrate_disable(void); -+ extern void migrate_enable(void); -+# else /* CONFIG_SMP */ -+# define migrate_disable() barrier() -+# define migrate_enable() barrier() -+# endif /* CONFIG_SMP */ -+#else -+# define preempt_disable_rt() barrier() -+# define preempt_enable_rt() barrier() -+# define preempt_disable_nort() preempt_disable() -+# define preempt_enable_nort() preempt_enable() -+# define migrate_disable() preempt_disable() -+# define migrate_enable() preempt_enable() -+#endif ++ if (!add_stripe_bio(sh, raid_bio, dd_idx, 0, 0)) { ++ release_stripe(sh); ++ raid5_set_bi_processed_stripes(raid_bio, scnt); ++ conf->retry_read_aligned = raid_bio; ++ return handled; ++ } + - #ifdef CONFIG_PREEMPT_NOTIFIERS - - struct preempt_notifier; -diff -Nur linux-4.1.6.orig/include/linux/preempt_mask.h linux-4.1.6/include/linux/preempt_mask.h ---- linux-4.1.6.orig/include/linux/preempt_mask.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/preempt_mask.h 2015-09-08 23:49:06.778014632 +0200 -@@ -44,16 +44,26 @@ - #define HARDIRQ_OFFSET (1UL << HARDIRQ_SHIFT) - #define NMI_OFFSET (1UL << NMI_SHIFT) - --#define SOFTIRQ_DISABLE_OFFSET (2 * SOFTIRQ_OFFSET) -+#ifndef CONFIG_PREEMPT_RT_FULL -+# define SOFTIRQ_DISABLE_OFFSET (2 * SOFTIRQ_OFFSET) -+#else -+# define SOFTIRQ_DISABLE_OFFSET (0) -+#endif - - #define PREEMPT_ACTIVE_BITS 1 - #define PREEMPT_ACTIVE_SHIFT (NMI_SHIFT + NMI_BITS) - #define PREEMPT_ACTIVE (__IRQ_MASK(PREEMPT_ACTIVE_BITS) << PREEMPT_ACTIVE_SHIFT) - - #define hardirq_count() (preempt_count() & HARDIRQ_MASK) --#define softirq_count() (preempt_count() & SOFTIRQ_MASK) - #define irq_count() (preempt_count() & (HARDIRQ_MASK | SOFTIRQ_MASK \ - | NMI_MASK)) -+#ifndef CONFIG_PREEMPT_RT_FULL -+# define softirq_count() (preempt_count() & SOFTIRQ_MASK) -+# define in_serving_softirq() (softirq_count() & SOFTIRQ_OFFSET) -+#else -+# define softirq_count() (0UL) -+extern int in_serving_softirq(void); -+#endif - - /* - * Are we doing bottom half or hardware interrupt processing? -@@ -64,7 +74,6 @@ - #define in_irq() (hardirq_count()) - #define in_softirq() (softirq_count()) - #define in_interrupt() (irq_count()) --#define in_serving_softirq() (softirq_count() & SOFTIRQ_OFFSET) - - /* - * Are we in NMI context? -diff -Nur linux-4.1.6.orig/include/linux/printk.h linux-4.1.6/include/linux/printk.h ---- linux-4.1.6.orig/include/linux/printk.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/printk.h 2015-09-08 23:49:06.778014632 +0200 -@@ -115,9 +115,11 @@ - #ifdef CONFIG_EARLY_PRINTK - extern asmlinkage __printf(1, 2) - void early_printk(const char *fmt, ...); -+extern void printk_kill(void); - #else - static inline __printf(1, 2) __cold - void early_printk(const char *s, ...) { } -+static inline void printk_kill(void) { } - #endif - - typedef int(*printk_func_t)(const char *fmt, va_list args); -diff -Nur linux-4.1.6.orig/include/linux/radix-tree.h linux-4.1.6/include/linux/radix-tree.h ---- linux-4.1.6.orig/include/linux/radix-tree.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/radix-tree.h 2015-09-08 23:49:06.778014632 +0200 -@@ -277,8 +277,13 @@ - unsigned int radix_tree_gang_lookup_slot(struct radix_tree_root *root, - void ***results, unsigned long *indices, - unsigned long first_index, unsigned int max_items); -+#ifndef CONFIG_PREEMPT_RT_FULL - int radix_tree_preload(gfp_t gfp_mask); - int radix_tree_maybe_preload(gfp_t gfp_mask); -+#else -+static inline int radix_tree_preload(gfp_t gm) { return 0; } -+static inline int radix_tree_maybe_preload(gfp_t gfp_mask) { return 0; } -+#endif - void radix_tree_init(void); - void *radix_tree_tag_set(struct radix_tree_root *root, - unsigned long index, unsigned int tag); -@@ -303,7 +308,7 @@ - - static inline void radix_tree_preload_end(void) - { -- preempt_enable(); -+ preempt_enable_nort(); - } - - /** -diff -Nur linux-4.1.6.orig/include/linux/random.h linux-4.1.6/include/linux/random.h ---- linux-4.1.6.orig/include/linux/random.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/random.h 2015-09-08 23:49:06.778014632 +0200 -@@ -11,7 +11,7 @@ - extern void add_device_randomness(const void *, unsigned int); - extern void add_input_randomness(unsigned int type, unsigned int code, - unsigned int value); --extern void add_interrupt_randomness(int irq, int irq_flags); -+extern void add_interrupt_randomness(int irq, int irq_flags, __u64 ip); - - extern void get_random_bytes(void *buf, int nbytes); - extern void get_random_bytes_arch(void *buf, int nbytes); -diff -Nur linux-4.1.6.orig/include/linux/rcupdate.h linux-4.1.6/include/linux/rcupdate.h ---- linux-4.1.6.orig/include/linux/rcupdate.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/rcupdate.h 2015-09-08 23:49:06.778014632 +0200 -@@ -167,6 +167,9 @@ - - #endif /* #else #ifdef CONFIG_PREEMPT_RCU */ - -+#ifdef CONFIG_PREEMPT_RT_FULL -+#define call_rcu_bh call_rcu -+#else - /** - * call_rcu_bh() - Queue an RCU for invocation after a quicker grace period. - * @head: structure to be used for queueing the RCU updates. -@@ -190,6 +193,7 @@ - */ - void call_rcu_bh(struct rcu_head *head, - void (*func)(struct rcu_head *head)); -+#endif - - /** - * call_rcu_sched() - Queue an RCU for invocation after sched grace period. -@@ -260,6 +264,11 @@ - * types of kernel builds, the rcu_read_lock() nesting depth is unknowable. - */ - #define rcu_preempt_depth() (current->rcu_read_lock_nesting) -+#ifndef CONFIG_PREEMPT_RT_FULL -+#define sched_rcu_preempt_depth() rcu_preempt_depth() -+#else -+static inline int sched_rcu_preempt_depth(void) { return 0; } -+#endif - - #else /* #ifdef CONFIG_PREEMPT_RCU */ - -@@ -283,6 +292,8 @@ - return 0; - } - -+#define sched_rcu_preempt_depth() rcu_preempt_depth() -+ - #endif /* #else #ifdef CONFIG_PREEMPT_RCU */ - - /* Internal to kernel */ -@@ -463,7 +474,14 @@ - int debug_lockdep_rcu_enabled(void); - - int rcu_read_lock_held(void); -+#ifdef CONFIG_PREEMPT_RT_FULL -+static inline int rcu_read_lock_bh_held(void) -+{ -+ return rcu_read_lock_held(); ++ set_bit(R5_ReadNoMerge, &sh->dev[dd_idx].flags); ++ handle_stripe(sh); ++ release_stripe(sh); ++ handled++; ++ } ++ remaining = raid5_dec_bi_active_stripes(raid_bio); ++ if (remaining == 0) { ++ trace_block_bio_complete(bdev_get_queue(raid_bio->bi_bdev), ++ raid_bio, 0); ++ bio_endio(raid_bio, 0); ++ } ++ if (atomic_dec_and_test(&conf->active_aligned_reads)) ++ wake_up(&conf->wait_for_stripe); ++ return handled; +} -+#else - int rcu_read_lock_bh_held(void); -+#endif - - /** - * rcu_read_lock_sched_held() - might we be in RCU-sched read-side critical section? -@@ -990,10 +1008,14 @@ - static inline void rcu_read_lock_bh(void) - { - local_bh_disable(); -+#ifdef CONFIG_PREEMPT_RT_FULL -+ rcu_read_lock(); -+#else - __acquire(RCU_BH); - rcu_lock_acquire(&rcu_bh_lock_map); - rcu_lockdep_assert(rcu_is_watching(), - "rcu_read_lock_bh() used illegally while idle"); -+#endif - } - - /* -@@ -1003,10 +1025,14 @@ - */ - static inline void rcu_read_unlock_bh(void) - { -+#ifdef CONFIG_PREEMPT_RT_FULL -+ rcu_read_unlock(); -+#else - rcu_lockdep_assert(rcu_is_watching(), - "rcu_read_unlock_bh() used illegally while idle"); - rcu_lock_release(&rcu_bh_lock_map); - __release(RCU_BH); -+#endif - local_bh_enable(); - } - -diff -Nur linux-4.1.6.orig/include/linux/rcutree.h linux-4.1.6/include/linux/rcutree.h ---- linux-4.1.6.orig/include/linux/rcutree.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/rcutree.h 2015-09-08 23:49:06.778014632 +0200 -@@ -46,7 +46,11 @@ - rcu_note_context_switch(); - } - -+#ifdef CONFIG_PREEMPT_RT_FULL -+# define synchronize_rcu_bh synchronize_rcu -+#else - void synchronize_rcu_bh(void); -+#endif - void synchronize_sched_expedited(void); - void synchronize_rcu_expedited(void); - -@@ -74,7 +78,11 @@ - } - - void rcu_barrier(void); -+#ifdef CONFIG_PREEMPT_RT_FULL -+# define rcu_barrier_bh rcu_barrier -+#else - void rcu_barrier_bh(void); -+#endif - void rcu_barrier_sched(void); - unsigned long get_state_synchronize_rcu(void); - void cond_synchronize_rcu(unsigned long oldstate); -@@ -85,12 +93,10 @@ - unsigned long rcu_batches_started_bh(void); - unsigned long rcu_batches_started_sched(void); - unsigned long rcu_batches_completed(void); --unsigned long rcu_batches_completed_bh(void); - unsigned long rcu_batches_completed_sched(void); - void show_rcu_gp_kthreads(void); - - void rcu_force_quiescent_state(void); --void rcu_bh_force_quiescent_state(void); - void rcu_sched_force_quiescent_state(void); - - void exit_rcu(void); -@@ -100,6 +106,14 @@ - - bool rcu_is_watching(void); - -+#ifndef CONFIG_PREEMPT_RT_FULL -+void rcu_bh_force_quiescent_state(void); -+unsigned long rcu_batches_completed_bh(void); -+#else -+# define rcu_bh_force_quiescent_state rcu_force_quiescent_state -+# define rcu_batches_completed_bh rcu_batches_completed -+#endif + - void rcu_all_qs(void); - - #endif /* __LINUX_RCUTREE_H */ -diff -Nur linux-4.1.6.orig/include/linux/rtmutex.h linux-4.1.6/include/linux/rtmutex.h ---- linux-4.1.6.orig/include/linux/rtmutex.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/rtmutex.h 2015-09-08 23:49:06.778014632 +0200 -@@ -14,10 +14,14 @@ - - #include - #include --#include -+#include - - extern int max_lock_depth; /* for sysctl */ - -+#ifdef CONFIG_DEBUG_MUTEXES -+#include -+#endif ++static int handle_active_stripes(struct r5conf *conf, int group, ++ struct r5worker *worker, ++ struct list_head *temp_inactive_list) ++{ ++ struct stripe_head *batch[MAX_STRIPE_BATCH], *sh; ++ int i, batch_size = 0, hash; ++ bool release_inactive = false; + - /** - * The rt_mutex structure - * -@@ -31,8 +35,8 @@ - struct rb_root waiters; - struct rb_node *waiters_leftmost; - struct task_struct *owner; --#ifdef CONFIG_DEBUG_RT_MUTEXES - int save_state; -+#ifdef CONFIG_DEBUG_RT_MUTEXES - const char *name, *file; - int line; - void *magic; -@@ -55,22 +59,33 @@ - # define rt_mutex_debug_check_no_locks_held(task) do { } while (0) - #endif - -+# define rt_mutex_init(mutex) \ -+ do { \ -+ raw_spin_lock_init(&(mutex)->wait_lock); \ -+ __rt_mutex_init(mutex, #mutex); \ -+ } while (0) ++ while (batch_size < MAX_STRIPE_BATCH && ++ (sh = __get_priority_stripe(conf, group)) != NULL) ++ batch[batch_size++] = sh; + - #ifdef CONFIG_DEBUG_RT_MUTEXES - # define __DEBUG_RT_MUTEX_INITIALIZER(mutexname) \ - , .name = #mutexname, .file = __FILE__, .line = __LINE__ --# define rt_mutex_init(mutex) __rt_mutex_init(mutex, __func__) - extern void rt_mutex_debug_task_free(struct task_struct *tsk); - #else - # define __DEBUG_RT_MUTEX_INITIALIZER(mutexname) --# define rt_mutex_init(mutex) __rt_mutex_init(mutex, NULL) - # define rt_mutex_debug_task_free(t) do { } while (0) - #endif - --#define __RT_MUTEX_INITIALIZER(mutexname) \ -- { .wait_lock = __RAW_SPIN_LOCK_UNLOCKED(mutexname.wait_lock) \ -+#define __RT_MUTEX_INITIALIZER_PLAIN(mutexname) \ -+ .wait_lock = __RAW_SPIN_LOCK_UNLOCKED(mutexname.wait_lock) \ - , .waiters = RB_ROOT \ - , .owner = NULL \ -- __DEBUG_RT_MUTEX_INITIALIZER(mutexname)} -+ __DEBUG_RT_MUTEX_INITIALIZER(mutexname) ++ if (batch_size == 0) { ++ for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++) ++ if (!list_empty(temp_inactive_list + i)) ++ break; ++ if (i == NR_STRIPE_HASH_LOCKS) ++ return batch_size; ++ release_inactive = true; ++ } ++ spin_unlock_irq(&conf->device_lock); + -+#define __RT_MUTEX_INITIALIZER(mutexname) \ -+ { __RT_MUTEX_INITIALIZER_PLAIN(mutexname) } ++ release_inactive_stripe_list(conf, temp_inactive_list, ++ NR_STRIPE_HASH_LOCKS); + -+#define __RT_MUTEX_INITIALIZER_SAVE_STATE(mutexname) \ -+ { __RT_MUTEX_INITIALIZER_PLAIN(mutexname) \ -+ , .save_state = 1 } - - #define DEFINE_RT_MUTEX(mutexname) \ - struct rt_mutex mutexname = __RT_MUTEX_INITIALIZER(mutexname) -@@ -91,6 +106,7 @@ - - extern void rt_mutex_lock(struct rt_mutex *lock); - extern int rt_mutex_lock_interruptible(struct rt_mutex *lock); -+extern int rt_mutex_lock_killable(struct rt_mutex *lock); - extern int rt_mutex_timed_lock(struct rt_mutex *lock, - struct hrtimer_sleeper *timeout); - -diff -Nur linux-4.1.6.orig/include/linux/rwlock_rt.h linux-4.1.6/include/linux/rwlock_rt.h ---- linux-4.1.6.orig/include/linux/rwlock_rt.h 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.6/include/linux/rwlock_rt.h 2015-09-08 23:49:06.778014632 +0200 -@@ -0,0 +1,99 @@ -+#ifndef __LINUX_RWLOCK_RT_H -+#define __LINUX_RWLOCK_RT_H ++ if (release_inactive) { ++ spin_lock_irq(&conf->device_lock); ++ return 0; ++ } + -+#ifndef __LINUX_SPINLOCK_H -+#error Do not include directly. Use spinlock.h -+#endif ++ for (i = 0; i < batch_size; i++) ++ handle_stripe(batch[i]); + -+#define rwlock_init(rwl) \ -+do { \ -+ static struct lock_class_key __key; \ -+ \ -+ rt_mutex_init(&(rwl)->lock); \ -+ __rt_rwlock_init(rwl, #rwl, &__key); \ -+} while (0) ++ cond_resched(); + -+extern void __lockfunc rt_write_lock(rwlock_t *rwlock); -+extern void __lockfunc rt_read_lock(rwlock_t *rwlock); -+extern int __lockfunc rt_write_trylock(rwlock_t *rwlock); -+extern int __lockfunc rt_write_trylock_irqsave(rwlock_t *trylock, unsigned long *flags); -+extern int __lockfunc rt_read_trylock(rwlock_t *rwlock); -+extern void __lockfunc rt_write_unlock(rwlock_t *rwlock); -+extern void __lockfunc rt_read_unlock(rwlock_t *rwlock); -+extern unsigned long __lockfunc rt_write_lock_irqsave(rwlock_t *rwlock); -+extern unsigned long __lockfunc rt_read_lock_irqsave(rwlock_t *rwlock); -+extern void __rt_rwlock_init(rwlock_t *rwlock, char *name, struct lock_class_key *key); ++ spin_lock_irq(&conf->device_lock); ++ for (i = 0; i < batch_size; i++) { ++ hash = batch[i]->hash_lock_index; ++ __release_stripe(conf, batch[i], &temp_inactive_list[hash]); ++ } ++ return batch_size; ++} + -+#define read_trylock(lock) __cond_lock(lock, rt_read_trylock(lock)) -+#define write_trylock(lock) __cond_lock(lock, rt_write_trylock(lock)) ++static void raid5_do_work(struct work_struct *work) ++{ ++ struct r5worker *worker = container_of(work, struct r5worker, work); ++ struct r5worker_group *group = worker->group; ++ struct r5conf *conf = group->conf; ++ int group_id = group - conf->worker_groups; ++ int handled; ++ struct blk_plug plug; ++ ++ pr_debug("+++ raid5worker active\n"); ++ ++ blk_start_plug(&plug); ++ handled = 0; ++ spin_lock_irq(&conf->device_lock); ++ while (1) { ++ int batch_size, released; + -+#define write_trylock_irqsave(lock, flags) \ -+ __cond_lock(lock, rt_write_trylock_irqsave(lock, &flags)) ++ released = release_stripe_list(conf, worker->temp_inactive_list); + -+#define read_lock_irqsave(lock, flags) \ -+ do { \ -+ typecheck(unsigned long, flags); \ -+ flags = rt_read_lock_irqsave(lock); \ -+ } while (0) ++ batch_size = handle_active_stripes(conf, group_id, worker, ++ worker->temp_inactive_list); ++ worker->working = false; ++ if (!batch_size && !released) ++ break; ++ handled += batch_size; ++ } ++ pr_debug("%d stripes handled\n", handled); + -+#define write_lock_irqsave(lock, flags) \ -+ do { \ -+ typecheck(unsigned long, flags); \ -+ flags = rt_write_lock_irqsave(lock); \ -+ } while (0) ++ spin_unlock_irq(&conf->device_lock); ++ blk_finish_plug(&plug); + -+#define read_lock(lock) rt_read_lock(lock) ++ pr_debug("--- raid5worker inactive\n"); ++} + -+#define read_lock_bh(lock) \ -+ do { \ -+ local_bh_disable(); \ -+ rt_read_lock(lock); \ -+ } while (0) ++/* ++ * This is our raid5 kernel thread. ++ * ++ * We scan the hash table for stripes which can be handled now. ++ * During the scan, completed stripes are saved for us by the interrupt ++ * handler, so that they will not have to wait for our next wakeup. ++ */ ++static void raid5d(struct md_thread *thread) ++{ ++ struct mddev *mddev = thread->mddev; ++ struct r5conf *conf = mddev->private; ++ int handled; ++ struct blk_plug plug; + -+#define read_lock_irq(lock) read_lock(lock) ++ pr_debug("+++ raid5d active\n"); + -+#define write_lock(lock) rt_write_lock(lock) ++ md_check_recovery(mddev); + -+#define write_lock_bh(lock) \ -+ do { \ -+ local_bh_disable(); \ -+ rt_write_lock(lock); \ -+ } while (0) ++ blk_start_plug(&plug); ++ handled = 0; ++ spin_lock_irq(&conf->device_lock); ++ while (1) { ++ struct bio *bio; ++ int batch_size, released; ++ ++ released = release_stripe_list(conf, conf->temp_inactive_list); ++ if (released) ++ clear_bit(R5_DID_ALLOC, &conf->cache_state); ++ ++ if ( ++ !list_empty(&conf->bitmap_list)) { ++ /* Now is a good time to flush some bitmap updates */ ++ conf->seq_flush++; ++ spin_unlock_irq(&conf->device_lock); ++ bitmap_unplug(mddev->bitmap); ++ spin_lock_irq(&conf->device_lock); ++ conf->seq_write = conf->seq_flush; ++ activate_bit_delay(conf, conf->temp_inactive_list); ++ } ++ raid5_activate_delayed(conf); ++ ++ while ((bio = remove_bio_from_retry(conf))) { ++ int ok; ++ spin_unlock_irq(&conf->device_lock); ++ ok = retry_aligned_read(conf, bio); ++ spin_lock_irq(&conf->device_lock); ++ if (!ok) ++ break; ++ handled++; ++ } + -+#define write_lock_irq(lock) write_lock(lock) ++ batch_size = handle_active_stripes(conf, ANY_GROUP, NULL, ++ conf->temp_inactive_list); ++ if (!batch_size && !released) ++ break; ++ handled += batch_size; + -+#define read_unlock(lock) rt_read_unlock(lock) ++ if (mddev->flags & ~(1<device_lock); ++ md_check_recovery(mddev); ++ spin_lock_irq(&conf->device_lock); ++ } ++ } ++ pr_debug("%d stripes handled\n", handled); ++ ++ spin_unlock_irq(&conf->device_lock); ++ if (test_and_clear_bit(R5_ALLOC_MORE, &conf->cache_state) && ++ mutex_trylock(&conf->cache_size_mutex)) { ++ grow_one_stripe(conf, __GFP_NOWARN); ++ /* Set flag even if allocation failed. This helps ++ * slow down allocation requests when mem is short ++ */ ++ set_bit(R5_DID_ALLOC, &conf->cache_state); ++ mutex_unlock(&conf->cache_size_mutex); ++ } + -+#define read_unlock_bh(lock) \ -+ do { \ -+ rt_read_unlock(lock); \ -+ local_bh_enable(); \ -+ } while (0) ++ async_tx_issue_pending_all(); ++ blk_finish_plug(&plug); + -+#define read_unlock_irq(lock) read_unlock(lock) ++ pr_debug("--- raid5d inactive\n"); ++} + -+#define write_unlock(lock) rt_write_unlock(lock) ++static ssize_t ++raid5_show_stripe_cache_size(struct mddev *mddev, char *page) ++{ ++ struct r5conf *conf; ++ int ret = 0; ++ spin_lock(&mddev->lock); ++ conf = mddev->private; ++ if (conf) ++ ret = sprintf(page, "%d\n", conf->min_nr_stripes); ++ spin_unlock(&mddev->lock); ++ return ret; ++} + -+#define write_unlock_bh(lock) \ -+ do { \ -+ rt_write_unlock(lock); \ -+ local_bh_enable(); \ -+ } while (0) ++int ++raid5_set_cache_size(struct mddev *mddev, int size) ++{ ++ struct r5conf *conf = mddev->private; ++ int err; + -+#define write_unlock_irq(lock) write_unlock(lock) ++ if (size <= 16 || size > 32768) ++ return -EINVAL; + -+#define read_unlock_irqrestore(lock, flags) \ -+ do { \ -+ typecheck(unsigned long, flags); \ -+ (void) flags; \ -+ rt_read_unlock(lock); \ -+ } while (0) ++ conf->min_nr_stripes = size; ++ mutex_lock(&conf->cache_size_mutex); ++ while (size < conf->max_nr_stripes && ++ drop_one_stripe(conf)) ++ ; ++ mutex_unlock(&conf->cache_size_mutex); + -+#define write_unlock_irqrestore(lock, flags) \ -+ do { \ -+ typecheck(unsigned long, flags); \ -+ (void) flags; \ -+ rt_write_unlock(lock); \ -+ } while (0) + -+#endif -diff -Nur linux-4.1.6.orig/include/linux/rwlock_types.h linux-4.1.6/include/linux/rwlock_types.h ---- linux-4.1.6.orig/include/linux/rwlock_types.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/rwlock_types.h 2015-09-08 23:49:06.778014632 +0200 -@@ -1,6 +1,10 @@ - #ifndef __LINUX_RWLOCK_TYPES_H - #define __LINUX_RWLOCK_TYPES_H - -+#if !defined(__LINUX_SPINLOCK_TYPES_H) -+# error "Do not include directly, include spinlock_types.h" -+#endif ++ err = md_allow_write(mddev); ++ if (err) ++ return err; + - /* - * include/linux/rwlock_types.h - generic rwlock type definitions - * and initializers -@@ -43,6 +47,7 @@ - RW_DEP_MAP_INIT(lockname) } - #endif - --#define DEFINE_RWLOCK(x) rwlock_t x = __RW_LOCK_UNLOCKED(x) -+#define DEFINE_RWLOCK(name) \ -+ rwlock_t name __cacheline_aligned_in_smp = __RW_LOCK_UNLOCKED(name) - - #endif /* __LINUX_RWLOCK_TYPES_H */ -diff -Nur linux-4.1.6.orig/include/linux/rwlock_types_rt.h linux-4.1.6/include/linux/rwlock_types_rt.h ---- linux-4.1.6.orig/include/linux/rwlock_types_rt.h 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.6/include/linux/rwlock_types_rt.h 2015-09-08 23:49:06.778014632 +0200 -@@ -0,0 +1,33 @@ -+#ifndef __LINUX_RWLOCK_TYPES_RT_H -+#define __LINUX_RWLOCK_TYPES_RT_H ++ mutex_lock(&conf->cache_size_mutex); ++ while (size > conf->max_nr_stripes) ++ if (!grow_one_stripe(conf, GFP_KERNEL)) ++ break; ++ mutex_unlock(&conf->cache_size_mutex); + -+#ifndef __LINUX_SPINLOCK_TYPES_H -+#error "Do not include directly. Include spinlock_types.h instead" -+#endif ++ return 0; ++} ++EXPORT_SYMBOL(raid5_set_cache_size); + -+/* -+ * rwlocks - rtmutex which allows single reader recursion -+ */ -+typedef struct { -+ struct rt_mutex lock; -+ int read_depth; -+ unsigned int break_lock; -+#ifdef CONFIG_DEBUG_LOCK_ALLOC -+ struct lockdep_map dep_map; -+#endif -+} rwlock_t; ++static ssize_t ++raid5_store_stripe_cache_size(struct mddev *mddev, const char *page, size_t len) ++{ ++ struct r5conf *conf; ++ unsigned long new; ++ int err; + -+#ifdef CONFIG_DEBUG_LOCK_ALLOC -+# define RW_DEP_MAP_INIT(lockname) .dep_map = { .name = #lockname } -+#else -+# define RW_DEP_MAP_INIT(lockname) -+#endif ++ if (len >= PAGE_SIZE) ++ return -EINVAL; ++ if (kstrtoul(page, 10, &new)) ++ return -EINVAL; ++ err = mddev_lock(mddev); ++ if (err) ++ return err; ++ conf = mddev->private; ++ if (!conf) ++ err = -ENODEV; ++ else ++ err = raid5_set_cache_size(mddev, new); ++ mddev_unlock(mddev); + -+#define __RW_LOCK_UNLOCKED(name) \ -+ { .lock = __RT_MUTEX_INITIALIZER_SAVE_STATE(name.lock), \ -+ RW_DEP_MAP_INIT(name) } ++ return err ?: len; ++} + -+#define DEFINE_RWLOCK(name) \ -+ rwlock_t name __cacheline_aligned_in_smp = __RW_LOCK_UNLOCKED(name) ++static struct md_sysfs_entry ++raid5_stripecache_size = __ATTR(stripe_cache_size, S_IRUGO | S_IWUSR, ++ raid5_show_stripe_cache_size, ++ raid5_store_stripe_cache_size); + -+#endif -diff -Nur linux-4.1.6.orig/include/linux/rwsem.h linux-4.1.6/include/linux/rwsem.h ---- linux-4.1.6.orig/include/linux/rwsem.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/rwsem.h 2015-09-08 23:49:06.778014632 +0200 -@@ -18,6 +18,10 @@ - #include - #endif - -+#ifdef CONFIG_PREEMPT_RT_FULL -+#include -+#else /* PREEMPT_RT_FULL */ ++static ssize_t ++raid5_show_rmw_level(struct mddev *mddev, char *page) ++{ ++ struct r5conf *conf = mddev->private; ++ if (conf) ++ return sprintf(page, "%d\n", conf->rmw_level); ++ else ++ return 0; ++} + - struct rw_semaphore; - - #ifdef CONFIG_RWSEM_GENERIC_SPINLOCK -@@ -177,4 +181,6 @@ - # define up_read_non_owner(sem) up_read(sem) - #endif - -+#endif /* !PREEMPT_RT_FULL */ ++static ssize_t ++raid5_store_rmw_level(struct mddev *mddev, const char *page, size_t len) ++{ ++ struct r5conf *conf = mddev->private; ++ unsigned long new; + - #endif /* _LINUX_RWSEM_H */ -diff -Nur linux-4.1.6.orig/include/linux/rwsem_rt.h linux-4.1.6/include/linux/rwsem_rt.h ---- linux-4.1.6.orig/include/linux/rwsem_rt.h 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.6/include/linux/rwsem_rt.h 2015-09-08 23:49:06.778014632 +0200 -@@ -0,0 +1,140 @@ -+#ifndef _LINUX_RWSEM_RT_H -+#define _LINUX_RWSEM_RT_H ++ if (!conf) ++ return -ENODEV; + -+#ifndef _LINUX_RWSEM_H -+#error "Include rwsem.h" -+#endif ++ if (len >= PAGE_SIZE) ++ return -EINVAL; + -+/* -+ * RW-semaphores are a spinlock plus a reader-depth count. -+ * -+ * Note that the semantics are different from the usual -+ * Linux rw-sems, in PREEMPT_RT mode we do not allow -+ * multiple readers to hold the lock at once, we only allow -+ * a read-lock owner to read-lock recursively. This is -+ * better for latency, makes the implementation inherently -+ * fair and makes it simpler as well. -+ */ ++ if (kstrtoul(page, 10, &new)) ++ return -EINVAL; + -+#include ++ if (new != PARITY_DISABLE_RMW && !raid6_call.xor_syndrome) ++ return -EINVAL; + -+struct rw_semaphore { -+ struct rt_mutex lock; -+ int read_depth; -+#ifdef CONFIG_DEBUG_LOCK_ALLOC -+ struct lockdep_map dep_map; -+#endif -+}; ++ if (new != PARITY_DISABLE_RMW && ++ new != PARITY_ENABLE_RMW && ++ new != PARITY_PREFER_RMW) ++ return -EINVAL; + -+#define __RWSEM_INITIALIZER(name) \ -+ { .lock = __RT_MUTEX_INITIALIZER(name.lock), \ -+ RW_DEP_MAP_INIT(name) } ++ conf->rmw_level = new; ++ return len; ++} + -+#define DECLARE_RWSEM(lockname) \ -+ struct rw_semaphore lockname = __RWSEM_INITIALIZER(lockname) ++static struct md_sysfs_entry ++raid5_rmw_level = __ATTR(rmw_level, S_IRUGO | S_IWUSR, ++ raid5_show_rmw_level, ++ raid5_store_rmw_level); + -+extern void __rt_rwsem_init(struct rw_semaphore *rwsem, const char *name, -+ struct lock_class_key *key); + -+#define __rt_init_rwsem(sem, name, key) \ -+ do { \ -+ rt_mutex_init(&(sem)->lock); \ -+ __rt_rwsem_init((sem), (name), (key));\ -+ } while (0) ++static ssize_t ++raid5_show_preread_threshold(struct mddev *mddev, char *page) ++{ ++ struct r5conf *conf; ++ int ret = 0; ++ spin_lock(&mddev->lock); ++ conf = mddev->private; ++ if (conf) ++ ret = sprintf(page, "%d\n", conf->bypass_threshold); ++ spin_unlock(&mddev->lock); ++ return ret; ++} + -+#define __init_rwsem(sem, name, key) __rt_init_rwsem(sem, name, key) ++static ssize_t ++raid5_store_preread_threshold(struct mddev *mddev, const char *page, size_t len) ++{ ++ struct r5conf *conf; ++ unsigned long new; ++ int err; + -+# define rt_init_rwsem(sem) \ -+do { \ -+ static struct lock_class_key __key; \ -+ \ -+ __rt_init_rwsem((sem), #sem, &__key); \ -+} while (0) ++ if (len >= PAGE_SIZE) ++ return -EINVAL; ++ if (kstrtoul(page, 10, &new)) ++ return -EINVAL; + -+extern void rt_down_write(struct rw_semaphore *rwsem); -+extern void rt_down_read_nested(struct rw_semaphore *rwsem, int subclass); -+extern void rt_down_write_nested(struct rw_semaphore *rwsem, int subclass); -+extern void rt_down_write_nested_lock(struct rw_semaphore *rwsem, -+ struct lockdep_map *nest); -+extern void rt_down_read(struct rw_semaphore *rwsem); -+extern int rt_down_write_trylock(struct rw_semaphore *rwsem); -+extern int rt_down_read_trylock(struct rw_semaphore *rwsem); -+extern void __rt_up_read(struct rw_semaphore *rwsem); -+extern void rt_up_read(struct rw_semaphore *rwsem); -+extern void rt_up_write(struct rw_semaphore *rwsem); -+extern void rt_downgrade_write(struct rw_semaphore *rwsem); ++ err = mddev_lock(mddev); ++ if (err) ++ return err; ++ conf = mddev->private; ++ if (!conf) ++ err = -ENODEV; ++ else if (new > conf->min_nr_stripes) ++ err = -EINVAL; ++ else ++ conf->bypass_threshold = new; ++ mddev_unlock(mddev); ++ return err ?: len; ++} + -+#define init_rwsem(sem) rt_init_rwsem(sem) -+#define rwsem_is_locked(s) rt_mutex_is_locked(&(s)->lock) ++static struct md_sysfs_entry ++raid5_preread_bypass_threshold = __ATTR(preread_bypass_threshold, ++ S_IRUGO | S_IWUSR, ++ raid5_show_preread_threshold, ++ raid5_store_preread_threshold); + -+static inline int rwsem_is_contended(struct rw_semaphore *sem) ++static ssize_t ++raid5_show_skip_copy(struct mddev *mddev, char *page) +{ -+ /* rt_mutex_has_waiters() */ -+ return !RB_EMPTY_ROOT(&sem->lock.waiters); ++ struct r5conf *conf; ++ int ret = 0; ++ spin_lock(&mddev->lock); ++ conf = mddev->private; ++ if (conf) ++ ret = sprintf(page, "%d\n", conf->skip_copy); ++ spin_unlock(&mddev->lock); ++ return ret; +} + -+static inline void down_read(struct rw_semaphore *sem) ++static ssize_t ++raid5_store_skip_copy(struct mddev *mddev, const char *page, size_t len) +{ -+ rt_down_read(sem); ++ struct r5conf *conf; ++ unsigned long new; ++ int err; ++ ++ if (len >= PAGE_SIZE) ++ return -EINVAL; ++ if (kstrtoul(page, 10, &new)) ++ return -EINVAL; ++ new = !!new; ++ ++ err = mddev_lock(mddev); ++ if (err) ++ return err; ++ conf = mddev->private; ++ if (!conf) ++ err = -ENODEV; ++ else if (new != conf->skip_copy) { ++ mddev_suspend(mddev); ++ conf->skip_copy = new; ++ if (new) ++ mddev->queue->backing_dev_info.capabilities |= ++ BDI_CAP_STABLE_WRITES; ++ else ++ mddev->queue->backing_dev_info.capabilities &= ++ ~BDI_CAP_STABLE_WRITES; ++ mddev_resume(mddev); ++ } ++ mddev_unlock(mddev); ++ return err ?: len; +} + -+static inline int down_read_trylock(struct rw_semaphore *sem) ++static struct md_sysfs_entry ++raid5_skip_copy = __ATTR(skip_copy, S_IRUGO | S_IWUSR, ++ raid5_show_skip_copy, ++ raid5_store_skip_copy); ++ ++static ssize_t ++stripe_cache_active_show(struct mddev *mddev, char *page) +{ -+ return rt_down_read_trylock(sem); ++ struct r5conf *conf = mddev->private; ++ if (conf) ++ return sprintf(page, "%d\n", atomic_read(&conf->active_stripes)); ++ else ++ return 0; +} + -+static inline void down_write(struct rw_semaphore *sem) ++static struct md_sysfs_entry ++raid5_stripecache_active = __ATTR_RO(stripe_cache_active); ++ ++static ssize_t ++raid5_show_group_thread_cnt(struct mddev *mddev, char *page) +{ -+ rt_down_write(sem); ++ struct r5conf *conf; ++ int ret = 0; ++ spin_lock(&mddev->lock); ++ conf = mddev->private; ++ if (conf) ++ ret = sprintf(page, "%d\n", conf->worker_cnt_per_group); ++ spin_unlock(&mddev->lock); ++ return ret; +} + -+static inline int down_write_trylock(struct rw_semaphore *sem) ++static int alloc_thread_groups(struct r5conf *conf, int cnt, ++ int *group_cnt, ++ int *worker_cnt_per_group, ++ struct r5worker_group **worker_groups); ++static ssize_t ++raid5_store_group_thread_cnt(struct mddev *mddev, const char *page, size_t len) +{ -+ return rt_down_write_trylock(sem); ++ struct r5conf *conf; ++ unsigned long new; ++ int err; ++ struct r5worker_group *new_groups, *old_groups; ++ int group_cnt, worker_cnt_per_group; ++ ++ if (len >= PAGE_SIZE) ++ return -EINVAL; ++ if (kstrtoul(page, 10, &new)) ++ return -EINVAL; ++ ++ err = mddev_lock(mddev); ++ if (err) ++ return err; ++ conf = mddev->private; ++ if (!conf) ++ err = -ENODEV; ++ else if (new != conf->worker_cnt_per_group) { ++ mddev_suspend(mddev); ++ ++ old_groups = conf->worker_groups; ++ if (old_groups) ++ flush_workqueue(raid5_wq); ++ ++ err = alloc_thread_groups(conf, new, ++ &group_cnt, &worker_cnt_per_group, ++ &new_groups); ++ if (!err) { ++ spin_lock_irq(&conf->device_lock); ++ conf->group_cnt = group_cnt; ++ conf->worker_cnt_per_group = worker_cnt_per_group; ++ conf->worker_groups = new_groups; ++ spin_unlock_irq(&conf->device_lock); ++ ++ if (old_groups) ++ kfree(old_groups[0].workers); ++ kfree(old_groups); ++ } ++ mddev_resume(mddev); ++ } ++ mddev_unlock(mddev); ++ ++ return err ?: len; +} + -+static inline void __up_read(struct rw_semaphore *sem) ++static struct md_sysfs_entry ++raid5_group_thread_cnt = __ATTR(group_thread_cnt, S_IRUGO | S_IWUSR, ++ raid5_show_group_thread_cnt, ++ raid5_store_group_thread_cnt); ++ ++static struct attribute *raid5_attrs[] = { ++ &raid5_stripecache_size.attr, ++ &raid5_stripecache_active.attr, ++ &raid5_preread_bypass_threshold.attr, ++ &raid5_group_thread_cnt.attr, ++ &raid5_skip_copy.attr, ++ &raid5_rmw_level.attr, ++ NULL, ++}; ++static struct attribute_group raid5_attrs_group = { ++ .name = NULL, ++ .attrs = raid5_attrs, ++}; ++ ++static int alloc_thread_groups(struct r5conf *conf, int cnt, ++ int *group_cnt, ++ int *worker_cnt_per_group, ++ struct r5worker_group **worker_groups) +{ -+ __rt_up_read(sem); ++ int i, j, k; ++ ssize_t size; ++ struct r5worker *workers; ++ ++ *worker_cnt_per_group = cnt; ++ if (cnt == 0) { ++ *group_cnt = 0; ++ *worker_groups = NULL; ++ return 0; ++ } ++ *group_cnt = num_possible_nodes(); ++ size = sizeof(struct r5worker) * cnt; ++ workers = kzalloc(size * *group_cnt, GFP_NOIO); ++ *worker_groups = kzalloc(sizeof(struct r5worker_group) * ++ *group_cnt, GFP_NOIO); ++ if (!*worker_groups || !workers) { ++ kfree(workers); ++ kfree(*worker_groups); ++ return -ENOMEM; ++ } ++ ++ for (i = 0; i < *group_cnt; i++) { ++ struct r5worker_group *group; ++ ++ group = &(*worker_groups)[i]; ++ INIT_LIST_HEAD(&group->handle_list); ++ group->conf = conf; ++ group->workers = workers + i * cnt; ++ ++ for (j = 0; j < cnt; j++) { ++ struct r5worker *worker = group->workers + j; ++ worker->group = group; ++ INIT_WORK(&worker->work, raid5_do_work); ++ ++ for (k = 0; k < NR_STRIPE_HASH_LOCKS; k++) ++ INIT_LIST_HEAD(worker->temp_inactive_list + k); ++ } ++ } ++ ++ return 0; +} + -+static inline void up_read(struct rw_semaphore *sem) ++static void free_thread_groups(struct r5conf *conf) +{ -+ rt_up_read(sem); ++ if (conf->worker_groups) ++ kfree(conf->worker_groups[0].workers); ++ kfree(conf->worker_groups); ++ conf->worker_groups = NULL; +} + -+static inline void up_write(struct rw_semaphore *sem) ++static sector_t ++raid5_size(struct mddev *mddev, sector_t sectors, int raid_disks) +{ -+ rt_up_write(sem); ++ struct r5conf *conf = mddev->private; ++ ++ if (!sectors) ++ sectors = mddev->dev_sectors; ++ if (!raid_disks) ++ /* size is defined by the smallest of previous and new size */ ++ raid_disks = min(conf->raid_disks, conf->previous_raid_disks); ++ ++ sectors &= ~((sector_t)mddev->chunk_sectors - 1); ++ sectors &= ~((sector_t)mddev->new_chunk_sectors - 1); ++ return sectors * (raid_disks - conf->max_degraded); +} + -+static inline void downgrade_write(struct rw_semaphore *sem) ++static void free_scratch_buffer(struct r5conf *conf, struct raid5_percpu *percpu) +{ -+ rt_downgrade_write(sem); ++ safe_put_page(percpu->spare_page); ++ if (percpu->scribble) ++ flex_array_free(percpu->scribble); ++ percpu->spare_page = NULL; ++ percpu->scribble = NULL; +} + -+static inline void down_read_nested(struct rw_semaphore *sem, int subclass) ++static int alloc_scratch_buffer(struct r5conf *conf, struct raid5_percpu *percpu) +{ -+ return rt_down_read_nested(sem, subclass); ++ if (conf->level == 6 && !percpu->spare_page) ++ percpu->spare_page = alloc_page(GFP_KERNEL); ++ if (!percpu->scribble) ++ percpu->scribble = scribble_alloc(max(conf->raid_disks, ++ conf->previous_raid_disks), ++ max(conf->chunk_sectors, ++ conf->prev_chunk_sectors) ++ / STRIPE_SECTORS, ++ GFP_KERNEL); ++ ++ if (!percpu->scribble || (conf->level == 6 && !percpu->spare_page)) { ++ free_scratch_buffer(conf, percpu); ++ return -ENOMEM; ++ } ++ ++ return 0; +} + -+static inline void down_write_nested(struct rw_semaphore *sem, int subclass) ++static void raid5_free_percpu(struct r5conf *conf) +{ -+ rt_down_write_nested(sem, subclass); ++ unsigned long cpu; ++ ++ if (!conf->percpu) ++ return; ++ ++#ifdef CONFIG_HOTPLUG_CPU ++ unregister_cpu_notifier(&conf->cpu_notify); ++#endif ++ ++ get_online_cpus(); ++ for_each_possible_cpu(cpu) ++ free_scratch_buffer(conf, per_cpu_ptr(conf->percpu, cpu)); ++ put_online_cpus(); ++ ++ free_percpu(conf->percpu); +} -+#ifdef CONFIG_DEBUG_LOCK_ALLOC -+static inline void down_write_nest_lock(struct rw_semaphore *sem, -+ struct rw_semaphore *nest_lock) ++ ++static void free_conf(struct r5conf *conf) +{ -+ rt_down_write_nested_lock(sem, &nest_lock->dep_map); ++ if (conf->shrinker.seeks) ++ unregister_shrinker(&conf->shrinker); ++ free_thread_groups(conf); ++ shrink_stripes(conf); ++ raid5_free_percpu(conf); ++ kfree(conf->disks); ++ kfree(conf->stripe_hashtbl); ++ kfree(conf); +} + -+#else -+ -+static inline void down_write_nest_lock(struct rw_semaphore *sem, -+ struct rw_semaphore *nest_lock) ++#ifdef CONFIG_HOTPLUG_CPU ++static int raid456_cpu_notify(struct notifier_block *nfb, unsigned long action, ++ void *hcpu) +{ -+ rt_down_write_nested_lock(sem, NULL); ++ struct r5conf *conf = container_of(nfb, struct r5conf, cpu_notify); ++ long cpu = (long)hcpu; ++ struct raid5_percpu *percpu = per_cpu_ptr(conf->percpu, cpu); ++ ++ switch (action) { ++ case CPU_UP_PREPARE: ++ case CPU_UP_PREPARE_FROZEN: ++ if (alloc_scratch_buffer(conf, percpu)) { ++ pr_err("%s: failed memory allocation for cpu%ld\n", ++ __func__, cpu); ++ return notifier_from_errno(-ENOMEM); ++ } ++ break; ++ case CPU_DEAD: ++ case CPU_DEAD_FROZEN: ++ free_scratch_buffer(conf, per_cpu_ptr(conf->percpu, cpu)); ++ break; ++ default: ++ break; ++ } ++ return NOTIFY_OK; +} +#endif -+#endif -diff -Nur linux-4.1.6.orig/include/linux/sched.h linux-4.1.6/include/linux/sched.h ---- linux-4.1.6.orig/include/linux/sched.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/sched.h 2015-09-08 23:49:06.778014632 +0200 -@@ -26,6 +26,7 @@ - #include - #include - #include -+#include - - #include - #include -@@ -234,10 +235,7 @@ - TASK_UNINTERRUPTIBLE | __TASK_STOPPED | \ - __TASK_TRACED | EXIT_ZOMBIE | EXIT_DEAD) - --#define task_is_traced(task) ((task->state & __TASK_TRACED) != 0) - #define task_is_stopped(task) ((task->state & __TASK_STOPPED) != 0) --#define task_is_stopped_or_traced(task) \ -- ((task->state & (__TASK_STOPPED | __TASK_TRACED)) != 0) - #define task_contributes_to_load(task) \ - ((task->state & TASK_UNINTERRUPTIBLE) != 0 && \ - (task->flags & PF_FROZEN) == 0) -@@ -302,6 +300,11 @@ - - #endif - -+#define __set_current_state_no_track(state_value) \ -+ do { current->state = (state_value); } while (0) -+#define set_current_state_no_track(state_value) \ -+ set_mb(current->state, (state_value)) -+ - /* Task command name length */ - #define TASK_COMM_LEN 16 - -@@ -900,6 +903,50 @@ - #define SCHED_CAPACITY_SCALE (1L << SCHED_CAPACITY_SHIFT) - - /* -+ * Wake-queues are lists of tasks with a pending wakeup, whose -+ * callers have already marked the task as woken internally, -+ * and can thus carry on. A common use case is being able to -+ * do the wakeups once the corresponding user lock as been -+ * released. -+ * -+ * We hold reference to each task in the list across the wakeup, -+ * thus guaranteeing that the memory is still valid by the time -+ * the actual wakeups are performed in wake_up_q(). -+ * -+ * One per task suffices, because there's never a need for a task to be -+ * in two wake queues simultaneously; it is forbidden to abandon a task -+ * in a wake queue (a call to wake_up_q() _must_ follow), so if a task is -+ * already in a wake queue, the wakeup will happen soon and the second -+ * waker can just skip it. -+ * -+ * The WAKE_Q macro declares and initializes the list head. -+ * wake_up_q() does NOT reinitialize the list; it's expected to be -+ * called near the end of a function, where the fact that the queue is -+ * not used again will be easy to see by inspection. -+ * -+ * Note that this can cause spurious wakeups. schedule() callers -+ * must ensure the call is done inside a loop, confirming that the -+ * wakeup condition has in fact occurred. -+ */ -+struct wake_q_node { -+ struct wake_q_node *next; -+}; -+ -+struct wake_q_head { -+ struct wake_q_node *first; -+ struct wake_q_node **lastp; -+}; + -+#define WAKE_Q_TAIL ((struct wake_q_node *) 0x01) -+ -+#define WAKE_Q(name) \ -+ struct wake_q_head name = { WAKE_Q_TAIL, &name.first } ++static int raid5_alloc_percpu(struct r5conf *conf) ++{ ++ unsigned long cpu; ++ int err = 0; + -+extern void wake_q_add(struct wake_q_head *head, -+ struct task_struct *task); -+extern void wake_up_q(struct wake_q_head *head); ++ conf->percpu = alloc_percpu(struct raid5_percpu); ++ if (!conf->percpu) ++ return -ENOMEM; + -+/* - * sched-domains (multiprocessor balancing) declarations: - */ - #ifdef CONFIG_SMP -@@ -1291,6 +1338,7 @@ - - struct task_struct { - volatile long state; /* -1 unrunnable, 0 runnable, >0 stopped */ -+ volatile long saved_state; /* saved state for "spinlock sleepers" */ - void *stack; - atomic_t usage; - unsigned int flags; /* per process flags, defined below */ -@@ -1327,6 +1375,12 @@ - #endif - - unsigned int policy; -+#ifdef CONFIG_PREEMPT_RT_FULL -+ int migrate_disable; -+# ifdef CONFIG_SCHED_DEBUG -+ int migrate_disable_atomic; -+# endif -+#endif - int nr_cpus_allowed; - cpumask_t cpus_allowed; - -@@ -1434,7 +1488,8 @@ - struct cputime prev_cputime; - #endif - #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN -- seqlock_t vtime_seqlock; -+ raw_spinlock_t vtime_lock; -+ seqcount_t vtime_seq; - unsigned long long vtime_snap; - enum { - VTIME_SLEEPING = 0, -@@ -1450,6 +1505,9 @@ - - struct task_cputime cputime_expires; - struct list_head cpu_timers[3]; -+#ifdef CONFIG_PREEMPT_RT_BASE -+ struct task_struct *posix_timer_list; -+#endif - - /* process credentials */ - const struct cred __rcu *real_cred; /* objective and real subjective task -@@ -1482,10 +1540,15 @@ - /* signal handlers */ - struct signal_struct *signal; - struct sighand_struct *sighand; -+ struct sigqueue *sigqueue_cache; - - sigset_t blocked, real_blocked; - sigset_t saved_sigmask; /* restored if set_restore_sigmask() was used */ - struct sigpending pending; -+#ifdef CONFIG_PREEMPT_RT_FULL -+ /* TODO: move me into ->restart_block ? */ -+ struct siginfo forced_info; ++#ifdef CONFIG_HOTPLUG_CPU ++ conf->cpu_notify.notifier_call = raid456_cpu_notify; ++ conf->cpu_notify.priority = 0; ++ err = register_cpu_notifier(&conf->cpu_notify); ++ if (err) ++ return err; +#endif - - unsigned long sas_ss_sp; - size_t sas_ss_size; -@@ -1511,6 +1574,8 @@ - /* Protection of the PI data structures: */ - raw_spinlock_t pi_lock; - -+ struct wake_q_node wake_q; + - #ifdef CONFIG_RT_MUTEXES - /* PI waiters blocked on a rt_mutex held by this task */ - struct rb_root pi_waiters; -@@ -1705,6 +1770,12 @@ - unsigned long trace; - /* bitmask and counter of trace recursion */ - unsigned long trace_recursion; -+#ifdef CONFIG_WAKEUP_LATENCY_HIST -+ u64 preempt_timestamp_hist; -+#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST -+ long timer_offset; -+#endif -+#endif - #endif /* CONFIG_TRACING */ - #ifdef CONFIG_MEMCG - struct memcg_oom_info { -@@ -1721,14 +1792,23 @@ - unsigned int sequential_io; - unsigned int sequential_io_avg; - #endif -+#ifdef CONFIG_PREEMPT_RT_BASE -+ struct rcu_head put_rcu; -+ int softirq_nestcnt; -+ unsigned int softirqs_raised; -+#endif -+#ifdef CONFIG_PREEMPT_RT_FULL -+# if defined CONFIG_HIGHMEM || defined CONFIG_X86_32 -+ int kmap_idx; -+ pte_t kmap_pte[KM_TYPE_NR]; -+# endif -+#endif - #ifdef CONFIG_DEBUG_ATOMIC_SLEEP - unsigned long task_state_change; - #endif -+ int pagefault_disabled; - }; - --/* Future-safe accessor for struct task_struct's cpus_allowed. */ --#define tsk_cpus_allowed(tsk) (&(tsk)->cpus_allowed) -- - #define TNF_MIGRATED 0x01 - #define TNF_NO_GROUP 0x02 - #define TNF_SHARED 0x04 -@@ -1917,6 +1997,15 @@ - extern void free_task(struct task_struct *tsk); - #define get_task_struct(tsk) do { atomic_inc(&(tsk)->usage); } while(0) - -+#ifdef CONFIG_PREEMPT_RT_BASE -+extern void __put_task_struct_cb(struct rcu_head *rhp); ++ get_online_cpus(); ++ for_each_present_cpu(cpu) { ++ err = alloc_scratch_buffer(conf, per_cpu_ptr(conf->percpu, cpu)); ++ if (err) { ++ pr_err("%s: failed memory allocation for cpu%ld\n", ++ __func__, cpu); ++ break; ++ } ++ } ++ put_online_cpus(); + -+static inline void put_task_struct(struct task_struct *t) -+{ -+ if (atomic_dec_and_test(&t->usage)) -+ call_rcu(&t->put_rcu, __put_task_struct_cb); ++ return err; +} -+#else - extern void __put_task_struct(struct task_struct *t); - - static inline void put_task_struct(struct task_struct *t) -@@ -1924,6 +2013,7 @@ - if (atomic_dec_and_test(&t->usage)) - __put_task_struct(t); - } -+#endif - - #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN - extern void task_cputime(struct task_struct *t, -@@ -1962,6 +2052,7 @@ - /* - * Per process flags - */ -+#define PF_IN_SOFTIRQ 0x00000001 /* Task is serving softirq */ - #define PF_EXITING 0x00000004 /* getting shut down */ - #define PF_EXITPIDONE 0x00000008 /* pi exit done on shut down */ - #define PF_VCPU 0x00000010 /* I'm a virtual CPU */ -@@ -2126,6 +2217,10 @@ - - extern int set_cpus_allowed_ptr(struct task_struct *p, - const struct cpumask *new_mask); -+int migrate_me(void); -+void tell_sched_cpu_down_begin(int cpu); -+void tell_sched_cpu_down_done(int cpu); -+ - #else - static inline void do_set_cpus_allowed(struct task_struct *p, - const struct cpumask *new_mask) -@@ -2138,6 +2233,9 @@ - return -EINVAL; - return 0; - } -+static inline int migrate_me(void) { return 0; } -+static inline void tell_sched_cpu_down_begin(int cpu) { } -+static inline void tell_sched_cpu_down_done(int cpu) { } - #endif - - #ifdef CONFIG_NO_HZ_COMMON -@@ -2354,6 +2452,7 @@ - - extern int wake_up_state(struct task_struct *tsk, unsigned int state); - extern int wake_up_process(struct task_struct *tsk); -+extern int wake_up_lock_sleeper(struct task_struct * tsk); - extern void wake_up_new_task(struct task_struct *tsk); - #ifdef CONFIG_SMP - extern void kick_process(struct task_struct *tsk); -@@ -2470,12 +2569,24 @@ - - /* mmdrop drops the mm and the page tables */ - extern void __mmdrop(struct mm_struct *); + - static inline void mmdrop(struct mm_struct * mm) - { - if (unlikely(atomic_dec_and_test(&mm->mm_count))) - __mmdrop(mm); - } - -+#ifdef CONFIG_PREEMPT_RT_BASE -+extern void __mmdrop_delayed(struct rcu_head *rhp); -+static inline void mmdrop_delayed(struct mm_struct *mm) ++static unsigned long raid5_cache_scan(struct shrinker *shrink, ++ struct shrink_control *sc) +{ -+ if (atomic_dec_and_test(&mm->mm_count)) -+ call_rcu(&mm->delayed_drop, __mmdrop_delayed); ++ struct r5conf *conf = container_of(shrink, struct r5conf, shrinker); ++ unsigned long ret = SHRINK_STOP; ++ ++ if (mutex_trylock(&conf->cache_size_mutex)) { ++ ret= 0; ++ while (ret < sc->nr_to_scan && ++ conf->max_nr_stripes > conf->min_nr_stripes) { ++ if (drop_one_stripe(conf) == 0) { ++ ret = SHRINK_STOP; ++ break; ++ } ++ ret++; ++ } ++ mutex_unlock(&conf->cache_size_mutex); ++ } ++ return ret; +} -+#else -+# define mmdrop_delayed(mm) mmdrop(mm) -+#endif + - /* mmput gets rid of the mappings and all user-space */ - extern void mmput(struct mm_struct *); - /* Grab a reference to a task's mm, if it is not already going away */ -@@ -2787,6 +2898,43 @@ - return unlikely(test_tsk_thread_flag(tsk,TIF_NEED_RESCHED)); - } - -+#ifdef CONFIG_PREEMPT_LAZY -+static inline void set_tsk_need_resched_lazy(struct task_struct *tsk) ++static unsigned long raid5_cache_count(struct shrinker *shrink, ++ struct shrink_control *sc) +{ -+ set_tsk_thread_flag(tsk,TIF_NEED_RESCHED_LAZY); ++ struct r5conf *conf = container_of(shrink, struct r5conf, shrinker); ++ ++ if (conf->max_nr_stripes < conf->min_nr_stripes) ++ /* unlikely, but not impossible */ ++ return 0; ++ return conf->max_nr_stripes - conf->min_nr_stripes; +} + -+static inline void clear_tsk_need_resched_lazy(struct task_struct *tsk) ++static struct r5conf *setup_conf(struct mddev *mddev) +{ -+ clear_tsk_thread_flag(tsk,TIF_NEED_RESCHED_LAZY); ++ struct r5conf *conf; ++ int raid_disk, memory, max_disks; ++ struct md_rdev *rdev; ++ struct disk_info *disk; ++ char pers_name[6]; ++ int i; ++ int group_cnt, worker_cnt_per_group; ++ struct r5worker_group *new_group; ++ ++ if (mddev->new_level != 5 ++ && mddev->new_level != 4 ++ && mddev->new_level != 6) { ++ printk(KERN_ERR "md/raid:%s: raid level not set to 4/5/6 (%d)\n", ++ mdname(mddev), mddev->new_level); ++ return ERR_PTR(-EIO); ++ } ++ if ((mddev->new_level == 5 ++ && !algorithm_valid_raid5(mddev->new_layout)) || ++ (mddev->new_level == 6 ++ && !algorithm_valid_raid6(mddev->new_layout))) { ++ printk(KERN_ERR "md/raid:%s: layout %d not supported\n", ++ mdname(mddev), mddev->new_layout); ++ return ERR_PTR(-EIO); ++ } ++ if (mddev->new_level == 6 && mddev->raid_disks < 4) { ++ printk(KERN_ERR "md/raid:%s: not enough configured devices (%d, minimum 4)\n", ++ mdname(mddev), mddev->raid_disks); ++ return ERR_PTR(-EINVAL); ++ } ++ ++ if (!mddev->new_chunk_sectors || ++ (mddev->new_chunk_sectors << 9) % PAGE_SIZE || ++ !is_power_of_2(mddev->new_chunk_sectors)) { ++ printk(KERN_ERR "md/raid:%s: invalid chunk size %d\n", ++ mdname(mddev), mddev->new_chunk_sectors << 9); ++ return ERR_PTR(-EINVAL); ++ } ++ ++ conf = kzalloc(sizeof(struct r5conf), GFP_KERNEL); ++ if (conf == NULL) ++ goto abort; ++ /* Don't enable multi-threading by default*/ ++ if (!alloc_thread_groups(conf, 0, &group_cnt, &worker_cnt_per_group, ++ &new_group)) { ++ conf->group_cnt = group_cnt; ++ conf->worker_cnt_per_group = worker_cnt_per_group; ++ conf->worker_groups = new_group; ++ } else ++ goto abort; ++ spin_lock_init(&conf->device_lock); ++ seqcount_init(&conf->gen_lock); ++ mutex_init(&conf->cache_size_mutex); ++ init_waitqueue_head(&conf->wait_for_stripe); ++ init_waitqueue_head(&conf->wait_for_overlap); ++ INIT_LIST_HEAD(&conf->handle_list); ++ INIT_LIST_HEAD(&conf->hold_list); ++ INIT_LIST_HEAD(&conf->delayed_list); ++ INIT_LIST_HEAD(&conf->bitmap_list); ++ init_llist_head(&conf->released_stripes); ++ atomic_set(&conf->active_stripes, 0); ++ atomic_set(&conf->preread_active_stripes, 0); ++ atomic_set(&conf->active_aligned_reads, 0); ++ conf->bypass_threshold = BYPASS_THRESHOLD; ++ conf->recovery_disabled = mddev->recovery_disabled - 1; ++ ++ conf->raid_disks = mddev->raid_disks; ++ if (mddev->reshape_position == MaxSector) ++ conf->previous_raid_disks = mddev->raid_disks; ++ else ++ conf->previous_raid_disks = mddev->raid_disks - mddev->delta_disks; ++ max_disks = max(conf->raid_disks, conf->previous_raid_disks); ++ ++ conf->disks = kzalloc(max_disks * sizeof(struct disk_info), ++ GFP_KERNEL); ++ if (!conf->disks) ++ goto abort; ++ ++ conf->mddev = mddev; ++ ++ if ((conf->stripe_hashtbl = kzalloc(PAGE_SIZE, GFP_KERNEL)) == NULL) ++ goto abort; ++ ++ /* We init hash_locks[0] separately to that it can be used ++ * as the reference lock in the spin_lock_nest_lock() call ++ * in lock_all_device_hash_locks_irq in order to convince ++ * lockdep that we know what we are doing. ++ */ ++ spin_lock_init(conf->hash_locks); ++ for (i = 1; i < NR_STRIPE_HASH_LOCKS; i++) ++ spin_lock_init(conf->hash_locks + i); ++ ++ for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++) ++ INIT_LIST_HEAD(conf->inactive_list + i); ++ ++ for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++) ++ INIT_LIST_HEAD(conf->temp_inactive_list + i); ++ ++ conf->level = mddev->new_level; ++ conf->chunk_sectors = mddev->new_chunk_sectors; ++ if (raid5_alloc_percpu(conf) != 0) ++ goto abort; ++ ++ pr_debug("raid456: run(%s) called.\n", mdname(mddev)); ++ ++ rdev_for_each(rdev, mddev) { ++ raid_disk = rdev->raid_disk; ++ if (raid_disk >= max_disks ++ || raid_disk < 0) ++ continue; ++ disk = conf->disks + raid_disk; ++ ++ if (test_bit(Replacement, &rdev->flags)) { ++ if (disk->replacement) ++ goto abort; ++ disk->replacement = rdev; ++ } else { ++ if (disk->rdev) ++ goto abort; ++ disk->rdev = rdev; ++ } ++ ++ if (test_bit(In_sync, &rdev->flags)) { ++ char b[BDEVNAME_SIZE]; ++ printk(KERN_INFO "md/raid:%s: device %s operational as raid" ++ " disk %d\n", ++ mdname(mddev), bdevname(rdev->bdev, b), raid_disk); ++ } else if (rdev->saved_raid_disk != raid_disk) ++ /* Cannot rely on bitmap to complete recovery */ ++ conf->fullsync = 1; ++ } ++ ++ conf->level = mddev->new_level; ++ if (conf->level == 6) { ++ conf->max_degraded = 2; ++ if (raid6_call.xor_syndrome) ++ conf->rmw_level = PARITY_ENABLE_RMW; ++ else ++ conf->rmw_level = PARITY_DISABLE_RMW; ++ } else { ++ conf->max_degraded = 1; ++ conf->rmw_level = PARITY_ENABLE_RMW; ++ } ++ conf->algorithm = mddev->new_layout; ++ conf->reshape_progress = mddev->reshape_position; ++ if (conf->reshape_progress != MaxSector) { ++ conf->prev_chunk_sectors = mddev->chunk_sectors; ++ conf->prev_algo = mddev->layout; ++ } ++ ++ conf->min_nr_stripes = NR_STRIPES; ++ memory = conf->min_nr_stripes * (sizeof(struct stripe_head) + ++ max_disks * ((sizeof(struct bio) + PAGE_SIZE))) / 1024; ++ atomic_set(&conf->empty_inactive_list_nr, NR_STRIPE_HASH_LOCKS); ++ if (grow_stripes(conf, conf->min_nr_stripes)) { ++ printk(KERN_ERR ++ "md/raid:%s: couldn't allocate %dkB for buffers\n", ++ mdname(mddev), memory); ++ goto abort; ++ } else ++ printk(KERN_INFO "md/raid:%s: allocated %dkB\n", ++ mdname(mddev), memory); ++ /* ++ * Losing a stripe head costs more than the time to refill it, ++ * it reduces the queue depth and so can hurt throughput. ++ * So set it rather large, scaled by number of devices. ++ */ ++ conf->shrinker.seeks = DEFAULT_SEEKS * conf->raid_disks * 4; ++ conf->shrinker.scan_objects = raid5_cache_scan; ++ conf->shrinker.count_objects = raid5_cache_count; ++ conf->shrinker.batch = 128; ++ conf->shrinker.flags = 0; ++ register_shrinker(&conf->shrinker); ++ ++ sprintf(pers_name, "raid%d", mddev->new_level); ++ conf->thread = md_register_thread(raid5d, mddev, pers_name); ++ if (!conf->thread) { ++ printk(KERN_ERR ++ "md/raid:%s: couldn't allocate thread.\n", ++ mdname(mddev)); ++ goto abort; ++ } ++ ++ return conf; ++ ++ abort: ++ if (conf) { ++ free_conf(conf); ++ return ERR_PTR(-EIO); ++ } else ++ return ERR_PTR(-ENOMEM); +} + -+static inline int test_tsk_need_resched_lazy(struct task_struct *tsk) ++static int only_parity(int raid_disk, int algo, int raid_disks, int max_degraded) +{ -+ return unlikely(test_tsk_thread_flag(tsk,TIF_NEED_RESCHED_LAZY)); ++ switch (algo) { ++ case ALGORITHM_PARITY_0: ++ if (raid_disk < max_degraded) ++ return 1; ++ break; ++ case ALGORITHM_PARITY_N: ++ if (raid_disk >= raid_disks - max_degraded) ++ return 1; ++ break; ++ case ALGORITHM_PARITY_0_6: ++ if (raid_disk == 0 || ++ raid_disk == raid_disks - 1) ++ return 1; ++ break; ++ case ALGORITHM_LEFT_ASYMMETRIC_6: ++ case ALGORITHM_RIGHT_ASYMMETRIC_6: ++ case ALGORITHM_LEFT_SYMMETRIC_6: ++ case ALGORITHM_RIGHT_SYMMETRIC_6: ++ if (raid_disk == raid_disks - 1) ++ return 1; ++ } ++ return 0; +} + -+static inline int need_resched_lazy(void) ++static int run(struct mddev *mddev) +{ -+ return test_thread_flag(TIF_NEED_RESCHED_LAZY); ++ struct r5conf *conf; ++ int working_disks = 0; ++ int dirty_parity_disks = 0; ++ struct md_rdev *rdev; ++ sector_t reshape_offset = 0; ++ int i; ++ long long min_offset_diff = 0; ++ int first = 1; ++ ++ if (mddev->recovery_cp != MaxSector) ++ printk(KERN_NOTICE "md/raid:%s: not clean" ++ " -- starting background reconstruction\n", ++ mdname(mddev)); ++ ++ rdev_for_each(rdev, mddev) { ++ long long diff; ++ if (rdev->raid_disk < 0) ++ continue; ++ diff = (rdev->new_data_offset - rdev->data_offset); ++ if (first) { ++ min_offset_diff = diff; ++ first = 0; ++ } else if (mddev->reshape_backwards && ++ diff < min_offset_diff) ++ min_offset_diff = diff; ++ else if (!mddev->reshape_backwards && ++ diff > min_offset_diff) ++ min_offset_diff = diff; ++ } ++ ++ if (mddev->reshape_position != MaxSector) { ++ /* Check that we can continue the reshape. ++ * Difficulties arise if the stripe we would write to ++ * next is at or after the stripe we would read from next. ++ * For a reshape that changes the number of devices, this ++ * is only possible for a very short time, and mdadm makes ++ * sure that time appears to have past before assembling ++ * the array. So we fail if that time hasn't passed. ++ * For a reshape that keeps the number of devices the same ++ * mdadm must be monitoring the reshape can keeping the ++ * critical areas read-only and backed up. It will start ++ * the array in read-only mode, so we check for that. ++ */ ++ sector_t here_new, here_old; ++ int old_disks; ++ int max_degraded = (mddev->level == 6 ? 2 : 1); ++ ++ if (mddev->new_level != mddev->level) { ++ printk(KERN_ERR "md/raid:%s: unsupported reshape " ++ "required - aborting.\n", ++ mdname(mddev)); ++ return -EINVAL; ++ } ++ old_disks = mddev->raid_disks - mddev->delta_disks; ++ /* reshape_position must be on a new-stripe boundary, and one ++ * further up in new geometry must map after here in old ++ * geometry. ++ */ ++ here_new = mddev->reshape_position; ++ if (sector_div(here_new, mddev->new_chunk_sectors * ++ (mddev->raid_disks - max_degraded))) { ++ printk(KERN_ERR "md/raid:%s: reshape_position not " ++ "on a stripe boundary\n", mdname(mddev)); ++ return -EINVAL; ++ } ++ reshape_offset = here_new * mddev->new_chunk_sectors; ++ /* here_new is the stripe we will write to */ ++ here_old = mddev->reshape_position; ++ sector_div(here_old, mddev->chunk_sectors * ++ (old_disks-max_degraded)); ++ /* here_old is the first stripe that we might need to read ++ * from */ ++ if (mddev->delta_disks == 0) { ++ if ((here_new * mddev->new_chunk_sectors != ++ here_old * mddev->chunk_sectors)) { ++ printk(KERN_ERR "md/raid:%s: reshape position is" ++ " confused - aborting\n", mdname(mddev)); ++ return -EINVAL; ++ } ++ /* We cannot be sure it is safe to start an in-place ++ * reshape. It is only safe if user-space is monitoring ++ * and taking constant backups. ++ * mdadm always starts a situation like this in ++ * readonly mode so it can take control before ++ * allowing any writes. So just check for that. ++ */ ++ if (abs(min_offset_diff) >= mddev->chunk_sectors && ++ abs(min_offset_diff) >= mddev->new_chunk_sectors) ++ /* not really in-place - so OK */; ++ else if (mddev->ro == 0) { ++ printk(KERN_ERR "md/raid:%s: in-place reshape " ++ "must be started in read-only mode " ++ "- aborting\n", ++ mdname(mddev)); ++ return -EINVAL; ++ } ++ } else if (mddev->reshape_backwards ++ ? (here_new * mddev->new_chunk_sectors + min_offset_diff <= ++ here_old * mddev->chunk_sectors) ++ : (here_new * mddev->new_chunk_sectors >= ++ here_old * mddev->chunk_sectors + (-min_offset_diff))) { ++ /* Reading from the same stripe as writing to - bad */ ++ printk(KERN_ERR "md/raid:%s: reshape_position too early for " ++ "auto-recovery - aborting.\n", ++ mdname(mddev)); ++ return -EINVAL; ++ } ++ printk(KERN_INFO "md/raid:%s: reshape will continue\n", ++ mdname(mddev)); ++ /* OK, we should be able to continue; */ ++ } else { ++ BUG_ON(mddev->level != mddev->new_level); ++ BUG_ON(mddev->layout != mddev->new_layout); ++ BUG_ON(mddev->chunk_sectors != mddev->new_chunk_sectors); ++ BUG_ON(mddev->delta_disks != 0); ++ } ++ ++ if (mddev->private == NULL) ++ conf = setup_conf(mddev); ++ else ++ conf = mddev->private; ++ ++ if (IS_ERR(conf)) ++ return PTR_ERR(conf); ++ ++ conf->min_offset_diff = min_offset_diff; ++ mddev->thread = conf->thread; ++ conf->thread = NULL; ++ mddev->private = conf; ++ ++ for (i = 0; i < conf->raid_disks && conf->previous_raid_disks; ++ i++) { ++ rdev = conf->disks[i].rdev; ++ if (!rdev && conf->disks[i].replacement) { ++ /* The replacement is all we have yet */ ++ rdev = conf->disks[i].replacement; ++ conf->disks[i].replacement = NULL; ++ clear_bit(Replacement, &rdev->flags); ++ conf->disks[i].rdev = rdev; ++ } ++ if (!rdev) ++ continue; ++ if (conf->disks[i].replacement && ++ conf->reshape_progress != MaxSector) { ++ /* replacements and reshape simply do not mix. */ ++ printk(KERN_ERR "md: cannot handle concurrent " ++ "replacement and reshape.\n"); ++ goto abort; ++ } ++ if (test_bit(In_sync, &rdev->flags)) { ++ working_disks++; ++ continue; ++ } ++ /* This disc is not fully in-sync. However if it ++ * just stored parity (beyond the recovery_offset), ++ * when we don't need to be concerned about the ++ * array being dirty. ++ * When reshape goes 'backwards', we never have ++ * partially completed devices, so we only need ++ * to worry about reshape going forwards. ++ */ ++ /* Hack because v0.91 doesn't store recovery_offset properly. */ ++ if (mddev->major_version == 0 && ++ mddev->minor_version > 90) ++ rdev->recovery_offset = reshape_offset; ++ ++ if (rdev->recovery_offset < reshape_offset) { ++ /* We need to check old and new layout */ ++ if (!only_parity(rdev->raid_disk, ++ conf->algorithm, ++ conf->raid_disks, ++ conf->max_degraded)) ++ continue; ++ } ++ if (!only_parity(rdev->raid_disk, ++ conf->prev_algo, ++ conf->previous_raid_disks, ++ conf->max_degraded)) ++ continue; ++ dirty_parity_disks++; ++ } ++ ++ /* ++ * 0 for a fully functional array, 1 or 2 for a degraded array. ++ */ ++ mddev->degraded = calc_degraded(conf); ++ ++ if (has_failed(conf)) { ++ printk(KERN_ERR "md/raid:%s: not enough operational devices" ++ " (%d/%d failed)\n", ++ mdname(mddev), mddev->degraded, conf->raid_disks); ++ goto abort; ++ } ++ ++ /* device size must be a multiple of chunk size */ ++ mddev->dev_sectors &= ~(mddev->chunk_sectors - 1); ++ mddev->resync_max_sectors = mddev->dev_sectors; ++ ++ if (mddev->degraded > dirty_parity_disks && ++ mddev->recovery_cp != MaxSector) { ++ if (mddev->ok_start_degraded) ++ printk(KERN_WARNING ++ "md/raid:%s: starting dirty degraded array" ++ " - data corruption possible.\n", ++ mdname(mddev)); ++ else { ++ printk(KERN_ERR ++ "md/raid:%s: cannot start dirty degraded array.\n", ++ mdname(mddev)); ++ goto abort; ++ } ++ } ++ ++ if (mddev->degraded == 0) ++ printk(KERN_INFO "md/raid:%s: raid level %d active with %d out of %d" ++ " devices, algorithm %d\n", mdname(mddev), conf->level, ++ mddev->raid_disks-mddev->degraded, mddev->raid_disks, ++ mddev->new_layout); ++ else ++ printk(KERN_ALERT "md/raid:%s: raid level %d active with %d" ++ " out of %d devices, algorithm %d\n", ++ mdname(mddev), conf->level, ++ mddev->raid_disks - mddev->degraded, ++ mddev->raid_disks, mddev->new_layout); ++ ++ print_raid5_conf(conf); ++ ++ if (conf->reshape_progress != MaxSector) { ++ conf->reshape_safe = conf->reshape_progress; ++ atomic_set(&conf->reshape_stripes, 0); ++ clear_bit(MD_RECOVERY_SYNC, &mddev->recovery); ++ clear_bit(MD_RECOVERY_CHECK, &mddev->recovery); ++ set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery); ++ set_bit(MD_RECOVERY_RUNNING, &mddev->recovery); ++ mddev->sync_thread = md_register_thread(md_do_sync, mddev, ++ "reshape"); ++ } ++ ++ /* Ok, everything is just fine now */ ++ if (mddev->to_remove == &raid5_attrs_group) ++ mddev->to_remove = NULL; ++ else if (mddev->kobj.sd && ++ sysfs_create_group(&mddev->kobj, &raid5_attrs_group)) ++ printk(KERN_WARNING ++ "raid5: failed to create sysfs attributes for %s\n", ++ mdname(mddev)); ++ md_set_array_sectors(mddev, raid5_size(mddev, 0, 0)); ++ ++ if (mddev->queue) { ++ int chunk_size; ++ bool discard_supported = true; ++ /* read-ahead size must cover two whole stripes, which ++ * is 2 * (datadisks) * chunksize where 'n' is the ++ * number of raid devices ++ */ ++ int data_disks = conf->previous_raid_disks - conf->max_degraded; ++ int stripe = data_disks * ++ ((mddev->chunk_sectors << 9) / PAGE_SIZE); ++ if (mddev->queue->backing_dev_info.ra_pages < 2 * stripe) ++ mddev->queue->backing_dev_info.ra_pages = 2 * stripe; ++ ++ chunk_size = mddev->chunk_sectors << 9; ++ blk_queue_io_min(mddev->queue, chunk_size); ++ blk_queue_io_opt(mddev->queue, chunk_size * ++ (conf->raid_disks - conf->max_degraded)); ++ mddev->queue->limits.raid_partial_stripes_expensive = 1; ++ /* ++ * We can only discard a whole stripe. It doesn't make sense to ++ * discard data disk but write parity disk ++ */ ++ stripe = stripe * PAGE_SIZE; ++ /* Round up to power of 2, as discard handling ++ * currently assumes that */ ++ while ((stripe-1) & stripe) ++ stripe = (stripe | (stripe-1)) + 1; ++ mddev->queue->limits.discard_alignment = stripe; ++ mddev->queue->limits.discard_granularity = stripe; ++ /* ++ * unaligned part of discard request will be ignored, so can't ++ * guarantee discard_zeroes_data ++ */ ++ mddev->queue->limits.discard_zeroes_data = 0; ++ ++ blk_queue_max_write_same_sectors(mddev->queue, 0); ++ ++ rdev_for_each(rdev, mddev) { ++ disk_stack_limits(mddev->gendisk, rdev->bdev, ++ rdev->data_offset << 9); ++ disk_stack_limits(mddev->gendisk, rdev->bdev, ++ rdev->new_data_offset << 9); ++ /* ++ * discard_zeroes_data is required, otherwise data ++ * could be lost. Consider a scenario: discard a stripe ++ * (the stripe could be inconsistent if ++ * discard_zeroes_data is 0); write one disk of the ++ * stripe (the stripe could be inconsistent again ++ * depending on which disks are used to calculate ++ * parity); the disk is broken; The stripe data of this ++ * disk is lost. ++ */ ++ if (!blk_queue_discard(bdev_get_queue(rdev->bdev)) || ++ !bdev_get_queue(rdev->bdev)-> ++ limits.discard_zeroes_data) ++ discard_supported = false; ++ /* Unfortunately, discard_zeroes_data is not currently ++ * a guarantee - just a hint. So we only allow DISCARD ++ * if the sysadmin has confirmed that only safe devices ++ * are in use by setting a module parameter. ++ */ ++ if (!devices_handle_discard_safely) { ++ if (discard_supported) { ++ pr_info("md/raid456: discard support disabled due to uncertainty.\n"); ++ pr_info("Set raid456.devices_handle_discard_safely=Y to override.\n"); ++ } ++ discard_supported = false; ++ } ++ } ++ ++ if (discard_supported && ++ mddev->queue->limits.max_discard_sectors >= stripe && ++ mddev->queue->limits.discard_granularity >= stripe) ++ queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, ++ mddev->queue); ++ else ++ queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD, ++ mddev->queue); ++ } ++ ++ return 0; ++abort: ++ md_unregister_thread(&mddev->thread); ++ print_raid5_conf(conf); ++ free_conf(conf); ++ mddev->private = NULL; ++ printk(KERN_ALERT "md/raid:%s: failed to run raid set.\n", mdname(mddev)); ++ return -EIO; +} + -+static inline int need_resched_now(void) ++static void raid5_free(struct mddev *mddev, void *priv) +{ -+ return test_thread_flag(TIF_NEED_RESCHED); -+} ++ struct r5conf *conf = priv; + -+#else -+static inline void clear_tsk_need_resched_lazy(struct task_struct *tsk) { } -+static inline int need_resched_lazy(void) { return 0; } ++ free_conf(conf); ++ mddev->to_remove = &raid5_attrs_group; ++} + -+static inline int need_resched_now(void) ++static void status(struct seq_file *seq, struct mddev *mddev) +{ -+ return test_thread_flag(TIF_NEED_RESCHED); -+} ++ struct r5conf *conf = mddev->private; ++ int i; + -+#endif ++ seq_printf(seq, " level %d, %dk chunk, algorithm %d", mddev->level, ++ mddev->chunk_sectors / 2, mddev->layout); ++ seq_printf (seq, " [%d/%d] [", conf->raid_disks, conf->raid_disks - mddev->degraded); ++ for (i = 0; i < conf->raid_disks; i++) ++ seq_printf (seq, "%s", ++ conf->disks[i].rdev && ++ test_bit(In_sync, &conf->disks[i].rdev->flags) ? "U" : "_"); ++ seq_printf (seq, "]"); ++} + - static inline int restart_syscall(void) - { - set_tsk_thread_flag(current, TIF_SIGPENDING); -@@ -2818,6 +2966,51 @@ - return (state & TASK_INTERRUPTIBLE) || __fatal_signal_pending(p); - } - -+static inline bool __task_is_stopped_or_traced(struct task_struct *task) ++static void print_raid5_conf (struct r5conf *conf) +{ -+ if (task->state & (__TASK_STOPPED | __TASK_TRACED)) -+ return true; -+#ifdef CONFIG_PREEMPT_RT_FULL -+ if (task->saved_state & (__TASK_STOPPED | __TASK_TRACED)) -+ return true; -+#endif -+ return false; ++ int i; ++ struct disk_info *tmp; ++ ++ printk(KERN_DEBUG "RAID conf printout:\n"); ++ if (!conf) { ++ printk("(conf==NULL)\n"); ++ return; ++ } ++ printk(KERN_DEBUG " --- level:%d rd:%d wd:%d\n", conf->level, ++ conf->raid_disks, ++ conf->raid_disks - conf->mddev->degraded); ++ ++ for (i = 0; i < conf->raid_disks; i++) { ++ char b[BDEVNAME_SIZE]; ++ tmp = conf->disks + i; ++ if (tmp->rdev) ++ printk(KERN_DEBUG " disk %d, o:%d, dev:%s\n", ++ i, !test_bit(Faulty, &tmp->rdev->flags), ++ bdevname(tmp->rdev->bdev, b)); ++ } +} + -+static inline bool task_is_stopped_or_traced(struct task_struct *task) ++static int raid5_spare_active(struct mddev *mddev) +{ -+ bool traced_stopped; -+ -+#ifdef CONFIG_PREEMPT_RT_FULL ++ int i; ++ struct r5conf *conf = mddev->private; ++ struct disk_info *tmp; ++ int count = 0; + unsigned long flags; + -+ raw_spin_lock_irqsave(&task->pi_lock, flags); -+ traced_stopped = __task_is_stopped_or_traced(task); -+ raw_spin_unlock_irqrestore(&task->pi_lock, flags); -+#else -+ traced_stopped = __task_is_stopped_or_traced(task); -+#endif -+ return traced_stopped; ++ for (i = 0; i < conf->raid_disks; i++) { ++ tmp = conf->disks + i; ++ if (tmp->replacement ++ && tmp->replacement->recovery_offset == MaxSector ++ && !test_bit(Faulty, &tmp->replacement->flags) ++ && !test_and_set_bit(In_sync, &tmp->replacement->flags)) { ++ /* Replacement has just become active. */ ++ if (!tmp->rdev ++ || !test_and_clear_bit(In_sync, &tmp->rdev->flags)) ++ count++; ++ if (tmp->rdev) { ++ /* Replaced device not technically faulty, ++ * but we need to be sure it gets removed ++ * and never re-added. ++ */ ++ set_bit(Faulty, &tmp->rdev->flags); ++ sysfs_notify_dirent_safe( ++ tmp->rdev->sysfs_state); ++ } ++ sysfs_notify_dirent_safe(tmp->replacement->sysfs_state); ++ } else if (tmp->rdev ++ && tmp->rdev->recovery_offset == MaxSector ++ && !test_bit(Faulty, &tmp->rdev->flags) ++ && !test_and_set_bit(In_sync, &tmp->rdev->flags)) { ++ count++; ++ sysfs_notify_dirent_safe(tmp->rdev->sysfs_state); ++ } ++ } ++ spin_lock_irqsave(&conf->device_lock, flags); ++ mddev->degraded = calc_degraded(conf); ++ spin_unlock_irqrestore(&conf->device_lock, flags); ++ print_raid5_conf(conf); ++ return count; +} + -+static inline bool task_is_traced(struct task_struct *task) ++static int raid5_remove_disk(struct mddev *mddev, struct md_rdev *rdev) +{ -+ bool traced = false; ++ struct r5conf *conf = mddev->private; ++ int err = 0; ++ int number = rdev->raid_disk; ++ struct md_rdev **rdevp; ++ struct disk_info *p = conf->disks + number; ++ ++ print_raid5_conf(conf); ++ if (rdev == p->rdev) ++ rdevp = &p->rdev; ++ else if (rdev == p->replacement) ++ rdevp = &p->replacement; ++ else ++ return 0; + -+ if (task->state & __TASK_TRACED) -+ return true; -+#ifdef CONFIG_PREEMPT_RT_FULL -+ /* in case the task is sleeping on tasklist_lock */ -+ raw_spin_lock_irq(&task->pi_lock); -+ if (task->state & __TASK_TRACED) -+ traced = true; -+ else if (task->saved_state & __TASK_TRACED) -+ traced = true; -+ raw_spin_unlock_irq(&task->pi_lock); -+#endif -+ return traced; -+} ++ if (number >= conf->raid_disks && ++ conf->reshape_progress == MaxSector) ++ clear_bit(In_sync, &rdev->flags); + - /* - * cond_resched() and cond_resched_lock(): latency reduction via - * explicit rescheduling in places that are safe. The return -@@ -2834,7 +3027,7 @@ - - extern int __cond_resched_lock(spinlock_t *lock); - --#ifdef CONFIG_PREEMPT_COUNT -+#if defined(CONFIG_PREEMPT_COUNT) && !defined(CONFIG_PREEMPT_RT_FULL) - #define PREEMPT_LOCK_OFFSET PREEMPT_OFFSET - #else - #define PREEMPT_LOCK_OFFSET 0 -@@ -2845,12 +3038,16 @@ - __cond_resched_lock(lock); \ - }) - -+#ifndef CONFIG_PREEMPT_RT_FULL - extern int __cond_resched_softirq(void); - - #define cond_resched_softirq() ({ \ - ___might_sleep(__FILE__, __LINE__, SOFTIRQ_DISABLE_OFFSET); \ - __cond_resched_softirq(); \ - }) -+#else -+# define cond_resched_softirq() cond_resched() -+#endif - - static inline void cond_resched_rcu(void) - { -@@ -3017,6 +3214,26 @@ - - #endif /* CONFIG_SMP */ - -+static inline int __migrate_disabled(struct task_struct *p) -+{ -+#ifdef CONFIG_PREEMPT_RT_FULL -+ return p->migrate_disable; -+#else -+ return 0; -+#endif ++ if (test_bit(In_sync, &rdev->flags) || ++ atomic_read(&rdev->nr_pending)) { ++ err = -EBUSY; ++ goto abort; ++ } ++ /* Only remove non-faulty devices if recovery ++ * isn't possible. ++ */ ++ if (!test_bit(Faulty, &rdev->flags) && ++ mddev->recovery_disabled != conf->recovery_disabled && ++ !has_failed(conf) && ++ (!p->replacement || p->replacement == rdev) && ++ number < conf->raid_disks) { ++ err = -EBUSY; ++ goto abort; ++ } ++ *rdevp = NULL; ++ synchronize_rcu(); ++ if (atomic_read(&rdev->nr_pending)) { ++ /* lost the race, try later */ ++ err = -EBUSY; ++ *rdevp = rdev; ++ } else if (p->replacement) { ++ /* We must have just cleared 'rdev' */ ++ p->rdev = p->replacement; ++ clear_bit(Replacement, &p->replacement->flags); ++ smp_mb(); /* Make sure other CPUs may see both as identical ++ * but will never see neither - if they are careful ++ */ ++ p->replacement = NULL; ++ clear_bit(WantReplacement, &rdev->flags); ++ } else ++ /* We might have just removed the Replacement as faulty- ++ * clear the bit just in case ++ */ ++ clear_bit(WantReplacement, &rdev->flags); ++abort: ++ ++ print_raid5_conf(conf); ++ return err; +} + -+/* Future-safe accessor for struct task_struct's cpus_allowed. */ -+static inline const struct cpumask *tsk_cpus_allowed(struct task_struct *p) ++static int raid5_add_disk(struct mddev *mddev, struct md_rdev *rdev) +{ -+#ifdef CONFIG_PREEMPT_RT_FULL -+ if (p->migrate_disable) -+ return cpumask_of(task_cpu(p)); -+#endif ++ struct r5conf *conf = mddev->private; ++ int err = -EEXIST; ++ int disk; ++ struct disk_info *p; ++ int first = 0; ++ int last = conf->raid_disks - 1; ++ ++ if (mddev->recovery_disabled == conf->recovery_disabled) ++ return -EBUSY; + -+ return &p->cpus_allowed; ++ if (rdev->saved_raid_disk < 0 && has_failed(conf)) ++ /* no point adding a device */ ++ return -EINVAL; ++ ++ if (rdev->raid_disk >= 0) ++ first = last = rdev->raid_disk; ++ ++ /* ++ * find the disk ... but prefer rdev->saved_raid_disk ++ * if possible. ++ */ ++ if (rdev->saved_raid_disk >= 0 && ++ rdev->saved_raid_disk >= first && ++ conf->disks[rdev->saved_raid_disk].rdev == NULL) ++ first = rdev->saved_raid_disk; ++ ++ for (disk = first; disk <= last; disk++) { ++ p = conf->disks + disk; ++ if (p->rdev == NULL) { ++ clear_bit(In_sync, &rdev->flags); ++ rdev->raid_disk = disk; ++ err = 0; ++ if (rdev->saved_raid_disk != disk) ++ conf->fullsync = 1; ++ rcu_assign_pointer(p->rdev, rdev); ++ goto out; ++ } ++ } ++ for (disk = first; disk <= last; disk++) { ++ p = conf->disks + disk; ++ if (test_bit(WantReplacement, &p->rdev->flags) && ++ p->replacement == NULL) { ++ clear_bit(In_sync, &rdev->flags); ++ set_bit(Replacement, &rdev->flags); ++ rdev->raid_disk = disk; ++ err = 0; ++ conf->fullsync = 1; ++ rcu_assign_pointer(p->replacement, rdev); ++ break; ++ } ++ } ++out: ++ print_raid5_conf(conf); ++ return err; +} + - extern long sched_setaffinity(pid_t pid, const struct cpumask *new_mask); - extern long sched_getaffinity(pid_t pid, struct cpumask *mask); - -diff -Nur linux-4.1.6.orig/include/linux/seqlock.h linux-4.1.6/include/linux/seqlock.h ---- linux-4.1.6.orig/include/linux/seqlock.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/seqlock.h 2015-09-08 23:49:06.778014632 +0200 -@@ -219,20 +219,30 @@ - return __read_seqcount_retry(s, start); - } - -- -- --static inline void raw_write_seqcount_begin(seqcount_t *s) -+static inline void __raw_write_seqcount_begin(seqcount_t *s) - { - s->sequence++; - smp_wmb(); - } - --static inline void raw_write_seqcount_end(seqcount_t *s) -+static inline void raw_write_seqcount_begin(seqcount_t *s) ++static int raid5_resize(struct mddev *mddev, sector_t sectors) +{ -+ preempt_disable_rt(); -+ __raw_write_seqcount_begin(s); ++ /* no resync is happening, and there is enough space ++ * on all devices, so we can resize. ++ * We need to make sure resync covers any new space. ++ * If the array is shrinking we should possibly wait until ++ * any io in the removed space completes, but it hardly seems ++ * worth it. ++ */ ++ sector_t newsize; ++ sectors &= ~((sector_t)mddev->chunk_sectors - 1); ++ newsize = raid5_size(mddev, sectors, mddev->raid_disks); ++ if (mddev->external_size && ++ mddev->array_sectors > newsize) ++ return -EINVAL; ++ if (mddev->bitmap) { ++ int ret = bitmap_resize(mddev->bitmap, sectors, 0, 0); ++ if (ret) ++ return ret; ++ } ++ md_set_array_sectors(mddev, newsize); ++ set_capacity(mddev->gendisk, mddev->array_sectors); ++ revalidate_disk(mddev->gendisk); ++ if (sectors > mddev->dev_sectors && ++ mddev->recovery_cp > mddev->dev_sectors) { ++ mddev->recovery_cp = mddev->dev_sectors; ++ set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); ++ } ++ mddev->dev_sectors = sectors; ++ mddev->resync_max_sectors = sectors; ++ return 0; +} + -+static inline void __raw_write_seqcount_end(seqcount_t *s) - { - smp_wmb(); - s->sequence++; - } - -+static inline void raw_write_seqcount_end(seqcount_t *s) ++static int check_stripe_cache(struct mddev *mddev) +{ -+ __raw_write_seqcount_end(s); -+ preempt_enable_rt(); ++ /* Can only proceed if there are plenty of stripe_heads. ++ * We need a minimum of one full stripe,, and for sensible progress ++ * it is best to have about 4 times that. ++ * If we require 4 times, then the default 256 4K stripe_heads will ++ * allow for chunk sizes up to 256K, which is probably OK. ++ * If the chunk size is greater, user-space should request more ++ * stripe_heads first. ++ */ ++ struct r5conf *conf = mddev->private; ++ if (((mddev->chunk_sectors << 9) / STRIPE_SIZE) * 4 ++ > conf->min_nr_stripes || ++ ((mddev->new_chunk_sectors << 9) / STRIPE_SIZE) * 4 ++ > conf->min_nr_stripes) { ++ printk(KERN_WARNING "md/raid:%s: reshape: not enough stripes. Needed %lu\n", ++ mdname(mddev), ++ ((max(mddev->chunk_sectors, mddev->new_chunk_sectors) << 9) ++ / STRIPE_SIZE)*4); ++ return 0; ++ } ++ return 1; +} + - /* - * raw_write_seqcount_latch - redirect readers to even/odd copy - * @s: pointer to seqcount_t -@@ -305,10 +315,32 @@ - /* - * Read side functions for starting and finalizing a read side section. - */ -+#ifndef CONFIG_PREEMPT_RT_FULL - static inline unsigned read_seqbegin(const seqlock_t *sl) - { - return read_seqcount_begin(&sl->seqcount); - } -+#else -+/* -+ * Starvation safe read side for RT -+ */ -+static inline unsigned read_seqbegin(seqlock_t *sl) ++static int check_reshape(struct mddev *mddev) +{ -+ unsigned ret; ++ struct r5conf *conf = mddev->private; + -+repeat: -+ ret = ACCESS_ONCE(sl->seqcount.sequence); -+ if (unlikely(ret & 1)) { -+ /* -+ * Take the lock and let the writer proceed (i.e. evtl -+ * boost it), otherwise we could loop here forever. ++ if (mddev->delta_disks == 0 && ++ mddev->new_layout == mddev->layout && ++ mddev->new_chunk_sectors == mddev->chunk_sectors) ++ return 0; /* nothing to do */ ++ if (has_failed(conf)) ++ return -EINVAL; ++ if (mddev->delta_disks < 0 && mddev->reshape_position == MaxSector) { ++ /* We might be able to shrink, but the devices must ++ * be made bigger first. ++ * For raid6, 4 is the minimum size. ++ * Otherwise 2 is the minimum + */ -+ spin_unlock_wait(&sl->lock); -+ goto repeat; ++ int min = 2; ++ if (mddev->level == 6) ++ min = 4; ++ if (mddev->raid_disks + mddev->delta_disks < min) ++ return -EINVAL; + } -+ return ret; -+} -+#endif - - static inline unsigned read_seqretry(const seqlock_t *sl, unsigned start) - { -@@ -323,36 +355,36 @@ - static inline void write_seqlock(seqlock_t *sl) - { - spin_lock(&sl->lock); -- write_seqcount_begin(&sl->seqcount); -+ __raw_write_seqcount_begin(&sl->seqcount); - } - - static inline void write_sequnlock(seqlock_t *sl) - { -- write_seqcount_end(&sl->seqcount); -+ __raw_write_seqcount_end(&sl->seqcount); - spin_unlock(&sl->lock); - } - - static inline void write_seqlock_bh(seqlock_t *sl) - { - spin_lock_bh(&sl->lock); -- write_seqcount_begin(&sl->seqcount); -+ __raw_write_seqcount_begin(&sl->seqcount); - } - - static inline void write_sequnlock_bh(seqlock_t *sl) - { -- write_seqcount_end(&sl->seqcount); -+ __raw_write_seqcount_end(&sl->seqcount); - spin_unlock_bh(&sl->lock); - } - - static inline void write_seqlock_irq(seqlock_t *sl) - { - spin_lock_irq(&sl->lock); -- write_seqcount_begin(&sl->seqcount); -+ __raw_write_seqcount_begin(&sl->seqcount); - } - - static inline void write_sequnlock_irq(seqlock_t *sl) - { -- write_seqcount_end(&sl->seqcount); -+ __raw_write_seqcount_end(&sl->seqcount); - spin_unlock_irq(&sl->lock); - } - -@@ -361,7 +393,7 @@ - unsigned long flags; - - spin_lock_irqsave(&sl->lock, flags); -- write_seqcount_begin(&sl->seqcount); -+ __raw_write_seqcount_begin(&sl->seqcount); - return flags; - } - -@@ -371,7 +403,7 @@ - static inline void - write_sequnlock_irqrestore(seqlock_t *sl, unsigned long flags) - { -- write_seqcount_end(&sl->seqcount); -+ __raw_write_seqcount_end(&sl->seqcount); - spin_unlock_irqrestore(&sl->lock, flags); - } - -diff -Nur linux-4.1.6.orig/include/linux/signal.h linux-4.1.6/include/linux/signal.h ---- linux-4.1.6.orig/include/linux/signal.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/signal.h 2015-09-08 23:49:06.778014632 +0200 -@@ -218,6 +218,7 @@ - } - - extern void flush_sigqueue(struct sigpending *queue); -+extern void flush_task_sigqueue(struct task_struct *tsk); - - /* Test if 'sig' is valid signal. Use this instead of testing _NSIG directly */ - static inline int valid_signal(unsigned long sig) -diff -Nur linux-4.1.6.orig/include/linux/skbuff.h linux-4.1.6/include/linux/skbuff.h ---- linux-4.1.6.orig/include/linux/skbuff.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/skbuff.h 2015-09-08 23:49:06.782014188 +0200 -@@ -187,6 +187,7 @@ - - __u32 qlen; - spinlock_t lock; -+ raw_spinlock_t raw_lock; - }; - - struct sk_buff; -@@ -1336,6 +1337,12 @@ - __skb_queue_head_init(list); - } - -+static inline void skb_queue_head_init_raw(struct sk_buff_head *list) -+{ -+ raw_spin_lock_init(&list->raw_lock); -+ __skb_queue_head_init(list); ++ ++ if (!check_stripe_cache(mddev)) ++ return -ENOSPC; ++ ++ if (mddev->new_chunk_sectors > mddev->chunk_sectors || ++ mddev->delta_disks > 0) ++ if (resize_chunks(conf, ++ conf->previous_raid_disks ++ + max(0, mddev->delta_disks), ++ max(mddev->new_chunk_sectors, ++ mddev->chunk_sectors) ++ ) < 0) ++ return -ENOMEM; ++ return resize_stripes(conf, (conf->previous_raid_disks ++ + mddev->delta_disks)); +} + - static inline void skb_queue_head_init_class(struct sk_buff_head *list, - struct lock_class_key *class) - { -diff -Nur linux-4.1.6.orig/include/linux/smp.h linux-4.1.6/include/linux/smp.h ---- linux-4.1.6.orig/include/linux/smp.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/smp.h 2015-09-08 23:49:06.782014188 +0200 -@@ -185,6 +185,9 @@ - #define get_cpu() ({ preempt_disable(); smp_processor_id(); }) - #define put_cpu() preempt_enable() - -+#define get_cpu_light() ({ migrate_disable(); smp_processor_id(); }) -+#define put_cpu_light() migrate_enable() ++static int raid5_start_reshape(struct mddev *mddev) ++{ ++ struct r5conf *conf = mddev->private; ++ struct md_rdev *rdev; ++ int spares = 0; ++ unsigned long flags; + - /* - * Callback to arch code if there's nosmp or maxcpus=0 on the - * boot command line: -diff -Nur linux-4.1.6.orig/include/linux/spinlock_api_smp.h linux-4.1.6/include/linux/spinlock_api_smp.h ---- linux-4.1.6.orig/include/linux/spinlock_api_smp.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/spinlock_api_smp.h 2015-09-08 23:49:06.782014188 +0200 -@@ -189,6 +189,8 @@ - return 0; - } - --#include -+#ifndef CONFIG_PREEMPT_RT_FULL -+# include -+#endif - - #endif /* __LINUX_SPINLOCK_API_SMP_H */ -diff -Nur linux-4.1.6.orig/include/linux/spinlock.h linux-4.1.6/include/linux/spinlock.h ---- linux-4.1.6.orig/include/linux/spinlock.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/spinlock.h 2015-09-08 23:49:06.782014188 +0200 -@@ -281,7 +281,11 @@ - #define raw_spin_can_lock(lock) (!raw_spin_is_locked(lock)) - - /* Include rwlock functions */ --#include -+#ifdef CONFIG_PREEMPT_RT_FULL -+# include -+#else -+# include -+#endif - - /* - * Pull the _spin_*()/_read_*()/_write_*() functions/declarations: -@@ -292,6 +296,10 @@ - # include - #endif - -+#ifdef CONFIG_PREEMPT_RT_FULL -+# include -+#else /* PREEMPT_RT_FULL */ ++ if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery)) ++ return -EBUSY; + - /* - * Map the spin_lock functions to the raw variants for PREEMPT_RT=n - */ -@@ -426,4 +434,6 @@ - #define atomic_dec_and_lock(atomic, lock) \ - __cond_lock(lock, _atomic_dec_and_lock(atomic, lock)) - -+#endif /* !PREEMPT_RT_FULL */ ++ if (!check_stripe_cache(mddev)) ++ return -ENOSPC; + - #endif /* __LINUX_SPINLOCK_H */ -diff -Nur linux-4.1.6.orig/include/linux/spinlock_rt.h linux-4.1.6/include/linux/spinlock_rt.h ---- linux-4.1.6.orig/include/linux/spinlock_rt.h 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.6/include/linux/spinlock_rt.h 2015-09-08 23:49:06.782014188 +0200 -@@ -0,0 +1,174 @@ -+#ifndef __LINUX_SPINLOCK_RT_H -+#define __LINUX_SPINLOCK_RT_H ++ if (has_failed(conf)) ++ return -EINVAL; + -+#ifndef __LINUX_SPINLOCK_H -+#error Do not include directly. Use spinlock.h -+#endif ++ rdev_for_each(rdev, mddev) { ++ if (!test_bit(In_sync, &rdev->flags) ++ && !test_bit(Faulty, &rdev->flags)) ++ spares++; ++ } + -+#include ++ if (spares - mddev->degraded < mddev->delta_disks - conf->max_degraded) ++ /* Not enough devices even to make a degraded array ++ * of that size ++ */ ++ return -EINVAL; + -+extern void -+__rt_spin_lock_init(spinlock_t *lock, char *name, struct lock_class_key *key); ++ /* Refuse to reduce size of the array. Any reductions in ++ * array size must be through explicit setting of array_size ++ * attribute. ++ */ ++ if (raid5_size(mddev, 0, conf->raid_disks + mddev->delta_disks) ++ < mddev->array_sectors) { ++ printk(KERN_ERR "md/raid:%s: array size must be reduced " ++ "before number of disks\n", mdname(mddev)); ++ return -EINVAL; ++ } + -+#define spin_lock_init(slock) \ -+do { \ -+ static struct lock_class_key __key; \ -+ \ -+ rt_mutex_init(&(slock)->lock); \ -+ __rt_spin_lock_init(slock, #slock, &__key); \ -+} while (0) ++ atomic_set(&conf->reshape_stripes, 0); ++ spin_lock_irq(&conf->device_lock); ++ write_seqcount_begin(&conf->gen_lock); ++ conf->previous_raid_disks = conf->raid_disks; ++ conf->raid_disks += mddev->delta_disks; ++ conf->prev_chunk_sectors = conf->chunk_sectors; ++ conf->chunk_sectors = mddev->new_chunk_sectors; ++ conf->prev_algo = conf->algorithm; ++ conf->algorithm = mddev->new_layout; ++ conf->generation++; ++ /* Code that selects data_offset needs to see the generation update ++ * if reshape_progress has been set - so a memory barrier needed. ++ */ ++ smp_mb(); ++ if (mddev->reshape_backwards) ++ conf->reshape_progress = raid5_size(mddev, 0, 0); ++ else ++ conf->reshape_progress = 0; ++ conf->reshape_safe = conf->reshape_progress; ++ write_seqcount_end(&conf->gen_lock); ++ spin_unlock_irq(&conf->device_lock); ++ ++ /* Now make sure any requests that proceeded on the assumption ++ * the reshape wasn't running - like Discard or Read - have ++ * completed. ++ */ ++ mddev_suspend(mddev); ++ mddev_resume(mddev); ++ ++ /* Add some new drives, as many as will fit. ++ * We know there are enough to make the newly sized array work. ++ * Don't add devices if we are reducing the number of ++ * devices in the array. This is because it is not possible ++ * to correctly record the "partially reconstructed" state of ++ * such devices during the reshape and confusion could result. ++ */ ++ if (mddev->delta_disks >= 0) { ++ rdev_for_each(rdev, mddev) ++ if (rdev->raid_disk < 0 && ++ !test_bit(Faulty, &rdev->flags)) { ++ if (raid5_add_disk(mddev, rdev) == 0) { ++ if (rdev->raid_disk ++ >= conf->previous_raid_disks) ++ set_bit(In_sync, &rdev->flags); ++ else ++ rdev->recovery_offset = 0; ++ ++ if (sysfs_link_rdev(mddev, rdev)) ++ /* Failure here is OK */; ++ } ++ } else if (rdev->raid_disk >= conf->previous_raid_disks ++ && !test_bit(Faulty, &rdev->flags)) { ++ /* This is a spare that was manually added */ ++ set_bit(In_sync, &rdev->flags); ++ } + -+extern void __lockfunc rt_spin_lock(spinlock_t *lock); -+extern unsigned long __lockfunc rt_spin_lock_trace_flags(spinlock_t *lock); -+extern void __lockfunc rt_spin_lock_nested(spinlock_t *lock, int subclass); -+extern void __lockfunc rt_spin_unlock(spinlock_t *lock); -+extern void __lockfunc rt_spin_unlock_wait(spinlock_t *lock); -+extern int __lockfunc rt_spin_trylock_irqsave(spinlock_t *lock, unsigned long *flags); -+extern int __lockfunc rt_spin_trylock_bh(spinlock_t *lock); -+extern int __lockfunc rt_spin_trylock(spinlock_t *lock); -+extern int atomic_dec_and_spin_lock(atomic_t *atomic, spinlock_t *lock); ++ /* When a reshape changes the number of devices, ++ * ->degraded is measured against the larger of the ++ * pre and post number of devices. ++ */ ++ spin_lock_irqsave(&conf->device_lock, flags); ++ mddev->degraded = calc_degraded(conf); ++ spin_unlock_irqrestore(&conf->device_lock, flags); ++ } ++ mddev->raid_disks = conf->raid_disks; ++ mddev->reshape_position = conf->reshape_progress; ++ set_bit(MD_CHANGE_DEVS, &mddev->flags); ++ ++ clear_bit(MD_RECOVERY_SYNC, &mddev->recovery); ++ clear_bit(MD_RECOVERY_CHECK, &mddev->recovery); ++ clear_bit(MD_RECOVERY_DONE, &mddev->recovery); ++ set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery); ++ set_bit(MD_RECOVERY_RUNNING, &mddev->recovery); ++ mddev->sync_thread = md_register_thread(md_do_sync, mddev, ++ "reshape"); ++ if (!mddev->sync_thread) { ++ mddev->recovery = 0; ++ spin_lock_irq(&conf->device_lock); ++ write_seqcount_begin(&conf->gen_lock); ++ mddev->raid_disks = conf->raid_disks = conf->previous_raid_disks; ++ mddev->new_chunk_sectors = ++ conf->chunk_sectors = conf->prev_chunk_sectors; ++ mddev->new_layout = conf->algorithm = conf->prev_algo; ++ rdev_for_each(rdev, mddev) ++ rdev->new_data_offset = rdev->data_offset; ++ smp_wmb(); ++ conf->generation --; ++ conf->reshape_progress = MaxSector; ++ mddev->reshape_position = MaxSector; ++ write_seqcount_end(&conf->gen_lock); ++ spin_unlock_irq(&conf->device_lock); ++ return -EAGAIN; ++ } ++ conf->reshape_checkpoint = jiffies; ++ md_wakeup_thread(mddev->sync_thread); ++ md_new_event(mddev); ++ return 0; ++} + -+/* -+ * lockdep-less calls, for derived types like rwlock: -+ * (for trylock they can use rt_mutex_trylock() directly. ++/* This is called from the reshape thread and should make any ++ * changes needed in 'conf' + */ -+extern void __lockfunc __rt_spin_lock(struct rt_mutex *lock); -+extern void __lockfunc __rt_spin_unlock(struct rt_mutex *lock); -+extern int __lockfunc __rt_spin_trylock(struct rt_mutex *lock); ++static void end_reshape(struct r5conf *conf) ++{ + -+#define spin_lock(lock) \ -+ do { \ -+ migrate_disable(); \ -+ rt_spin_lock(lock); \ -+ } while (0) ++ if (!test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery)) { ++ struct md_rdev *rdev; + -+#define spin_lock_bh(lock) \ -+ do { \ -+ local_bh_disable(); \ -+ migrate_disable(); \ -+ rt_spin_lock(lock); \ -+ } while (0) ++ spin_lock_irq(&conf->device_lock); ++ conf->previous_raid_disks = conf->raid_disks; ++ rdev_for_each(rdev, conf->mddev) ++ rdev->data_offset = rdev->new_data_offset; ++ smp_wmb(); ++ conf->reshape_progress = MaxSector; ++ spin_unlock_irq(&conf->device_lock); ++ wake_up(&conf->wait_for_overlap); + -+#define spin_lock_irq(lock) spin_lock(lock) ++ /* read-ahead size must cover two whole stripes, which is ++ * 2 * (datadisks) * chunksize where 'n' is the number of raid devices ++ */ ++ if (conf->mddev->queue) { ++ int data_disks = conf->raid_disks - conf->max_degraded; ++ int stripe = data_disks * ((conf->chunk_sectors << 9) ++ / PAGE_SIZE); ++ if (conf->mddev->queue->backing_dev_info.ra_pages < 2 * stripe) ++ conf->mddev->queue->backing_dev_info.ra_pages = 2 * stripe; ++ } ++ } ++} + -+#define spin_do_trylock(lock) __cond_lock(lock, rt_spin_trylock(lock)) ++/* This is called from the raid5d thread with mddev_lock held. ++ * It makes config changes to the device. ++ */ ++static void raid5_finish_reshape(struct mddev *mddev) ++{ ++ struct r5conf *conf = mddev->private; + -+#define spin_trylock(lock) \ -+({ \ -+ int __locked; \ -+ migrate_disable(); \ -+ __locked = spin_do_trylock(lock); \ -+ if (!__locked) \ -+ migrate_enable(); \ -+ __locked; \ -+}) ++ if (!test_bit(MD_RECOVERY_INTR, &mddev->recovery)) { + -+#ifdef CONFIG_LOCKDEP -+# define spin_lock_nested(lock, subclass) \ -+ do { \ -+ migrate_disable(); \ -+ rt_spin_lock_nested(lock, subclass); \ -+ } while (0) ++ if (mddev->delta_disks > 0) { ++ md_set_array_sectors(mddev, raid5_size(mddev, 0, 0)); ++ set_capacity(mddev->gendisk, mddev->array_sectors); ++ revalidate_disk(mddev->gendisk); ++ } else { ++ int d; ++ spin_lock_irq(&conf->device_lock); ++ mddev->degraded = calc_degraded(conf); ++ spin_unlock_irq(&conf->device_lock); ++ for (d = conf->raid_disks ; ++ d < conf->raid_disks - mddev->delta_disks; ++ d++) { ++ struct md_rdev *rdev = conf->disks[d].rdev; ++ if (rdev) ++ clear_bit(In_sync, &rdev->flags); ++ rdev = conf->disks[d].replacement; ++ if (rdev) ++ clear_bit(In_sync, &rdev->flags); ++ } ++ } ++ mddev->layout = conf->algorithm; ++ mddev->chunk_sectors = conf->chunk_sectors; ++ mddev->reshape_position = MaxSector; ++ mddev->delta_disks = 0; ++ mddev->reshape_backwards = 0; ++ } ++} + -+#define spin_lock_bh_nested(lock, subclass) \ -+ do { \ -+ local_bh_disable(); \ -+ migrate_disable(); \ -+ rt_spin_lock_nested(lock, subclass); \ -+ } while (0) ++static void raid5_quiesce(struct mddev *mddev, int state) ++{ ++ struct r5conf *conf = mddev->private; + -+# define spin_lock_irqsave_nested(lock, flags, subclass) \ -+ do { \ -+ typecheck(unsigned long, flags); \ -+ flags = 0; \ -+ migrate_disable(); \ -+ rt_spin_lock_nested(lock, subclass); \ -+ } while (0) -+#else -+# define spin_lock_nested(lock, subclass) spin_lock(lock) -+# define spin_lock_bh_nested(lock, subclass) spin_lock_bh(lock) ++ switch(state) { ++ case 2: /* resume for a suspend */ ++ wake_up(&conf->wait_for_overlap); ++ break; + -+# define spin_lock_irqsave_nested(lock, flags, subclass) \ -+ do { \ -+ typecheck(unsigned long, flags); \ -+ flags = 0; \ -+ spin_lock(lock); \ -+ } while (0) -+#endif ++ case 1: /* stop all writes */ ++ lock_all_device_hash_locks_irq(conf); ++ /* '2' tells resync/reshape to pause so that all ++ * active stripes can drain ++ */ ++ conf->quiesce = 2; ++ wait_event_cmd(conf->wait_for_stripe, ++ atomic_read(&conf->active_stripes) == 0 && ++ atomic_read(&conf->active_aligned_reads) == 0, ++ unlock_all_device_hash_locks_irq(conf), ++ lock_all_device_hash_locks_irq(conf)); ++ conf->quiesce = 1; ++ unlock_all_device_hash_locks_irq(conf); ++ /* allow reshape to continue */ ++ wake_up(&conf->wait_for_overlap); ++ break; + -+#define spin_lock_irqsave(lock, flags) \ -+ do { \ -+ typecheck(unsigned long, flags); \ -+ flags = 0; \ -+ spin_lock(lock); \ -+ } while (0) ++ case 0: /* re-enable writes */ ++ lock_all_device_hash_locks_irq(conf); ++ conf->quiesce = 0; ++ wake_up(&conf->wait_for_stripe); ++ wake_up(&conf->wait_for_overlap); ++ unlock_all_device_hash_locks_irq(conf); ++ break; ++ } ++} + -+static inline unsigned long spin_lock_trace_flags(spinlock_t *lock) ++static void *raid45_takeover_raid0(struct mddev *mddev, int level) +{ -+ unsigned long flags = 0; -+#ifdef CONFIG_TRACE_IRQFLAGS -+ flags = rt_spin_lock_trace_flags(lock); -+#else -+ spin_lock(lock); /* lock_local */ -+#endif -+ return flags; ++ struct r0conf *raid0_conf = mddev->private; ++ sector_t sectors; ++ ++ /* for raid0 takeover only one zone is supported */ ++ if (raid0_conf->nr_strip_zones > 1) { ++ printk(KERN_ERR "md/raid:%s: cannot takeover raid0 with more than one zone.\n", ++ mdname(mddev)); ++ return ERR_PTR(-EINVAL); ++ } ++ ++ sectors = raid0_conf->strip_zone[0].zone_end; ++ sector_div(sectors, raid0_conf->strip_zone[0].nb_dev); ++ mddev->dev_sectors = sectors; ++ mddev->new_level = level; ++ mddev->new_layout = ALGORITHM_PARITY_N; ++ mddev->new_chunk_sectors = mddev->chunk_sectors; ++ mddev->raid_disks += 1; ++ mddev->delta_disks = 1; ++ /* make sure it will be not marked as dirty */ ++ mddev->recovery_cp = MaxSector; ++ ++ return setup_conf(mddev); +} + -+/* FIXME: we need rt_spin_lock_nest_lock */ -+#define spin_lock_nest_lock(lock, nest_lock) spin_lock_nested(lock, 0) ++static void *raid5_takeover_raid1(struct mddev *mddev) ++{ ++ int chunksect; + -+#define spin_unlock(lock) \ -+ do { \ -+ rt_spin_unlock(lock); \ -+ migrate_enable(); \ -+ } while (0) ++ if (mddev->raid_disks != 2 || ++ mddev->degraded > 1) ++ return ERR_PTR(-EINVAL); + -+#define spin_unlock_bh(lock) \ -+ do { \ -+ rt_spin_unlock(lock); \ -+ migrate_enable(); \ -+ local_bh_enable(); \ -+ } while (0) ++ /* Should check if there are write-behind devices? */ + -+#define spin_unlock_irq(lock) spin_unlock(lock) ++ chunksect = 64*2; /* 64K by default */ + -+#define spin_unlock_irqrestore(lock, flags) \ -+ do { \ -+ typecheck(unsigned long, flags); \ -+ (void) flags; \ -+ spin_unlock(lock); \ -+ } while (0) ++ /* The array must be an exact multiple of chunksize */ ++ while (chunksect && (mddev->array_sectors & (chunksect-1))) ++ chunksect >>= 1; + -+#define spin_trylock_bh(lock) __cond_lock(lock, rt_spin_trylock_bh(lock)) -+#define spin_trylock_irq(lock) spin_trylock(lock) ++ if ((chunksect<<9) < STRIPE_SIZE) ++ /* array size does not allow a suitable chunk size */ ++ return ERR_PTR(-EINVAL); + -+#define spin_trylock_irqsave(lock, flags) \ -+ rt_spin_trylock_irqsave(lock, &(flags)) ++ mddev->new_level = 5; ++ mddev->new_layout = ALGORITHM_LEFT_SYMMETRIC; ++ mddev->new_chunk_sectors = chunksect; + -+#define spin_unlock_wait(lock) rt_spin_unlock_wait(lock) ++ return setup_conf(mddev); ++} + -+#ifdef CONFIG_GENERIC_LOCKBREAK -+# define spin_is_contended(lock) ((lock)->break_lock) -+#else -+# define spin_is_contended(lock) (((void)(lock), 0)) -+#endif ++static void *raid5_takeover_raid6(struct mddev *mddev) ++{ ++ int new_layout; + -+static inline int spin_can_lock(spinlock_t *lock) ++ switch (mddev->layout) { ++ case ALGORITHM_LEFT_ASYMMETRIC_6: ++ new_layout = ALGORITHM_LEFT_ASYMMETRIC; ++ break; ++ case ALGORITHM_RIGHT_ASYMMETRIC_6: ++ new_layout = ALGORITHM_RIGHT_ASYMMETRIC; ++ break; ++ case ALGORITHM_LEFT_SYMMETRIC_6: ++ new_layout = ALGORITHM_LEFT_SYMMETRIC; ++ break; ++ case ALGORITHM_RIGHT_SYMMETRIC_6: ++ new_layout = ALGORITHM_RIGHT_SYMMETRIC; ++ break; ++ case ALGORITHM_PARITY_0_6: ++ new_layout = ALGORITHM_PARITY_0; ++ break; ++ case ALGORITHM_PARITY_N: ++ new_layout = ALGORITHM_PARITY_N; ++ break; ++ default: ++ return ERR_PTR(-EINVAL); ++ } ++ mddev->new_level = 5; ++ mddev->new_layout = new_layout; ++ mddev->delta_disks = -1; ++ mddev->raid_disks -= 1; ++ return setup_conf(mddev); ++} ++ ++static int raid5_check_reshape(struct mddev *mddev) +{ -+ return !rt_mutex_is_locked(&lock->lock); ++ /* For a 2-drive array, the layout and chunk size can be changed ++ * immediately as not restriping is needed. ++ * For larger arrays we record the new value - after validation ++ * to be used by a reshape pass. ++ */ ++ struct r5conf *conf = mddev->private; ++ int new_chunk = mddev->new_chunk_sectors; ++ ++ if (mddev->new_layout >= 0 && !algorithm_valid_raid5(mddev->new_layout)) ++ return -EINVAL; ++ if (new_chunk > 0) { ++ if (!is_power_of_2(new_chunk)) ++ return -EINVAL; ++ if (new_chunk < (PAGE_SIZE>>9)) ++ return -EINVAL; ++ if (mddev->array_sectors & (new_chunk-1)) ++ /* not factor of array size */ ++ return -EINVAL; ++ } ++ ++ /* They look valid */ ++ ++ if (mddev->raid_disks == 2) { ++ /* can make the change immediately */ ++ if (mddev->new_layout >= 0) { ++ conf->algorithm = mddev->new_layout; ++ mddev->layout = mddev->new_layout; ++ } ++ if (new_chunk > 0) { ++ conf->chunk_sectors = new_chunk ; ++ mddev->chunk_sectors = new_chunk; ++ } ++ set_bit(MD_CHANGE_DEVS, &mddev->flags); ++ md_wakeup_thread(mddev->thread); ++ } ++ return check_reshape(mddev); +} + -+static inline int spin_is_locked(spinlock_t *lock) ++static int raid6_check_reshape(struct mddev *mddev) +{ -+ return rt_mutex_is_locked(&lock->lock); ++ int new_chunk = mddev->new_chunk_sectors; ++ ++ if (mddev->new_layout >= 0 && !algorithm_valid_raid6(mddev->new_layout)) ++ return -EINVAL; ++ if (new_chunk > 0) { ++ if (!is_power_of_2(new_chunk)) ++ return -EINVAL; ++ if (new_chunk < (PAGE_SIZE >> 9)) ++ return -EINVAL; ++ if (mddev->array_sectors & (new_chunk-1)) ++ /* not factor of array size */ ++ return -EINVAL; ++ } ++ ++ /* They look valid */ ++ return check_reshape(mddev); +} + -+static inline void assert_spin_locked(spinlock_t *lock) ++static void *raid5_takeover(struct mddev *mddev) +{ -+ BUG_ON(!spin_is_locked(lock)); ++ /* raid5 can take over: ++ * raid0 - if there is only one strip zone - make it a raid4 layout ++ * raid1 - if there are two drives. We need to know the chunk size ++ * raid4 - trivial - just use a raid4 layout. ++ * raid6 - Providing it is a *_6 layout ++ */ ++ if (mddev->level == 0) ++ return raid45_takeover_raid0(mddev, 5); ++ if (mddev->level == 1) ++ return raid5_takeover_raid1(mddev); ++ if (mddev->level == 4) { ++ mddev->new_layout = ALGORITHM_PARITY_N; ++ mddev->new_level = 5; ++ return setup_conf(mddev); ++ } ++ if (mddev->level == 6) ++ return raid5_takeover_raid6(mddev); ++ ++ return ERR_PTR(-EINVAL); +} + -+#define atomic_dec_and_lock(atomic, lock) \ -+ atomic_dec_and_spin_lock(atomic, lock) ++static void *raid4_takeover(struct mddev *mddev) ++{ ++ /* raid4 can take over: ++ * raid0 - if there is only one strip zone ++ * raid5 - if layout is right ++ */ ++ if (mddev->level == 0) ++ return raid45_takeover_raid0(mddev, 4); ++ if (mddev->level == 5 && ++ mddev->layout == ALGORITHM_PARITY_N) { ++ mddev->new_layout = 0; ++ mddev->new_level = 4; ++ return setup_conf(mddev); ++ } ++ return ERR_PTR(-EINVAL); ++} + -+#endif -diff -Nur linux-4.1.6.orig/include/linux/spinlock_types.h linux-4.1.6/include/linux/spinlock_types.h ---- linux-4.1.6.orig/include/linux/spinlock_types.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/spinlock_types.h 2015-09-08 23:49:06.782014188 +0200 -@@ -9,80 +9,15 @@ - * Released under the General Public License (GPL). - */ - --#if defined(CONFIG_SMP) --# include --#else --# include --#endif -- --#include -- --typedef struct raw_spinlock { -- arch_spinlock_t raw_lock; --#ifdef CONFIG_GENERIC_LOCKBREAK -- unsigned int break_lock; --#endif --#ifdef CONFIG_DEBUG_SPINLOCK -- unsigned int magic, owner_cpu; -- void *owner; --#endif --#ifdef CONFIG_DEBUG_LOCK_ALLOC -- struct lockdep_map dep_map; --#endif --} raw_spinlock_t; -- --#define SPINLOCK_MAGIC 0xdead4ead -- --#define SPINLOCK_OWNER_INIT ((void *)-1L) -- --#ifdef CONFIG_DEBUG_LOCK_ALLOC --# define SPIN_DEP_MAP_INIT(lockname) .dep_map = { .name = #lockname } --#else --# define SPIN_DEP_MAP_INIT(lockname) --#endif -+#include - --#ifdef CONFIG_DEBUG_SPINLOCK --# define SPIN_DEBUG_INIT(lockname) \ -- .magic = SPINLOCK_MAGIC, \ -- .owner_cpu = -1, \ -- .owner = SPINLOCK_OWNER_INIT, -+#ifndef CONFIG_PREEMPT_RT_FULL -+# include -+# include - #else --# define SPIN_DEBUG_INIT(lockname) -+# include -+# include -+# include - #endif - --#define __RAW_SPIN_LOCK_INITIALIZER(lockname) \ -- { \ -- .raw_lock = __ARCH_SPIN_LOCK_UNLOCKED, \ -- SPIN_DEBUG_INIT(lockname) \ -- SPIN_DEP_MAP_INIT(lockname) } -- --#define __RAW_SPIN_LOCK_UNLOCKED(lockname) \ -- (raw_spinlock_t) __RAW_SPIN_LOCK_INITIALIZER(lockname) -- --#define DEFINE_RAW_SPINLOCK(x) raw_spinlock_t x = __RAW_SPIN_LOCK_UNLOCKED(x) -- --typedef struct spinlock { -- union { -- struct raw_spinlock rlock; -- --#ifdef CONFIG_DEBUG_LOCK_ALLOC --# define LOCK_PADSIZE (offsetof(struct raw_spinlock, dep_map)) -- struct { -- u8 __padding[LOCK_PADSIZE]; -- struct lockdep_map dep_map; -- }; --#endif -- }; --} spinlock_t; -- --#define __SPIN_LOCK_INITIALIZER(lockname) \ -- { { .rlock = __RAW_SPIN_LOCK_INITIALIZER(lockname) } } -- --#define __SPIN_LOCK_UNLOCKED(lockname) \ -- (spinlock_t ) __SPIN_LOCK_INITIALIZER(lockname) -- --#define DEFINE_SPINLOCK(x) spinlock_t x = __SPIN_LOCK_UNLOCKED(x) -- --#include -- - #endif /* __LINUX_SPINLOCK_TYPES_H */ -diff -Nur linux-4.1.6.orig/include/linux/spinlock_types_nort.h linux-4.1.6/include/linux/spinlock_types_nort.h ---- linux-4.1.6.orig/include/linux/spinlock_types_nort.h 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.6/include/linux/spinlock_types_nort.h 2015-09-08 23:49:06.782014188 +0200 -@@ -0,0 +1,33 @@ -+#ifndef __LINUX_SPINLOCK_TYPES_NORT_H -+#define __LINUX_SPINLOCK_TYPES_NORT_H -+ -+#ifndef __LINUX_SPINLOCK_TYPES_H -+#error "Do not include directly. Include spinlock_types.h instead" -+#endif -+ -+/* -+ * The non RT version maps spinlocks to raw_spinlocks -+ */ -+typedef struct spinlock { -+ union { -+ struct raw_spinlock rlock; -+ -+#ifdef CONFIG_DEBUG_LOCK_ALLOC -+# define LOCK_PADSIZE (offsetof(struct raw_spinlock, dep_map)) -+ struct { -+ u8 __padding[LOCK_PADSIZE]; -+ struct lockdep_map dep_map; -+ }; -+#endif -+ }; -+} spinlock_t; -+ -+#define __SPIN_LOCK_INITIALIZER(lockname) \ -+ { { .rlock = __RAW_SPIN_LOCK_INITIALIZER(lockname) } } -+ -+#define __SPIN_LOCK_UNLOCKED(lockname) \ -+ (spinlock_t ) __SPIN_LOCK_INITIALIZER(lockname) -+ -+#define DEFINE_SPINLOCK(x) spinlock_t x = __SPIN_LOCK_UNLOCKED(x) -+ -+#endif -diff -Nur linux-4.1.6.orig/include/linux/spinlock_types_raw.h linux-4.1.6/include/linux/spinlock_types_raw.h ---- linux-4.1.6.orig/include/linux/spinlock_types_raw.h 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.6/include/linux/spinlock_types_raw.h 2015-09-08 23:49:06.782014188 +0200 -@@ -0,0 +1,56 @@ -+#ifndef __LINUX_SPINLOCK_TYPES_RAW_H -+#define __LINUX_SPINLOCK_TYPES_RAW_H -+ -+#if defined(CONFIG_SMP) -+# include -+#else -+# include -+#endif -+ -+#include -+ -+typedef struct raw_spinlock { -+ arch_spinlock_t raw_lock; -+#ifdef CONFIG_GENERIC_LOCKBREAK -+ unsigned int break_lock; -+#endif -+#ifdef CONFIG_DEBUG_SPINLOCK -+ unsigned int magic, owner_cpu; -+ void *owner; -+#endif -+#ifdef CONFIG_DEBUG_LOCK_ALLOC -+ struct lockdep_map dep_map; -+#endif -+} raw_spinlock_t; ++static struct md_personality raid5_personality; + -+#define SPINLOCK_MAGIC 0xdead4ead -+ -+#define SPINLOCK_OWNER_INIT ((void *)-1L) -+ -+#ifdef CONFIG_DEBUG_LOCK_ALLOC -+# define SPIN_DEP_MAP_INIT(lockname) .dep_map = { .name = #lockname } -+#else -+# define SPIN_DEP_MAP_INIT(lockname) -+#endif -+ -+#ifdef CONFIG_DEBUG_SPINLOCK -+# define SPIN_DEBUG_INIT(lockname) \ -+ .magic = SPINLOCK_MAGIC, \ -+ .owner_cpu = -1, \ -+ .owner = SPINLOCK_OWNER_INIT, -+#else -+# define SPIN_DEBUG_INIT(lockname) -+#endif ++static void *raid6_takeover(struct mddev *mddev) ++{ ++ /* Currently can only take over a raid5. We map the ++ * personality to an equivalent raid6 personality ++ * with the Q block at the end. ++ */ ++ int new_layout; ++ ++ if (mddev->pers != &raid5_personality) ++ return ERR_PTR(-EINVAL); ++ if (mddev->degraded > 1) ++ return ERR_PTR(-EINVAL); ++ if (mddev->raid_disks > 253) ++ return ERR_PTR(-EINVAL); ++ if (mddev->raid_disks < 3) ++ return ERR_PTR(-EINVAL); ++ ++ switch (mddev->layout) { ++ case ALGORITHM_LEFT_ASYMMETRIC: ++ new_layout = ALGORITHM_LEFT_ASYMMETRIC_6; ++ break; ++ case ALGORITHM_RIGHT_ASYMMETRIC: ++ new_layout = ALGORITHM_RIGHT_ASYMMETRIC_6; ++ break; ++ case ALGORITHM_LEFT_SYMMETRIC: ++ new_layout = ALGORITHM_LEFT_SYMMETRIC_6; ++ break; ++ case ALGORITHM_RIGHT_SYMMETRIC: ++ new_layout = ALGORITHM_RIGHT_SYMMETRIC_6; ++ break; ++ case ALGORITHM_PARITY_0: ++ new_layout = ALGORITHM_PARITY_0_6; ++ break; ++ case ALGORITHM_PARITY_N: ++ new_layout = ALGORITHM_PARITY_N; ++ break; ++ default: ++ return ERR_PTR(-EINVAL); ++ } ++ mddev->new_level = 6; ++ mddev->new_layout = new_layout; ++ mddev->delta_disks = 1; ++ mddev->raid_disks += 1; ++ return setup_conf(mddev); ++} + -+#define __RAW_SPIN_LOCK_INITIALIZER(lockname) \ -+ { \ -+ .raw_lock = __ARCH_SPIN_LOCK_UNLOCKED, \ -+ SPIN_DEBUG_INIT(lockname) \ -+ SPIN_DEP_MAP_INIT(lockname) } ++static struct md_personality raid6_personality = ++{ ++ .name = "raid6", ++ .level = 6, ++ .owner = THIS_MODULE, ++ .make_request = make_request, ++ .run = run, ++ .free = raid5_free, ++ .status = status, ++ .error_handler = error, ++ .hot_add_disk = raid5_add_disk, ++ .hot_remove_disk= raid5_remove_disk, ++ .spare_active = raid5_spare_active, ++ .sync_request = sync_request, ++ .resize = raid5_resize, ++ .size = raid5_size, ++ .check_reshape = raid6_check_reshape, ++ .start_reshape = raid5_start_reshape, ++ .finish_reshape = raid5_finish_reshape, ++ .quiesce = raid5_quiesce, ++ .takeover = raid6_takeover, ++ .congested = raid5_congested, ++ .mergeable_bvec = raid5_mergeable_bvec, ++}; ++static struct md_personality raid5_personality = ++{ ++ .name = "raid5", ++ .level = 5, ++ .owner = THIS_MODULE, ++ .make_request = make_request, ++ .run = run, ++ .free = raid5_free, ++ .status = status, ++ .error_handler = error, ++ .hot_add_disk = raid5_add_disk, ++ .hot_remove_disk= raid5_remove_disk, ++ .spare_active = raid5_spare_active, ++ .sync_request = sync_request, ++ .resize = raid5_resize, ++ .size = raid5_size, ++ .check_reshape = raid5_check_reshape, ++ .start_reshape = raid5_start_reshape, ++ .finish_reshape = raid5_finish_reshape, ++ .quiesce = raid5_quiesce, ++ .takeover = raid5_takeover, ++ .congested = raid5_congested, ++ .mergeable_bvec = raid5_mergeable_bvec, ++}; + -+#define __RAW_SPIN_LOCK_UNLOCKED(lockname) \ -+ (raw_spinlock_t) __RAW_SPIN_LOCK_INITIALIZER(lockname) ++static struct md_personality raid4_personality = ++{ ++ .name = "raid4", ++ .level = 4, ++ .owner = THIS_MODULE, ++ .make_request = make_request, ++ .run = run, ++ .free = raid5_free, ++ .status = status, ++ .error_handler = error, ++ .hot_add_disk = raid5_add_disk, ++ .hot_remove_disk= raid5_remove_disk, ++ .spare_active = raid5_spare_active, ++ .sync_request = sync_request, ++ .resize = raid5_resize, ++ .size = raid5_size, ++ .check_reshape = raid5_check_reshape, ++ .start_reshape = raid5_start_reshape, ++ .finish_reshape = raid5_finish_reshape, ++ .quiesce = raid5_quiesce, ++ .takeover = raid4_takeover, ++ .congested = raid5_congested, ++ .mergeable_bvec = raid5_mergeable_bvec, ++}; + -+#define DEFINE_RAW_SPINLOCK(x) raw_spinlock_t x = __RAW_SPIN_LOCK_UNLOCKED(x) ++static int __init raid5_init(void) ++{ ++ raid5_wq = alloc_workqueue("raid5wq", ++ WQ_UNBOUND|WQ_MEM_RECLAIM|WQ_CPU_INTENSIVE|WQ_SYSFS, 0); ++ if (!raid5_wq) ++ return -ENOMEM; ++ register_md_personality(&raid6_personality); ++ register_md_personality(&raid5_personality); ++ register_md_personality(&raid4_personality); ++ return 0; ++} + -+#endif -diff -Nur linux-4.1.6.orig/include/linux/spinlock_types_rt.h linux-4.1.6/include/linux/spinlock_types_rt.h ---- linux-4.1.6.orig/include/linux/spinlock_types_rt.h 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.6/include/linux/spinlock_types_rt.h 2015-09-08 23:49:06.782014188 +0200 -@@ -0,0 +1,51 @@ -+#ifndef __LINUX_SPINLOCK_TYPES_RT_H -+#define __LINUX_SPINLOCK_TYPES_RT_H ++static void raid5_exit(void) ++{ ++ unregister_md_personality(&raid6_personality); ++ unregister_md_personality(&raid5_personality); ++ unregister_md_personality(&raid4_personality); ++ destroy_workqueue(raid5_wq); ++} + -+#ifndef __LINUX_SPINLOCK_TYPES_H -+#error "Do not include directly. Include spinlock_types.h instead" -+#endif ++module_init(raid5_init); ++module_exit(raid5_exit); ++MODULE_LICENSE("GPL"); ++MODULE_DESCRIPTION("RAID4/5/6 (striping with parity) personality for MD"); ++MODULE_ALIAS("md-personality-4"); /* RAID5 */ ++MODULE_ALIAS("md-raid5"); ++MODULE_ALIAS("md-raid4"); ++MODULE_ALIAS("md-level-5"); ++MODULE_ALIAS("md-level-4"); ++MODULE_ALIAS("md-personality-8"); /* RAID6 */ ++MODULE_ALIAS("md-raid6"); ++MODULE_ALIAS("md-level-6"); ++ ++/* This used to be two separate modules, they were: */ ++MODULE_ALIAS("raid5"); ++MODULE_ALIAS("raid6"); +diff -Nur linux-4.1.10.orig/drivers/md/raid5.h linux-4.1.10/drivers/md/raid5.h +--- linux-4.1.10.orig/drivers/md/raid5.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/md/raid5.h 2015-10-07 18:00:08.000000000 +0200 +@@ -495,6 +495,7 @@ + int recovery_disabled; + /* per cpu variables */ + struct raid5_percpu { ++ spinlock_t lock; /* Protection for -RT */ + struct page *spare_page; /* Used when checking P/Q in raid6 */ + struct flex_array *scribble; /* space for constructing buffer + * lists and performing address +diff -Nur linux-4.1.10.orig/drivers/md/raid5.h.orig linux-4.1.10/drivers/md/raid5.h.orig +--- linux-4.1.10.orig/drivers/md/raid5.h.orig 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/drivers/md/raid5.h.orig 2015-10-03 13:49:38.000000000 +0200 +@@ -0,0 +1,608 @@ ++#ifndef _RAID5_H ++#define _RAID5_H + -+#include ++#include ++#include + +/* -+ * PREEMPT_RT: spinlocks - an RT mutex plus lock-break field: ++ * ++ * Each stripe contains one buffer per device. Each buffer can be in ++ * one of a number of states stored in "flags". Changes between ++ * these states happen *almost* exclusively under the protection of the ++ * STRIPE_ACTIVE flag. Some very specific changes can happen in bi_end_io, and ++ * these are not protected by STRIPE_ACTIVE. ++ * ++ * The flag bits that are used to represent these states are: ++ * R5_UPTODATE and R5_LOCKED ++ * ++ * State Empty == !UPTODATE, !LOCK ++ * We have no data, and there is no active request ++ * State Want == !UPTODATE, LOCK ++ * A read request is being submitted for this block ++ * State Dirty == UPTODATE, LOCK ++ * Some new data is in this buffer, and it is being written out ++ * State Clean == UPTODATE, !LOCK ++ * We have valid data which is the same as on disc ++ * ++ * The possible state transitions are: ++ * ++ * Empty -> Want - on read or write to get old data for parity calc ++ * Empty -> Dirty - on compute_parity to satisfy write/sync request. ++ * Empty -> Clean - on compute_block when computing a block for failed drive ++ * Want -> Empty - on failed read ++ * Want -> Clean - on successful completion of read request ++ * Dirty -> Clean - on successful completion of write request ++ * Dirty -> Clean - on failed write ++ * Clean -> Dirty - on compute_parity to satisfy write/sync (RECONSTRUCT or RMW) ++ * ++ * The Want->Empty, Want->Clean, Dirty->Clean, transitions ++ * all happen in b_end_io at interrupt time. ++ * Each sets the Uptodate bit before releasing the Lock bit. ++ * This leaves one multi-stage transition: ++ * Want->Dirty->Clean ++ * This is safe because thinking that a Clean buffer is actually dirty ++ * will at worst delay some action, and the stripe will be scheduled ++ * for attention after the transition is complete. ++ * ++ * There is one possibility that is not covered by these states. That ++ * is if one drive has failed and there is a spare being rebuilt. We ++ * can't distinguish between a clean block that has been generated ++ * from parity calculations, and a clean block that has been ++ * successfully written to the spare ( or to parity when resyncing). ++ * To distinguish these states we have a stripe bit STRIPE_INSYNC that ++ * is set whenever a write is scheduled to the spare, or to the parity ++ * disc if there is no spare. A sync request clears this bit, and ++ * when we find it set with no buffers locked, we know the sync is ++ * complete. ++ * ++ * Buffers for the md device that arrive via make_request are attached ++ * to the appropriate stripe in one of two lists linked on b_reqnext. ++ * One list (bh_read) for read requests, one (bh_write) for write. ++ * There should never be more than one buffer on the two lists ++ * together, but we are not guaranteed of that so we allow for more. ++ * ++ * If a buffer is on the read list when the associated cache buffer is ++ * Uptodate, the data is copied into the read buffer and it's b_end_io ++ * routine is called. This may happen in the end_request routine only ++ * if the buffer has just successfully been read. end_request should ++ * remove the buffers from the list and then set the Uptodate bit on ++ * the buffer. Other threads may do this only if they first check ++ * that the Uptodate bit is set. Once they have checked that they may ++ * take buffers off the read queue. ++ * ++ * When a buffer on the write list is committed for write it is copied ++ * into the cache buffer, which is then marked dirty, and moved onto a ++ * third list, the written list (bh_written). Once both the parity ++ * block and the cached buffer are successfully written, any buffer on ++ * a written list can be returned with b_end_io. ++ * ++ * The write list and read list both act as fifos. The read list, ++ * write list and written list are protected by the device_lock. ++ * The device_lock is only for list manipulations and will only be ++ * held for a very short time. It can be claimed from interrupts. ++ * ++ * ++ * Stripes in the stripe cache can be on one of two lists (or on ++ * neither). The "inactive_list" contains stripes which are not ++ * currently being used for any request. They can freely be reused ++ * for another stripe. The "handle_list" contains stripes that need ++ * to be handled in some way. Both of these are fifo queues. Each ++ * stripe is also (potentially) linked to a hash bucket in the hash ++ * table so that it can be found by sector number. Stripes that are ++ * not hashed must be on the inactive_list, and will normally be at ++ * the front. All stripes start life this way. ++ * ++ * The inactive_list, handle_list and hash bucket lists are all protected by the ++ * device_lock. ++ * - stripes have a reference counter. If count==0, they are on a list. ++ * - If a stripe might need handling, STRIPE_HANDLE is set. ++ * - When refcount reaches zero, then if STRIPE_HANDLE it is put on ++ * handle_list else inactive_list ++ * ++ * This, combined with the fact that STRIPE_HANDLE is only ever ++ * cleared while a stripe has a non-zero count means that if the ++ * refcount is 0 and STRIPE_HANDLE is set, then it is on the ++ * handle_list and if recount is 0 and STRIPE_HANDLE is not set, then ++ * the stripe is on inactive_list. ++ * ++ * The possible transitions are: ++ * activate an unhashed/inactive stripe (get_active_stripe()) ++ * lockdev check-hash unlink-stripe cnt++ clean-stripe hash-stripe unlockdev ++ * activate a hashed, possibly active stripe (get_active_stripe()) ++ * lockdev check-hash if(!cnt++)unlink-stripe unlockdev ++ * attach a request to an active stripe (add_stripe_bh()) ++ * lockdev attach-buffer unlockdev ++ * handle a stripe (handle_stripe()) ++ * setSTRIPE_ACTIVE, clrSTRIPE_HANDLE ... ++ * (lockdev check-buffers unlockdev) .. ++ * change-state .. ++ * record io/ops needed clearSTRIPE_ACTIVE schedule io/ops ++ * release an active stripe (release_stripe()) ++ * lockdev if (!--cnt) { if STRIPE_HANDLE, add to handle_list else add to inactive-list } unlockdev ++ * ++ * The refcount counts each thread that have activated the stripe, ++ * plus raid5d if it is handling it, plus one for each active request ++ * on a cached buffer, and plus one if the stripe is undergoing stripe ++ * operations. ++ * ++ * The stripe operations are: ++ * -copying data between the stripe cache and user application buffers ++ * -computing blocks to save a disk access, or to recover a missing block ++ * -updating the parity on a write operation (reconstruct write and ++ * read-modify-write) ++ * -checking parity correctness ++ * -running i/o to disk ++ * These operations are carried out by raid5_run_ops which uses the async_tx ++ * api to (optionally) offload operations to dedicated hardware engines. ++ * When requesting an operation handle_stripe sets the pending bit for the ++ * operation and increments the count. raid5_run_ops is then run whenever ++ * the count is non-zero. ++ * There are some critical dependencies between the operations that prevent some ++ * from being requested while another is in flight. ++ * 1/ Parity check operations destroy the in cache version of the parity block, ++ * so we prevent parity dependent operations like writes and compute_blocks ++ * from starting while a check is in progress. Some dma engines can perform ++ * the check without damaging the parity block, in these cases the parity ++ * block is re-marked up to date (assuming the check was successful) and is ++ * not re-read from disk. ++ * 2/ When a write operation is requested we immediately lock the affected ++ * blocks, and mark them as not up to date. This causes new read requests ++ * to be held off, as well as parity checks and compute block operations. ++ * 3/ Once a compute block operation has been requested handle_stripe treats ++ * that block as if it is up to date. raid5_run_ops guaruntees that any ++ * operation that is dependent on the compute block result is initiated after ++ * the compute block completes. + */ -+typedef struct spinlock { -+ struct rt_mutex lock; -+ unsigned int break_lock; -+#ifdef CONFIG_DEBUG_LOCK_ALLOC -+ struct lockdep_map dep_map; -+#endif -+} spinlock_t; -+ -+#ifdef CONFIG_DEBUG_RT_MUTEXES -+# define __RT_SPIN_INITIALIZER(name) \ -+ { \ -+ .wait_lock = __RAW_SPIN_LOCK_UNLOCKED(name.wait_lock), \ -+ .save_state = 1, \ -+ .file = __FILE__, \ -+ .line = __LINE__ , \ -+ } -+#else -+# define __RT_SPIN_INITIALIZER(name) \ -+ { \ -+ .wait_lock = __RAW_SPIN_LOCK_UNLOCKED(name.wait_lock), \ -+ .save_state = 1, \ -+ } -+#endif + +/* -+.wait_list = PLIST_HEAD_INIT_RAW((name).lock.wait_list, (name).lock.wait_lock) -+*/ ++ * Operations state - intermediate states that are visible outside of ++ * STRIPE_ACTIVE. ++ * In general _idle indicates nothing is running, _run indicates a data ++ * processing operation is active, and _result means the data processing result ++ * is stable and can be acted upon. For simple operations like biofill and ++ * compute that only have an _idle and _run state they are indicated with ++ * sh->state flags (STRIPE_BIOFILL_RUN and STRIPE_COMPUTE_RUN) ++ */ ++/** ++ * enum check_states - handles syncing / repairing a stripe ++ * @check_state_idle - check operations are quiesced ++ * @check_state_run - check operation is running ++ * @check_state_result - set outside lock when check result is valid ++ * @check_state_compute_run - check failed and we are repairing ++ * @check_state_compute_result - set outside lock when compute result is valid ++ */ ++enum check_states { ++ check_state_idle = 0, ++ check_state_run, /* xor parity check */ ++ check_state_run_q, /* q-parity check */ ++ check_state_run_pq, /* pq dual parity check */ ++ check_state_check_result, ++ check_state_compute_run, /* parity repair */ ++ check_state_compute_result, ++}; + -+#define __SPIN_LOCK_UNLOCKED(name) \ -+ { .lock = __RT_SPIN_INITIALIZER(name.lock), \ -+ SPIN_DEP_MAP_INIT(name) } ++/** ++ * enum reconstruct_states - handles writing or expanding a stripe ++ */ ++enum reconstruct_states { ++ reconstruct_state_idle = 0, ++ reconstruct_state_prexor_drain_run, /* prexor-write */ ++ reconstruct_state_drain_run, /* write */ ++ reconstruct_state_run, /* expand */ ++ reconstruct_state_prexor_drain_result, ++ reconstruct_state_drain_result, ++ reconstruct_state_result, ++}; + -+#define __DEFINE_SPINLOCK(name) \ -+ spinlock_t name = __SPIN_LOCK_UNLOCKED(name) ++struct stripe_head { ++ struct hlist_node hash; ++ struct list_head lru; /* inactive_list or handle_list */ ++ struct llist_node release_list; ++ struct r5conf *raid_conf; ++ short generation; /* increments with every ++ * reshape */ ++ sector_t sector; /* sector of this row */ ++ short pd_idx; /* parity disk index */ ++ short qd_idx; /* 'Q' disk index for raid6 */ ++ short ddf_layout;/* use DDF ordering to calculate Q */ ++ short hash_lock_index; ++ unsigned long state; /* state flags */ ++ atomic_t count; /* nr of active thread/requests */ ++ int bm_seq; /* sequence number for bitmap flushes */ ++ int disks; /* disks in stripe */ ++ int overwrite_disks; /* total overwrite disks in stripe, ++ * this is only checked when stripe ++ * has STRIPE_BATCH_READY ++ */ ++ enum check_states check_state; ++ enum reconstruct_states reconstruct_state; ++ spinlock_t stripe_lock; ++ int cpu; ++ struct r5worker_group *group; ++ ++ struct stripe_head *batch_head; /* protected by stripe lock */ ++ spinlock_t batch_lock; /* only header's lock is useful */ ++ struct list_head batch_list; /* protected by head's batch lock*/ ++ /** ++ * struct stripe_operations ++ * @target - STRIPE_OP_COMPUTE_BLK target ++ * @target2 - 2nd compute target in the raid6 case ++ * @zero_sum_result - P and Q verification flags ++ * @request - async service request flags for raid_run_ops ++ */ ++ struct stripe_operations { ++ int target, target2; ++ enum sum_check_flags zero_sum_result; ++ } ops; ++ struct r5dev { ++ /* rreq and rvec are used for the replacement device when ++ * writing data to both devices. ++ */ ++ struct bio req, rreq; ++ struct bio_vec vec, rvec; ++ struct page *page, *orig_page; ++ struct bio *toread, *read, *towrite, *written; ++ sector_t sector; /* sector of this page */ ++ unsigned long flags; ++ } dev[1]; /* allocated with extra space depending of RAID geometry */ ++}; + -+#define DEFINE_SPINLOCK(name) \ -+ spinlock_t name __cacheline_aligned_in_smp = __SPIN_LOCK_UNLOCKED(name) ++/* stripe_head_state - collects and tracks the dynamic state of a stripe_head ++ * for handle_stripe. ++ */ ++struct stripe_head_state { ++ /* 'syncing' means that we need to read all devices, either ++ * to check/correct parity, or to reconstruct a missing device. ++ * 'replacing' means we are replacing one or more drives and ++ * the source is valid at this point so we don't need to ++ * read all devices, just the replacement targets. ++ */ ++ int syncing, expanding, expanded, replacing; ++ int locked, uptodate, to_read, to_write, failed, written; ++ int to_fill, compute, req_compute, non_overwrite; ++ int failed_num[2]; ++ int p_failed, q_failed; ++ int dec_preread_active; ++ unsigned long ops_request; ++ ++ struct bio *return_bi; ++ struct md_rdev *blocked_rdev; ++ int handle_bad_blocks; ++}; + -+#endif -diff -Nur linux-4.1.6.orig/include/linux/srcu.h linux-4.1.6/include/linux/srcu.h ---- linux-4.1.6.orig/include/linux/srcu.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/srcu.h 2015-09-08 23:49:06.782014188 +0200 -@@ -84,10 +84,10 @@ - - void process_srcu(struct work_struct *work); - --#define __SRCU_STRUCT_INIT(name) \ -+#define __SRCU_STRUCT_INIT(name, pcpu_name) \ - { \ - .completed = -300, \ -- .per_cpu_ref = &name##_srcu_array, \ -+ .per_cpu_ref = &pcpu_name, \ - .queue_lock = __SPIN_LOCK_UNLOCKED(name.queue_lock), \ - .running = false, \ - .batch_queue = RCU_BATCH_INIT(name.batch_queue), \ -@@ -104,7 +104,7 @@ - */ - #define __DEFINE_SRCU(name, is_static) \ - static DEFINE_PER_CPU(struct srcu_struct_array, name##_srcu_array);\ -- is_static struct srcu_struct name = __SRCU_STRUCT_INIT(name) -+ is_static struct srcu_struct name = __SRCU_STRUCT_INIT(name, name##_srcu_array) - #define DEFINE_SRCU(name) __DEFINE_SRCU(name, /* not static */) - #define DEFINE_STATIC_SRCU(name) __DEFINE_SRCU(name, static) - -diff -Nur linux-4.1.6.orig/include/linux/swap.h linux-4.1.6/include/linux/swap.h ---- linux-4.1.6.orig/include/linux/swap.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/swap.h 2015-09-08 23:49:06.782014188 +0200 -@@ -11,6 +11,7 @@ - #include - #include - #include -+#include - #include - - struct notifier_block; -@@ -252,7 +253,8 @@ - void *workingset_eviction(struct address_space *mapping, struct page *page); - bool workingset_refault(void *shadow); - void workingset_activation(struct page *page); --extern struct list_lru workingset_shadow_nodes; -+extern struct list_lru __workingset_shadow_nodes; -+DECLARE_LOCAL_IRQ_LOCK(workingset_shadow_lock); - - static inline unsigned int workingset_node_pages(struct radix_tree_node *node) - { -@@ -296,6 +298,7 @@ - - - /* linux/mm/swap.c */ -+DECLARE_LOCAL_IRQ_LOCK(swapvec_lock); - extern void lru_cache_add(struct page *); - extern void lru_cache_add_anon(struct page *page); - extern void lru_cache_add_file(struct page *page); -diff -Nur linux-4.1.6.orig/include/linux/thread_info.h linux-4.1.6/include/linux/thread_info.h ---- linux-4.1.6.orig/include/linux/thread_info.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/thread_info.h 2015-09-08 23:49:06.782014188 +0200 -@@ -102,7 +102,17 @@ - #define test_thread_flag(flag) \ - test_ti_thread_flag(current_thread_info(), flag) - --#define tif_need_resched() test_thread_flag(TIF_NEED_RESCHED) -+#ifdef CONFIG_PREEMPT_LAZY -+#define tif_need_resched() (test_thread_flag(TIF_NEED_RESCHED) || \ -+ test_thread_flag(TIF_NEED_RESCHED_LAZY)) -+#define tif_need_resched_now() (test_thread_flag(TIF_NEED_RESCHED)) -+#define tif_need_resched_lazy() test_thread_flag(TIF_NEED_RESCHED_LAZY)) ++/* Flags for struct r5dev.flags */ ++enum r5dev_flags { ++ R5_UPTODATE, /* page contains current data */ ++ R5_LOCKED, /* IO has been submitted on "req" */ ++ R5_DOUBLE_LOCKED,/* Cannot clear R5_LOCKED until 2 writes complete */ ++ R5_OVERWRITE, /* towrite covers whole page */ ++/* and some that are internal to handle_stripe */ ++ R5_Insync, /* rdev && rdev->in_sync at start */ ++ R5_Wantread, /* want to schedule a read */ ++ R5_Wantwrite, ++ R5_Overlap, /* There is a pending overlapping request ++ * on this block */ ++ R5_ReadNoMerge, /* prevent bio from merging in block-layer */ ++ R5_ReadError, /* seen a read error here recently */ ++ R5_ReWrite, /* have tried to over-write the readerror */ ++ ++ R5_Expanded, /* This block now has post-expand data */ ++ R5_Wantcompute, /* compute_block in progress treat as ++ * uptodate ++ */ ++ R5_Wantfill, /* dev->toread contains a bio that needs ++ * filling ++ */ ++ R5_Wantdrain, /* dev->towrite needs to be drained */ ++ R5_WantFUA, /* Write should be FUA */ ++ R5_SyncIO, /* The IO is sync */ ++ R5_WriteError, /* got a write error - need to record it */ ++ R5_MadeGood, /* A bad block has been fixed by writing to it */ ++ R5_ReadRepl, /* Will/did read from replacement rather than orig */ ++ R5_MadeGoodRepl,/* A bad block on the replacement device has been ++ * fixed by writing to it */ ++ R5_NeedReplace, /* This device has a replacement which is not ++ * up-to-date at this stripe. */ ++ R5_WantReplace, /* We need to update the replacement, we have read ++ * data in, and now is a good time to write it out. ++ */ ++ R5_Discard, /* Discard the stripe */ ++ R5_SkipCopy, /* Don't copy data from bio to stripe cache */ ++}; + -+#else -+#define tif_need_resched() test_thread_flag(TIF_NEED_RESCHED) -+#define tif_need_resched_now() test_thread_flag(TIF_NEED_RESCHED) -+#define tif_need_resched_lazy() 0 -+#endif - - #if defined TIF_RESTORE_SIGMASK && !defined HAVE_SET_RESTORE_SIGMASK - /* -diff -Nur linux-4.1.6.orig/include/linux/timer.h linux-4.1.6/include/linux/timer.h ---- linux-4.1.6.orig/include/linux/timer.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/timer.h 2015-09-08 23:49:06.782014188 +0200 -@@ -241,7 +241,7 @@ - - extern int try_to_del_timer_sync(struct timer_list *timer); - --#ifdef CONFIG_SMP -+#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT_FULL) - extern int del_timer_sync(struct timer_list *timer); - #else - # define del_timer_sync(t) del_timer(t) -diff -Nur linux-4.1.6.orig/include/linux/uaccess.h linux-4.1.6/include/linux/uaccess.h ---- linux-4.1.6.orig/include/linux/uaccess.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/uaccess.h 2015-09-08 23:49:06.782014188 +0200 -@@ -1,21 +1,31 @@ - #ifndef __LINUX_UACCESS_H__ - #define __LINUX_UACCESS_H__ - --#include -+#include - #include - -+static __always_inline void pagefault_disabled_inc(void) -+{ -+ current->pagefault_disabled++; -+} ++/* ++ * Stripe state ++ */ ++enum { ++ STRIPE_ACTIVE, ++ STRIPE_HANDLE, ++ STRIPE_SYNC_REQUESTED, ++ STRIPE_SYNCING, ++ STRIPE_INSYNC, ++ STRIPE_REPLACED, ++ STRIPE_PREREAD_ACTIVE, ++ STRIPE_DELAYED, ++ STRIPE_DEGRADED, ++ STRIPE_BIT_DELAY, ++ STRIPE_EXPANDING, ++ STRIPE_EXPAND_SOURCE, ++ STRIPE_EXPAND_READY, ++ STRIPE_IO_STARTED, /* do not count towards 'bypass_count' */ ++ STRIPE_FULL_WRITE, /* all blocks are set to be overwritten */ ++ STRIPE_BIOFILL_RUN, ++ STRIPE_COMPUTE_RUN, ++ STRIPE_OPS_REQ_PENDING, ++ STRIPE_ON_UNPLUG_LIST, ++ STRIPE_DISCARD, ++ STRIPE_ON_RELEASE_LIST, ++ STRIPE_BATCH_READY, ++ STRIPE_BATCH_ERR, ++ STRIPE_BITMAP_PENDING, /* Being added to bitmap, don't add ++ * to batch yet. ++ */ ++}; + -+static __always_inline void pagefault_disabled_dec(void) -+{ -+ current->pagefault_disabled--; -+ WARN_ON(current->pagefault_disabled < 0); -+} ++#define STRIPE_EXPAND_SYNC_FLAGS \ ++ ((1 << STRIPE_EXPAND_SOURCE) |\ ++ (1 << STRIPE_EXPAND_READY) |\ ++ (1 << STRIPE_EXPANDING) |\ ++ (1 << STRIPE_SYNC_REQUESTED)) ++/* ++ * Operation request flags ++ */ ++enum { ++ STRIPE_OP_BIOFILL, ++ STRIPE_OP_COMPUTE_BLK, ++ STRIPE_OP_PREXOR, ++ STRIPE_OP_BIODRAIN, ++ STRIPE_OP_RECONSTRUCT, ++ STRIPE_OP_CHECK, ++}; + - /* -- * These routines enable/disable the pagefault handler in that -- * it will not take any locks and go straight to the fixup table. -+ * These routines enable/disable the pagefault handler. If disabled, it will -+ * not take any locks and go straight to the fixup table. - * -- * They have great resemblance to the preempt_disable/enable calls -- * and in fact they are identical; this is because currently there is -- * no other way to make the pagefault handlers do this. So we do -- * disable preemption but we don't necessarily care about that. -+ * User access methods will not sleep when called from a pagefault_disabled() -+ * environment. - */ - static inline void pagefault_disable(void) - { -- preempt_count_inc(); -+ migrate_disable(); -+ pagefault_disabled_inc(); - /* - * make sure to have issued the store before a pagefault - * can hit. -@@ -25,18 +35,32 @@ - - static inline void pagefault_enable(void) - { --#ifndef CONFIG_PREEMPT - /* - * make sure to issue those last loads/stores before enabling - * the pagefault handler again. - */ - barrier(); -- preempt_count_dec(); --#else -- preempt_enable(); --#endif -+ pagefault_disabled_dec(); -+ migrate_enable(); - } - +/* -+ * Is the pagefault handler disabled? If so, user access methods will not sleep. ++ * RAID parity calculation preferences + */ -+#define pagefault_disabled() (current->pagefault_disabled != 0) ++enum { ++ PARITY_DISABLE_RMW = 0, ++ PARITY_ENABLE_RMW, ++ PARITY_PREFER_RMW, ++}; + +/* -+ * The pagefault handler is in general disabled by pagefault_disable() or -+ * when in irq context (via in_atomic()). ++ * Pages requested from set_syndrome_sources() ++ */ ++enum { ++ SYNDROME_SRC_ALL, ++ SYNDROME_SRC_WANT_DRAIN, ++ SYNDROME_SRC_WRITTEN, ++}; ++/* ++ * Plugging: + * -+ * This function should only be used by the fault handlers. Other users should -+ * stick to pagefault_disabled(). -+ * Please NEVER use preempt_disable() to disable the fault handler. With -+ * !CONFIG_PREEMPT_COUNT, this is like a NOP. So the handler won't be disabled. -+ * in_atomic() will report different values based on !CONFIG_PREEMPT_COUNT. ++ * To improve write throughput, we need to delay the handling of some ++ * stripes until there has been a chance that several write requests ++ * for the one stripe have all been collected. ++ * In particular, any write request that would require pre-reading ++ * is put on a "delayed" queue until there are no stripes currently ++ * in a pre-read phase. Further, if the "delayed" queue is empty when ++ * a stripe is put on it then we "plug" the queue and do not process it ++ * until an unplug call is made. (the unplug_io_fn() is called). ++ * ++ * When preread is initiated on a stripe, we set PREREAD_ACTIVE and add ++ * it to the count of prereading stripes. ++ * When write is initiated, or the stripe refcnt == 0 (just in case) we ++ * clear the PREREAD_ACTIVE flag and decrement the count ++ * Whenever the 'handle' queue is empty and the device is not plugged, we ++ * move any strips from delayed to handle and clear the DELAYED flag and set ++ * PREREAD_ACTIVE. ++ * In stripe_handle, if we find pre-reading is necessary, we do it if ++ * PREREAD_ACTIVE is set, else we set DELAYED which will send it to the delayed queue. ++ * HANDLE gets cleared if stripe_handle leaves nothing locked. + */ -+#define faulthandler_disabled() (pagefault_disabled() || in_atomic()) -+ - #ifndef ARCH_HAS_NOCACHE_UACCESS - - static inline unsigned long __copy_from_user_inatomic_nocache(void *to, -diff -Nur linux-4.1.6.orig/include/linux/uprobes.h linux-4.1.6/include/linux/uprobes.h ---- linux-4.1.6.orig/include/linux/uprobes.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/uprobes.h 2015-09-08 23:49:06.782014188 +0200 -@@ -27,6 +27,7 @@ - #include - #include - #include -+#include - - struct vm_area_struct; - struct mm_struct; -diff -Nur linux-4.1.6.orig/include/linux/vmstat.h linux-4.1.6/include/linux/vmstat.h ---- linux-4.1.6.orig/include/linux/vmstat.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/vmstat.h 2015-09-08 23:49:06.782014188 +0200 -@@ -33,7 +33,9 @@ - */ - static inline void __count_vm_event(enum vm_event_item item) - { -+ preempt_disable_rt(); - raw_cpu_inc(vm_event_states.event[item]); -+ preempt_enable_rt(); - } - - static inline void count_vm_event(enum vm_event_item item) -@@ -43,7 +45,9 @@ - - static inline void __count_vm_events(enum vm_event_item item, long delta) - { -+ preempt_disable_rt(); - raw_cpu_add(vm_event_states.event[item], delta); -+ preempt_enable_rt(); - } - - static inline void count_vm_events(enum vm_event_item item, long delta) -diff -Nur linux-4.1.6.orig/include/linux/wait.h linux-4.1.6/include/linux/wait.h ---- linux-4.1.6.orig/include/linux/wait.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/linux/wait.h 2015-09-08 23:49:06.786013744 +0200 -@@ -8,6 +8,7 @@ - #include - #include - #include -+#include - - typedef struct __wait_queue wait_queue_t; - typedef int (*wait_queue_func_t)(wait_queue_t *wait, unsigned mode, int flags, void *key); -diff -Nur linux-4.1.6.orig/include/linux/wait-simple.h linux-4.1.6/include/linux/wait-simple.h ---- linux-4.1.6.orig/include/linux/wait-simple.h 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.6/include/linux/wait-simple.h 2015-09-08 23:49:06.786013744 +0200 -@@ -0,0 +1,207 @@ -+#ifndef _LINUX_WAIT_SIMPLE_H -+#define _LINUX_WAIT_SIMPLE_H + -+#include -+#include ++struct disk_info { ++ struct md_rdev *rdev, *replacement; ++}; + -+#include ++/* NOTE NR_STRIPE_HASH_LOCKS must remain below 64. ++ * This is because we sometimes take all the spinlocks ++ * and creating that much locking depth can cause ++ * problems. ++ */ ++#define NR_STRIPE_HASH_LOCKS 8 ++#define STRIPE_HASH_LOCKS_MASK (NR_STRIPE_HASH_LOCKS - 1) ++ ++struct r5worker { ++ struct work_struct work; ++ struct r5worker_group *group; ++ struct list_head temp_inactive_list[NR_STRIPE_HASH_LOCKS]; ++ bool working; ++}; + -+struct swaiter { -+ struct task_struct *task; -+ struct list_head node; ++struct r5worker_group { ++ struct list_head handle_list; ++ struct r5conf *conf; ++ struct r5worker *workers; ++ int stripes_cnt; +}; + -+#define DEFINE_SWAITER(name) \ -+ struct swaiter name = { \ -+ .task = current, \ -+ .node = LIST_HEAD_INIT((name).node), \ -+ } ++struct r5conf { ++ struct hlist_head *stripe_hashtbl; ++ /* only protect corresponding hash list and inactive_list */ ++ spinlock_t hash_locks[NR_STRIPE_HASH_LOCKS]; ++ struct mddev *mddev; ++ int chunk_sectors; ++ int level, algorithm, rmw_level; ++ int max_degraded; ++ int raid_disks; ++ int max_nr_stripes; ++ int min_nr_stripes; ++ ++ /* reshape_progress is the leading edge of a 'reshape' ++ * It has value MaxSector when no reshape is happening ++ * If delta_disks < 0, it is the last sector we started work on, ++ * else is it the next sector to work on. ++ */ ++ sector_t reshape_progress; ++ /* reshape_safe is the trailing edge of a reshape. We know that ++ * before (or after) this address, all reshape has completed. ++ */ ++ sector_t reshape_safe; ++ int previous_raid_disks; ++ int prev_chunk_sectors; ++ int prev_algo; ++ short generation; /* increments with every reshape */ ++ seqcount_t gen_lock; /* lock against generation changes */ ++ unsigned long reshape_checkpoint; /* Time we last updated ++ * metadata */ ++ long long min_offset_diff; /* minimum difference between ++ * data_offset and ++ * new_data_offset across all ++ * devices. May be negative, ++ * but is closest to zero. ++ */ ++ ++ struct list_head handle_list; /* stripes needing handling */ ++ struct list_head hold_list; /* preread ready stripes */ ++ struct list_head delayed_list; /* stripes that have plugged requests */ ++ struct list_head bitmap_list; /* stripes delaying awaiting bitmap update */ ++ struct bio *retry_read_aligned; /* currently retrying aligned bios */ ++ struct bio *retry_read_aligned_list; /* aligned bios retry list */ ++ atomic_t preread_active_stripes; /* stripes with scheduled io */ ++ atomic_t active_aligned_reads; ++ atomic_t pending_full_writes; /* full write backlog */ ++ int bypass_count; /* bypassed prereads */ ++ int bypass_threshold; /* preread nice */ ++ int skip_copy; /* Don't copy data from bio to stripe cache */ ++ struct list_head *last_hold; /* detect hold_list promotions */ ++ ++ atomic_t reshape_stripes; /* stripes with pending writes for reshape */ ++ /* unfortunately we need two cache names as we temporarily have ++ * two caches. ++ */ ++ int active_name; ++ char cache_name[2][32]; ++ struct kmem_cache *slab_cache; /* for allocating stripes */ ++ struct mutex cache_size_mutex; /* Protect changes to cache size */ ++ ++ int seq_flush, seq_write; ++ int quiesce; ++ ++ int fullsync; /* set to 1 if a full sync is needed, ++ * (fresh device added). ++ * Cleared when a sync completes. ++ */ ++ int recovery_disabled; ++ /* per cpu variables */ ++ struct raid5_percpu { ++ struct page *spare_page; /* Used when checking P/Q in raid6 */ ++ struct flex_array *scribble; /* space for constructing buffer ++ * lists and performing address ++ * conversions ++ */ ++ } __percpu *percpu; ++#ifdef CONFIG_HOTPLUG_CPU ++ struct notifier_block cpu_notify; ++#endif + -+struct swait_head { -+ raw_spinlock_t lock; -+ struct list_head list; ++ /* ++ * Free stripes pool ++ */ ++ atomic_t active_stripes; ++ struct list_head inactive_list[NR_STRIPE_HASH_LOCKS]; ++ atomic_t empty_inactive_list_nr; ++ struct llist_head released_stripes; ++ wait_queue_head_t wait_for_stripe; ++ wait_queue_head_t wait_for_overlap; ++ unsigned long cache_state; ++#define R5_INACTIVE_BLOCKED 1 /* release of inactive stripes blocked, ++ * waiting for 25% to be free ++ */ ++#define R5_ALLOC_MORE 2 /* It might help to allocate another ++ * stripe. ++ */ ++#define R5_DID_ALLOC 4 /* A stripe was allocated, don't allocate ++ * more until at least one has been ++ * released. This avoids flooding ++ * the cache. ++ */ ++ struct shrinker shrinker; ++ int pool_size; /* number of disks in stripeheads in pool */ ++ spinlock_t device_lock; ++ struct disk_info *disks; ++ ++ /* When taking over an array from a different personality, we store ++ * the new thread here until we fully activate the array. ++ */ ++ struct md_thread *thread; ++ struct list_head temp_inactive_list[NR_STRIPE_HASH_LOCKS]; ++ struct r5worker_group *worker_groups; ++ int group_cnt; ++ int worker_cnt_per_group; +}; + -+#define SWAIT_HEAD_INITIALIZER(name) { \ -+ .lock = __RAW_SPIN_LOCK_UNLOCKED(name.lock), \ -+ .list = LIST_HEAD_INIT((name).list), \ -+ } + -+#define DEFINE_SWAIT_HEAD(name) \ -+ struct swait_head name = SWAIT_HEAD_INITIALIZER(name) ++/* ++ * Our supported algorithms ++ */ ++#define ALGORITHM_LEFT_ASYMMETRIC 0 /* Rotating Parity N with Data Restart */ ++#define ALGORITHM_RIGHT_ASYMMETRIC 1 /* Rotating Parity 0 with Data Restart */ ++#define ALGORITHM_LEFT_SYMMETRIC 2 /* Rotating Parity N with Data Continuation */ ++#define ALGORITHM_RIGHT_SYMMETRIC 3 /* Rotating Parity 0 with Data Continuation */ + -+extern void __init_swait_head(struct swait_head *h, struct lock_class_key *key); ++/* Define non-rotating (raid4) algorithms. These allow ++ * conversion of raid4 to raid5. ++ */ ++#define ALGORITHM_PARITY_0 4 /* P or P,Q are initial devices */ ++#define ALGORITHM_PARITY_N 5 /* P or P,Q are final devices. */ ++ ++/* DDF RAID6 layouts differ from md/raid6 layouts in two ways. ++ * Firstly, the exact positioning of the parity block is slightly ++ * different between the 'LEFT_*' modes of md and the "_N_*" modes ++ * of DDF. ++ * Secondly, or order of datablocks over which the Q syndrome is computed ++ * is different. ++ * Consequently we have different layouts for DDF/raid6 than md/raid6. ++ * These layouts are from the DDFv1.2 spec. ++ * Interestingly DDFv1.2-Errata-A does not specify N_CONTINUE but ++ * leaves RLQ=3 as 'Vendor Specific' ++ */ + -+#define init_swait_head(swh) \ -+ do { \ -+ static struct lock_class_key __key; \ -+ \ -+ __init_swait_head((swh), &__key); \ -+ } while (0) ++#define ALGORITHM_ROTATING_ZERO_RESTART 8 /* DDF PRL=6 RLQ=1 */ ++#define ALGORITHM_ROTATING_N_RESTART 9 /* DDF PRL=6 RLQ=2 */ ++#define ALGORITHM_ROTATING_N_CONTINUE 10 /*DDF PRL=6 RLQ=3 */ + -+/* -+ * Waiter functions ++/* For every RAID5 algorithm we define a RAID6 algorithm ++ * with exactly the same layout for data and parity, and ++ * with the Q block always on the last device (N-1). ++ * This allows trivial conversion from RAID5 to RAID6 + */ -+extern void swait_prepare_locked(struct swait_head *head, struct swaiter *w); -+extern void swait_prepare(struct swait_head *head, struct swaiter *w, int state); -+extern void swait_finish_locked(struct swait_head *head, struct swaiter *w); -+extern void swait_finish(struct swait_head *head, struct swaiter *w); ++#define ALGORITHM_LEFT_ASYMMETRIC_6 16 ++#define ALGORITHM_RIGHT_ASYMMETRIC_6 17 ++#define ALGORITHM_LEFT_SYMMETRIC_6 18 ++#define ALGORITHM_RIGHT_SYMMETRIC_6 19 ++#define ALGORITHM_PARITY_0_6 20 ++#define ALGORITHM_PARITY_N_6 ALGORITHM_PARITY_N ++ ++static inline int algorithm_valid_raid5(int layout) ++{ ++ return (layout >= 0) && ++ (layout <= 5); ++} ++static inline int algorithm_valid_raid6(int layout) ++{ ++ return (layout >= 0 && layout <= 5) ++ || ++ (layout >= 8 && layout <= 10) ++ || ++ (layout >= 16 && layout <= 20); ++} + -+/* Check whether a head has waiters enqueued */ -+static inline bool swaitqueue_active(struct swait_head *h) ++static inline int algorithm_is_DDF(int layout) +{ -+ /* Make sure the condition is visible before checking list_empty() */ -+ smp_mb(); -+ return !list_empty(&h->list); ++ return layout >= 8 && layout <= 10; +} + ++extern void md_raid5_kick_device(struct r5conf *conf); ++extern int raid5_set_cache_size(struct mddev *mddev, int size); ++#endif +diff -Nur linux-4.1.10.orig/drivers/misc/hwlat_detector.c linux-4.1.10/drivers/misc/hwlat_detector.c +--- linux-4.1.10.orig/drivers/misc/hwlat_detector.c 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/drivers/misc/hwlat_detector.c 2015-10-07 18:00:08.000000000 +0200 +@@ -0,0 +1,1240 @@ +/* -+ * Wakeup functions ++ * hwlat_detector.c - A simple Hardware Latency detector. ++ * ++ * Use this module to detect large system latencies induced by the behavior of ++ * certain underlying system hardware or firmware, independent of Linux itself. ++ * The code was developed originally to detect the presence of SMIs on Intel ++ * and AMD systems, although there is no dependency upon x86 herein. ++ * ++ * The classical example usage of this module is in detecting the presence of ++ * SMIs or System Management Interrupts on Intel and AMD systems. An SMI is a ++ * somewhat special form of hardware interrupt spawned from earlier CPU debug ++ * modes in which the (BIOS/EFI/etc.) firmware arranges for the South Bridge ++ * LPC (or other device) to generate a special interrupt under certain ++ * circumstances, for example, upon expiration of a special SMI timer device, ++ * due to certain external thermal readings, on certain I/O address accesses, ++ * and other situations. An SMI hits a special CPU pin, triggers a special ++ * SMI mode (complete with special memory map), and the OS is unaware. ++ * ++ * Although certain hardware-inducing latencies are necessary (for example, ++ * a modern system often requires an SMI handler for correct thermal control ++ * and remote management) they can wreak havoc upon any OS-level performance ++ * guarantees toward low-latency, especially when the OS is not even made ++ * aware of the presence of these interrupts. For this reason, we need a ++ * somewhat brute force mechanism to detect these interrupts. In this case, ++ * we do it by hogging all of the CPU(s) for configurable timer intervals, ++ * sampling the built-in CPU timer, looking for discontiguous readings. ++ * ++ * WARNING: This implementation necessarily introduces latencies. Therefore, ++ * you should NEVER use this module in a production environment ++ * requiring any kind of low-latency performance guarantee(s). ++ * ++ * Copyright (C) 2008-2009 Jon Masters, Red Hat, Inc. ++ * ++ * Includes useful feedback from Clark Williams ++ * ++ * This file is licensed under the terms of the GNU General Public ++ * License version 2. This program is licensed "as is" without any ++ * warranty of any kind, whether express or implied. + */ -+extern unsigned int __swait_wake(struct swait_head *head, unsigned int state, unsigned int num); -+extern unsigned int __swait_wake_locked(struct swait_head *head, unsigned int state, unsigned int num); + -+#define swait_wake(head) __swait_wake(head, TASK_NORMAL, 1) -+#define swait_wake_interruptible(head) __swait_wake(head, TASK_INTERRUPTIBLE, 1) -+#define swait_wake_all(head) __swait_wake(head, TASK_NORMAL, 0) -+#define swait_wake_all_interruptible(head) __swait_wake(head, TASK_INTERRUPTIBLE, 0) ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include + -+/* -+ * Event API -+ */ -+#define __swait_event(wq, condition) \ -+do { \ -+ DEFINE_SWAITER(__wait); \ -+ \ -+ for (;;) { \ -+ swait_prepare(&wq, &__wait, TASK_UNINTERRUPTIBLE); \ -+ if (condition) \ -+ break; \ -+ schedule(); \ -+ } \ -+ swait_finish(&wq, &__wait); \ -+} while (0) ++#define BUF_SIZE_DEFAULT 262144UL /* 8K*(sizeof(entry)) */ ++#define BUF_FLAGS (RB_FL_OVERWRITE) /* no block on full */ ++#define U64STR_SIZE 22 /* 20 digits max */ + -+/** -+ * swait_event - sleep until a condition gets true -+ * @wq: the waitqueue to wait on -+ * @condition: a C expression for the event to wait for -+ * -+ * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the -+ * @condition evaluates to true. The @condition is checked each time -+ * the waitqueue @wq is woken up. -+ * -+ * wake_up() has to be called after changing any variable that could -+ * change the result of the wait condition. -+ */ -+#define swait_event(wq, condition) \ -+do { \ -+ if (condition) \ -+ break; \ -+ __swait_event(wq, condition); \ -+} while (0) ++#define VERSION "1.0.0" ++#define BANNER "hwlat_detector: " ++#define DRVNAME "hwlat_detector" ++#define DEFAULT_SAMPLE_WINDOW 1000000 /* 1s */ ++#define DEFAULT_SAMPLE_WIDTH 500000 /* 0.5s */ ++#define DEFAULT_LAT_THRESHOLD 10 /* 10us */ + -+#define __swait_event_interruptible(wq, condition, ret) \ -+do { \ -+ DEFINE_SWAITER(__wait); \ -+ \ -+ for (;;) { \ -+ swait_prepare(&wq, &__wait, TASK_INTERRUPTIBLE); \ -+ if (condition) \ -+ break; \ -+ if (signal_pending(current)) { \ -+ ret = -ERESTARTSYS; \ -+ break; \ -+ } \ -+ schedule(); \ -+ } \ -+ swait_finish(&wq, &__wait); \ -+} while (0) ++/* Module metadata */ + -+#define __swait_event_interruptible_timeout(wq, condition, ret) \ -+do { \ -+ DEFINE_SWAITER(__wait); \ -+ \ -+ for (;;) { \ -+ swait_prepare(&wq, &__wait, TASK_INTERRUPTIBLE); \ -+ if (condition) \ -+ break; \ -+ if (signal_pending(current)) { \ -+ ret = -ERESTARTSYS; \ -+ break; \ -+ } \ -+ ret = schedule_timeout(ret); \ -+ if (!ret) \ -+ break; \ -+ } \ -+ swait_finish(&wq, &__wait); \ -+} while (0) ++MODULE_LICENSE("GPL"); ++MODULE_AUTHOR("Jon Masters "); ++MODULE_DESCRIPTION("A simple hardware latency detector"); ++MODULE_VERSION(VERSION); + -+/** -+ * swait_event_interruptible - sleep until a condition gets true -+ * @wq: the waitqueue to wait on -+ * @condition: a C expression for the event to wait for -+ * -+ * The process is put to sleep (TASK_INTERRUPTIBLE) until the -+ * @condition evaluates to true. The @condition is checked each time -+ * the waitqueue @wq is woken up. -+ * -+ * wake_up() has to be called after changing any variable that could -+ * change the result of the wait condition. -+ */ -+#define swait_event_interruptible(wq, condition) \ -+({ \ -+ int __ret = 0; \ -+ if (!(condition)) \ -+ __swait_event_interruptible(wq, condition, __ret); \ -+ __ret; \ -+}) ++/* Module parameters */ + -+#define swait_event_interruptible_timeout(wq, condition, timeout) \ -+({ \ -+ int __ret = timeout; \ -+ if (!(condition)) \ -+ __swait_event_interruptible_timeout(wq, condition, __ret); \ -+ __ret; \ -+}) ++static int debug; ++static int enabled; ++static int threshold; + -+#define __swait_event_timeout(wq, condition, ret) \ -+do { \ -+ DEFINE_SWAITER(__wait); \ -+ \ -+ for (;;) { \ -+ swait_prepare(&wq, &__wait, TASK_UNINTERRUPTIBLE); \ -+ if (condition) \ -+ break; \ -+ ret = schedule_timeout(ret); \ -+ if (!ret) \ -+ break; \ -+ } \ -+ swait_finish(&wq, &__wait); \ -+} while (0) ++module_param(debug, int, 0); /* enable debug */ ++module_param(enabled, int, 0); /* enable detector */ ++module_param(threshold, int, 0); /* latency threshold */ + -+/** -+ * swait_event_timeout - sleep until a condition gets true or a timeout elapses -+ * @wq: the waitqueue to wait on -+ * @condition: a C expression for the event to wait for -+ * @timeout: timeout, in jiffies -+ * -+ * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the -+ * @condition evaluates to true. The @condition is checked each time -+ * the waitqueue @wq is woken up. -+ * -+ * wake_up() has to be called after changing any variable that could -+ * change the result of the wait condition. -+ * -+ * The function returns 0 if the @timeout elapsed, and the remaining -+ * jiffies if the condition evaluated to true before the timeout elapsed. -+ */ -+#define swait_event_timeout(wq, condition, timeout) \ -+({ \ -+ long __ret = timeout; \ -+ if (!(condition)) \ -+ __swait_event_timeout(wq, condition, __ret); \ -+ __ret; \ -+}) ++/* Buffering and sampling */ + -+#endif -diff -Nur linux-4.1.6.orig/include/linux/work-simple.h linux-4.1.6/include/linux/work-simple.h ---- linux-4.1.6.orig/include/linux/work-simple.h 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.6/include/linux/work-simple.h 2015-09-08 23:49:06.786013744 +0200 -@@ -0,0 +1,24 @@ -+#ifndef _LINUX_SWORK_H -+#define _LINUX_SWORK_H ++static struct ring_buffer *ring_buffer; /* sample buffer */ ++static DEFINE_MUTEX(ring_buffer_mutex); /* lock changes */ ++static unsigned long buf_size = BUF_SIZE_DEFAULT; ++static struct task_struct *kthread; /* sampling thread */ + -+#include ++/* DebugFS filesystem entries */ + -+struct swork_event { -+ struct list_head item; -+ unsigned long flags; -+ void (*func)(struct swork_event *); -+}; ++static struct dentry *debug_dir; /* debugfs directory */ ++static struct dentry *debug_max; /* maximum TSC delta */ ++static struct dentry *debug_count; /* total detect count */ ++static struct dentry *debug_sample_width; /* sample width us */ ++static struct dentry *debug_sample_window; /* sample window us */ ++static struct dentry *debug_sample; /* raw samples us */ ++static struct dentry *debug_threshold; /* threshold us */ ++static struct dentry *debug_enable; /* enable/disable */ + -+static inline void INIT_SWORK(struct swork_event *event, -+ void (*func)(struct swork_event *)) -+{ -+ event->flags = 0; -+ event->func = func; -+} ++/* Individual samples and global state */ + -+bool swork_queue(struct swork_event *sev); ++struct sample; /* latency sample */ ++struct data; /* Global state */ + -+int swork_get(void); -+void swork_put(void); ++/* Sampling functions */ ++static int __buffer_add_sample(struct sample *sample); ++static struct sample *buffer_get_sample(struct sample *sample); + -+#endif /* _LINUX_SWORK_H */ -diff -Nur linux-4.1.6.orig/include/net/dst.h linux-4.1.6/include/net/dst.h ---- linux-4.1.6.orig/include/net/dst.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/net/dst.h 2015-09-08 23:49:06.786013744 +0200 -@@ -403,7 +403,7 @@ - static inline int dst_neigh_output(struct dst_entry *dst, struct neighbour *n, - struct sk_buff *skb) - { -- const struct hh_cache *hh; -+ struct hh_cache *hh; - - if (dst->pending_confirm) { - unsigned long now = jiffies; -diff -Nur linux-4.1.6.orig/include/net/neighbour.h linux-4.1.6/include/net/neighbour.h ---- linux-4.1.6.orig/include/net/neighbour.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/net/neighbour.h 2015-09-08 23:49:07.409944556 +0200 -@@ -445,7 +445,7 @@ - } - #endif - --static inline int neigh_hh_output(const struct hh_cache *hh, struct sk_buff *skb) -+static inline int neigh_hh_output(struct hh_cache *hh, struct sk_buff *skb) - { - unsigned int seq; - int hh_len; -@@ -500,7 +500,7 @@ - - #define NEIGH_CB(skb) ((struct neighbour_cb *)(skb)->cb) - --static inline void neigh_ha_snapshot(char *dst, const struct neighbour *n, -+static inline void neigh_ha_snapshot(char *dst, struct neighbour *n, - const struct net_device *dev) - { - unsigned int seq; -diff -Nur linux-4.1.6.orig/include/net/netns/ipv4.h linux-4.1.6/include/net/netns/ipv4.h ---- linux-4.1.6.orig/include/net/netns/ipv4.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/include/net/netns/ipv4.h 2015-09-08 23:49:08.077870489 +0200 -@@ -69,6 +69,7 @@ - - int sysctl_icmp_echo_ignore_all; - int sysctl_icmp_echo_ignore_broadcasts; -+ int sysctl_icmp_echo_sysrq; - int sysctl_icmp_ignore_bogus_error_responses; - int sysctl_icmp_ratelimit; - int sysctl_icmp_ratemask; -diff -Nur linux-4.1.6.orig/include/trace/events/hist.h linux-4.1.6/include/trace/events/hist.h ---- linux-4.1.6.orig/include/trace/events/hist.h 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.6/include/trace/events/hist.h 2015-09-08 23:49:08.077870489 +0200 -@@ -0,0 +1,72 @@ -+#undef TRACE_SYSTEM -+#define TRACE_SYSTEM hist ++/* Threading and state */ ++static int kthread_fn(void *unused); ++static int start_kthread(void); ++static int stop_kthread(void); ++static void __reset_stats(void); ++static int init_stats(void); + -+#if !defined(_TRACE_HIST_H) || defined(TRACE_HEADER_MULTI_READ) -+#define _TRACE_HIST_H ++/* Debugfs interface */ ++static ssize_t simple_data_read(struct file *filp, char __user *ubuf, ++ size_t cnt, loff_t *ppos, const u64 *entry); ++static ssize_t simple_data_write(struct file *filp, const char __user *ubuf, ++ size_t cnt, loff_t *ppos, u64 *entry); ++static int debug_sample_fopen(struct inode *inode, struct file *filp); ++static ssize_t debug_sample_fread(struct file *filp, char __user *ubuf, ++ size_t cnt, loff_t *ppos); ++static int debug_sample_release(struct inode *inode, struct file *filp); ++static int debug_enable_fopen(struct inode *inode, struct file *filp); ++static ssize_t debug_enable_fread(struct file *filp, char __user *ubuf, ++ size_t cnt, loff_t *ppos); ++static ssize_t debug_enable_fwrite(struct file *file, ++ const char __user *user_buffer, ++ size_t user_size, loff_t *offset); + -+#include "latency_hist.h" -+#include ++/* Initialization functions */ ++static int init_debugfs(void); ++static void free_debugfs(void); ++static int detector_init(void); ++static void detector_exit(void); + -+#if !defined(CONFIG_PREEMPT_OFF_HIST) && !defined(CONFIG_INTERRUPT_OFF_HIST) -+#define trace_preemptirqsoff_hist(a, b) -+#else -+TRACE_EVENT(preemptirqsoff_hist, ++/* Individual latency samples are stored here when detected and packed into ++ * the ring_buffer circular buffer, where they are overwritten when ++ * more than buf_size/sizeof(sample) samples are received. */ ++struct sample { ++ u64 seqnum; /* unique sequence */ ++ u64 duration; /* ktime delta */ ++ u64 outer_duration; /* ktime delta (outer loop) */ ++ struct timespec timestamp; /* wall time */ ++ unsigned long lost; ++}; + -+ TP_PROTO(int reason, int starthist), ++/* keep the global state somewhere. */ ++static struct data { + -+ TP_ARGS(reason, starthist), ++ struct mutex lock; /* protect changes */ + -+ TP_STRUCT__entry( -+ __field(int, reason) -+ __field(int, starthist) -+ ), ++ u64 count; /* total since reset */ ++ u64 max_sample; /* max hardware latency */ ++ u64 threshold; /* sample threshold level */ + -+ TP_fast_assign( -+ __entry->reason = reason; -+ __entry->starthist = starthist; -+ ), ++ u64 sample_window; /* total sampling window (on+off) */ ++ u64 sample_width; /* active sampling portion of window */ + -+ TP_printk("reason=%s starthist=%s", getaction(__entry->reason), -+ __entry->starthist ? "start" : "stop") -+); -+#endif ++ atomic_t sample_open; /* whether the sample file is open */ + -+#ifndef CONFIG_MISSED_TIMER_OFFSETS_HIST -+#define trace_hrtimer_interrupt(a, b, c, d) -+#else -+TRACE_EVENT(hrtimer_interrupt, ++ wait_queue_head_t wq; /* waitqeue for new sample values */ + -+ TP_PROTO(int cpu, long long offset, struct task_struct *curr, -+ struct task_struct *task), ++} data; + -+ TP_ARGS(cpu, offset, curr, task), ++/** ++ * __buffer_add_sample - add a new latency sample recording to the ring buffer ++ * @sample: The new latency sample value ++ * ++ * This receives a new latency sample and records it in a global ring buffer. ++ * No additional locking is used in this case. ++ */ ++static int __buffer_add_sample(struct sample *sample) ++{ ++ return ring_buffer_write(ring_buffer, ++ sizeof(struct sample), sample); ++} + -+ TP_STRUCT__entry( -+ __field(int, cpu) -+ __field(long long, offset) -+ __array(char, ccomm, TASK_COMM_LEN) -+ __field(int, cprio) -+ __array(char, tcomm, TASK_COMM_LEN) -+ __field(int, tprio) -+ ), ++/** ++ * buffer_get_sample - remove a hardware latency sample from the ring buffer ++ * @sample: Pre-allocated storage for the sample ++ * ++ * This retrieves a hardware latency sample from the global circular buffer ++ */ ++static struct sample *buffer_get_sample(struct sample *sample) ++{ ++ struct ring_buffer_event *e = NULL; ++ struct sample *s = NULL; ++ unsigned int cpu = 0; + -+ TP_fast_assign( -+ __entry->cpu = cpu; -+ __entry->offset = offset; -+ memcpy(__entry->ccomm, curr->comm, TASK_COMM_LEN); -+ __entry->cprio = curr->prio; -+ memcpy(__entry->tcomm, task != NULL ? task->comm : "", -+ task != NULL ? TASK_COMM_LEN : 7); -+ __entry->tprio = task != NULL ? task->prio : -1; -+ ), ++ if (!sample) ++ return NULL; + -+ TP_printk("cpu=%d offset=%lld curr=%s[%d] thread=%s[%d]", -+ __entry->cpu, __entry->offset, __entry->ccomm, -+ __entry->cprio, __entry->tcomm, __entry->tprio) -+); ++ mutex_lock(&ring_buffer_mutex); ++ for_each_online_cpu(cpu) { ++ e = ring_buffer_consume(ring_buffer, cpu, NULL, &sample->lost); ++ if (e) ++ break; ++ } ++ ++ if (e) { ++ s = ring_buffer_event_data(e); ++ memcpy(sample, s, sizeof(struct sample)); ++ } else ++ sample = NULL; ++ mutex_unlock(&ring_buffer_mutex); ++ ++ return sample; ++} ++ ++#ifndef CONFIG_TRACING ++#define time_type ktime_t ++#define time_get() ktime_get() ++#define time_to_us(x) ktime_to_us(x) ++#define time_sub(a, b) ktime_sub(a, b) ++#define init_time(a, b) (a).tv64 = b ++#define time_u64(a) ((a).tv64) ++#else ++#define time_type u64 ++#define time_get() trace_clock_local() ++#define time_to_us(x) div_u64(x, 1000) ++#define time_sub(a, b) ((a) - (b)) ++#define init_time(a, b) (a = b) ++#define time_u64(a) a +#endif ++/** ++ * get_sample - sample the CPU TSC and look for likely hardware latencies ++ * ++ * Used to repeatedly capture the CPU TSC (or similar), looking for potential ++ * hardware-induced latency. Called with interrupts disabled and with ++ * data.lock held. ++ */ ++static int get_sample(void) ++{ ++ time_type start, t1, t2, last_t2; ++ s64 diff, total = 0; ++ u64 sample = 0; ++ u64 outer_sample = 0; ++ int ret = -1; + -+#endif /* _TRACE_HIST_H */ ++ init_time(last_t2, 0); ++ start = time_get(); /* start timestamp */ + -+/* This part must be outside protection */ -+#include -diff -Nur linux-4.1.6.orig/include/trace/events/latency_hist.h linux-4.1.6/include/trace/events/latency_hist.h ---- linux-4.1.6.orig/include/trace/events/latency_hist.h 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.6/include/trace/events/latency_hist.h 2015-09-08 23:49:08.077870489 +0200 -@@ -0,0 +1,29 @@ -+#ifndef _LATENCY_HIST_H -+#define _LATENCY_HIST_H ++ do { + -+enum hist_action { -+ IRQS_ON, -+ PREEMPT_ON, -+ TRACE_STOP, -+ IRQS_OFF, -+ PREEMPT_OFF, -+ TRACE_START, -+}; ++ t1 = time_get(); /* we'll look for a discontinuity */ ++ t2 = time_get(); + -+static char *actions[] = { -+ "IRQS_ON", -+ "PREEMPT_ON", -+ "TRACE_STOP", -+ "IRQS_OFF", -+ "PREEMPT_OFF", -+ "TRACE_START", -+}; ++ if (time_u64(last_t2)) { ++ /* Check the delta from outer loop (t2 to next t1) */ ++ diff = time_to_us(time_sub(t1, last_t2)); ++ /* This shouldn't happen */ ++ if (diff < 0) { ++ pr_err(BANNER "time running backwards\n"); ++ goto out; ++ } ++ if (diff > outer_sample) ++ outer_sample = diff; ++ } ++ last_t2 = t2; + -+static inline char *getaction(int action) -+{ -+ if (action >= 0 && action <= sizeof(actions)/sizeof(actions[0])) -+ return actions[action]; -+ return "unknown"; -+} ++ total = time_to_us(time_sub(t2, start)); /* sample width */ + -+#endif /* _LATENCY_HIST_H */ -diff -Nur linux-4.1.6.orig/init/Kconfig linux-4.1.6/init/Kconfig ---- linux-4.1.6.orig/init/Kconfig 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/init/Kconfig 2015-09-08 23:49:08.077870489 +0200 -@@ -637,7 +637,7 @@ - - config RCU_FAST_NO_HZ - bool "Accelerate last non-dyntick-idle CPU's grace periods" -- depends on NO_HZ_COMMON && SMP -+ depends on NO_HZ_COMMON && SMP && !PREEMPT_RT_FULL - default n - help - This option permits CPUs to enter dynticks-idle state even if -@@ -664,7 +664,7 @@ - config RCU_BOOST - bool "Enable RCU priority boosting" - depends on RT_MUTEXES && PREEMPT_RCU -- default n -+ default y if PREEMPT_RT_FULL - help - This option boosts the priority of preempted RCU readers that - block the current preemptible RCU grace period for too long. -@@ -1101,6 +1101,7 @@ - config RT_GROUP_SCHED - bool "Group scheduling for SCHED_RR/FIFO" - depends on CGROUP_SCHED -+ depends on !PREEMPT_RT_FULL - default n - help - This feature lets you explicitly allocate real CPU bandwidth -@@ -1688,6 +1689,7 @@ - - config SLAB - bool "SLAB" -+ depends on !PREEMPT_RT_FULL - help - The regular slab allocator that is established and known to work - well in all environments. It organizes cache hot objects in -@@ -1706,6 +1708,7 @@ - config SLOB - depends on EXPERT - bool "SLOB (Simple Allocator)" -+ depends on !PREEMPT_RT_FULL - help - SLOB replaces the stock allocator with a drastically simpler - allocator. SLOB is generally more space efficient but -@@ -1715,7 +1718,7 @@ - - config SLUB_CPU_PARTIAL - default y -- depends on SLUB && SMP -+ depends on SLUB && SMP && !PREEMPT_RT_FULL - bool "SLUB per cpu partial cache" - help - Per cpu partial caches accellerate objects allocation and freeing -diff -Nur linux-4.1.6.orig/init/main.c linux-4.1.6/init/main.c ---- linux-4.1.6.orig/init/main.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/init/main.c 2015-09-08 23:49:08.081870045 +0200 -@@ -525,6 +525,7 @@ - setup_command_line(command_line); - setup_nr_cpu_ids(); - setup_per_cpu_areas(); -+ softirq_early_init(); - smp_prepare_boot_cpu(); /* arch-specific boot-cpu hooks */ - - build_all_zonelists(NULL, NULL); -diff -Nur linux-4.1.6.orig/init/Makefile linux-4.1.6/init/Makefile ---- linux-4.1.6.orig/init/Makefile 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/init/Makefile 2015-09-08 23:49:08.077870489 +0200 -@@ -33,4 +33,4 @@ - include/generated/compile.h: FORCE - @$($(quiet)chk_compile.h) - $(Q)$(CONFIG_SHELL) $(srctree)/scripts/mkcompile_h $@ \ -- "$(UTS_MACHINE)" "$(CONFIG_SMP)" "$(CONFIG_PREEMPT)" "$(CC) $(KBUILD_CFLAGS)" -+ "$(UTS_MACHINE)" "$(CONFIG_SMP)" "$(CONFIG_PREEMPT)" "$(CONFIG_PREEMPT_RT_FULL)" "$(CC) $(KBUILD_CFLAGS)" -diff -Nur linux-4.1.6.orig/ipc/mqueue.c linux-4.1.6/ipc/mqueue.c ---- linux-4.1.6.orig/ipc/mqueue.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/ipc/mqueue.c 2015-09-08 23:49:08.081870045 +0200 -@@ -47,8 +47,7 @@ - #define RECV 1 - - #define STATE_NONE 0 --#define STATE_PENDING 1 --#define STATE_READY 2 -+#define STATE_READY 1 - - struct posix_msg_tree_node { - struct rb_node rb_node; -@@ -568,15 +567,12 @@ - wq_add(info, sr, ewp); - - for (;;) { -- set_current_state(TASK_INTERRUPTIBLE); -+ __set_current_state(TASK_INTERRUPTIBLE); - - spin_unlock(&info->lock); - time = schedule_hrtimeout_range_clock(timeout, 0, - HRTIMER_MODE_ABS, CLOCK_REALTIME); - -- while (ewp->state == STATE_PENDING) -- cpu_relax(); -- - if (ewp->state == STATE_READY) { - retval = 0; - goto out; -@@ -904,11 +900,15 @@ - * list of waiting receivers. A sender checks that list before adding the new - * message into the message array. If there is a waiting receiver, then it - * bypasses the message array and directly hands the message over to the -- * receiver. -- * The receiver accepts the message and returns without grabbing the queue -- * spinlock. Therefore an intermediate STATE_PENDING state and memory barriers -- * are necessary. The same algorithm is used for sysv semaphores, see -- * ipc/sem.c for more details. -+ * receiver. The receiver accepts the message and returns without grabbing the -+ * queue spinlock: -+ * -+ * - Set pointer to message. -+ * - Queue the receiver task for later wakeup (without the info->lock). -+ * - Update its state to STATE_READY. Now the receiver can continue. -+ * - Wake up the process after the lock is dropped. Should the process wake up -+ * before this wakeup (due to a timeout or a signal) it will either see -+ * STATE_READY and continue or acquire the lock to check the state again. - * - * The same algorithm is used for senders. - */ -@@ -916,21 +916,29 @@ - /* pipelined_send() - send a message directly to the task waiting in - * sys_mq_timedreceive() (without inserting message into a queue). - */ --static inline void pipelined_send(struct mqueue_inode_info *info, -+static inline void pipelined_send(struct wake_q_head *wake_q, -+ struct mqueue_inode_info *info, - struct msg_msg *message, - struct ext_wait_queue *receiver) - { - receiver->msg = message; - list_del(&receiver->list); -- receiver->state = STATE_PENDING; -- wake_up_process(receiver->task); -- smp_wmb(); -+ wake_q_add(wake_q, receiver->task); -+ /* -+ * Rely on the implicit cmpxchg barrier from wake_q_add such -+ * that we can ensure that updating receiver->state is the last -+ * write operation: As once set, the receiver can continue, -+ * and if we don't have the reference count from the wake_q, -+ * yet, at that point we can later have a use-after-free -+ * condition and bogus wakeup. -+ */ - receiver->state = STATE_READY; - } - - /* pipelined_receive() - if there is task waiting in sys_mq_timedsend() - * gets its message and put to the queue (we have one free place for sure). */ --static inline void pipelined_receive(struct mqueue_inode_info *info) -+static inline void pipelined_receive(struct wake_q_head *wake_q, -+ struct mqueue_inode_info *info) - { - struct ext_wait_queue *sender = wq_get_first_waiter(info, SEND); - -@@ -941,10 +949,9 @@ - } - if (msg_insert(sender->msg, info)) - return; ++ /* This checks the inner loop (t1 to t2) */ ++ diff = time_to_us(time_sub(t2, t1)); /* current diff */ + - list_del(&sender->list); -- sender->state = STATE_PENDING; -- wake_up_process(sender->task); -- smp_wmb(); -+ wake_q_add(wake_q, sender->task); - sender->state = STATE_READY; - } - -@@ -962,6 +969,7 @@ - struct timespec ts; - struct posix_msg_tree_node *new_leaf = NULL; - int ret = 0; -+ WAKE_Q(wake_q); - - if (u_abs_timeout) { - int res = prepare_timeout(u_abs_timeout, &expires, &ts); -@@ -1045,7 +1053,7 @@ - } else { - receiver = wq_get_first_waiter(info, RECV); - if (receiver) { -- pipelined_send(info, msg_ptr, receiver); -+ pipelined_send(&wake_q, info, msg_ptr, receiver); - } else { - /* adds message to the queue */ - ret = msg_insert(msg_ptr, info); -@@ -1058,6 +1066,7 @@ - } - out_unlock: - spin_unlock(&info->lock); -+ wake_up_q(&wake_q); - out_free: - if (ret) - free_msg(msg_ptr); -@@ -1144,14 +1153,17 @@ - msg_ptr = wait.msg; - } - } else { -+ WAKE_Q(wake_q); ++ /* This shouldn't happen */ ++ if (diff < 0) { ++ pr_err(BANNER "time running backwards\n"); ++ goto out; ++ } + - msg_ptr = msg_get(info); - - inode->i_atime = inode->i_mtime = inode->i_ctime = - CURRENT_TIME; - - /* There is now free space in queue. */ -- pipelined_receive(info); -+ pipelined_receive(&wake_q, info); - spin_unlock(&info->lock); -+ wake_up_q(&wake_q); - ret = 0; - } - if (ret == 0) { -diff -Nur linux-4.1.6.orig/ipc/msg.c linux-4.1.6/ipc/msg.c ---- linux-4.1.6.orig/ipc/msg.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/ipc/msg.c 2015-09-08 23:49:08.081870045 +0200 -@@ -188,6 +188,12 @@ - struct msg_receiver *msr, *t; - - list_for_each_entry_safe(msr, t, &msq->q_receivers, r_list) { -+ /* -+ * Make sure that the wakeup doesnt preempt -+ * this CPU prematurely. (on PREEMPT_RT) -+ */ -+ preempt_disable_rt(); ++ if (diff > sample) ++ sample = diff; /* only want highest value */ + - msr->r_msg = NULL; /* initialize expunge ordering */ - wake_up_process(msr->r_tsk); - /* -@@ -198,6 +204,8 @@ - */ - smp_mb(); - msr->r_msg = ERR_PTR(res); ++ } while (total <= data.sample_width); + -+ preempt_enable_rt(); - } - } - -@@ -574,6 +582,11 @@ - if (testmsg(msg, msr->r_msgtype, msr->r_mode) && - !security_msg_queue_msgrcv(msq, msg, msr->r_tsk, - msr->r_msgtype, msr->r_mode)) { -+ /* -+ * Make sure that the wakeup doesnt preempt -+ * this CPU prematurely. (on PREEMPT_RT) -+ */ -+ preempt_disable_rt(); - - list_del(&msr->r_list); - if (msr->r_maxsize < msg->m_ts) { -@@ -595,12 +608,13 @@ - */ - smp_mb(); - msr->r_msg = msg; -+ preempt_enable_rt(); - - return 1; - } -+ preempt_enable_rt(); - } - } -- - return 0; - } - -diff -Nur linux-4.1.6.orig/ipc/sem.c linux-4.1.6/ipc/sem.c ---- linux-4.1.6.orig/ipc/sem.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/ipc/sem.c 2015-09-08 23:49:08.085869602 +0200 -@@ -680,6 +680,13 @@ - static void wake_up_sem_queue_prepare(struct list_head *pt, - struct sem_queue *q, int error) - { -+#ifdef CONFIG_PREEMPT_RT_BASE -+ struct task_struct *p = q->sleeper; -+ get_task_struct(p); -+ q->status = error; -+ wake_up_process(p); -+ put_task_struct(p); -+#else - if (list_empty(pt)) { - /* - * Hold preempt off so that we don't get preempted and have the -@@ -691,6 +698,7 @@ - q->pid = error; - - list_add_tail(&q->list, pt); -+#endif - } - - /** -@@ -704,6 +712,7 @@ - */ - static void wake_up_sem_queue_do(struct list_head *pt) - { -+#ifndef CONFIG_PREEMPT_RT_BASE - struct sem_queue *q, *t; - int did_something; - -@@ -716,6 +725,7 @@ - } - if (did_something) - preempt_enable(); -+#endif - } - - static void unlink_queue(struct sem_array *sma, struct sem_queue *q) -diff -Nur linux-4.1.6.orig/kernel/cgroup.c linux-4.1.6/kernel/cgroup.c ---- linux-4.1.6.orig/kernel/cgroup.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/cgroup.c 2015-09-08 23:49:08.089869160 +0200 -@@ -4421,10 +4421,10 @@ - queue_work(cgroup_destroy_wq, &css->destroy_work); - } - --static void css_release_work_fn(struct work_struct *work) -+static void css_release_work_fn(struct swork_event *sev) - { - struct cgroup_subsys_state *css = -- container_of(work, struct cgroup_subsys_state, destroy_work); -+ container_of(sev, struct cgroup_subsys_state, destroy_swork); - struct cgroup_subsys *ss = css->ss; - struct cgroup *cgrp = css->cgroup; - -@@ -4463,8 +4463,8 @@ - struct cgroup_subsys_state *css = - container_of(ref, struct cgroup_subsys_state, refcnt); - -- INIT_WORK(&css->destroy_work, css_release_work_fn); -- queue_work(cgroup_destroy_wq, &css->destroy_work); -+ INIT_SWORK(&css->destroy_swork, css_release_work_fn); -+ swork_queue(&css->destroy_swork); - } - - static void init_and_link_css(struct cgroup_subsys_state *css, -@@ -5068,6 +5068,7 @@ - */ - cgroup_destroy_wq = alloc_workqueue("cgroup_destroy", 0, 1); - BUG_ON(!cgroup_destroy_wq); -+ BUG_ON(swork_get()); - - /* - * Used to destroy pidlists and separate to serve as flush domain. -diff -Nur linux-4.1.6.orig/kernel/cpu.c linux-4.1.6/kernel/cpu.c ---- linux-4.1.6.orig/kernel/cpu.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/cpu.c 2015-09-08 23:49:08.089869160 +0200 -@@ -74,8 +74,8 @@ - #endif - } cpu_hotplug = { - .active_writer = NULL, -- .wq = __WAIT_QUEUE_HEAD_INITIALIZER(cpu_hotplug.wq), - .lock = __MUTEX_INITIALIZER(cpu_hotplug.lock), -+ .wq = __WAIT_QUEUE_HEAD_INITIALIZER(cpu_hotplug.wq), - #ifdef CONFIG_DEBUG_LOCK_ALLOC - .dep_map = {.name = "cpu_hotplug.lock" }, - #endif -@@ -88,6 +88,289 @@ - #define cpuhp_lock_acquire() lock_map_acquire(&cpu_hotplug.dep_map) - #define cpuhp_lock_release() lock_map_release(&cpu_hotplug.dep_map) - -+/** -+ * hotplug_pcp - per cpu hotplug descriptor -+ * @unplug: set when pin_current_cpu() needs to sync tasks -+ * @sync_tsk: the task that waits for tasks to finish pinned sections -+ * @refcount: counter of tasks in pinned sections -+ * @grab_lock: set when the tasks entering pinned sections should wait -+ * @synced: notifier for @sync_tsk to tell cpu_down it's finished -+ * @mutex: the mutex to make tasks wait (used when @grab_lock is true) -+ * @mutex_init: zero if the mutex hasn't been initialized yet. -+ * -+ * Although @unplug and @sync_tsk may point to the same task, the @unplug -+ * is used as a flag and still exists after @sync_tsk has exited and -+ * @sync_tsk set to NULL. -+ */ -+struct hotplug_pcp { -+ struct task_struct *unplug; -+ struct task_struct *sync_tsk; -+ int refcount; -+ int grab_lock; -+ struct completion synced; -+ struct completion unplug_wait; -+#ifdef CONFIG_PREEMPT_RT_FULL -+ /* -+ * Note, on PREEMPT_RT, the hotplug lock must save the state of -+ * the task, otherwise the mutex will cause the task to fail -+ * to sleep when required. (Because it's called from migrate_disable()) -+ * -+ * The spinlock_t on PREEMPT_RT is a mutex that saves the task's -+ * state. -+ */ -+ spinlock_t lock; -+#else -+ struct mutex mutex; -+#endif -+ int mutex_init; -+}; ++ ret = 0; + -+#ifdef CONFIG_PREEMPT_RT_FULL -+# define hotplug_lock(hp) rt_spin_lock(&(hp)->lock) -+# define hotplug_unlock(hp) rt_spin_unlock(&(hp)->lock) -+#else -+# define hotplug_lock(hp) mutex_lock(&(hp)->mutex) -+# define hotplug_unlock(hp) mutex_unlock(&(hp)->mutex) -+#endif ++ /* If we exceed the threshold value, we have found a hardware latency */ ++ if (sample > data.threshold || outer_sample > data.threshold) { ++ struct sample s; + -+static DEFINE_PER_CPU(struct hotplug_pcp, hotplug_pcp); ++ ret = 1; + -+/** -+ * pin_current_cpu - Prevent the current cpu from being unplugged ++ data.count++; ++ s.seqnum = data.count; ++ s.duration = sample; ++ s.outer_duration = outer_sample; ++ s.timestamp = CURRENT_TIME; ++ __buffer_add_sample(&s); ++ ++ /* Keep a running maximum ever recorded hardware latency */ ++ if (sample > data.max_sample) ++ data.max_sample = sample; ++ } ++ ++out: ++ return ret; ++} ++ ++/* ++ * kthread_fn - The CPU time sampling/hardware latency detection kernel thread ++ * @unused: A required part of the kthread API. + * -+ * Lightweight version of get_online_cpus() to prevent cpu from being -+ * unplugged when code runs in a migration disabled region. ++ * Used to periodically sample the CPU TSC via a call to get_sample. We ++ * disable interrupts, which does (intentionally) introduce latency since we ++ * need to ensure nothing else might be running (and thus pre-empting). ++ * Obviously this should never be used in production environments. + * -+ * Must be called with preemption disabled (preempt_count = 1)! ++ * Currently this runs on which ever CPU it was scheduled on, but most ++ * real-worald hardware latency situations occur across several CPUs, ++ * but we might later generalize this if we find there are any actualy ++ * systems with alternate SMI delivery or other hardware latencies. + */ -+void pin_current_cpu(void) ++static int kthread_fn(void *unused) +{ -+ struct hotplug_pcp *hp; -+ int force = 0; ++ int ret; ++ u64 interval; + -+retry: -+ hp = this_cpu_ptr(&hotplug_pcp); ++ while (!kthread_should_stop()) { + -+ if (!hp->unplug || hp->refcount || force || preempt_count() > 1 || -+ hp->unplug == current) { -+ hp->refcount++; -+ return; -+ } -+ if (hp->grab_lock) { -+ preempt_enable(); -+ hotplug_lock(hp); -+ hotplug_unlock(hp); -+ } else { -+ preempt_enable(); -+ /* -+ * Try to push this task off of this CPU. -+ */ -+ if (!migrate_me()) { -+ preempt_disable(); -+ hp = this_cpu_ptr(&hotplug_pcp); -+ if (!hp->grab_lock) { -+ /* -+ * Just let it continue it's already pinned -+ * or about to sleep. -+ */ -+ force = 1; -+ goto retry; -+ } -+ preempt_enable(); -+ } ++ mutex_lock(&data.lock); ++ ++ local_irq_disable(); ++ ret = get_sample(); ++ local_irq_enable(); ++ ++ if (ret > 0) ++ wake_up(&data.wq); /* wake up reader(s) */ ++ ++ interval = data.sample_window - data.sample_width; ++ do_div(interval, USEC_PER_MSEC); /* modifies interval value */ ++ ++ mutex_unlock(&data.lock); ++ ++ if (msleep_interruptible(interval)) ++ break; + } -+ preempt_disable(); -+ goto retry; ++ ++ return 0; +} + +/** -+ * unpin_current_cpu - Allow unplug of current cpu ++ * start_kthread - Kick off the hardware latency sampling/detector kthread + * -+ * Must be called with preemption or interrupts disabled! ++ * This starts a kernel thread that will sit and sample the CPU timestamp ++ * counter (TSC or similar) and look for potential hardware latencies. + */ -+void unpin_current_cpu(void) ++static int start_kthread(void) +{ -+ struct hotplug_pcp *hp = this_cpu_ptr(&hotplug_pcp); -+ -+ WARN_ON(hp->refcount <= 0); ++ kthread = kthread_run(kthread_fn, NULL, ++ DRVNAME); ++ if (IS_ERR(kthread)) { ++ pr_err(BANNER "could not start sampling thread\n"); ++ enabled = 0; ++ return -ENOMEM; ++ } + -+ /* This is safe. sync_unplug_thread is pinned to this cpu */ -+ if (!--hp->refcount && hp->unplug && hp->unplug != current) -+ wake_up_process(hp->unplug); ++ return 0; +} + -+static void wait_for_pinned_cpus(struct hotplug_pcp *hp) ++/** ++ * stop_kthread - Inform the hardware latency samping/detector kthread to stop ++ * ++ * This kicks the running hardware latency sampling/detector kernel thread and ++ * tells it to stop sampling now. Use this on unload and at system shutdown. ++ */ ++static int stop_kthread(void) +{ -+ set_current_state(TASK_UNINTERRUPTIBLE); -+ while (hp->refcount) { -+ schedule_preempt_disabled(); -+ set_current_state(TASK_UNINTERRUPTIBLE); -+ } -+} ++ int ret; + -+static int sync_unplug_thread(void *data) -+{ -+ struct hotplug_pcp *hp = data; ++ ret = kthread_stop(kthread); + -+ wait_for_completion(&hp->unplug_wait); -+ preempt_disable(); -+ hp->unplug = current; -+ wait_for_pinned_cpus(hp); ++ return ret; ++} + -+ /* -+ * This thread will synchronize the cpu_down() with threads -+ * that have pinned the CPU. When the pinned CPU count reaches -+ * zero, we inform the cpu_down code to continue to the next step. -+ */ -+ set_current_state(TASK_UNINTERRUPTIBLE); -+ preempt_enable(); -+ complete(&hp->synced); ++/** ++ * __reset_stats - Reset statistics for the hardware latency detector ++ * ++ * We use data to store various statistics and global state. We call this ++ * function in order to reset those when "enable" is toggled on or off, and ++ * also at initialization. Should be called with data.lock held. ++ */ ++static void __reset_stats(void) ++{ ++ data.count = 0; ++ data.max_sample = 0; ++ ring_buffer_reset(ring_buffer); /* flush out old sample entries */ ++} + -+ /* -+ * If all succeeds, the next step will need tasks to wait till -+ * the CPU is offline before continuing. To do this, the grab_lock -+ * is set and tasks going into pin_current_cpu() will block on the -+ * mutex. But we still need to wait for those that are already in -+ * pinned CPU sections. If the cpu_down() failed, the kthread_should_stop() -+ * will kick this thread out. -+ */ -+ while (!hp->grab_lock && !kthread_should_stop()) { -+ schedule(); -+ set_current_state(TASK_UNINTERRUPTIBLE); -+ } ++/** ++ * init_stats - Setup global state statistics for the hardware latency detector ++ * ++ * We use data to store various statistics and global state. We also use ++ * a global ring buffer (ring_buffer) to keep raw samples of detected hardware ++ * induced system latencies. This function initializes these structures and ++ * allocates the global ring buffer also. ++ */ ++static int init_stats(void) ++{ ++ int ret = -ENOMEM; + -+ /* Make sure grab_lock is seen before we see a stale completion */ -+ smp_mb(); ++ mutex_init(&data.lock); ++ init_waitqueue_head(&data.wq); ++ atomic_set(&data.sample_open, 0); + -+ /* -+ * Now just before cpu_down() enters stop machine, we need to make -+ * sure all tasks that are in pinned CPU sections are out, and new -+ * tasks will now grab the lock, keeping them from entering pinned -+ * CPU sections. -+ */ -+ if (!kthread_should_stop()) { -+ preempt_disable(); -+ wait_for_pinned_cpus(hp); -+ preempt_enable(); -+ complete(&hp->synced); -+ } ++ ring_buffer = ring_buffer_alloc(buf_size, BUF_FLAGS); + -+ set_current_state(TASK_UNINTERRUPTIBLE); -+ while (!kthread_should_stop()) { -+ schedule(); -+ set_current_state(TASK_UNINTERRUPTIBLE); -+ } -+ set_current_state(TASK_RUNNING); ++ if (WARN(!ring_buffer, KERN_ERR BANNER ++ "failed to allocate ring buffer!\n")) ++ goto out; + -+ /* -+ * Force this thread off this CPU as it's going down and -+ * we don't want any more work on this CPU. -+ */ -+ current->flags &= ~PF_NO_SETAFFINITY; -+ set_cpus_allowed_ptr(current, cpu_present_mask); -+ migrate_me(); -+ return 0; -+} ++ __reset_stats(); ++ data.threshold = threshold ?: DEFAULT_LAT_THRESHOLD; /* threshold us */ ++ data.sample_window = DEFAULT_SAMPLE_WINDOW; /* window us */ ++ data.sample_width = DEFAULT_SAMPLE_WIDTH; /* width us */ + -+static void __cpu_unplug_sync(struct hotplug_pcp *hp) -+{ -+ wake_up_process(hp->sync_tsk); -+ wait_for_completion(&hp->synced); -+} ++ ret = 0; + -+static void __cpu_unplug_wait(unsigned int cpu) -+{ -+ struct hotplug_pcp *hp = &per_cpu(hotplug_pcp, cpu); ++out: ++ return ret; + -+ complete(&hp->unplug_wait); -+ wait_for_completion(&hp->synced); +} + +/* -+ * Start the sync_unplug_thread on the target cpu and wait for it to -+ * complete. ++ * simple_data_read - Wrapper read function for global state debugfs entries ++ * @filp: The active open file structure for the debugfs "file" ++ * @ubuf: The userspace provided buffer to read value into ++ * @cnt: The maximum number of bytes to read ++ * @ppos: The current "file" position ++ * @entry: The entry to read from ++ * ++ * This function provides a generic read implementation for the global state ++ * "data" structure debugfs filesystem entries. It would be nice to use ++ * simple_attr_read directly, but we need to make sure that the data.lock ++ * is held during the actual read. + */ -+static int cpu_unplug_begin(unsigned int cpu) ++static ssize_t simple_data_read(struct file *filp, char __user *ubuf, ++ size_t cnt, loff_t *ppos, const u64 *entry) +{ -+ struct hotplug_pcp *hp = &per_cpu(hotplug_pcp, cpu); -+ int err; ++ char buf[U64STR_SIZE]; ++ u64 val = 0; ++ int len = 0; + -+ /* Protected by cpu_hotplug.lock */ -+ if (!hp->mutex_init) { -+#ifdef CONFIG_PREEMPT_RT_FULL -+ spin_lock_init(&hp->lock); -+#else -+ mutex_init(&hp->mutex); -+#endif -+ hp->mutex_init = 1; -+ } ++ memset(buf, 0, sizeof(buf)); + -+ /* Inform the scheduler to migrate tasks off this CPU */ -+ tell_sched_cpu_down_begin(cpu); ++ if (!entry) ++ return -EFAULT; + -+ init_completion(&hp->synced); -+ init_completion(&hp->unplug_wait); ++ mutex_lock(&data.lock); ++ val = *entry; ++ mutex_unlock(&data.lock); + -+ hp->sync_tsk = kthread_create(sync_unplug_thread, hp, "sync_unplug/%d", cpu); -+ if (IS_ERR(hp->sync_tsk)) { -+ err = PTR_ERR(hp->sync_tsk); -+ hp->sync_tsk = NULL; -+ return err; -+ } -+ kthread_bind(hp->sync_tsk, cpu); ++ len = snprintf(buf, sizeof(buf), "%llu\n", (unsigned long long)val); ++ ++ return simple_read_from_buffer(ubuf, cnt, ppos, buf, len); + -+ /* -+ * Wait for tasks to get out of the pinned sections, -+ * it's still OK if new tasks enter. Some CPU notifiers will -+ * wait for tasks that are going to enter these sections and -+ * we must not have them block. -+ */ -+ wake_up_process(hp->sync_tsk); -+ return 0; +} + -+static void cpu_unplug_sync(unsigned int cpu) ++/* ++ * simple_data_write - Wrapper write function for global state debugfs entries ++ * @filp: The active open file structure for the debugfs "file" ++ * @ubuf: The userspace provided buffer to write value from ++ * @cnt: The maximum number of bytes to write ++ * @ppos: The current "file" position ++ * @entry: The entry to write to ++ * ++ * This function provides a generic write implementation for the global state ++ * "data" structure debugfs filesystem entries. It would be nice to use ++ * simple_attr_write directly, but we need to make sure that the data.lock ++ * is held during the actual write. ++ */ ++static ssize_t simple_data_write(struct file *filp, const char __user *ubuf, ++ size_t cnt, loff_t *ppos, u64 *entry) +{ -+ struct hotplug_pcp *hp = &per_cpu(hotplug_pcp, cpu); ++ char buf[U64STR_SIZE]; ++ int csize = min(cnt, sizeof(buf)); ++ u64 val = 0; ++ int err = 0; + -+ init_completion(&hp->synced); -+ /* The completion needs to be initialzied before setting grab_lock */ -+ smp_wmb(); ++ memset(buf, '\0', sizeof(buf)); ++ if (copy_from_user(buf, ubuf, csize)) ++ return -EFAULT; + -+ /* Grab the mutex before setting grab_lock */ -+ hotplug_lock(hp); -+ hp->grab_lock = 1; ++ buf[U64STR_SIZE-1] = '\0'; /* just in case */ ++ err = kstrtoull(buf, 10, &val); ++ if (err) ++ return -EINVAL; + -+ /* -+ * The CPU notifiers have been completed. -+ * Wait for tasks to get out of pinned CPU sections and have new -+ * tasks block until the CPU is completely down. -+ */ -+ __cpu_unplug_sync(hp); ++ mutex_lock(&data.lock); ++ *entry = val; ++ mutex_unlock(&data.lock); + -+ /* All done with the sync thread */ -+ kthread_stop(hp->sync_tsk); -+ hp->sync_tsk = NULL; ++ return csize; +} + -+static void cpu_unplug_done(unsigned int cpu) ++/** ++ * debug_count_fopen - Open function for "count" debugfs entry ++ * @inode: The in-kernel inode representation of the debugfs "file" ++ * @filp: The active open file structure for the debugfs "file" ++ * ++ * This function provides an open implementation for the "count" debugfs ++ * interface to the hardware latency detector. ++ */ ++static int debug_count_fopen(struct inode *inode, struct file *filp) +{ -+ struct hotplug_pcp *hp = &per_cpu(hotplug_pcp, cpu); -+ -+ hp->unplug = NULL; -+ /* Let all tasks know cpu unplug is finished before cleaning up */ -+ smp_wmb(); -+ -+ if (hp->sync_tsk) -+ kthread_stop(hp->sync_tsk); ++ return 0; ++} + -+ if (hp->grab_lock) { -+ hotplug_unlock(hp); -+ /* protected by cpu_hotplug.lock */ -+ hp->grab_lock = 0; -+ } -+ tell_sched_cpu_down_done(cpu); ++/** ++ * debug_count_fread - Read function for "count" debugfs entry ++ * @filp: The active open file structure for the debugfs "file" ++ * @ubuf: The userspace provided buffer to read value into ++ * @cnt: The maximum number of bytes to read ++ * @ppos: The current "file" position ++ * ++ * This function provides a read implementation for the "count" debugfs ++ * interface to the hardware latency detector. Can be used to read the ++ * number of latency readings exceeding the configured threshold since ++ * the detector was last reset (e.g. by writing a zero into "count"). ++ */ ++static ssize_t debug_count_fread(struct file *filp, char __user *ubuf, ++ size_t cnt, loff_t *ppos) ++{ ++ return simple_data_read(filp, ubuf, cnt, ppos, &data.count); +} - - void get_online_cpus(void) - { -@@ -349,13 +632,15 @@ - /* Requires cpu_add_remove_lock to be held */ - static int __ref _cpu_down(unsigned int cpu, int tasks_frozen) - { -- int err, nr_calls = 0; -+ int mycpu, err, nr_calls = 0; - void *hcpu = (void *)(long)cpu; - unsigned long mod = tasks_frozen ? CPU_TASKS_FROZEN : 0; - struct take_cpu_down_param tcd_param = { - .mod = mod, - .hcpu = hcpu, - }; -+ cpumask_var_t cpumask; -+ cpumask_var_t cpumask_org; - - if (num_online_cpus() == 1) - return -EBUSY; -@@ -363,7 +648,34 @@ - if (!cpu_online(cpu)) - return -EINVAL; - -+ /* Move the downtaker off the unplug cpu */ -+ if (!alloc_cpumask_var(&cpumask, GFP_KERNEL)) -+ return -ENOMEM; -+ if (!alloc_cpumask_var(&cpumask_org, GFP_KERNEL)) { -+ free_cpumask_var(cpumask); -+ return -ENOMEM; -+ } + -+ cpumask_copy(cpumask_org, tsk_cpus_allowed(current)); -+ cpumask_andnot(cpumask, cpu_online_mask, cpumask_of(cpu)); -+ set_cpus_allowed_ptr(current, cpumask); -+ free_cpumask_var(cpumask); -+ migrate_disable(); -+ mycpu = smp_processor_id(); -+ if (mycpu == cpu) { -+ printk(KERN_ERR "Yuck! Still on unplug CPU\n!"); -+ migrate_enable(); -+ err = -EBUSY; -+ goto restore_cpus; -+ } -+ migrate_enable(); ++/** ++ * debug_count_fwrite - Write function for "count" debugfs entry ++ * @filp: The active open file structure for the debugfs "file" ++ * @ubuf: The user buffer that contains the value to write ++ * @cnt: The maximum number of bytes to write to "file" ++ * @ppos: The current position in the debugfs "file" ++ * ++ * This function provides a write implementation for the "count" debugfs ++ * interface to the hardware latency detector. Can be used to write a ++ * desired value, especially to zero the total count. ++ */ ++static ssize_t debug_count_fwrite(struct file *filp, ++ const char __user *ubuf, ++ size_t cnt, ++ loff_t *ppos) ++{ ++ return simple_data_write(filp, ubuf, cnt, ppos, &data.count); ++} + - cpu_hotplug_begin(); -+ err = cpu_unplug_begin(cpu); -+ if (err) { -+ printk("cpu_unplug_begin(%d) failed\n", cpu); -+ goto out_cancel; -+ } - - err = __cpu_notify(CPU_DOWN_PREPARE | mod, hcpu, -1, &nr_calls); - if (err) { -@@ -389,8 +701,12 @@ - #endif - synchronize_rcu(); - -+ __cpu_unplug_wait(cpu); - smpboot_park_threads(cpu); - -+ /* Notifiers are done. Don't let any more tasks pin this CPU. */ -+ cpu_unplug_sync(cpu); ++/** ++ * debug_enable_fopen - Dummy open function for "enable" debugfs interface ++ * @inode: The in-kernel inode representation of the debugfs "file" ++ * @filp: The active open file structure for the debugfs "file" ++ * ++ * This function provides an open implementation for the "enable" debugfs ++ * interface to the hardware latency detector. ++ */ ++static int debug_enable_fopen(struct inode *inode, struct file *filp) ++{ ++ return 0; ++} + - /* - * So now all preempt/rcu users must observe !cpu_active(). - */ -@@ -427,9 +743,14 @@ - check_for_tasks(cpu); - - out_release: -+ cpu_unplug_done(cpu); -+out_cancel: - cpu_hotplug_done(); - if (!err) - cpu_notify_nofail(CPU_POST_DEAD | mod, hcpu); -+restore_cpus: -+ set_cpus_allowed_ptr(current, cpumask_org); -+ free_cpumask_var(cpumask_org); - return err; - } - -diff -Nur linux-4.1.6.orig/kernel/debug/kdb/kdb_io.c linux-4.1.6/kernel/debug/kdb/kdb_io.c ---- linux-4.1.6.orig/kernel/debug/kdb/kdb_io.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/debug/kdb/kdb_io.c 2015-09-08 23:49:08.089869160 +0200 -@@ -554,7 +554,6 @@ - int linecount; - int colcount; - int logging, saved_loglevel = 0; -- int saved_trap_printk; - int got_printf_lock = 0; - int retlen = 0; - int fnd, len; -@@ -565,8 +564,6 @@ - unsigned long uninitialized_var(flags); - - preempt_disable(); -- saved_trap_printk = kdb_trap_printk; -- kdb_trap_printk = 0; - - /* Serialize kdb_printf if multiple cpus try to write at once. - * But if any cpu goes recursive in kdb, just print the output, -@@ -855,7 +852,6 @@ - } else { - __release(kdb_printf_lock); - } -- kdb_trap_printk = saved_trap_printk; - preempt_enable(); - return retlen; - } -@@ -865,9 +861,11 @@ - va_list ap; - int r; - -+ kdb_trap_printk++; - va_start(ap, fmt); - r = vkdb_printf(KDB_MSGSRC_INTERNAL, fmt, ap); - va_end(ap); -+ kdb_trap_printk--; - - return r; - } -diff -Nur linux-4.1.6.orig/kernel/events/core.c linux-4.1.6/kernel/events/core.c ---- linux-4.1.6.orig/kernel/events/core.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/events/core.c 2015-09-08 23:49:08.089869160 +0200 -@@ -6890,6 +6890,7 @@ - - hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); - hwc->hrtimer.function = perf_swevent_hrtimer; -+ hwc->hrtimer.irqsafe = 1; - - /* - * Since hrtimers have a fixed rate, we can do a static freq->period -diff -Nur linux-4.1.6.orig/kernel/exit.c linux-4.1.6/kernel/exit.c ---- linux-4.1.6.orig/kernel/exit.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/exit.c 2015-09-08 23:49:08.089869160 +0200 -@@ -144,7 +144,7 @@ - * Do this under ->siglock, we can race with another thread - * doing sigqueue_free() if we have SIGQUEUE_PREALLOC signals. - */ -- flush_sigqueue(&tsk->pending); -+ flush_task_sigqueue(tsk); - tsk->sighand = NULL; - spin_unlock(&sighand->siglock); - -diff -Nur linux-4.1.6.orig/kernel/fork.c linux-4.1.6/kernel/fork.c ---- linux-4.1.6.orig/kernel/fork.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/fork.c 2015-09-08 23:49:08.089869160 +0200 -@@ -108,7 +108,7 @@ - - DEFINE_PER_CPU(unsigned long, process_counts) = 0; - --__cacheline_aligned DEFINE_RWLOCK(tasklist_lock); /* outer */ -+DEFINE_RWLOCK(tasklist_lock); /* outer */ - - #ifdef CONFIG_PROVE_RCU - int lockdep_tasklist_lock_is_held(void) -@@ -244,7 +244,9 @@ - if (atomic_dec_and_test(&sig->sigcnt)) - free_signal_struct(sig); - } -- -+#ifdef CONFIG_PREEMPT_RT_BASE -+static -+#endif - void __put_task_struct(struct task_struct *tsk) - { - WARN_ON(!tsk->exit_state); -@@ -260,7 +262,18 @@ - if (!profile_handoff_task(tsk)) - free_task(tsk); - } -+#ifndef CONFIG_PREEMPT_RT_BASE - EXPORT_SYMBOL_GPL(__put_task_struct); -+#else -+void __put_task_struct_cb(struct rcu_head *rhp) ++/** ++ * debug_enable_fread - Read function for "enable" debugfs interface ++ * @filp: The active open file structure for the debugfs "file" ++ * @ubuf: The userspace provided buffer to read value into ++ * @cnt: The maximum number of bytes to read ++ * @ppos: The current "file" position ++ * ++ * This function provides a read implementation for the "enable" debugfs ++ * interface to the hardware latency detector. Can be used to determine ++ * whether the detector is currently enabled ("0\n" or "1\n" returned). ++ */ ++static ssize_t debug_enable_fread(struct file *filp, char __user *ubuf, ++ size_t cnt, loff_t *ppos) +{ -+ struct task_struct *tsk = container_of(rhp, struct task_struct, put_rcu); ++ char buf[4]; + -+ __put_task_struct(tsk); ++ if ((cnt < sizeof(buf)) || (*ppos)) ++ return 0; + ++ buf[0] = enabled ? '1' : '0'; ++ buf[1] = '\n'; ++ buf[2] = '\0'; ++ if (copy_to_user(ubuf, buf, strlen(buf))) ++ return -EFAULT; ++ return *ppos = strlen(buf); +} -+EXPORT_SYMBOL_GPL(__put_task_struct_cb); -+#endif - - void __init __weak arch_task_cache_init(void) { } - -@@ -680,6 +693,19 @@ - } - EXPORT_SYMBOL_GPL(__mmdrop); - -+#ifdef CONFIG_PREEMPT_RT_BASE -+/* -+ * RCU callback for delayed mm drop. Not strictly rcu, but we don't -+ * want another facility to make this work. ++ ++/** ++ * debug_enable_fwrite - Write function for "enable" debugfs interface ++ * @filp: The active open file structure for the debugfs "file" ++ * @ubuf: The user buffer that contains the value to write ++ * @cnt: The maximum number of bytes to write to "file" ++ * @ppos: The current position in the debugfs "file" ++ * ++ * This function provides a write implementation for the "enable" debugfs ++ * interface to the hardware latency detector. Can be used to enable or ++ * disable the detector, which will have the side-effect of possibly ++ * also resetting the global stats and kicking off the measuring ++ * kthread (on an enable) or the converse (upon a disable). + */ -+void __mmdrop_delayed(struct rcu_head *rhp) ++static ssize_t debug_enable_fwrite(struct file *filp, ++ const char __user *ubuf, ++ size_t cnt, ++ loff_t *ppos) +{ -+ struct mm_struct *mm = container_of(rhp, struct mm_struct, delayed_drop); ++ char buf[4]; ++ int csize = min(cnt, sizeof(buf)); ++ long val = 0; ++ int err = 0; + -+ __mmdrop(mm); -+} -+#endif ++ memset(buf, '\0', sizeof(buf)); ++ if (copy_from_user(buf, ubuf, csize)) ++ return -EFAULT; + - /* - * Decrement the use count and release all resources for an mm. - */ -@@ -1214,6 +1240,9 @@ - */ - static void posix_cpu_timers_init(struct task_struct *tsk) - { -+#ifdef CONFIG_PREEMPT_RT_BASE -+ tsk->posix_timer_list = NULL; -+#endif - tsk->cputime_expires.prof_exp = 0; - tsk->cputime_expires.virt_exp = 0; - tsk->cputime_expires.sched_exp = 0; -@@ -1338,6 +1367,7 @@ - spin_lock_init(&p->alloc_lock); - - init_sigpending(&p->pending); -+ p->sigqueue_cache = NULL; - - p->utime = p->stime = p->gtime = 0; - p->utimescaled = p->stimescaled = 0; -@@ -1345,7 +1375,8 @@ - p->prev_cputime.utime = p->prev_cputime.stime = 0; - #endif - #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN -- seqlock_init(&p->vtime_seqlock); -+ raw_spin_lock_init(&p->vtime_lock); -+ seqcount_init(&p->vtime_seq); - p->vtime_snap = 0; - p->vtime_snap_whence = VTIME_SLEEPING; - #endif -@@ -1396,6 +1427,9 @@ - p->hardirq_context = 0; - p->softirq_context = 0; - #endif ++ buf[sizeof(buf)-1] = '\0'; /* just in case */ ++ err = kstrtoul(buf, 10, &val); ++ if (0 != err) ++ return -EINVAL; + -+ p->pagefault_disabled = 0; ++ if (val) { ++ if (enabled) ++ goto unlock; ++ enabled = 1; ++ __reset_stats(); ++ if (start_kthread()) ++ return -EFAULT; ++ } else { ++ if (!enabled) ++ goto unlock; ++ enabled = 0; ++ err = stop_kthread(); ++ if (err) { ++ pr_err(BANNER "cannot stop kthread\n"); ++ return -EFAULT; ++ } ++ wake_up(&data.wq); /* reader(s) should return */ ++ } ++unlock: ++ return csize; ++} + - #ifdef CONFIG_LOCKDEP - p->lockdep_depth = 0; /* no locks held yet */ - p->curr_chain_key = 0; -diff -Nur linux-4.1.6.orig/kernel/futex.c linux-4.1.6/kernel/futex.c ---- linux-4.1.6.orig/kernel/futex.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/futex.c 2015-09-08 23:49:08.089869160 +0200 -@@ -738,7 +738,9 @@ - * task still owns the PI-state: - */ - if (head->next != next) { -+ raw_spin_unlock_irq(&curr->pi_lock); - spin_unlock(&hb->lock); -+ raw_spin_lock_irq(&curr->pi_lock); - continue; - } - -@@ -1090,9 +1092,11 @@ - - /* - * The hash bucket lock must be held when this is called. -- * Afterwards, the futex_q must not be accessed. -+ * Afterwards, the futex_q must not be accessed. Callers -+ * must ensure to later call wake_up_q() for the actual -+ * wakeups to occur. - */ --static void wake_futex(struct futex_q *q) -+static void mark_wake_futex(struct wake_q_head *wake_q, struct futex_q *q) - { - struct task_struct *p = q->task; - -@@ -1100,14 +1104,10 @@ - return; - - /* -- * We set q->lock_ptr = NULL _before_ we wake up the task. If -- * a non-futex wake up happens on another CPU then the task -- * might exit and p would dereference a non-existing task -- * struct. Prevent this by holding a reference on p across the -- * wake up. -+ * Queue the task for later wakeup for after we've released -+ * the hb->lock. wake_q_add() grabs reference to p. - */ -- get_task_struct(p); -- -+ wake_q_add(wake_q, p); - __unqueue_futex(q); - /* - * The waiting task can free the futex_q as soon as -@@ -1117,16 +1117,15 @@ - */ - smp_wmb(); - q->lock_ptr = NULL; -- -- wake_up_state(p, TASK_NORMAL); -- put_task_struct(p); - } - --static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_q *this) -+static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_q *this, -+ struct futex_hash_bucket *hb) - { - struct task_struct *new_owner; - struct futex_pi_state *pi_state = this->pi_state; - u32 uninitialized_var(curval), newval; -+ bool deboost; - int ret = 0; - - if (!pi_state) -@@ -1178,7 +1177,17 @@ - raw_spin_unlock_irq(&new_owner->pi_lock); - - raw_spin_unlock(&pi_state->pi_mutex.wait_lock); -- rt_mutex_unlock(&pi_state->pi_mutex); ++/** ++ * debug_max_fopen - Open function for "max" debugfs entry ++ * @inode: The in-kernel inode representation of the debugfs "file" ++ * @filp: The active open file structure for the debugfs "file" ++ * ++ * This function provides an open implementation for the "max" debugfs ++ * interface to the hardware latency detector. ++ */ ++static int debug_max_fopen(struct inode *inode, struct file *filp) ++{ ++ return 0; ++} + -+ deboost = rt_mutex_futex_unlock(&pi_state->pi_mutex); ++/** ++ * debug_max_fread - Read function for "max" debugfs entry ++ * @filp: The active open file structure for the debugfs "file" ++ * @ubuf: The userspace provided buffer to read value into ++ * @cnt: The maximum number of bytes to read ++ * @ppos: The current "file" position ++ * ++ * This function provides a read implementation for the "max" debugfs ++ * interface to the hardware latency detector. Can be used to determine ++ * the maximum latency value observed since it was last reset. ++ */ ++static ssize_t debug_max_fread(struct file *filp, char __user *ubuf, ++ size_t cnt, loff_t *ppos) ++{ ++ return simple_data_read(filp, ubuf, cnt, ppos, &data.max_sample); ++} + -+ /* -+ * We deboost after dropping hb->lock. That prevents a double -+ * wakeup on RT. -+ */ -+ spin_unlock(&hb->lock); ++/** ++ * debug_max_fwrite - Write function for "max" debugfs entry ++ * @filp: The active open file structure for the debugfs "file" ++ * @ubuf: The user buffer that contains the value to write ++ * @cnt: The maximum number of bytes to write to "file" ++ * @ppos: The current position in the debugfs "file" ++ * ++ * This function provides a write implementation for the "max" debugfs ++ * interface to the hardware latency detector. Can be used to reset the ++ * maximum or set it to some other desired value - if, then, subsequent ++ * measurements exceed this value, the maximum will be updated. ++ */ ++static ssize_t debug_max_fwrite(struct file *filp, ++ const char __user *ubuf, ++ size_t cnt, ++ loff_t *ppos) ++{ ++ return simple_data_write(filp, ubuf, cnt, ppos, &data.max_sample); ++} + -+ if (deboost) -+ rt_mutex_adjust_prio(current); - - return 0; - } -@@ -1217,6 +1226,7 @@ - struct futex_q *this, *next; - union futex_key key = FUTEX_KEY_INIT; - int ret; -+ WAKE_Q(wake_q); - - if (!bitset) - return -EINVAL; -@@ -1244,13 +1254,14 @@ - if (!(this->bitset & bitset)) - continue; - -- wake_futex(this); -+ mark_wake_futex(&wake_q, this); - if (++ret >= nr_wake) - break; - } - } - - spin_unlock(&hb->lock); -+ wake_up_q(&wake_q); - out_put_key: - put_futex_key(&key); - out: -@@ -1269,6 +1280,7 @@ - struct futex_hash_bucket *hb1, *hb2; - struct futex_q *this, *next; - int ret, op_ret; -+ WAKE_Q(wake_q); - - retry: - ret = get_futex_key(uaddr1, flags & FLAGS_SHARED, &key1, VERIFY_READ); -@@ -1320,7 +1332,7 @@ - ret = -EINVAL; - goto out_unlock; - } -- wake_futex(this); -+ mark_wake_futex(&wake_q, this); - if (++ret >= nr_wake) - break; - } -@@ -1334,7 +1346,7 @@ - ret = -EINVAL; - goto out_unlock; - } -- wake_futex(this); -+ mark_wake_futex(&wake_q, this); - if (++op_ret >= nr_wake2) - break; - } -@@ -1344,6 +1356,7 @@ - - out_unlock: - double_unlock_hb(hb1, hb2); -+ wake_up_q(&wake_q); - out_put_keys: - put_futex_key(&key2); - out_put_key1: -@@ -1503,6 +1516,7 @@ - struct futex_pi_state *pi_state = NULL; - struct futex_hash_bucket *hb1, *hb2; - struct futex_q *this, *next; -+ WAKE_Q(wake_q); - - if (requeue_pi) { - /* -@@ -1679,7 +1693,7 @@ - * woken by futex_unlock_pi(). - */ - if (++task_count <= nr_wake && !requeue_pi) { -- wake_futex(this); -+ mark_wake_futex(&wake_q, this); - continue; - } - -@@ -1705,6 +1719,16 @@ - requeue_pi_wake_futex(this, &key2, hb2); - drop_count++; - continue; -+ } else if (ret == -EAGAIN) { -+ /* -+ * Waiter was woken by timeout or -+ * signal and has set pi_blocked_on to -+ * PI_WAKEUP_INPROGRESS before we -+ * tried to enqueue it on the rtmutex. -+ */ -+ this->pi_state = NULL; -+ free_pi_state(pi_state); -+ continue; - } else if (ret) { - /* -EDEADLK */ - this->pi_state = NULL; -@@ -1719,6 +1743,7 @@ - out_unlock: - free_pi_state(pi_state); - double_unlock_hb(hb1, hb2); -+ wake_up_q(&wake_q); - hb_waiters_dec(hb2); - - /* -@@ -2412,13 +2437,26 @@ - */ - match = futex_top_waiter(hb, &key); - if (match) { -- ret = wake_futex_pi(uaddr, uval, match); -+ ret = wake_futex_pi(uaddr, uval, match, hb); + -+ /* -+ * In case of success wake_futex_pi dropped the hash -+ * bucket lock. -+ */ -+ if (!ret) -+ goto out_putkey; ++/** ++ * debug_sample_fopen - An open function for "sample" debugfs interface ++ * @inode: The in-kernel inode representation of this debugfs "file" ++ * @filp: The active open file structure for the debugfs "file" ++ * ++ * This function handles opening the "sample" file within the hardware ++ * latency detector debugfs directory interface. This file is used to read ++ * raw samples from the global ring_buffer and allows the user to see a ++ * running latency history. Can be opened blocking or non-blocking, ++ * affecting whether it behaves as a buffer read pipe, or does not. ++ * Implements simple locking to prevent multiple simultaneous use. ++ */ ++static int debug_sample_fopen(struct inode *inode, struct file *filp) ++{ ++ if (!atomic_add_unless(&data.sample_open, 1, 1)) ++ return -EBUSY; ++ else ++ return 0; ++} + - /* - * The atomic access to the futex value generated a - * pagefault, so retry the user-access and the wakeup: - */ - if (ret == -EFAULT) - goto pi_faulted; ++/** ++ * debug_sample_fread - A read function for "sample" debugfs interface ++ * @filp: The active open file structure for the debugfs "file" ++ * @ubuf: The user buffer that will contain the samples read ++ * @cnt: The maximum bytes to read from the debugfs "file" ++ * @ppos: The current position in the debugfs "file" ++ * ++ * This function handles reading from the "sample" file within the hardware ++ * latency detector debugfs directory interface. This file is used to read ++ * raw samples from the global ring_buffer and allows the user to see a ++ * running latency history. By default this will block pending a new ++ * value written into the sample buffer, unless there are already a ++ * number of value(s) waiting in the buffer, or the sample file was ++ * previously opened in a non-blocking mode of operation. ++ */ ++static ssize_t debug_sample_fread(struct file *filp, char __user *ubuf, ++ size_t cnt, loff_t *ppos) ++{ ++ int len = 0; ++ char buf[64]; ++ struct sample *sample = NULL; + -+ /* -+ * wake_futex_pi has detected invalid state. Tell user -+ * space. -+ */ - goto out_unlock; - } - -@@ -2439,6 +2477,7 @@ - - out_unlock: - spin_unlock(&hb->lock); -+out_putkey: - put_futex_key(&key); - return ret; - -@@ -2549,7 +2588,7 @@ - struct hrtimer_sleeper timeout, *to = NULL; - struct rt_mutex_waiter rt_waiter; - struct rt_mutex *pi_mutex = NULL; -- struct futex_hash_bucket *hb; -+ struct futex_hash_bucket *hb, *hb2; - union futex_key key2 = FUTEX_KEY_INIT; - struct futex_q q = futex_q_init; - int res, ret; -@@ -2574,10 +2613,7 @@ - * The waiter is allocated on our stack, manipulated by the requeue - * code while we sleep on uaddr. - */ -- debug_rt_mutex_init_waiter(&rt_waiter); -- RB_CLEAR_NODE(&rt_waiter.pi_tree_entry); -- RB_CLEAR_NODE(&rt_waiter.tree_entry); -- rt_waiter.task = NULL; -+ rt_mutex_init_waiter(&rt_waiter, false); - - ret = get_futex_key(uaddr2, flags & FLAGS_SHARED, &key2, VERIFY_WRITE); - if (unlikely(ret != 0)) -@@ -2608,20 +2644,55 @@ - /* Queue the futex_q, drop the hb lock, wait for wakeup. */ - futex_wait_queue_me(hb, &q, to); - -- spin_lock(&hb->lock); -- ret = handle_early_requeue_pi_wakeup(hb, &q, &key2, to); -- spin_unlock(&hb->lock); -- if (ret) -- goto out_put_keys; -+ /* -+ * On RT we must avoid races with requeue and trying to block -+ * on two mutexes (hb->lock and uaddr2's rtmutex) by -+ * serializing access to pi_blocked_on with pi_lock. -+ */ -+ raw_spin_lock_irq(¤t->pi_lock); -+ if (current->pi_blocked_on) { -+ /* -+ * We have been requeued or are in the process of -+ * being requeued. -+ */ -+ raw_spin_unlock_irq(¤t->pi_lock); -+ } else { -+ /* -+ * Setting pi_blocked_on to PI_WAKEUP_INPROGRESS -+ * prevents a concurrent requeue from moving us to the -+ * uaddr2 rtmutex. After that we can safely acquire -+ * (and possibly block on) hb->lock. -+ */ -+ current->pi_blocked_on = PI_WAKEUP_INPROGRESS; -+ raw_spin_unlock_irq(¤t->pi_lock); ++ if (!enabled) ++ return 0; + -+ spin_lock(&hb->lock); ++ sample = kzalloc(sizeof(struct sample), GFP_KERNEL); ++ if (!sample) ++ return -ENOMEM; + -+ /* -+ * Clean up pi_blocked_on. We might leak it otherwise -+ * when we succeeded with the hb->lock in the fast -+ * path. -+ */ -+ raw_spin_lock_irq(¤t->pi_lock); -+ current->pi_blocked_on = NULL; -+ raw_spin_unlock_irq(¤t->pi_lock); ++ while (!buffer_get_sample(sample)) { + -+ ret = handle_early_requeue_pi_wakeup(hb, &q, &key2, to); -+ spin_unlock(&hb->lock); -+ if (ret) -+ goto out_put_keys; ++ DEFINE_WAIT(wait); ++ ++ if (filp->f_flags & O_NONBLOCK) { ++ len = -EAGAIN; ++ goto out; ++ } ++ ++ prepare_to_wait(&data.wq, &wait, TASK_INTERRUPTIBLE); ++ schedule(); ++ finish_wait(&data.wq, &wait); ++ ++ if (signal_pending(current)) { ++ len = -EINTR; ++ goto out; ++ } ++ ++ if (!enabled) { /* enable was toggled */ ++ len = 0; ++ goto out; ++ } + } - - /* -- * In order for us to be here, we know our q.key == key2, and since -- * we took the hb->lock above, we also know that futex_requeue() has -- * completed and we no longer have to concern ourselves with a wakeup -- * race with the atomic proxy lock acquisition by the requeue code. The -- * futex_requeue dropped our key1 reference and incremented our key2 -- * reference count. -+ * In order to be here, we have either been requeued, are in -+ * the process of being requeued, or requeue successfully -+ * acquired uaddr2 on our behalf. If pi_blocked_on was -+ * non-null above, we may be racing with a requeue. Do not -+ * rely on q->lock_ptr to be hb2->lock until after blocking on -+ * hb->lock or hb2->lock. The futex_requeue dropped our key1 -+ * reference and incremented our key2 reference count. - */ -+ hb2 = hash_futex(&key2); - - /* Check if the requeue code acquired the second futex for us. */ - if (!q.rt_waiter) { -@@ -2630,9 +2701,10 @@ - * did a lock-steal - fix up the PI-state in that case. - */ - if (q.pi_state && (q.pi_state->owner != current)) { -- spin_lock(q.lock_ptr); -+ spin_lock(&hb2->lock); -+ BUG_ON(&hb2->lock != q.lock_ptr); - ret = fixup_pi_state_owner(uaddr2, &q, current); -- spin_unlock(q.lock_ptr); -+ spin_unlock(&hb2->lock); - } - } else { - /* -@@ -2645,7 +2717,8 @@ - ret = rt_mutex_finish_proxy_lock(pi_mutex, to, &rt_waiter); - debug_rt_mutex_free_waiter(&rt_waiter); - -- spin_lock(q.lock_ptr); -+ spin_lock(&hb2->lock); -+ BUG_ON(&hb2->lock != q.lock_ptr); - /* - * Fixup the pi_state owner and possibly acquire the lock if we - * haven't already. -diff -Nur linux-4.1.6.orig/kernel/irq/handle.c linux-4.1.6/kernel/irq/handle.c ---- linux-4.1.6.orig/kernel/irq/handle.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/irq/handle.c 2015-09-08 23:49:08.093868717 +0200 -@@ -133,6 +133,8 @@ - irqreturn_t - handle_irq_event_percpu(struct irq_desc *desc, struct irqaction *action) - { -+ struct pt_regs *regs = get_irq_regs(); -+ u64 ip = regs ? instruction_pointer(regs) : 0; - irqreturn_t retval = IRQ_NONE; - unsigned int flags = 0, irq = desc->irq_data.irq; - -@@ -173,7 +175,11 @@ - action = action->next; - } while (action); - -- add_interrupt_randomness(irq, flags); -+#ifndef CONFIG_PREEMPT_RT_FULL -+ add_interrupt_randomness(irq, flags, ip); -+#else -+ desc->random_ip = ip; -+#endif - - if (!noirqdebug) - note_interrupt(irq, desc, retval); -diff -Nur linux-4.1.6.orig/kernel/irq/manage.c linux-4.1.6/kernel/irq/manage.c ---- linux-4.1.6.orig/kernel/irq/manage.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/irq/manage.c 2015-09-08 23:49:08.093868717 +0200 -@@ -22,6 +22,7 @@ - #include "internals.h" - - #ifdef CONFIG_IRQ_FORCED_THREADING -+# ifndef CONFIG_PREEMPT_RT_BASE - __read_mostly bool force_irqthreads; - - static int __init setup_forced_irqthreads(char *arg) -@@ -30,6 +31,7 @@ - return 0; - } - early_param("threadirqs", setup_forced_irqthreads); -+# endif - #endif - - static void __synchronize_hardirq(struct irq_desc *desc) -@@ -179,6 +181,62 @@ - irq_get_pending(struct cpumask *mask, struct irq_desc *desc) { } - #endif - -+#ifdef CONFIG_PREEMPT_RT_FULL -+static void _irq_affinity_notify(struct irq_affinity_notify *notify); -+static struct task_struct *set_affinity_helper; -+static LIST_HEAD(affinity_list); -+static DEFINE_RAW_SPINLOCK(affinity_list_lock); + -+static int set_affinity_thread(void *unused) -+{ -+ while (1) { -+ struct irq_affinity_notify *notify; -+ int empty; ++ len = snprintf(buf, sizeof(buf), "%010lu.%010lu\t%llu\t%llu\n", ++ sample->timestamp.tv_sec, ++ sample->timestamp.tv_nsec, ++ sample->duration, ++ sample->outer_duration); + -+ set_current_state(TASK_INTERRUPTIBLE); + -+ raw_spin_lock_irq(&affinity_list_lock); -+ empty = list_empty(&affinity_list); -+ raw_spin_unlock_irq(&affinity_list_lock); ++ /* handling partial reads is more trouble than it's worth */ ++ if (len > cnt) ++ goto out; + -+ if (empty) -+ schedule(); -+ if (kthread_should_stop()) -+ break; -+ set_current_state(TASK_RUNNING); -+try_next: -+ notify = NULL; ++ if (copy_to_user(ubuf, buf, len)) ++ len = -EFAULT; + -+ raw_spin_lock_irq(&affinity_list_lock); -+ if (!list_empty(&affinity_list)) { -+ notify = list_first_entry(&affinity_list, -+ struct irq_affinity_notify, list); -+ list_del_init(¬ify->list); -+ } -+ raw_spin_unlock_irq(&affinity_list_lock); ++out: ++ kfree(sample); ++ return len; ++} ++ ++/** ++ * debug_sample_release - Release function for "sample" debugfs interface ++ * @inode: The in-kernel inode represenation of the debugfs "file" ++ * @filp: The active open file structure for the debugfs "file" ++ * ++ * This function completes the close of the debugfs interface "sample" file. ++ * Frees the sample_open "lock" so that other users may open the interface. ++ */ ++static int debug_sample_release(struct inode *inode, struct file *filp) ++{ ++ atomic_dec(&data.sample_open); + -+ if (!notify) -+ continue; -+ _irq_affinity_notify(notify); -+ goto try_next; -+ } + return 0; +} + -+static void init_helper_thread(void) ++/** ++ * debug_threshold_fopen - Open function for "threshold" debugfs entry ++ * @inode: The in-kernel inode representation of the debugfs "file" ++ * @filp: The active open file structure for the debugfs "file" ++ * ++ * This function provides an open implementation for the "threshold" debugfs ++ * interface to the hardware latency detector. ++ */ ++static int debug_threshold_fopen(struct inode *inode, struct file *filp) +{ -+ if (set_affinity_helper) -+ return; -+ set_affinity_helper = kthread_run(set_affinity_thread, NULL, -+ "affinity-cb"); -+ WARN_ON(IS_ERR(set_affinity_helper)); ++ return 0; +} -+#else + -+static inline void init_helper_thread(void) { } ++/** ++ * debug_threshold_fread - Read function for "threshold" debugfs entry ++ * @filp: The active open file structure for the debugfs "file" ++ * @ubuf: The userspace provided buffer to read value into ++ * @cnt: The maximum number of bytes to read ++ * @ppos: The current "file" position ++ * ++ * This function provides a read implementation for the "threshold" debugfs ++ * interface to the hardware latency detector. It can be used to determine ++ * the current threshold level at which a latency will be recorded in the ++ * global ring buffer, typically on the order of 10us. ++ */ ++static ssize_t debug_threshold_fread(struct file *filp, char __user *ubuf, ++ size_t cnt, loff_t *ppos) ++{ ++ return simple_data_read(filp, ubuf, cnt, ppos, &data.threshold); ++} + -+#endif ++/** ++ * debug_threshold_fwrite - Write function for "threshold" debugfs entry ++ * @filp: The active open file structure for the debugfs "file" ++ * @ubuf: The user buffer that contains the value to write ++ * @cnt: The maximum number of bytes to write to "file" ++ * @ppos: The current position in the debugfs "file" ++ * ++ * This function provides a write implementation for the "threshold" debugfs ++ * interface to the hardware latency detector. It can be used to configure ++ * the threshold level at which any subsequently detected latencies will ++ * be recorded into the global ring buffer. ++ */ ++static ssize_t debug_threshold_fwrite(struct file *filp, ++ const char __user *ubuf, ++ size_t cnt, ++ loff_t *ppos) ++{ ++ int ret; + - int irq_do_set_affinity(struct irq_data *data, const struct cpumask *mask, - bool force) - { -@@ -218,7 +276,17 @@ - - if (desc->affinity_notify) { - kref_get(&desc->affinity_notify->kref); ++ ret = simple_data_write(filp, ubuf, cnt, ppos, &data.threshold); + -+#ifdef CONFIG_PREEMPT_RT_FULL -+ raw_spin_lock(&affinity_list_lock); -+ if (list_empty(&desc->affinity_notify->list)) -+ list_add_tail(&affinity_list, -+ &desc->affinity_notify->list); -+ raw_spin_unlock(&affinity_list_lock); -+ wake_up_process(set_affinity_helper); -+#else - schedule_work(&desc->affinity_notify->work); -+#endif - } - irqd_set(data, IRQD_AFFINITY_SET); - -@@ -256,10 +324,8 @@ - } - EXPORT_SYMBOL_GPL(irq_set_affinity_hint); - --static void irq_affinity_notify(struct work_struct *work) -+static void _irq_affinity_notify(struct irq_affinity_notify *notify) - { -- struct irq_affinity_notify *notify = -- container_of(work, struct irq_affinity_notify, work); - struct irq_desc *desc = irq_to_desc(notify->irq); - cpumask_var_t cpumask; - unsigned long flags; -@@ -281,6 +347,13 @@ - kref_put(¬ify->kref, notify->release); - } - -+static void irq_affinity_notify(struct work_struct *work) -+{ -+ struct irq_affinity_notify *notify = -+ container_of(work, struct irq_affinity_notify, work); -+ _irq_affinity_notify(notify); -+} ++ if (enabled) ++ wake_up_process(kthread); + - /** - * irq_set_affinity_notifier - control notification of IRQ affinity changes - * @irq: Interrupt for which to enable/disable notification -@@ -310,6 +383,8 @@ - notify->irq = irq; - kref_init(¬ify->kref); - INIT_WORK(¬ify->work, irq_affinity_notify); -+ INIT_LIST_HEAD(¬ify->list); -+ init_helper_thread(); - } - - raw_spin_lock_irqsave(&desc->lock, flags); -@@ -825,7 +900,15 @@ - local_bh_disable(); - ret = action->thread_fn(action->irq, action->dev_id); - irq_finalize_oneshot(desc, action); -- local_bh_enable(); -+ /* -+ * Interrupts which have real time requirements can be set up -+ * to avoid softirq processing in the thread handler. This is -+ * safe as these interrupts do not raise soft interrupts. -+ */ -+ if (irq_settings_no_softirq_call(desc)) -+ _local_bh_enable(); -+ else -+ local_bh_enable(); - return ret; - } - -@@ -908,6 +991,12 @@ - if (action_ret == IRQ_HANDLED) - atomic_inc(&desc->threads_handled); - -+#ifdef CONFIG_PREEMPT_RT_FULL -+ migrate_disable(); -+ add_interrupt_randomness(action->irq, 0, -+ desc->random_ip ^ (unsigned long) action); -+ migrate_enable(); -+#endif - wake_threads_waitq(desc); - } - -@@ -1221,6 +1310,9 @@ - irqd_set(&desc->irq_data, IRQD_NO_BALANCING); - } - -+ if (new->flags & IRQF_NO_SOFTIRQ_CALL) -+ irq_settings_set_no_softirq_call(desc); ++ return ret; ++} + - /* Set default affinity mask once everything is setup */ - setup_affinity(irq, desc, mask); - -diff -Nur linux-4.1.6.orig/kernel/irq/settings.h linux-4.1.6/kernel/irq/settings.h ---- linux-4.1.6.orig/kernel/irq/settings.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/irq/settings.h 2015-09-08 23:49:08.093868717 +0200 -@@ -15,6 +15,7 @@ - _IRQ_NESTED_THREAD = IRQ_NESTED_THREAD, - _IRQ_PER_CPU_DEVID = IRQ_PER_CPU_DEVID, - _IRQ_IS_POLLED = IRQ_IS_POLLED, -+ _IRQ_NO_SOFTIRQ_CALL = IRQ_NO_SOFTIRQ_CALL, - _IRQF_MODIFY_MASK = IRQF_MODIFY_MASK, - }; - -@@ -28,6 +29,7 @@ - #define IRQ_NESTED_THREAD GOT_YOU_MORON - #define IRQ_PER_CPU_DEVID GOT_YOU_MORON - #define IRQ_IS_POLLED GOT_YOU_MORON -+#define IRQ_NO_SOFTIRQ_CALL GOT_YOU_MORON - #undef IRQF_MODIFY_MASK - #define IRQF_MODIFY_MASK GOT_YOU_MORON - -@@ -38,6 +40,16 @@ - desc->status_use_accessors |= (set & _IRQF_MODIFY_MASK); - } - -+static inline bool irq_settings_no_softirq_call(struct irq_desc *desc) ++/** ++ * debug_width_fopen - Open function for "width" debugfs entry ++ * @inode: The in-kernel inode representation of the debugfs "file" ++ * @filp: The active open file structure for the debugfs "file" ++ * ++ * This function provides an open implementation for the "width" debugfs ++ * interface to the hardware latency detector. ++ */ ++static int debug_width_fopen(struct inode *inode, struct file *filp) +{ -+ return desc->status_use_accessors & _IRQ_NO_SOFTIRQ_CALL; ++ return 0; +} + -+static inline void irq_settings_set_no_softirq_call(struct irq_desc *desc) ++/** ++ * debug_width_fread - Read function for "width" debugfs entry ++ * @filp: The active open file structure for the debugfs "file" ++ * @ubuf: The userspace provided buffer to read value into ++ * @cnt: The maximum number of bytes to read ++ * @ppos: The current "file" position ++ * ++ * This function provides a read implementation for the "width" debugfs ++ * interface to the hardware latency detector. It can be used to determine ++ * for how many us of the total window us we will actively sample for any ++ * hardware-induced latecy periods. Obviously, it is not possible to ++ * sample constantly and have the system respond to a sample reader, or, ++ * worse, without having the system appear to have gone out to lunch. ++ */ ++static ssize_t debug_width_fread(struct file *filp, char __user *ubuf, ++ size_t cnt, loff_t *ppos) +{ -+ desc->status_use_accessors |= _IRQ_NO_SOFTIRQ_CALL; ++ return simple_data_read(filp, ubuf, cnt, ppos, &data.sample_width); +} + - static inline bool irq_settings_is_per_cpu(struct irq_desc *desc) - { - return desc->status_use_accessors & _IRQ_PER_CPU; -diff -Nur linux-4.1.6.orig/kernel/irq/spurious.c linux-4.1.6/kernel/irq/spurious.c ---- linux-4.1.6.orig/kernel/irq/spurious.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/irq/spurious.c 2015-09-08 23:49:08.093868717 +0200 -@@ -444,6 +444,10 @@ - - static int __init irqfixup_setup(char *str) - { -+#ifdef CONFIG_PREEMPT_RT_BASE -+ pr_warn("irqfixup boot option not supported w/ CONFIG_PREEMPT_RT_BASE\n"); -+ return 1; -+#endif - irqfixup = 1; - printk(KERN_WARNING "Misrouted IRQ fixup support enabled.\n"); - printk(KERN_WARNING "This may impact system performance.\n"); -@@ -456,6 +460,10 @@ - - static int __init irqpoll_setup(char *str) - { -+#ifdef CONFIG_PREEMPT_RT_BASE -+ pr_warn("irqpoll boot option not supported w/ CONFIG_PREEMPT_RT_BASE\n"); -+ return 1; -+#endif - irqfixup = 2; - printk(KERN_WARNING "Misrouted IRQ fixup and polling support " - "enabled\n"); -diff -Nur linux-4.1.6.orig/kernel/irq_work.c linux-4.1.6/kernel/irq_work.c ---- linux-4.1.6.orig/kernel/irq_work.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/irq_work.c 2015-09-08 23:49:08.101867828 +0200 -@@ -17,6 +17,7 @@ - #include - #include - #include -+#include - #include - - -@@ -65,6 +66,8 @@ - */ - bool irq_work_queue_on(struct irq_work *work, int cpu) - { -+ struct llist_head *list; ++/** ++ * debug_width_fwrite - Write function for "width" debugfs entry ++ * @filp: The active open file structure for the debugfs "file" ++ * @ubuf: The user buffer that contains the value to write ++ * @cnt: The maximum number of bytes to write to "file" ++ * @ppos: The current position in the debugfs "file" ++ * ++ * This function provides a write implementation for the "width" debugfs ++ * interface to the hardware latency detector. It can be used to configure ++ * for how many us of the total window us we will actively sample for any ++ * hardware-induced latency periods. Obviously, it is not possible to ++ * sample constantly and have the system respond to a sample reader, or, ++ * worse, without having the system appear to have gone out to lunch. It ++ * is enforced that width is less that the total window size. ++ */ ++static ssize_t debug_width_fwrite(struct file *filp, ++ const char __user *ubuf, ++ size_t cnt, ++ loff_t *ppos) ++{ ++ char buf[U64STR_SIZE]; ++ int csize = min(cnt, sizeof(buf)); ++ u64 val = 0; ++ int err = 0; + - /* All work should have been flushed before going offline */ - WARN_ON_ONCE(cpu_is_offline(cpu)); - -@@ -75,7 +78,12 @@ - if (!irq_work_claim(work)) - return false; - -- if (llist_add(&work->llnode, &per_cpu(raised_list, cpu))) -+ if (IS_ENABLED(CONFIG_PREEMPT_RT_FULL) && !(work->flags & IRQ_WORK_HARD_IRQ)) -+ list = &per_cpu(lazy_list, cpu); -+ else -+ list = &per_cpu(raised_list, cpu); ++ memset(buf, '\0', sizeof(buf)); ++ if (copy_from_user(buf, ubuf, csize)) ++ return -EFAULT; + -+ if (llist_add(&work->llnode, list)) - arch_send_call_function_single_ipi(cpu); - - return true; -@@ -86,6 +94,9 @@ - /* Enqueue the irq work @work on the current CPU */ - bool irq_work_queue(struct irq_work *work) - { -+ struct llist_head *list; -+ bool lazy_work, realtime = IS_ENABLED(CONFIG_PREEMPT_RT_FULL); ++ buf[U64STR_SIZE-1] = '\0'; /* just in case */ ++ err = kstrtoull(buf, 10, &val); ++ if (0 != err) ++ return -EINVAL; + - /* Only queue if not already pending */ - if (!irq_work_claim(work)) - return false; -@@ -93,13 +104,15 @@ - /* Queue the entry and raise the IPI if needed. */ - preempt_disable(); - -- /* If the work is "lazy", handle it from next tick if any */ -- if (work->flags & IRQ_WORK_LAZY) { -- if (llist_add(&work->llnode, this_cpu_ptr(&lazy_list)) && -- tick_nohz_tick_stopped()) -- arch_irq_work_raise(); -- } else { -- if (llist_add(&work->llnode, this_cpu_ptr(&raised_list))) -+ lazy_work = work->flags & IRQ_WORK_LAZY; ++ mutex_lock(&data.lock); ++ if (val < data.sample_window) ++ data.sample_width = val; ++ else { ++ mutex_unlock(&data.lock); ++ return -EINVAL; ++ } ++ mutex_unlock(&data.lock); + -+ if (lazy_work || (realtime && !(work->flags & IRQ_WORK_HARD_IRQ))) -+ list = this_cpu_ptr(&lazy_list); -+ else -+ list = this_cpu_ptr(&raised_list); ++ if (enabled) ++ wake_up_process(kthread); + -+ if (llist_add(&work->llnode, list)) { -+ if (!lazy_work || tick_nohz_tick_stopped()) - arch_irq_work_raise(); - } - -@@ -116,9 +129,8 @@ - raised = this_cpu_ptr(&raised_list); - lazy = this_cpu_ptr(&lazy_list); - -- if (llist_empty(raised) || arch_irq_work_has_interrupt()) -- if (llist_empty(lazy)) -- return false; -+ if (llist_empty(raised) && llist_empty(lazy)) -+ return false; - - /* All work should have been flushed before going offline */ - WARN_ON_ONCE(cpu_is_offline(smp_processor_id())); -@@ -132,7 +144,7 @@ - struct irq_work *work; - struct llist_node *llnode; - -- BUG_ON(!irqs_disabled()); -+ BUG_ON_NONRT(!irqs_disabled()); - - if (llist_empty(list)) - return; -@@ -169,7 +181,16 @@ - void irq_work_run(void) - { - irq_work_run_list(this_cpu_ptr(&raised_list)); -- irq_work_run_list(this_cpu_ptr(&lazy_list)); -+ if (IS_ENABLED(CONFIG_PREEMPT_RT_FULL)) { -+ /* -+ * NOTE: we raise softirq via IPI for safety, -+ * and execute in irq_work_tick() to move the -+ * overhead from hard to soft irq context. -+ */ -+ if (!llist_empty(this_cpu_ptr(&lazy_list))) -+ raise_softirq(TIMER_SOFTIRQ); -+ } else -+ irq_work_run_list(this_cpu_ptr(&lazy_list)); - } - EXPORT_SYMBOL_GPL(irq_work_run); - -diff -Nur linux-4.1.6.orig/kernel/Kconfig.locks linux-4.1.6/kernel/Kconfig.locks ---- linux-4.1.6.orig/kernel/Kconfig.locks 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/Kconfig.locks 2015-09-08 23:49:08.085869602 +0200 -@@ -225,11 +225,11 @@ - - config MUTEX_SPIN_ON_OWNER - def_bool y -- depends on SMP && !DEBUG_MUTEXES && ARCH_SUPPORTS_ATOMIC_RMW -+ depends on SMP && !DEBUG_MUTEXES && ARCH_SUPPORTS_ATOMIC_RMW && !PREEMPT_RT_FULL - - config RWSEM_SPIN_ON_OWNER - def_bool y -- depends on SMP && RWSEM_XCHGADD_ALGORITHM && ARCH_SUPPORTS_ATOMIC_RMW -+ depends on SMP && RWSEM_XCHGADD_ALGORITHM && ARCH_SUPPORTS_ATOMIC_RMW && !PREEMPT_RT_FULL - - config LOCK_SPIN_ON_OWNER - def_bool y -diff -Nur linux-4.1.6.orig/kernel/Kconfig.preempt linux-4.1.6/kernel/Kconfig.preempt ---- linux-4.1.6.orig/kernel/Kconfig.preempt 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/Kconfig.preempt 2015-09-08 23:49:08.085869602 +0200 -@@ -1,3 +1,16 @@ -+config PREEMPT -+ bool -+ select PREEMPT_COUNT ++ return csize; ++} + -+config PREEMPT_RT_BASE -+ bool -+ select PREEMPT ++/** ++ * debug_window_fopen - Open function for "window" debugfs entry ++ * @inode: The in-kernel inode representation of the debugfs "file" ++ * @filp: The active open file structure for the debugfs "file" ++ * ++ * This function provides an open implementation for the "window" debugfs ++ * interface to the hardware latency detector. The window is the total time ++ * in us that will be considered one sample period. Conceptually, windows ++ * occur back-to-back and contain a sample width period during which ++ * actual sampling occurs. ++ */ ++static int debug_window_fopen(struct inode *inode, struct file *filp) ++{ ++ return 0; ++} + -+config HAVE_PREEMPT_LAZY -+ bool ++/** ++ * debug_window_fread - Read function for "window" debugfs entry ++ * @filp: The active open file structure for the debugfs "file" ++ * @ubuf: The userspace provided buffer to read value into ++ * @cnt: The maximum number of bytes to read ++ * @ppos: The current "file" position ++ * ++ * This function provides a read implementation for the "window" debugfs ++ * interface to the hardware latency detector. The window is the total time ++ * in us that will be considered one sample period. Conceptually, windows ++ * occur back-to-back and contain a sample width period during which ++ * actual sampling occurs. Can be used to read the total window size. ++ */ ++static ssize_t debug_window_fread(struct file *filp, char __user *ubuf, ++ size_t cnt, loff_t *ppos) ++{ ++ return simple_data_read(filp, ubuf, cnt, ppos, &data.sample_window); ++} + -+config PREEMPT_LAZY -+ def_bool y if HAVE_PREEMPT_LAZY && PREEMPT_RT_FULL - - choice - prompt "Preemption Model" -@@ -33,9 +46,9 @@ - - Select this if you are building a kernel for a desktop system. - --config PREEMPT -+config PREEMPT__LL - bool "Preemptible Kernel (Low-Latency Desktop)" -- select PREEMPT_COUNT -+ select PREEMPT - select UNINLINE_SPIN_UNLOCK if !ARCH_INLINE_SPIN_UNLOCK - help - This option reduces the latency of the kernel by making -@@ -52,6 +65,22 @@ - embedded system with latency requirements in the milliseconds - range. - -+config PREEMPT_RTB -+ bool "Preemptible Kernel (Basic RT)" -+ select PREEMPT_RT_BASE -+ help -+ This option is basically the same as (Low-Latency Desktop) but -+ enables changes which are preliminary for the full preemptible -+ RT kernel. -+ -+config PREEMPT_RT_FULL -+ bool "Fully Preemptible Kernel (RT)" -+ depends on IRQ_FORCED_THREADING -+ select PREEMPT_RT_BASE -+ select PREEMPT_RCU -+ help -+ All and everything -+ - endchoice - - config PREEMPT_COUNT -diff -Nur linux-4.1.6.orig/kernel/ksysfs.c linux-4.1.6/kernel/ksysfs.c ---- linux-4.1.6.orig/kernel/ksysfs.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/ksysfs.c 2015-09-08 23:49:08.101867828 +0200 -@@ -136,6 +136,15 @@ - - #endif /* CONFIG_KEXEC */ - -+#if defined(CONFIG_PREEMPT_RT_FULL) -+static ssize_t realtime_show(struct kobject *kobj, -+ struct kobj_attribute *attr, char *buf) ++/** ++ * debug_window_fwrite - Write function for "window" debugfs entry ++ * @filp: The active open file structure for the debugfs "file" ++ * @ubuf: The user buffer that contains the value to write ++ * @cnt: The maximum number of bytes to write to "file" ++ * @ppos: The current position in the debugfs "file" ++ * ++ * This function provides a write implementation for the "window" debufds ++ * interface to the hardware latency detetector. The window is the total time ++ * in us that will be considered one sample period. Conceptually, windows ++ * occur back-to-back and contain a sample width period during which ++ * actual sampling occurs. Can be used to write a new total window size. It ++ * is enfoced that any value written must be greater than the sample width ++ * size, or an error results. ++ */ ++static ssize_t debug_window_fwrite(struct file *filp, ++ const char __user *ubuf, ++ size_t cnt, ++ loff_t *ppos) +{ -+ return sprintf(buf, "%d\n", 1); -+} -+KERNEL_ATTR_RO(realtime); -+#endif ++ char buf[U64STR_SIZE]; ++ int csize = min(cnt, sizeof(buf)); ++ u64 val = 0; ++ int err = 0; + - /* whether file capabilities are enabled */ - static ssize_t fscaps_show(struct kobject *kobj, - struct kobj_attribute *attr, char *buf) -@@ -203,6 +212,9 @@ - &vmcoreinfo_attr.attr, - #endif - &rcu_expedited_attr.attr, -+#ifdef CONFIG_PREEMPT_RT_FULL -+ &realtime_attr.attr, -+#endif - NULL - }; - -diff -Nur linux-4.1.6.orig/kernel/locking/lglock.c linux-4.1.6/kernel/locking/lglock.c ---- linux-4.1.6.orig/kernel/locking/lglock.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/locking/lglock.c 2015-09-08 23:49:08.101867828 +0200 -@@ -4,6 +4,15 @@ - #include - #include - -+#ifndef CONFIG_PREEMPT_RT_FULL -+# define lg_lock_ptr arch_spinlock_t -+# define lg_do_lock(l) arch_spin_lock(l) -+# define lg_do_unlock(l) arch_spin_unlock(l) -+#else -+# define lg_lock_ptr struct rt_mutex -+# define lg_do_lock(l) __rt_spin_lock(l) -+# define lg_do_unlock(l) __rt_spin_unlock(l) -+#endif - /* - * Note there is no uninit, so lglocks cannot be defined in - * modules (but it's fine to use them from there) -@@ -12,51 +21,60 @@ - - void lg_lock_init(struct lglock *lg, char *name) - { -+#ifdef CONFIG_PREEMPT_RT_FULL -+ int i; ++ memset(buf, '\0', sizeof(buf)); ++ if (copy_from_user(buf, ubuf, csize)) ++ return -EFAULT; + -+ for_each_possible_cpu(i) { -+ struct rt_mutex *lock = per_cpu_ptr(lg->lock, i); ++ buf[U64STR_SIZE-1] = '\0'; /* just in case */ ++ err = kstrtoull(buf, 10, &val); ++ if (0 != err) ++ return -EINVAL; + -+ rt_mutex_init(lock); ++ mutex_lock(&data.lock); ++ if (data.sample_width < val) ++ data.sample_window = val; ++ else { ++ mutex_unlock(&data.lock); ++ return -EINVAL; + } -+#endif - LOCKDEP_INIT_MAP(&lg->lock_dep_map, name, &lg->lock_key, 0); - } - EXPORT_SYMBOL(lg_lock_init); - - void lg_local_lock(struct lglock *lg) - { -- arch_spinlock_t *lock; -+ lg_lock_ptr *lock; - -- preempt_disable(); -+ migrate_disable(); - lock_acquire_shared(&lg->lock_dep_map, 0, 0, NULL, _RET_IP_); - lock = this_cpu_ptr(lg->lock); -- arch_spin_lock(lock); -+ lg_do_lock(lock); - } - EXPORT_SYMBOL(lg_local_lock); - - void lg_local_unlock(struct lglock *lg) - { -- arch_spinlock_t *lock; -+ lg_lock_ptr *lock; - - lock_release(&lg->lock_dep_map, 1, _RET_IP_); - lock = this_cpu_ptr(lg->lock); -- arch_spin_unlock(lock); -- preempt_enable(); -+ lg_do_unlock(lock); -+ migrate_enable(); - } - EXPORT_SYMBOL(lg_local_unlock); - - void lg_local_lock_cpu(struct lglock *lg, int cpu) - { -- arch_spinlock_t *lock; -+ lg_lock_ptr *lock; - -- preempt_disable(); -+ preempt_disable_nort(); - lock_acquire_shared(&lg->lock_dep_map, 0, 0, NULL, _RET_IP_); - lock = per_cpu_ptr(lg->lock, cpu); -- arch_spin_lock(lock); -+ lg_do_lock(lock); - } - EXPORT_SYMBOL(lg_local_lock_cpu); - - void lg_local_unlock_cpu(struct lglock *lg, int cpu) - { -- arch_spinlock_t *lock; -+ lg_lock_ptr *lock; - - lock_release(&lg->lock_dep_map, 1, _RET_IP_); - lock = per_cpu_ptr(lg->lock, cpu); -- arch_spin_unlock(lock); -- preempt_enable(); -+ lg_do_unlock(lock); -+ preempt_enable_nort(); - } - EXPORT_SYMBOL(lg_local_unlock_cpu); - -@@ -64,12 +82,12 @@ - { - int i; - -- preempt_disable(); -+ preempt_disable_nort(); - lock_acquire_exclusive(&lg->lock_dep_map, 0, 0, NULL, _RET_IP_); - for_each_possible_cpu(i) { -- arch_spinlock_t *lock; -+ lg_lock_ptr *lock; - lock = per_cpu_ptr(lg->lock, i); -- arch_spin_lock(lock); -+ lg_do_lock(lock); - } - } - EXPORT_SYMBOL(lg_global_lock); -@@ -80,10 +98,35 @@ - - lock_release(&lg->lock_dep_map, 1, _RET_IP_); - for_each_possible_cpu(i) { -- arch_spinlock_t *lock; -+ lg_lock_ptr *lock; - lock = per_cpu_ptr(lg->lock, i); -- arch_spin_unlock(lock); -+ lg_do_unlock(lock); - } -- preempt_enable(); -+ preempt_enable_nort(); - } - EXPORT_SYMBOL(lg_global_unlock); ++ mutex_unlock(&data.lock); ++ ++ return csize; ++} + -+#ifdef CONFIG_PREEMPT_RT_FULL +/* -+ * HACK: If you use this, you get to keep the pieces. -+ * Used in queue_stop_cpus_work() when stop machinery -+ * is called from inactive CPU, so we can't schedule. ++ * Function pointers for the "count" debugfs file operations + */ -+# define lg_do_trylock_relax(l) \ -+ do { \ -+ while (!__rt_spin_trylock(l)) \ -+ cpu_relax(); \ -+ } while (0) ++static const struct file_operations count_fops = { ++ .open = debug_count_fopen, ++ .read = debug_count_fread, ++ .write = debug_count_fwrite, ++ .owner = THIS_MODULE, ++}; + -+void lg_global_trylock_relax(struct lglock *lg) -+{ -+ int i; ++/* ++ * Function pointers for the "enable" debugfs file operations ++ */ ++static const struct file_operations enable_fops = { ++ .open = debug_enable_fopen, ++ .read = debug_enable_fread, ++ .write = debug_enable_fwrite, ++ .owner = THIS_MODULE, ++}; + -+ lock_acquire_exclusive(&lg->lock_dep_map, 0, 0, NULL, _RET_IP_); -+ for_each_possible_cpu(i) { -+ lg_lock_ptr *lock; -+ lock = per_cpu_ptr(lg->lock, i); -+ lg_do_trylock_relax(lock); -+ } -+} -+#endif -diff -Nur linux-4.1.6.orig/kernel/locking/lockdep.c linux-4.1.6/kernel/locking/lockdep.c ---- linux-4.1.6.orig/kernel/locking/lockdep.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/locking/lockdep.c 2015-09-08 23:49:08.101867828 +0200 -@@ -3563,6 +3563,7 @@ - } - } - -+#ifndef CONFIG_PREEMPT_RT_FULL - /* - * We dont accurately track softirq state in e.g. - * hardirq contexts (such as on 4KSTACKS), so only -@@ -3577,6 +3578,7 @@ - DEBUG_LOCKS_WARN_ON(!current->softirqs_enabled); - } - } -+#endif - - if (!debug_locks) - print_irqtrace_events(current); -diff -Nur linux-4.1.6.orig/kernel/locking/locktorture.c linux-4.1.6/kernel/locking/locktorture.c ---- linux-4.1.6.orig/kernel/locking/locktorture.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/locking/locktorture.c 2015-09-08 23:49:08.101867828 +0200 -@@ -24,7 +24,6 @@ - #include - #include - #include --#include - #include - #include - #include -diff -Nur linux-4.1.6.orig/kernel/locking/Makefile linux-4.1.6/kernel/locking/Makefile ---- linux-4.1.6.orig/kernel/locking/Makefile 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/locking/Makefile 2015-09-08 23:49:08.101867828 +0200 -@@ -1,5 +1,5 @@ - --obj-y += mutex.o semaphore.o rwsem.o -+obj-y += semaphore.o - - ifdef CONFIG_FUNCTION_TRACER - CFLAGS_REMOVE_lockdep.o = $(CC_FLAGS_FTRACE) -@@ -8,7 +8,11 @@ - CFLAGS_REMOVE_rtmutex-debug.o = $(CC_FLAGS_FTRACE) - endif - -+ifneq ($(CONFIG_PREEMPT_RT_FULL),y) -+obj-y += mutex.o - obj-$(CONFIG_DEBUG_MUTEXES) += mutex-debug.o -+obj-y += rwsem.o -+endif - obj-$(CONFIG_LOCKDEP) += lockdep.o - ifeq ($(CONFIG_PROC_FS),y) - obj-$(CONFIG_LOCKDEP) += lockdep_proc.o -@@ -22,8 +26,11 @@ - obj-$(CONFIG_RT_MUTEX_TESTER) += rtmutex-tester.o - obj-$(CONFIG_DEBUG_SPINLOCK) += spinlock.o - obj-$(CONFIG_DEBUG_SPINLOCK) += spinlock_debug.o -+ifneq ($(CONFIG_PREEMPT_RT_FULL),y) - obj-$(CONFIG_RWSEM_GENERIC_SPINLOCK) += rwsem-spinlock.o - obj-$(CONFIG_RWSEM_XCHGADD_ALGORITHM) += rwsem-xadd.o -+endif - obj-$(CONFIG_PERCPU_RWSEM) += percpu-rwsem.o -+obj-$(CONFIG_PREEMPT_RT_FULL) += rt.o - obj-$(CONFIG_QUEUE_RWLOCK) += qrwlock.o - obj-$(CONFIG_LOCK_TORTURE_TEST) += locktorture.o -diff -Nur linux-4.1.6.orig/kernel/locking/rt.c linux-4.1.6/kernel/locking/rt.c ---- linux-4.1.6.orig/kernel/locking/rt.c 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.6/kernel/locking/rt.c 2015-09-08 23:49:08.101867828 +0200 -@@ -0,0 +1,461 @@ +/* -+ * kernel/rt.c -+ * -+ * Real-Time Preemption Support -+ * -+ * started by Ingo Molnar: -+ * -+ * Copyright (C) 2004-2006 Red Hat, Inc., Ingo Molnar -+ * Copyright (C) 2006, Timesys Corp., Thomas Gleixner -+ * -+ * historic credit for proving that Linux spinlocks can be implemented via -+ * RT-aware mutexes goes to many people: The Pmutex project (Dirk Grambow -+ * and others) who prototyped it on 2.4 and did lots of comparative -+ * research and analysis; TimeSys, for proving that you can implement a -+ * fully preemptible kernel via the use of IRQ threading and mutexes; -+ * Bill Huey for persuasively arguing on lkml that the mutex model is the -+ * right one; and to MontaVista, who ported pmutexes to 2.6. -+ * -+ * This code is a from-scratch implementation and is not based on pmutexes, -+ * but the idea of converting spinlocks to mutexes is used here too. -+ * -+ * lock debugging, locking tree, deadlock detection: -+ * -+ * Copyright (C) 2004, LynuxWorks, Inc., Igor Manyilov, Bill Huey -+ * Released under the General Public License (GPL). -+ * -+ * Includes portions of the generic R/W semaphore implementation from: -+ * -+ * Copyright (c) 2001 David Howells (dhowells@redhat.com). -+ * - Derived partially from idea by Andrea Arcangeli -+ * - Derived also from comments by Linus -+ * -+ * Pending ownership of locks and ownership stealing: -+ * -+ * Copyright (C) 2005, Kihon Technologies Inc., Steven Rostedt -+ * -+ * (also by Steven Rostedt) -+ * - Converted single pi_lock to individual task locks. -+ * -+ * By Esben Nielsen: -+ * Doing priority inheritance with help of the scheduler. -+ * -+ * Copyright (C) 2006, Timesys Corp., Thomas Gleixner -+ * - major rework based on Esben Nielsens initial patch -+ * - replaced thread_info references by task_struct refs -+ * - removed task->pending_owner dependency -+ * - BKL drop/reacquire for semaphore style locks to avoid deadlocks -+ * in the scheduler return path as discussed with Steven Rostedt -+ * -+ * Copyright (C) 2006, Kihon Technologies Inc. -+ * Steven Rostedt -+ * - debugged and patched Thomas Gleixner's rework. -+ * - added back the cmpxchg to the rework. -+ * - turned atomic require back on for SMP. ++ * Function pointers for the "max" debugfs file operations + */ ++static const struct file_operations max_fops = { ++ .open = debug_max_fopen, ++ .read = debug_max_fread, ++ .write = debug_max_fwrite, ++ .owner = THIS_MODULE, ++}; + -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include ++/* ++ * Function pointers for the "sample" debugfs file operations ++ */ ++static const struct file_operations sample_fops = { ++ .open = debug_sample_fopen, ++ .read = debug_sample_fread, ++ .release = debug_sample_release, ++ .owner = THIS_MODULE, ++}; + -+#include "rtmutex_common.h" ++/* ++ * Function pointers for the "threshold" debugfs file operations ++ */ ++static const struct file_operations threshold_fops = { ++ .open = debug_threshold_fopen, ++ .read = debug_threshold_fread, ++ .write = debug_threshold_fwrite, ++ .owner = THIS_MODULE, ++}; + +/* -+ * struct mutex functions ++ * Function pointers for the "width" debugfs file operations + */ -+void __mutex_do_init(struct mutex *mutex, const char *name, -+ struct lock_class_key *key) -+{ -+#ifdef CONFIG_DEBUG_LOCK_ALLOC -+ /* -+ * Make sure we are not reinitializing a held lock: -+ */ -+ debug_check_no_locks_freed((void *)mutex, sizeof(*mutex)); -+ lockdep_init_map(&mutex->dep_map, name, key, 0); -+#endif -+ mutex->lock.save_state = 0; -+} -+EXPORT_SYMBOL(__mutex_do_init); ++static const struct file_operations width_fops = { ++ .open = debug_width_fopen, ++ .read = debug_width_fread, ++ .write = debug_width_fwrite, ++ .owner = THIS_MODULE, ++}; + -+void __lockfunc _mutex_lock(struct mutex *lock) -+{ -+ mutex_acquire(&lock->dep_map, 0, 0, _RET_IP_); -+ rt_mutex_lock(&lock->lock); -+} -+EXPORT_SYMBOL(_mutex_lock); ++/* ++ * Function pointers for the "window" debugfs file operations ++ */ ++static const struct file_operations window_fops = { ++ .open = debug_window_fopen, ++ .read = debug_window_fread, ++ .write = debug_window_fwrite, ++ .owner = THIS_MODULE, ++}; + -+int __lockfunc _mutex_lock_interruptible(struct mutex *lock) ++/** ++ * init_debugfs - A function to initialize the debugfs interface files ++ * ++ * This function creates entries in debugfs for "hwlat_detector", including ++ * files to read values from the detector, current samples, and the ++ * maximum sample that has been captured since the hardware latency ++ * dectector was started. ++ */ ++static int init_debugfs(void) +{ -+ int ret; -+ -+ mutex_acquire(&lock->dep_map, 0, 0, _RET_IP_); -+ ret = rt_mutex_lock_interruptible(&lock->lock); -+ if (ret) -+ mutex_release(&lock->dep_map, 1, _RET_IP_); -+ return ret; -+} -+EXPORT_SYMBOL(_mutex_lock_interruptible); ++ int ret = -ENOMEM; + -+int __lockfunc _mutex_lock_killable(struct mutex *lock) -+{ -+ int ret; ++ debug_dir = debugfs_create_dir(DRVNAME, NULL); ++ if (!debug_dir) ++ goto err_debug_dir; + -+ mutex_acquire(&lock->dep_map, 0, 0, _RET_IP_); -+ ret = rt_mutex_lock_killable(&lock->lock); -+ if (ret) -+ mutex_release(&lock->dep_map, 1, _RET_IP_); -+ return ret; -+} -+EXPORT_SYMBOL(_mutex_lock_killable); ++ debug_sample = debugfs_create_file("sample", 0444, ++ debug_dir, NULL, ++ &sample_fops); ++ if (!debug_sample) ++ goto err_sample; + -+#ifdef CONFIG_DEBUG_LOCK_ALLOC -+void __lockfunc _mutex_lock_nested(struct mutex *lock, int subclass) -+{ -+ mutex_acquire_nest(&lock->dep_map, subclass, 0, NULL, _RET_IP_); -+ rt_mutex_lock(&lock->lock); -+} -+EXPORT_SYMBOL(_mutex_lock_nested); ++ debug_count = debugfs_create_file("count", 0444, ++ debug_dir, NULL, ++ &count_fops); ++ if (!debug_count) ++ goto err_count; + -+void __lockfunc _mutex_lock_nest_lock(struct mutex *lock, struct lockdep_map *nest) -+{ -+ mutex_acquire_nest(&lock->dep_map, 0, 0, nest, _RET_IP_); -+ rt_mutex_lock(&lock->lock); -+} -+EXPORT_SYMBOL(_mutex_lock_nest_lock); ++ debug_max = debugfs_create_file("max", 0444, ++ debug_dir, NULL, ++ &max_fops); ++ if (!debug_max) ++ goto err_max; + -+int __lockfunc _mutex_lock_interruptible_nested(struct mutex *lock, int subclass) -+{ -+ int ret; ++ debug_sample_window = debugfs_create_file("window", 0644, ++ debug_dir, NULL, ++ &window_fops); ++ if (!debug_sample_window) ++ goto err_window; + -+ mutex_acquire_nest(&lock->dep_map, subclass, 0, NULL, _RET_IP_); -+ ret = rt_mutex_lock_interruptible(&lock->lock); -+ if (ret) -+ mutex_release(&lock->dep_map, 1, _RET_IP_); -+ return ret; -+} -+EXPORT_SYMBOL(_mutex_lock_interruptible_nested); ++ debug_sample_width = debugfs_create_file("width", 0644, ++ debug_dir, NULL, ++ &width_fops); ++ if (!debug_sample_width) ++ goto err_width; + -+int __lockfunc _mutex_lock_killable_nested(struct mutex *lock, int subclass) -+{ -+ int ret; -+ -+ mutex_acquire(&lock->dep_map, subclass, 0, _RET_IP_); -+ ret = rt_mutex_lock_killable(&lock->lock); -+ if (ret) -+ mutex_release(&lock->dep_map, 1, _RET_IP_); -+ return ret; -+} -+EXPORT_SYMBOL(_mutex_lock_killable_nested); -+#endif ++ debug_threshold = debugfs_create_file("threshold", 0644, ++ debug_dir, NULL, ++ &threshold_fops); ++ if (!debug_threshold) ++ goto err_threshold; + -+int __lockfunc _mutex_trylock(struct mutex *lock) -+{ -+ int ret = rt_mutex_trylock(&lock->lock); ++ debug_enable = debugfs_create_file("enable", 0644, ++ debug_dir, &enabled, ++ &enable_fops); ++ if (!debug_enable) ++ goto err_enable; + -+ if (ret) -+ mutex_acquire(&lock->dep_map, 0, 1, _RET_IP_); ++ else { ++ ret = 0; ++ goto out; ++ } + ++err_enable: ++ debugfs_remove(debug_threshold); ++err_threshold: ++ debugfs_remove(debug_sample_width); ++err_width: ++ debugfs_remove(debug_sample_window); ++err_window: ++ debugfs_remove(debug_max); ++err_max: ++ debugfs_remove(debug_count); ++err_count: ++ debugfs_remove(debug_sample); ++err_sample: ++ debugfs_remove(debug_dir); ++err_debug_dir: ++out: + return ret; +} -+EXPORT_SYMBOL(_mutex_trylock); + -+void __lockfunc _mutex_unlock(struct mutex *lock) ++/** ++ * free_debugfs - A function to cleanup the debugfs file interface ++ */ ++static void free_debugfs(void) +{ -+ mutex_release(&lock->dep_map, 1, _RET_IP_); -+ rt_mutex_unlock(&lock->lock); ++ /* could also use a debugfs_remove_recursive */ ++ debugfs_remove(debug_enable); ++ debugfs_remove(debug_threshold); ++ debugfs_remove(debug_sample_width); ++ debugfs_remove(debug_sample_window); ++ debugfs_remove(debug_max); ++ debugfs_remove(debug_count); ++ debugfs_remove(debug_sample); ++ debugfs_remove(debug_dir); +} -+EXPORT_SYMBOL(_mutex_unlock); + -+/* -+ * rwlock_t functions ++/** ++ * detector_init - Standard module initialization code + */ -+int __lockfunc rt_write_trylock(rwlock_t *rwlock) ++static int detector_init(void) +{ -+ int ret; ++ int ret = -ENOMEM; + -+ migrate_disable(); -+ ret = rt_mutex_trylock(&rwlock->lock); -+ if (ret) -+ rwlock_acquire(&rwlock->dep_map, 0, 1, _RET_IP_); -+ else -+ migrate_enable(); ++ pr_info(BANNER "version %s\n", VERSION); + ++ ret = init_stats(); ++ if (0 != ret) ++ goto out; ++ ++ ret = init_debugfs(); ++ if (0 != ret) ++ goto err_stats; ++ ++ if (enabled) ++ ret = start_kthread(); ++ ++ goto out; ++ ++err_stats: ++ ring_buffer_free(ring_buffer); ++out: + return ret; ++ +} -+EXPORT_SYMBOL(rt_write_trylock); + -+int __lockfunc rt_write_trylock_irqsave(rwlock_t *rwlock, unsigned long *flags) ++/** ++ * detector_exit - Standard module cleanup code ++ */ ++static void detector_exit(void) +{ -+ int ret; ++ int err; ++ ++ if (enabled) { ++ enabled = 0; ++ err = stop_kthread(); ++ if (err) ++ pr_err(BANNER "cannot stop kthread\n"); ++ } ++ ++ free_debugfs(); ++ ring_buffer_free(ring_buffer); /* free up the ring buffer */ + -+ *flags = 0; -+ ret = rt_write_trylock(rwlock); -+ return ret; +} -+EXPORT_SYMBOL(rt_write_trylock_irqsave); + -+int __lockfunc rt_read_trylock(rwlock_t *rwlock) -+{ -+ struct rt_mutex *lock = &rwlock->lock; -+ int ret = 1; ++module_init(detector_init); ++module_exit(detector_exit); +diff -Nur linux-4.1.10.orig/drivers/misc/Kconfig linux-4.1.10/drivers/misc/Kconfig +--- linux-4.1.10.orig/drivers/misc/Kconfig 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/misc/Kconfig 2015-10-07 18:00:08.000000000 +0200 +@@ -54,6 +54,7 @@ + config ATMEL_TCLIB + bool "Atmel AT32/AT91 Timer/Counter Library" + depends on (AVR32 || ARCH_AT91) ++ default y if PREEMPT_RT_FULL + help + Select this if you want a library to allocate the Timer/Counter + blocks found on many Atmel processors. This facilitates using +@@ -69,8 +70,7 @@ + are combined to make a single 32-bit timer. + + When GENERIC_CLOCKEVENTS is defined, the third timer channel +- may be used as a clock event device supporting oneshot mode +- (delays of up to two seconds) based on the 32 KiHz clock. ++ may be used as a clock event device supporting oneshot mode. + + config ATMEL_TCB_CLKSRC_BLOCK + int +@@ -84,6 +84,15 @@ + TC can be used for other purposes, such as PWM generation and + interval timing. + ++config ATMEL_TCB_CLKSRC_USE_SLOW_CLOCK ++ bool "TC Block use 32 KiHz clock" ++ depends on ATMEL_TCB_CLKSRC ++ default y if !PREEMPT_RT_FULL ++ help ++ Select this to use 32 KiHz base clock rate as TC block clock ++ source for clock events. + -+ /* -+ * recursive read locks succeed when current owns the lock, -+ * but not when read_depth == 0 which means that the lock is -+ * write locked. -+ */ -+ if (rt_mutex_owner(lock) != current) { -+ migrate_disable(); -+ ret = rt_mutex_trylock(lock); -+ if (ret) -+ rwlock_acquire(&rwlock->dep_map, 0, 1, _RET_IP_); -+ else -+ migrate_enable(); + -+ } else if (!rwlock->read_depth) { -+ ret = 0; -+ } + config DUMMY_IRQ + tristate "Dummy IRQ handler" + default n +@@ -113,6 +122,35 @@ + for information on the specific driver level and support statement + for your IBM server. + ++config HWLAT_DETECTOR ++ tristate "Testing module to detect hardware-induced latencies" ++ depends on DEBUG_FS ++ depends on RING_BUFFER ++ default m ++ ---help--- ++ A simple hardware latency detector. Use this module to detect ++ large latencies introduced by the behavior of the underlying ++ system firmware external to Linux. We do this using periodic ++ use of stop_machine to grab all available CPUs and measure ++ for unexplainable gaps in the CPU timestamp counter(s). By ++ default, the module is not enabled until the "enable" file ++ within the "hwlat_detector" debugfs directory is toggled. + -+ if (ret) -+ rwlock->read_depth++; ++ This module is often used to detect SMI (System Management ++ Interrupts) on x86 systems, though is not x86 specific. To ++ this end, we default to using a sample window of 1 second, ++ during which we will sample for 0.5 seconds. If an SMI or ++ similar event occurs during that time, it is recorded ++ into an 8K samples global ring buffer until retreived. + -+ return ret; ++ WARNING: This software should never be enabled (it can be built ++ but should not be turned on after it is loaded) in a production ++ environment where high latencies are a concern since the ++ sampling mechanism actually introduces latencies for ++ regular tasks while the CPU(s) are being held. ++ ++ If unsure, say N ++ + config PHANTOM + tristate "Sensable PHANToM (PCI)" + depends on PCI +diff -Nur linux-4.1.10.orig/drivers/misc/Makefile linux-4.1.10/drivers/misc/Makefile +--- linux-4.1.10.orig/drivers/misc/Makefile 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/misc/Makefile 2015-10-07 18:00:08.000000000 +0200 +@@ -38,6 +38,7 @@ + obj-$(CONFIG_HMC6352) += hmc6352.o + obj-y += eeprom/ + obj-y += cb710/ ++obj-$(CONFIG_HWLAT_DETECTOR) += hwlat_detector.o + obj-$(CONFIG_SPEAR13XX_PCIE_GADGET) += spear13xx_pcie_gadget.o + obj-$(CONFIG_VMWARE_BALLOON) += vmw_balloon.o + obj-$(CONFIG_ARM_CHARLCD) += arm-charlcd.o +diff -Nur linux-4.1.10.orig/drivers/mmc/host/mmci.c linux-4.1.10/drivers/mmc/host/mmci.c +--- linux-4.1.10.orig/drivers/mmc/host/mmci.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/mmc/host/mmci.c 2015-10-07 18:00:08.000000000 +0200 +@@ -1155,15 +1155,12 @@ + struct sg_mapping_iter *sg_miter = &host->sg_miter; + struct variant_data *variant = host->variant; + void __iomem *base = host->base; +- unsigned long flags; + u32 status; + + status = readl(base + MMCISTATUS); + + dev_dbg(mmc_dev(host->mmc), "irq1 (pio) %08x\n", status); + +- local_irq_save(flags); +- + do { + unsigned int remain, len; + char *buffer; +@@ -1203,8 +1200,6 @@ + + sg_miter_stop(sg_miter); + +- local_irq_restore(flags); +- + /* + * If we have less than the fifo 'half-full' threshold to transfer, + * trigger a PIO interrupt as soon as any data is available. +diff -Nur linux-4.1.10.orig/drivers/net/ethernet/3com/3c59x.c linux-4.1.10/drivers/net/ethernet/3com/3c59x.c +--- linux-4.1.10.orig/drivers/net/ethernet/3com/3c59x.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/net/ethernet/3com/3c59x.c 2015-10-07 18:00:08.000000000 +0200 +@@ -842,9 +842,9 @@ + { + struct vortex_private *vp = netdev_priv(dev); + unsigned long flags; +- local_irq_save(flags); ++ local_irq_save_nort(flags); + (vp->full_bus_master_rx ? boomerang_interrupt:vortex_interrupt)(dev->irq,dev); +- local_irq_restore(flags); ++ local_irq_restore_nort(flags); + } + #endif + +@@ -1916,12 +1916,12 @@ + * Block interrupts because vortex_interrupt does a bare spin_lock() + */ + unsigned long flags; +- local_irq_save(flags); ++ local_irq_save_nort(flags); + if (vp->full_bus_master_tx) + boomerang_interrupt(dev->irq, dev); + else + vortex_interrupt(dev->irq, dev); +- local_irq_restore(flags); ++ local_irq_restore_nort(flags); + } + } + +diff -Nur linux-4.1.10.orig/drivers/net/ethernet/atheros/atl1c/atl1c_main.c linux-4.1.10/drivers/net/ethernet/atheros/atl1c/atl1c_main.c +--- linux-4.1.10.orig/drivers/net/ethernet/atheros/atl1c/atl1c_main.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/net/ethernet/atheros/atl1c/atl1c_main.c 2015-10-07 18:00:08.000000000 +0200 +@@ -2213,11 +2213,7 @@ + } + + tpd_req = atl1c_cal_tpd_req(skb); +- if (!spin_trylock_irqsave(&adapter->tx_lock, flags)) { +- if (netif_msg_pktdata(adapter)) +- dev_info(&adapter->pdev->dev, "tx locked\n"); +- return NETDEV_TX_LOCKED; +- } ++ spin_lock_irqsave(&adapter->tx_lock, flags); + + if (atl1c_tpd_avail(adapter, type) < tpd_req) { + /* no enough descriptor, just stop queue */ +diff -Nur linux-4.1.10.orig/drivers/net/ethernet/atheros/atl1e/atl1e_main.c linux-4.1.10/drivers/net/ethernet/atheros/atl1e/atl1e_main.c +--- linux-4.1.10.orig/drivers/net/ethernet/atheros/atl1e/atl1e_main.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/net/ethernet/atheros/atl1e/atl1e_main.c 2015-10-07 18:00:08.000000000 +0200 +@@ -1880,8 +1880,7 @@ + return NETDEV_TX_OK; + } + tpd_req = atl1e_cal_tdp_req(skb); +- if (!spin_trylock_irqsave(&adapter->tx_lock, flags)) +- return NETDEV_TX_LOCKED; ++ spin_lock_irqsave(&adapter->tx_lock, flags); + + if (atl1e_tpd_avail(adapter) < tpd_req) { + /* no enough descriptor, just stop queue */ +diff -Nur linux-4.1.10.orig/drivers/net/ethernet/chelsio/cxgb/sge.c linux-4.1.10/drivers/net/ethernet/chelsio/cxgb/sge.c +--- linux-4.1.10.orig/drivers/net/ethernet/chelsio/cxgb/sge.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/net/ethernet/chelsio/cxgb/sge.c 2015-10-07 18:00:08.000000000 +0200 +@@ -1664,8 +1664,7 @@ + struct cmdQ *q = &sge->cmdQ[qid]; + unsigned int credits, pidx, genbit, count, use_sched_skb = 0; + +- if (!spin_trylock(&q->lock)) +- return NETDEV_TX_LOCKED; ++ spin_lock(&q->lock); + + reclaim_completed_tx(sge, q); + +diff -Nur linux-4.1.10.orig/drivers/net/ethernet/freescale/gianfar.c linux-4.1.10/drivers/net/ethernet/freescale/gianfar.c +--- linux-4.1.10.orig/drivers/net/ethernet/freescale/gianfar.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/net/ethernet/freescale/gianfar.c 2015-10-07 18:00:08.000000000 +0200 +@@ -1540,7 +1540,7 @@ + + if (netif_running(ndev)) { + +- local_irq_save(flags); ++ local_irq_save_nort(flags); + lock_tx_qs(priv); + + gfar_halt_nodisable(priv); +@@ -1556,7 +1556,7 @@ + gfar_write(®s->maccfg1, tempval); + + unlock_tx_qs(priv); +- local_irq_restore(flags); ++ local_irq_restore_nort(flags); + + disable_napi(priv); + +@@ -1598,7 +1598,7 @@ + /* Disable Magic Packet mode, in case something + * else woke us up. + */ +- local_irq_save(flags); ++ local_irq_save_nort(flags); + lock_tx_qs(priv); + + tempval = gfar_read(®s->maccfg2); +@@ -1608,7 +1608,7 @@ + gfar_start(priv); + + unlock_tx_qs(priv); +- local_irq_restore(flags); ++ local_irq_restore_nort(flags); + + netif_device_attach(ndev); + +@@ -3418,14 +3418,14 @@ + dev->stats.tx_dropped++; + atomic64_inc(&priv->extra_stats.tx_underrun); + +- local_irq_save(flags); ++ local_irq_save_nort(flags); + lock_tx_qs(priv); + + /* Reactivate the Tx Queues */ + gfar_write(®s->tstat, gfargrp->tstat); + + unlock_tx_qs(priv); +- local_irq_restore(flags); ++ local_irq_restore_nort(flags); + } + netif_dbg(priv, tx_err, dev, "Transmit Error\n"); + } +diff -Nur linux-4.1.10.orig/drivers/net/ethernet/neterion/s2io.c linux-4.1.10/drivers/net/ethernet/neterion/s2io.c +--- linux-4.1.10.orig/drivers/net/ethernet/neterion/s2io.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/net/ethernet/neterion/s2io.c 2015-10-07 18:00:08.000000000 +0200 +@@ -4084,12 +4084,7 @@ + [skb->priority & (MAX_TX_FIFOS - 1)]; + fifo = &mac_control->fifos[queue]; + +- if (do_spin_lock) +- spin_lock_irqsave(&fifo->tx_lock, flags); +- else { +- if (unlikely(!spin_trylock_irqsave(&fifo->tx_lock, flags))) +- return NETDEV_TX_LOCKED; +- } ++ spin_lock_irqsave(&fifo->tx_lock, flags); + + if (sp->config.multiq) { + if (__netif_subqueue_stopped(dev, fifo->fifo_no)) { +diff -Nur linux-4.1.10.orig/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c linux-4.1.10/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c +--- linux-4.1.10.orig/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c 2015-10-07 18:00:08.000000000 +0200 +@@ -2137,10 +2137,8 @@ + struct pch_gbe_tx_ring *tx_ring = adapter->tx_ring; + unsigned long flags; + +- if (!spin_trylock_irqsave(&tx_ring->tx_lock, flags)) { +- /* Collision - tell upper layer to requeue */ +- return NETDEV_TX_LOCKED; +- } ++ spin_lock_irqsave(&tx_ring->tx_lock, flags); ++ + if (unlikely(!PCH_GBE_DESC_UNUSED(tx_ring))) { + netif_stop_queue(netdev); + spin_unlock_irqrestore(&tx_ring->tx_lock, flags); +diff -Nur linux-4.1.10.orig/drivers/net/ethernet/realtek/8139too.c linux-4.1.10/drivers/net/ethernet/realtek/8139too.c +--- linux-4.1.10.orig/drivers/net/ethernet/realtek/8139too.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/net/ethernet/realtek/8139too.c 2015-10-07 18:00:08.000000000 +0200 +@@ -2229,7 +2229,7 @@ + struct rtl8139_private *tp = netdev_priv(dev); + const int irq = tp->pci_dev->irq; + +- disable_irq(irq); ++ disable_irq_nosync(irq); + rtl8139_interrupt(irq, dev); + enable_irq(irq); + } +diff -Nur linux-4.1.10.orig/drivers/net/ethernet/tehuti/tehuti.c linux-4.1.10/drivers/net/ethernet/tehuti/tehuti.c +--- linux-4.1.10.orig/drivers/net/ethernet/tehuti/tehuti.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/net/ethernet/tehuti/tehuti.c 2015-10-07 18:00:08.000000000 +0200 +@@ -1629,13 +1629,8 @@ + unsigned long flags; + + ENTER; +- local_irq_save(flags); +- if (!spin_trylock(&priv->tx_lock)) { +- local_irq_restore(flags); +- DBG("%s[%s]: TX locked, returning NETDEV_TX_LOCKED\n", +- BDX_DRV_NAME, ndev->name); +- return NETDEV_TX_LOCKED; +- } ++ ++ spin_lock_irqsave(&priv->tx_lock, flags); + + /* build tx descriptor */ + BDX_ASSERT(f->m.wptr >= f->m.memsz); /* started with valid wptr */ +diff -Nur linux-4.1.10.orig/drivers/net/rionet.c linux-4.1.10/drivers/net/rionet.c +--- linux-4.1.10.orig/drivers/net/rionet.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/net/rionet.c 2015-10-07 18:00:08.000000000 +0200 +@@ -174,11 +174,7 @@ + unsigned long flags; + int add_num = 1; + +- local_irq_save(flags); +- if (!spin_trylock(&rnet->tx_lock)) { +- local_irq_restore(flags); +- return NETDEV_TX_LOCKED; +- } ++ spin_lock_irqsave(&rnet->tx_lock, flags); + + if (is_multicast_ether_addr(eth->h_dest)) + add_num = nets[rnet->mport->id].nact; +diff -Nur linux-4.1.10.orig/drivers/net/wireless/orinoco/orinoco_usb.c linux-4.1.10/drivers/net/wireless/orinoco/orinoco_usb.c +--- linux-4.1.10.orig/drivers/net/wireless/orinoco/orinoco_usb.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/net/wireless/orinoco/orinoco_usb.c 2015-10-07 18:00:08.000000000 +0200 +@@ -697,7 +697,7 @@ + while (!ctx->done.done && msecs--) + udelay(1000); + } else { +- wait_event_interruptible(ctx->done.wait, ++ swait_event_interruptible(ctx->done.wait, + ctx->done.done); + } + break; +diff -Nur linux-4.1.10.orig/drivers/pci/access.c linux-4.1.10/drivers/pci/access.c +--- linux-4.1.10.orig/drivers/pci/access.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/pci/access.c 2015-10-07 18:00:08.000000000 +0200 +@@ -580,7 +580,7 @@ + WARN_ON(!dev->block_cfg_access); + + dev->block_cfg_access = 0; +- wake_up_all(&pci_cfg_wait); ++ wake_up_all_locked(&pci_cfg_wait); + raw_spin_unlock_irqrestore(&pci_lock, flags); + } + EXPORT_SYMBOL_GPL(pci_cfg_access_unlock); +diff -Nur linux-4.1.10.orig/drivers/pci/access.c.orig linux-4.1.10/drivers/pci/access.c.orig +--- linux-4.1.10.orig/drivers/pci/access.c.orig 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/drivers/pci/access.c.orig 2015-10-03 13:49:38.000000000 +0200 +@@ -0,0 +1,784 @@ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "pci.h" ++ ++/* ++ * This interrupt-safe spinlock protects all accesses to PCI ++ * configuration space. ++ */ ++ ++DEFINE_RAW_SPINLOCK(pci_lock); ++ ++/* ++ * Wrappers for all PCI configuration access functions. They just check ++ * alignment, do locking and call the low-level functions pointed to ++ * by pci_dev->ops. ++ */ ++ ++#define PCI_byte_BAD 0 ++#define PCI_word_BAD (pos & 1) ++#define PCI_dword_BAD (pos & 3) ++ ++#define PCI_OP_READ(size,type,len) \ ++int pci_bus_read_config_##size \ ++ (struct pci_bus *bus, unsigned int devfn, int pos, type *value) \ ++{ \ ++ int res; \ ++ unsigned long flags; \ ++ u32 data = 0; \ ++ if (PCI_##size##_BAD) return PCIBIOS_BAD_REGISTER_NUMBER; \ ++ raw_spin_lock_irqsave(&pci_lock, flags); \ ++ res = bus->ops->read(bus, devfn, pos, len, &data); \ ++ *value = (type)data; \ ++ raw_spin_unlock_irqrestore(&pci_lock, flags); \ ++ return res; \ +} -+EXPORT_SYMBOL(rt_read_trylock); + -+void __lockfunc rt_write_lock(rwlock_t *rwlock) -+{ -+ rwlock_acquire(&rwlock->dep_map, 0, 0, _RET_IP_); -+ migrate_disable(); -+ __rt_spin_lock(&rwlock->lock); ++#define PCI_OP_WRITE(size,type,len) \ ++int pci_bus_write_config_##size \ ++ (struct pci_bus *bus, unsigned int devfn, int pos, type value) \ ++{ \ ++ int res; \ ++ unsigned long flags; \ ++ if (PCI_##size##_BAD) return PCIBIOS_BAD_REGISTER_NUMBER; \ ++ raw_spin_lock_irqsave(&pci_lock, flags); \ ++ res = bus->ops->write(bus, devfn, pos, len, value); \ ++ raw_spin_unlock_irqrestore(&pci_lock, flags); \ ++ return res; \ +} -+EXPORT_SYMBOL(rt_write_lock); + -+void __lockfunc rt_read_lock(rwlock_t *rwlock) ++PCI_OP_READ(byte, u8, 1) ++PCI_OP_READ(word, u16, 2) ++PCI_OP_READ(dword, u32, 4) ++PCI_OP_WRITE(byte, u8, 1) ++PCI_OP_WRITE(word, u16, 2) ++PCI_OP_WRITE(dword, u32, 4) ++ ++EXPORT_SYMBOL(pci_bus_read_config_byte); ++EXPORT_SYMBOL(pci_bus_read_config_word); ++EXPORT_SYMBOL(pci_bus_read_config_dword); ++EXPORT_SYMBOL(pci_bus_write_config_byte); ++EXPORT_SYMBOL(pci_bus_write_config_word); ++EXPORT_SYMBOL(pci_bus_write_config_dword); ++ ++int pci_generic_config_read(struct pci_bus *bus, unsigned int devfn, ++ int where, int size, u32 *val) +{ -+ struct rt_mutex *lock = &rwlock->lock; ++ void __iomem *addr; + -+ -+ /* -+ * recursive read locks succeed when current owns the lock -+ */ -+ if (rt_mutex_owner(lock) != current) { -+ migrate_disable(); -+ rwlock_acquire(&rwlock->dep_map, 0, 0, _RET_IP_); -+ __rt_spin_lock(lock); ++ addr = bus->ops->map_bus(bus, devfn, where); ++ if (!addr) { ++ *val = ~0; ++ return PCIBIOS_DEVICE_NOT_FOUND; + } -+ rwlock->read_depth++; -+} + -+EXPORT_SYMBOL(rt_read_lock); ++ if (size == 1) ++ *val = readb(addr); ++ else if (size == 2) ++ *val = readw(addr); ++ else ++ *val = readl(addr); + -+void __lockfunc rt_write_unlock(rwlock_t *rwlock) ++ return PCIBIOS_SUCCESSFUL; ++} ++EXPORT_SYMBOL_GPL(pci_generic_config_read); ++ ++int pci_generic_config_write(struct pci_bus *bus, unsigned int devfn, ++ int where, int size, u32 val) +{ -+ /* NOTE: we always pass in '1' for nested, for simplicity */ -+ rwlock_release(&rwlock->dep_map, 1, _RET_IP_); -+ __rt_spin_unlock(&rwlock->lock); -+ migrate_enable(); ++ void __iomem *addr; ++ ++ addr = bus->ops->map_bus(bus, devfn, where); ++ if (!addr) ++ return PCIBIOS_DEVICE_NOT_FOUND; ++ ++ if (size == 1) ++ writeb(val, addr); ++ else if (size == 2) ++ writew(val, addr); ++ else ++ writel(val, addr); ++ ++ return PCIBIOS_SUCCESSFUL; +} -+EXPORT_SYMBOL(rt_write_unlock); ++EXPORT_SYMBOL_GPL(pci_generic_config_write); + -+void __lockfunc rt_read_unlock(rwlock_t *rwlock) ++int pci_generic_config_read32(struct pci_bus *bus, unsigned int devfn, ++ int where, int size, u32 *val) +{ -+ /* Release the lock only when read_depth is down to 0 */ -+ if (--rwlock->read_depth == 0) { -+ rwlock_release(&rwlock->dep_map, 1, _RET_IP_); -+ __rt_spin_unlock(&rwlock->lock); -+ migrate_enable(); ++ void __iomem *addr; ++ ++ addr = bus->ops->map_bus(bus, devfn, where & ~0x3); ++ if (!addr) { ++ *val = ~0; ++ return PCIBIOS_DEVICE_NOT_FOUND; + } ++ ++ *val = readl(addr); ++ ++ if (size <= 2) ++ *val = (*val >> (8 * (where & 3))) & ((1 << (size * 8)) - 1); ++ ++ return PCIBIOS_SUCCESSFUL; +} -+EXPORT_SYMBOL(rt_read_unlock); ++EXPORT_SYMBOL_GPL(pci_generic_config_read32); + -+unsigned long __lockfunc rt_write_lock_irqsave(rwlock_t *rwlock) ++int pci_generic_config_write32(struct pci_bus *bus, unsigned int devfn, ++ int where, int size, u32 val) +{ -+ rt_write_lock(rwlock); ++ void __iomem *addr; ++ u32 mask, tmp; + -+ return 0; ++ addr = bus->ops->map_bus(bus, devfn, where & ~0x3); ++ if (!addr) ++ return PCIBIOS_DEVICE_NOT_FOUND; ++ ++ if (size == 4) { ++ writel(val, addr); ++ return PCIBIOS_SUCCESSFUL; ++ } else { ++ mask = ~(((1 << (size * 8)) - 1) << ((where & 0x3) * 8)); ++ } ++ ++ tmp = readl(addr) & mask; ++ tmp |= val << ((where & 0x3) * 8); ++ writel(tmp, addr); ++ ++ return PCIBIOS_SUCCESSFUL; +} -+EXPORT_SYMBOL(rt_write_lock_irqsave); ++EXPORT_SYMBOL_GPL(pci_generic_config_write32); + -+unsigned long __lockfunc rt_read_lock_irqsave(rwlock_t *rwlock) ++/** ++ * pci_bus_set_ops - Set raw operations of pci bus ++ * @bus: pci bus struct ++ * @ops: new raw operations ++ * ++ * Return previous raw operations ++ */ ++struct pci_ops *pci_bus_set_ops(struct pci_bus *bus, struct pci_ops *ops) +{ -+ rt_read_lock(rwlock); ++ struct pci_ops *old_ops; ++ unsigned long flags; + -+ return 0; ++ raw_spin_lock_irqsave(&pci_lock, flags); ++ old_ops = bus->ops; ++ bus->ops = ops; ++ raw_spin_unlock_irqrestore(&pci_lock, flags); ++ return old_ops; +} -+EXPORT_SYMBOL(rt_read_lock_irqsave); ++EXPORT_SYMBOL(pci_bus_set_ops); + -+void __rt_rwlock_init(rwlock_t *rwlock, char *name, struct lock_class_key *key) ++/** ++ * pci_read_vpd - Read one entry from Vital Product Data ++ * @dev: pci device struct ++ * @pos: offset in vpd space ++ * @count: number of bytes to read ++ * @buf: pointer to where to store result ++ * ++ */ ++ssize_t pci_read_vpd(struct pci_dev *dev, loff_t pos, size_t count, void *buf) +{ -+#ifdef CONFIG_DEBUG_LOCK_ALLOC -+ /* -+ * Make sure we are not reinitializing a held lock: -+ */ -+ debug_check_no_locks_freed((void *)rwlock, sizeof(*rwlock)); -+ lockdep_init_map(&rwlock->dep_map, name, key, 0); -+#endif -+ rwlock->lock.save_state = 1; -+ rwlock->read_depth = 0; ++ if (!dev->vpd || !dev->vpd->ops) ++ return -ENODEV; ++ return dev->vpd->ops->read(dev, pos, count, buf); +} -+EXPORT_SYMBOL(__rt_rwlock_init); ++EXPORT_SYMBOL(pci_read_vpd); + -+/* -+ * rw_semaphores ++/** ++ * pci_write_vpd - Write entry to Vital Product Data ++ * @dev: pci device struct ++ * @pos: offset in vpd space ++ * @count: number of bytes to write ++ * @buf: buffer containing write data ++ * + */ -+ -+void rt_up_write(struct rw_semaphore *rwsem) ++ssize_t pci_write_vpd(struct pci_dev *dev, loff_t pos, size_t count, const void *buf) +{ -+ rwsem_release(&rwsem->dep_map, 1, _RET_IP_); -+ rt_mutex_unlock(&rwsem->lock); ++ if (!dev->vpd || !dev->vpd->ops) ++ return -ENODEV; ++ return dev->vpd->ops->write(dev, pos, count, buf); +} -+EXPORT_SYMBOL(rt_up_write); ++EXPORT_SYMBOL(pci_write_vpd); + -+void __rt_up_read(struct rw_semaphore *rwsem) -+{ -+ if (--rwsem->read_depth == 0) -+ rt_mutex_unlock(&rwsem->lock); -+} ++/* ++ * The following routines are to prevent the user from accessing PCI config ++ * space when it's unsafe to do so. Some devices require this during BIST and ++ * we're required to prevent it during D-state transitions. ++ * ++ * We have a bit per device to indicate it's blocked and a global wait queue ++ * for callers to sleep on until devices are unblocked. ++ */ ++static DECLARE_WAIT_QUEUE_HEAD(pci_cfg_wait); + -+void rt_up_read(struct rw_semaphore *rwsem) ++static noinline void pci_wait_cfg(struct pci_dev *dev) +{ -+ rwsem_release(&rwsem->dep_map, 1, _RET_IP_); -+ __rt_up_read(rwsem); ++ DECLARE_WAITQUEUE(wait, current); ++ ++ __add_wait_queue(&pci_cfg_wait, &wait); ++ do { ++ set_current_state(TASK_UNINTERRUPTIBLE); ++ raw_spin_unlock_irq(&pci_lock); ++ schedule(); ++ raw_spin_lock_irq(&pci_lock); ++ } while (dev->block_cfg_access); ++ __remove_wait_queue(&pci_cfg_wait, &wait); +} -+EXPORT_SYMBOL(rt_up_read); ++ ++/* Returns 0 on success, negative values indicate error. */ ++#define PCI_USER_READ_CONFIG(size,type) \ ++int pci_user_read_config_##size \ ++ (struct pci_dev *dev, int pos, type *val) \ ++{ \ ++ int ret = PCIBIOS_SUCCESSFUL; \ ++ u32 data = -1; \ ++ if (PCI_##size##_BAD) \ ++ return -EINVAL; \ ++ raw_spin_lock_irq(&pci_lock); \ ++ if (unlikely(dev->block_cfg_access)) \ ++ pci_wait_cfg(dev); \ ++ ret = dev->bus->ops->read(dev->bus, dev->devfn, \ ++ pos, sizeof(type), &data); \ ++ raw_spin_unlock_irq(&pci_lock); \ ++ *val = (type)data; \ ++ return pcibios_err_to_errno(ret); \ ++} \ ++EXPORT_SYMBOL_GPL(pci_user_read_config_##size); ++ ++/* Returns 0 on success, negative values indicate error. */ ++#define PCI_USER_WRITE_CONFIG(size,type) \ ++int pci_user_write_config_##size \ ++ (struct pci_dev *dev, int pos, type val) \ ++{ \ ++ int ret = PCIBIOS_SUCCESSFUL; \ ++ if (PCI_##size##_BAD) \ ++ return -EINVAL; \ ++ raw_spin_lock_irq(&pci_lock); \ ++ if (unlikely(dev->block_cfg_access)) \ ++ pci_wait_cfg(dev); \ ++ ret = dev->bus->ops->write(dev->bus, dev->devfn, \ ++ pos, sizeof(type), val); \ ++ raw_spin_unlock_irq(&pci_lock); \ ++ return pcibios_err_to_errno(ret); \ ++} \ ++EXPORT_SYMBOL_GPL(pci_user_write_config_##size); ++ ++PCI_USER_READ_CONFIG(byte, u8) ++PCI_USER_READ_CONFIG(word, u16) ++PCI_USER_READ_CONFIG(dword, u32) ++PCI_USER_WRITE_CONFIG(byte, u8) ++PCI_USER_WRITE_CONFIG(word, u16) ++PCI_USER_WRITE_CONFIG(dword, u32) ++ ++/* VPD access through PCI 2.2+ VPD capability */ ++ ++#define PCI_VPD_PCI22_SIZE (PCI_VPD_ADDR_MASK + 1) ++ ++struct pci_vpd_pci22 { ++ struct pci_vpd base; ++ struct mutex lock; ++ u16 flag; ++ bool busy; ++ u8 cap; ++}; + +/* -+ * downgrade a write lock into a read lock -+ * - just wake up any readers at the front of the queue ++ * Wait for last operation to complete. ++ * This code has to spin since there is no other notification from the PCI ++ * hardware. Since the VPD is often implemented by serial attachment to an ++ * EEPROM, it may take many milliseconds to complete. ++ * ++ * Returns 0 on success, negative values indicate error. + */ -+void rt_downgrade_write(struct rw_semaphore *rwsem) ++static int pci_vpd_pci22_wait(struct pci_dev *dev) +{ -+ BUG_ON(rt_mutex_owner(&rwsem->lock) != current); -+ rwsem->read_depth = 1; ++ struct pci_vpd_pci22 *vpd = ++ container_of(dev->vpd, struct pci_vpd_pci22, base); ++ unsigned long timeout = jiffies + HZ/20 + 2; ++ u16 status; ++ int ret; ++ ++ if (!vpd->busy) ++ return 0; ++ ++ for (;;) { ++ ret = pci_user_read_config_word(dev, vpd->cap + PCI_VPD_ADDR, ++ &status); ++ if (ret < 0) ++ return ret; ++ ++ if ((status & PCI_VPD_ADDR_F) == vpd->flag) { ++ vpd->busy = false; ++ return 0; ++ } ++ ++ if (time_after(jiffies, timeout)) { ++ dev_printk(KERN_DEBUG, &dev->dev, "vpd r/w failed. This is likely a firmware bug on this device. Contact the card vendor for a firmware update\n"); ++ return -ETIMEDOUT; ++ } ++ if (fatal_signal_pending(current)) ++ return -EINTR; ++ if (!cond_resched()) ++ udelay(10); ++ } +} -+EXPORT_SYMBOL(rt_downgrade_write); + -+int rt_down_write_trylock(struct rw_semaphore *rwsem) ++static ssize_t pci_vpd_pci22_read(struct pci_dev *dev, loff_t pos, size_t count, ++ void *arg) +{ -+ int ret = rt_mutex_trylock(&rwsem->lock); ++ struct pci_vpd_pci22 *vpd = ++ container_of(dev->vpd, struct pci_vpd_pci22, base); ++ int ret; ++ loff_t end = pos + count; ++ u8 *buf = arg; + -+ if (ret) -+ rwsem_acquire(&rwsem->dep_map, 0, 1, _RET_IP_); -+ return ret; ++ if (pos < 0 || pos > vpd->base.len || end > vpd->base.len) ++ return -EINVAL; ++ ++ if (mutex_lock_killable(&vpd->lock)) ++ return -EINTR; ++ ++ ret = pci_vpd_pci22_wait(dev); ++ if (ret < 0) ++ goto out; ++ ++ while (pos < end) { ++ u32 val; ++ unsigned int i, skip; ++ ++ ret = pci_user_write_config_word(dev, vpd->cap + PCI_VPD_ADDR, ++ pos & ~3); ++ if (ret < 0) ++ break; ++ vpd->busy = true; ++ vpd->flag = PCI_VPD_ADDR_F; ++ ret = pci_vpd_pci22_wait(dev); ++ if (ret < 0) ++ break; ++ ++ ret = pci_user_read_config_dword(dev, vpd->cap + PCI_VPD_DATA, &val); ++ if (ret < 0) ++ break; ++ ++ skip = pos & 3; ++ for (i = 0; i < sizeof(u32); i++) { ++ if (i >= skip) { ++ *buf++ = val; ++ if (++pos == end) ++ break; ++ } ++ val >>= 8; ++ } ++ } ++out: ++ mutex_unlock(&vpd->lock); ++ return ret ? ret : count; +} -+EXPORT_SYMBOL(rt_down_write_trylock); + -+void rt_down_write(struct rw_semaphore *rwsem) ++static ssize_t pci_vpd_pci22_write(struct pci_dev *dev, loff_t pos, size_t count, ++ const void *arg) +{ -+ rwsem_acquire(&rwsem->dep_map, 0, 0, _RET_IP_); -+ rt_mutex_lock(&rwsem->lock); ++ struct pci_vpd_pci22 *vpd = ++ container_of(dev->vpd, struct pci_vpd_pci22, base); ++ const u8 *buf = arg; ++ loff_t end = pos + count; ++ int ret = 0; ++ ++ if (pos < 0 || (pos & 3) || (count & 3) || end > vpd->base.len) ++ return -EINVAL; ++ ++ if (mutex_lock_killable(&vpd->lock)) ++ return -EINTR; ++ ++ ret = pci_vpd_pci22_wait(dev); ++ if (ret < 0) ++ goto out; ++ ++ while (pos < end) { ++ u32 val; ++ ++ val = *buf++; ++ val |= *buf++ << 8; ++ val |= *buf++ << 16; ++ val |= *buf++ << 24; ++ ++ ret = pci_user_write_config_dword(dev, vpd->cap + PCI_VPD_DATA, val); ++ if (ret < 0) ++ break; ++ ret = pci_user_write_config_word(dev, vpd->cap + PCI_VPD_ADDR, ++ pos | PCI_VPD_ADDR_F); ++ if (ret < 0) ++ break; ++ ++ vpd->busy = true; ++ vpd->flag = 0; ++ ret = pci_vpd_pci22_wait(dev); ++ if (ret < 0) ++ break; ++ ++ pos += sizeof(u32); ++ } ++out: ++ mutex_unlock(&vpd->lock); ++ return ret ? ret : count; +} -+EXPORT_SYMBOL(rt_down_write); + -+void rt_down_write_nested(struct rw_semaphore *rwsem, int subclass) ++static void pci_vpd_pci22_release(struct pci_dev *dev) +{ -+ rwsem_acquire(&rwsem->dep_map, subclass, 0, _RET_IP_); -+ rt_mutex_lock(&rwsem->lock); ++ kfree(container_of(dev->vpd, struct pci_vpd_pci22, base)); +} -+EXPORT_SYMBOL(rt_down_write_nested); + -+void rt_down_write_nested_lock(struct rw_semaphore *rwsem, -+ struct lockdep_map *nest) ++static const struct pci_vpd_ops pci_vpd_pci22_ops = { ++ .read = pci_vpd_pci22_read, ++ .write = pci_vpd_pci22_write, ++ .release = pci_vpd_pci22_release, ++}; ++ ++static ssize_t pci_vpd_f0_read(struct pci_dev *dev, loff_t pos, size_t count, ++ void *arg) +{ -+ rwsem_acquire_nest(&rwsem->dep_map, 0, 0, nest, _RET_IP_); -+ rt_mutex_lock(&rwsem->lock); ++ struct pci_dev *tdev = pci_get_slot(dev->bus, PCI_SLOT(dev->devfn)); ++ ssize_t ret; ++ ++ if (!tdev) ++ return -ENODEV; ++ ++ ret = pci_read_vpd(tdev, pos, count, arg); ++ pci_dev_put(tdev); ++ return ret; +} -+EXPORT_SYMBOL(rt_down_write_nested_lock); + -+int rt_down_read_trylock(struct rw_semaphore *rwsem) ++static ssize_t pci_vpd_f0_write(struct pci_dev *dev, loff_t pos, size_t count, ++ const void *arg) +{ -+ struct rt_mutex *lock = &rwsem->lock; -+ int ret = 1; ++ struct pci_dev *tdev = pci_get_slot(dev->bus, PCI_SLOT(dev->devfn)); ++ ssize_t ret; + -+ /* -+ * recursive read locks succeed when current owns the rwsem, -+ * but not when read_depth == 0 which means that the rwsem is -+ * write locked. -+ */ -+ if (rt_mutex_owner(lock) != current) -+ ret = rt_mutex_trylock(&rwsem->lock); -+ else if (!rwsem->read_depth) -+ ret = 0; ++ if (!tdev) ++ return -ENODEV; + -+ if (ret) { -+ rwsem->read_depth++; -+ rwsem_acquire(&rwsem->dep_map, 0, 1, _RET_IP_); -+ } ++ ret = pci_write_vpd(tdev, pos, count, arg); ++ pci_dev_put(tdev); + return ret; +} -+EXPORT_SYMBOL(rt_down_read_trylock); + -+static void __rt_down_read(struct rw_semaphore *rwsem, int subclass) ++static const struct pci_vpd_ops pci_vpd_f0_ops = { ++ .read = pci_vpd_f0_read, ++ .write = pci_vpd_f0_write, ++ .release = pci_vpd_pci22_release, ++}; ++ ++static int pci_vpd_f0_dev_check(struct pci_dev *dev) +{ -+ struct rt_mutex *lock = &rwsem->lock; ++ struct pci_dev *tdev = pci_get_slot(dev->bus, PCI_SLOT(dev->devfn)); ++ int ret = 0; + -+ rwsem_acquire_read(&rwsem->dep_map, subclass, 0, _RET_IP_); ++ if (!tdev) ++ return -ENODEV; ++ if (!tdev->vpd || !tdev->multifunction || ++ dev->class != tdev->class || dev->vendor != tdev->vendor || ++ dev->device != tdev->device) ++ ret = -ENODEV; + -+ if (rt_mutex_owner(lock) != current) -+ rt_mutex_lock(&rwsem->lock); -+ rwsem->read_depth++; ++ pci_dev_put(tdev); ++ return ret; +} + -+void rt_down_read(struct rw_semaphore *rwsem) ++int pci_vpd_pci22_init(struct pci_dev *dev) +{ -+ __rt_down_read(rwsem, 0); ++ struct pci_vpd_pci22 *vpd; ++ u8 cap; ++ ++ cap = pci_find_capability(dev, PCI_CAP_ID_VPD); ++ if (!cap) ++ return -ENODEV; ++ if (dev->dev_flags & PCI_DEV_FLAGS_VPD_REF_F0) { ++ int ret = pci_vpd_f0_dev_check(dev); ++ ++ if (ret) ++ return ret; ++ } ++ vpd = kzalloc(sizeof(*vpd), GFP_ATOMIC); ++ if (!vpd) ++ return -ENOMEM; ++ ++ vpd->base.len = PCI_VPD_PCI22_SIZE; ++ if (dev->dev_flags & PCI_DEV_FLAGS_VPD_REF_F0) ++ vpd->base.ops = &pci_vpd_f0_ops; ++ else ++ vpd->base.ops = &pci_vpd_pci22_ops; ++ mutex_init(&vpd->lock); ++ vpd->cap = cap; ++ vpd->busy = false; ++ dev->vpd = &vpd->base; ++ return 0; +} -+EXPORT_SYMBOL(rt_down_read); + -+void rt_down_read_nested(struct rw_semaphore *rwsem, int subclass) ++/** ++ * pci_cfg_access_lock - Lock PCI config reads/writes ++ * @dev: pci device struct ++ * ++ * When access is locked, any userspace reads or writes to config ++ * space and concurrent lock requests will sleep until access is ++ * allowed via pci_cfg_access_unlocked again. ++ */ ++void pci_cfg_access_lock(struct pci_dev *dev) +{ -+ __rt_down_read(rwsem, subclass); ++ might_sleep(); ++ ++ raw_spin_lock_irq(&pci_lock); ++ if (dev->block_cfg_access) ++ pci_wait_cfg(dev); ++ dev->block_cfg_access = 1; ++ raw_spin_unlock_irq(&pci_lock); +} -+EXPORT_SYMBOL(rt_down_read_nested); ++EXPORT_SYMBOL_GPL(pci_cfg_access_lock); + -+void __rt_rwsem_init(struct rw_semaphore *rwsem, const char *name, -+ struct lock_class_key *key) ++/** ++ * pci_cfg_access_trylock - try to lock PCI config reads/writes ++ * @dev: pci device struct ++ * ++ * Same as pci_cfg_access_lock, but will return 0 if access is ++ * already locked, 1 otherwise. This function can be used from ++ * atomic contexts. ++ */ ++bool pci_cfg_access_trylock(struct pci_dev *dev) +{ -+#ifdef CONFIG_DEBUG_LOCK_ALLOC -+ /* -+ * Make sure we are not reinitializing a held lock: -+ */ -+ debug_check_no_locks_freed((void *)rwsem, sizeof(*rwsem)); -+ lockdep_init_map(&rwsem->dep_map, name, key, 0); -+#endif -+ rwsem->read_depth = 0; -+ rwsem->lock.save_state = 0; ++ unsigned long flags; ++ bool locked = true; ++ ++ raw_spin_lock_irqsave(&pci_lock, flags); ++ if (dev->block_cfg_access) ++ locked = false; ++ else ++ dev->block_cfg_access = 1; ++ raw_spin_unlock_irqrestore(&pci_lock, flags); ++ ++ return locked; +} -+EXPORT_SYMBOL(__rt_rwsem_init); ++EXPORT_SYMBOL_GPL(pci_cfg_access_trylock); + +/** -+ * atomic_dec_and_mutex_lock - return holding mutex if we dec to 0 -+ * @cnt: the atomic which we are to dec -+ * @lock: the mutex to return holding if we dec to 0 ++ * pci_cfg_access_unlock - Unlock PCI config reads/writes ++ * @dev: pci device struct + * -+ * return true and hold lock if we dec to 0, return false otherwise ++ * This function allows PCI config accesses to resume. + */ -+int atomic_dec_and_mutex_lock(atomic_t *cnt, struct mutex *lock) ++void pci_cfg_access_unlock(struct pci_dev *dev) +{ -+ /* dec if we can't possibly hit 0 */ -+ if (atomic_add_unless(cnt, -1, 1)) -+ return 0; -+ /* we might hit 0, so take the lock */ -+ mutex_lock(lock); -+ if (!atomic_dec_and_test(cnt)) { -+ /* when we actually did the dec, we didn't hit 0 */ -+ mutex_unlock(lock); -+ return 0; -+ } -+ /* we hit 0, and we hold the lock */ -+ return 1; ++ unsigned long flags; ++ ++ raw_spin_lock_irqsave(&pci_lock, flags); ++ ++ /* This indicates a problem in the caller, but we don't need ++ * to kill them, unlike a double-block above. */ ++ WARN_ON(!dev->block_cfg_access); ++ ++ dev->block_cfg_access = 0; ++ wake_up_all(&pci_cfg_wait); ++ raw_spin_unlock_irqrestore(&pci_lock, flags); +} -+EXPORT_SYMBOL(atomic_dec_and_mutex_lock); -diff -Nur linux-4.1.6.orig/kernel/locking/rtmutex.c linux-4.1.6/kernel/locking/rtmutex.c ---- linux-4.1.6.orig/kernel/locking/rtmutex.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/locking/rtmutex.c 2015-09-08 23:49:08.101867828 +0200 -@@ -7,6 +7,11 @@ - * Copyright (C) 2005-2006 Timesys Corp., Thomas Gleixner - * Copyright (C) 2005 Kihon Technologies Inc., Steven Rostedt - * Copyright (C) 2006 Esben Nielsen -+ * Adaptive Spinlocks: -+ * Copyright (C) 2008 Novell, Inc., Gregory Haskins, Sven Dietrich, -+ * and Peter Morreale, -+ * Adaptive Spinlocks simplification: -+ * Copyright (C) 2008 Red Hat, Inc., Steven Rostedt - * - * See Documentation/locking/rt-mutex-design.txt for details. - */ -@@ -16,6 +21,7 @@ - #include - #include - #include -+#include - - #include "rtmutex_common.h" - -@@ -69,6 +75,12 @@ - clear_rt_mutex_waiters(lock); - } - -+static int rt_mutex_real_waiter(struct rt_mutex_waiter *waiter) ++EXPORT_SYMBOL_GPL(pci_cfg_access_unlock); ++ ++static inline int pcie_cap_version(const struct pci_dev *dev) +{ -+ return waiter && waiter != PI_WAKEUP_INPROGRESS && -+ waiter != PI_REQUEUE_INPROGRESS; ++ return pcie_caps_reg(dev) & PCI_EXP_FLAGS_VERS; +} + - /* - * We can speed up the acquire/release, if the architecture - * supports cmpxchg and if there's no debugging state to be set up -@@ -300,7 +312,7 @@ - * of task. We do not use the spin_xx_mutex() variants here as we are - * outside of the debug path.) - */ --static void rt_mutex_adjust_prio(struct task_struct *task) -+void rt_mutex_adjust_prio(struct task_struct *task) - { - unsigned long flags; - -@@ -335,6 +347,14 @@ - return debug_rt_mutex_detect_deadlock(waiter, chwalk); - } - -+static void rt_mutex_wake_waiter(struct rt_mutex_waiter *waiter) ++bool pcie_cap_has_lnkctl(const struct pci_dev *dev) +{ -+ if (waiter->savestate) -+ wake_up_lock_sleeper(waiter->task); -+ else -+ wake_up_process(waiter->task); ++ int type = pci_pcie_type(dev); ++ ++ return type == PCI_EXP_TYPE_ENDPOINT || ++ type == PCI_EXP_TYPE_LEG_END || ++ type == PCI_EXP_TYPE_ROOT_PORT || ++ type == PCI_EXP_TYPE_UPSTREAM || ++ type == PCI_EXP_TYPE_DOWNSTREAM || ++ type == PCI_EXP_TYPE_PCI_BRIDGE || ++ type == PCI_EXP_TYPE_PCIE_BRIDGE; +} + - /* - * Max number of times we'll walk the boosting chain: - */ -@@ -342,7 +362,8 @@ - - static inline struct rt_mutex *task_blocked_on_lock(struct task_struct *p) - { -- return p->pi_blocked_on ? p->pi_blocked_on->lock : NULL; -+ return rt_mutex_real_waiter(p->pi_blocked_on) ? -+ p->pi_blocked_on->lock : NULL; - } - - /* -@@ -479,7 +500,7 @@ - * reached or the state of the chain has changed while we - * dropped the locks. - */ -- if (!waiter) -+ if (!rt_mutex_real_waiter(waiter)) - goto out_unlock_pi; - - /* -@@ -641,13 +662,16 @@ - * follow here. This is the end of the chain we are walking. - */ - if (!rt_mutex_owner(lock)) { -+ struct rt_mutex_waiter *lock_top_waiter; -+ - /* - * If the requeue [7] above changed the top waiter, - * then we need to wake the new top waiter up to try - * to get the lock. - */ -- if (prerequeue_top_waiter != rt_mutex_top_waiter(lock)) -- wake_up_process(rt_mutex_top_waiter(lock)->task); -+ lock_top_waiter = rt_mutex_top_waiter(lock); -+ if (prerequeue_top_waiter != lock_top_waiter) -+ rt_mutex_wake_waiter(lock_top_waiter); - raw_spin_unlock(&lock->wait_lock); - return 0; - } -@@ -740,6 +764,25 @@ - return ret; - } - -+ -+#define STEAL_NORMAL 0 -+#define STEAL_LATERAL 1 -+ -+/* -+ * Note that RT tasks are excluded from lateral-steals to prevent the -+ * introduction of an unbounded latency -+ */ -+static inline int lock_is_stealable(struct task_struct *task, -+ struct task_struct *pendowner, int mode) ++static inline bool pcie_cap_has_sltctl(const struct pci_dev *dev) +{ -+ if (mode == STEAL_NORMAL || rt_task(task)) { -+ if (task->prio >= pendowner->prio) -+ return 0; -+ } else if (task->prio > pendowner->prio) -+ return 0; -+ return 1; ++ int type = pci_pcie_type(dev); ++ ++ return (type == PCI_EXP_TYPE_ROOT_PORT || ++ type == PCI_EXP_TYPE_DOWNSTREAM) && ++ pcie_caps_reg(dev) & PCI_EXP_FLAGS_SLOT; +} + - /* - * Try to take an rt-mutex - * -@@ -750,8 +793,9 @@ - * @waiter: The waiter that is queued to the lock's wait list if the - * callsite called task_blocked_on_lock(), otherwise NULL - */ --static int try_to_take_rt_mutex(struct rt_mutex *lock, struct task_struct *task, -- struct rt_mutex_waiter *waiter) -+static int __try_to_take_rt_mutex(struct rt_mutex *lock, -+ struct task_struct *task, -+ struct rt_mutex_waiter *waiter, int mode) - { - unsigned long flags; - -@@ -790,8 +834,10 @@ - * If waiter is not the highest priority waiter of - * @lock, give up. - */ -- if (waiter != rt_mutex_top_waiter(lock)) -+ if (waiter != rt_mutex_top_waiter(lock)) { -+ /* XXX lock_is_stealable() ? */ - return 0; -+ } - - /* - * We can acquire the lock. Remove the waiter from the -@@ -809,14 +855,10 @@ - * not need to be dequeued. - */ - if (rt_mutex_has_waiters(lock)) { -- /* -- * If @task->prio is greater than or equal to -- * the top waiter priority (kernel view), -- * @task lost. -- */ -- if (task->prio >= rt_mutex_top_waiter(lock)->prio) -- return 0; -+ struct task_struct *pown = rt_mutex_top_waiter(lock)->task; - -+ if (task != pown && !lock_is_stealable(task, pown, mode)) -+ return 0; - /* - * The current top waiter stays enqueued. We - * don't have to change anything in the lock -@@ -865,6 +907,347 @@ - return 1; - } - -+#ifdef CONFIG_PREEMPT_RT_FULL -+/* -+ * preemptible spin_lock functions: -+ */ -+static inline void rt_spin_lock_fastlock(struct rt_mutex *lock, -+ void (*slowfn)(struct rt_mutex *lock)) ++static inline bool pcie_cap_has_rtctl(const struct pci_dev *dev) +{ -+ might_sleep_no_state_check(); ++ int type = pci_pcie_type(dev); + -+ if (likely(rt_mutex_cmpxchg(lock, NULL, current))) -+ rt_mutex_deadlock_account_lock(lock, current); -+ else -+ slowfn(lock); ++ return type == PCI_EXP_TYPE_ROOT_PORT || ++ type == PCI_EXP_TYPE_RC_EC; +} + -+static inline void rt_spin_lock_fastunlock(struct rt_mutex *lock, -+ void (*slowfn)(struct rt_mutex *lock)) ++static bool pcie_capability_reg_implemented(struct pci_dev *dev, int pos) +{ -+ if (likely(rt_mutex_cmpxchg(lock, current, NULL))) -+ rt_mutex_deadlock_account_unlock(current); -+ else -+ slowfn(lock); ++ if (!pci_is_pcie(dev)) ++ return false; ++ ++ switch (pos) { ++ case PCI_EXP_FLAGS: ++ return true; ++ case PCI_EXP_DEVCAP: ++ case PCI_EXP_DEVCTL: ++ case PCI_EXP_DEVSTA: ++ return true; ++ case PCI_EXP_LNKCAP: ++ case PCI_EXP_LNKCTL: ++ case PCI_EXP_LNKSTA: ++ return pcie_cap_has_lnkctl(dev); ++ case PCI_EXP_SLTCAP: ++ case PCI_EXP_SLTCTL: ++ case PCI_EXP_SLTSTA: ++ return pcie_cap_has_sltctl(dev); ++ case PCI_EXP_RTCTL: ++ case PCI_EXP_RTCAP: ++ case PCI_EXP_RTSTA: ++ return pcie_cap_has_rtctl(dev); ++ case PCI_EXP_DEVCAP2: ++ case PCI_EXP_DEVCTL2: ++ case PCI_EXP_LNKCAP2: ++ case PCI_EXP_LNKCTL2: ++ case PCI_EXP_LNKSTA2: ++ return pcie_cap_version(dev) > 1; ++ default: ++ return false; ++ } +} -+#ifdef CONFIG_SMP ++ +/* -+ * Note that owner is a speculative pointer and dereferencing relies -+ * on rcu_read_lock() and the check against the lock owner. ++ * Note that these accessor functions are only for the "PCI Express ++ * Capability" (see PCIe spec r3.0, sec 7.8). They do not apply to the ++ * other "PCI Express Extended Capabilities" (AER, VC, ACS, MFVC, etc.) + */ -+static int adaptive_wait(struct rt_mutex *lock, -+ struct task_struct *owner) ++int pcie_capability_read_word(struct pci_dev *dev, int pos, u16 *val) +{ -+ int res = 0; ++ int ret; + -+ rcu_read_lock(); -+ for (;;) { -+ if (owner != rt_mutex_owner(lock)) -+ break; ++ *val = 0; ++ if (pos & 1) ++ return -EINVAL; ++ ++ if (pcie_capability_reg_implemented(dev, pos)) { ++ ret = pci_read_config_word(dev, pci_pcie_cap(dev) + pos, val); + /* -+ * Ensure that owner->on_cpu is dereferenced _after_ -+ * checking the above to be valid. ++ * Reset *val to 0 if pci_read_config_word() fails, it may ++ * have been written as 0xFFFF if hardware error happens ++ * during pci_read_config_word(). + */ -+ barrier(); -+ if (!owner->on_cpu) { -+ res = 1; -+ break; -+ } -+ cpu_relax(); ++ if (ret) ++ *val = 0; ++ return ret; + } -+ rcu_read_unlock(); -+ return res; -+} -+#else -+static int adaptive_wait(struct rt_mutex *lock, -+ struct task_struct *orig_owner) -+{ -+ return 1; -+} -+#endif + -+# define pi_lock(lock) raw_spin_lock_irq(lock) -+# define pi_unlock(lock) raw_spin_unlock_irq(lock) ++ /* ++ * For Functions that do not implement the Slot Capabilities, ++ * Slot Status, and Slot Control registers, these spaces must ++ * be hardwired to 0b, with the exception of the Presence Detect ++ * State bit in the Slot Status register of Downstream Ports, ++ * which must be hardwired to 1b. (PCIe Base Spec 3.0, sec 7.8) ++ */ ++ if (pci_is_pcie(dev) && pos == PCI_EXP_SLTSTA && ++ pci_pcie_type(dev) == PCI_EXP_TYPE_DOWNSTREAM) { ++ *val = PCI_EXP_SLTSTA_PDS; ++ } + -+static int task_blocks_on_rt_mutex(struct rt_mutex *lock, -+ struct rt_mutex_waiter *waiter, -+ struct task_struct *task, -+ enum rtmutex_chainwalk chwalk); -+/* -+ * Slow path lock function spin_lock style: this variant is very -+ * careful not to miss any non-lock wakeups. -+ * -+ * We store the current state under p->pi_lock in p->saved_state and -+ * the try_to_wake_up() code handles this accordingly. -+ */ -+static void noinline __sched rt_spin_lock_slowlock(struct rt_mutex *lock) ++ return 0; ++} ++EXPORT_SYMBOL(pcie_capability_read_word); ++ ++int pcie_capability_read_dword(struct pci_dev *dev, int pos, u32 *val) +{ -+ struct task_struct *lock_owner, *self = current; -+ struct rt_mutex_waiter waiter, *top_waiter; + int ret; + -+ rt_mutex_init_waiter(&waiter, true); -+ -+ raw_spin_lock(&lock->wait_lock); ++ *val = 0; ++ if (pos & 3) ++ return -EINVAL; + -+ if (__try_to_take_rt_mutex(lock, self, NULL, STEAL_LATERAL)) { -+ raw_spin_unlock(&lock->wait_lock); -+ return; ++ if (pcie_capability_reg_implemented(dev, pos)) { ++ ret = pci_read_config_dword(dev, pci_pcie_cap(dev) + pos, val); ++ /* ++ * Reset *val to 0 if pci_read_config_dword() fails, it may ++ * have been written as 0xFFFFFFFF if hardware error happens ++ * during pci_read_config_dword(). ++ */ ++ if (ret) ++ *val = 0; ++ return ret; + } + -+ BUG_ON(rt_mutex_owner(lock) == self); ++ if (pci_is_pcie(dev) && pos == PCI_EXP_SLTCTL && ++ pci_pcie_type(dev) == PCI_EXP_TYPE_DOWNSTREAM) { ++ *val = PCI_EXP_SLTSTA_PDS; ++ } + -+ /* -+ * We save whatever state the task is in and we'll restore it -+ * after acquiring the lock taking real wakeups into account -+ * as well. We are serialized via pi_lock against wakeups. See -+ * try_to_wake_up(). -+ */ -+ pi_lock(&self->pi_lock); -+ self->saved_state = self->state; -+ __set_current_state_no_track(TASK_UNINTERRUPTIBLE); -+ pi_unlock(&self->pi_lock); ++ return 0; ++} ++EXPORT_SYMBOL(pcie_capability_read_dword); + -+ ret = task_blocks_on_rt_mutex(lock, &waiter, self, 0); -+ BUG_ON(ret); ++int pcie_capability_write_word(struct pci_dev *dev, int pos, u16 val) ++{ ++ if (pos & 1) ++ return -EINVAL; + -+ for (;;) { -+ /* Try to acquire the lock again. */ -+ if (__try_to_take_rt_mutex(lock, self, &waiter, STEAL_LATERAL)) -+ break; ++ if (!pcie_capability_reg_implemented(dev, pos)) ++ return 0; + -+ top_waiter = rt_mutex_top_waiter(lock); -+ lock_owner = rt_mutex_owner(lock); ++ return pci_write_config_word(dev, pci_pcie_cap(dev) + pos, val); ++} ++EXPORT_SYMBOL(pcie_capability_write_word); + -+ raw_spin_unlock(&lock->wait_lock); ++int pcie_capability_write_dword(struct pci_dev *dev, int pos, u32 val) ++{ ++ if (pos & 3) ++ return -EINVAL; + -+ debug_rt_mutex_print_deadlock(&waiter); ++ if (!pcie_capability_reg_implemented(dev, pos)) ++ return 0; + -+ if (top_waiter != &waiter || adaptive_wait(lock, lock_owner)) -+ schedule_rt_mutex(lock); ++ return pci_write_config_dword(dev, pci_pcie_cap(dev) + pos, val); ++} ++EXPORT_SYMBOL(pcie_capability_write_dword); + -+ raw_spin_lock(&lock->wait_lock); ++int pcie_capability_clear_and_set_word(struct pci_dev *dev, int pos, ++ u16 clear, u16 set) ++{ ++ int ret; ++ u16 val; + -+ pi_lock(&self->pi_lock); -+ __set_current_state_no_track(TASK_UNINTERRUPTIBLE); -+ pi_unlock(&self->pi_lock); ++ ret = pcie_capability_read_word(dev, pos, &val); ++ if (!ret) { ++ val &= ~clear; ++ val |= set; ++ ret = pcie_capability_write_word(dev, pos, val); + } + -+ /* -+ * Restore the task state to current->saved_state. We set it -+ * to the original state above and the try_to_wake_up() code -+ * has possibly updated it when a real (non-rtmutex) wakeup -+ * happened while we were blocked. Clear saved_state so -+ * try_to_wakeup() does not get confused. -+ */ -+ pi_lock(&self->pi_lock); -+ __set_current_state_no_track(self->saved_state); -+ self->saved_state = TASK_RUNNING; -+ pi_unlock(&self->pi_lock); ++ return ret; ++} ++EXPORT_SYMBOL(pcie_capability_clear_and_set_word); + -+ /* -+ * try_to_take_rt_mutex() sets the waiter bit -+ * unconditionally. We might have to fix that up: -+ */ -+ fixup_rt_mutex_waiters(lock); -+ -+ BUG_ON(rt_mutex_has_waiters(lock) && &waiter == rt_mutex_top_waiter(lock)); -+ BUG_ON(!RB_EMPTY_NODE(&waiter.tree_entry)); -+ -+ raw_spin_unlock(&lock->wait_lock); -+ -+ debug_rt_mutex_free_waiter(&waiter); -+} -+ -+static void wakeup_next_waiter(struct rt_mutex *lock); -+/* -+ * Slow path to release a rt_mutex spin_lock style -+ */ -+static void noinline __sched rt_spin_lock_slowunlock(struct rt_mutex *lock) ++int pcie_capability_clear_and_set_dword(struct pci_dev *dev, int pos, ++ u32 clear, u32 set) +{ -+ raw_spin_lock(&lock->wait_lock); -+ -+ debug_rt_mutex_unlock(lock); -+ -+ rt_mutex_deadlock_account_unlock(current); ++ int ret; ++ u32 val; + -+ if (!rt_mutex_has_waiters(lock)) { -+ lock->owner = NULL; -+ raw_spin_unlock(&lock->wait_lock); -+ return; ++ ret = pcie_capability_read_dword(dev, pos, &val); ++ if (!ret) { ++ val &= ~clear; ++ val |= set; ++ ret = pcie_capability_write_dword(dev, pos, val); + } + -+ wakeup_next_waiter(lock); -+ -+ raw_spin_unlock(&lock->wait_lock); -+ -+ /* Undo pi boosting.when necessary */ -+ rt_mutex_adjust_prio(current); -+} -+ -+void __lockfunc rt_spin_lock(spinlock_t *lock) -+{ -+ rt_spin_lock_fastlock(&lock->lock, rt_spin_lock_slowlock); -+ spin_acquire(&lock->dep_map, 0, 0, _RET_IP_); ++ return ret; +} -+EXPORT_SYMBOL(rt_spin_lock); -+ -+void __lockfunc __rt_spin_lock(struct rt_mutex *lock) -+{ -+ rt_spin_lock_fastlock(lock, rt_spin_lock_slowlock); ++EXPORT_SYMBOL(pcie_capability_clear_and_set_dword); +diff -Nur linux-4.1.10.orig/drivers/scsi/fcoe/fcoe.c linux-4.1.10/drivers/scsi/fcoe/fcoe.c +--- linux-4.1.10.orig/drivers/scsi/fcoe/fcoe.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/scsi/fcoe/fcoe.c 2015-10-07 18:00:08.000000000 +0200 +@@ -1287,7 +1287,7 @@ + struct sk_buff *skb; + #ifdef CONFIG_SMP + struct fcoe_percpu_s *p0; +- unsigned targ_cpu = get_cpu(); ++ unsigned targ_cpu = get_cpu_light(); + #endif /* CONFIG_SMP */ + + FCOE_DBG("Destroying receive thread for CPU %d\n", cpu); +@@ -1343,7 +1343,7 @@ + kfree_skb(skb); + spin_unlock_bh(&p->fcoe_rx_list.lock); + } +- put_cpu(); ++ put_cpu_light(); + #else + /* + * This a non-SMP scenario where the singular Rx thread is +@@ -1567,11 +1567,11 @@ + static int fcoe_alloc_paged_crc_eof(struct sk_buff *skb, int tlen) + { + struct fcoe_percpu_s *fps; +- int rc; ++ int rc, cpu = get_cpu_light(); + +- fps = &get_cpu_var(fcoe_percpu); ++ fps = &per_cpu(fcoe_percpu, cpu); + rc = fcoe_get_paged_crc_eof(skb, tlen, fps); +- put_cpu_var(fcoe_percpu); ++ put_cpu_light(); + + return rc; + } +@@ -1767,11 +1767,11 @@ + return 0; + } + +- stats = per_cpu_ptr(lport->stats, get_cpu()); ++ stats = per_cpu_ptr(lport->stats, get_cpu_light()); + stats->InvalidCRCCount++; + if (stats->InvalidCRCCount < 5) + printk(KERN_WARNING "fcoe: dropping frame with CRC error\n"); +- put_cpu(); ++ put_cpu_light(); + return -EINVAL; + } + +@@ -1847,13 +1847,13 @@ + goto drop; + + if (!fcoe_filter_frames(lport, fp)) { +- put_cpu(); ++ put_cpu_light(); + fc_exch_recv(lport, fp); + return; + } + drop: + stats->ErrorFrames++; +- put_cpu(); ++ put_cpu_light(); + kfree_skb(skb); + } + +diff -Nur linux-4.1.10.orig/drivers/scsi/fcoe/fcoe_ctlr.c linux-4.1.10/drivers/scsi/fcoe/fcoe_ctlr.c +--- linux-4.1.10.orig/drivers/scsi/fcoe/fcoe_ctlr.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/scsi/fcoe/fcoe_ctlr.c 2015-10-07 18:00:08.000000000 +0200 +@@ -831,7 +831,7 @@ + + INIT_LIST_HEAD(&del_list); + +- stats = per_cpu_ptr(fip->lp->stats, get_cpu()); ++ stats = per_cpu_ptr(fip->lp->stats, get_cpu_light()); + + list_for_each_entry_safe(fcf, next, &fip->fcfs, list) { + deadline = fcf->time + fcf->fka_period + fcf->fka_period / 2; +@@ -867,7 +867,7 @@ + sel_time = fcf->time; + } + } +- put_cpu(); ++ put_cpu_light(); + + list_for_each_entry_safe(fcf, next, &del_list, list) { + /* Removes fcf from current list */ +diff -Nur linux-4.1.10.orig/drivers/scsi/libfc/fc_exch.c linux-4.1.10/drivers/scsi/libfc/fc_exch.c +--- linux-4.1.10.orig/drivers/scsi/libfc/fc_exch.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/scsi/libfc/fc_exch.c 2015-10-07 18:00:08.000000000 +0200 +@@ -814,10 +814,10 @@ + } + memset(ep, 0, sizeof(*ep)); + +- cpu = get_cpu(); ++ cpu = get_cpu_light(); + pool = per_cpu_ptr(mp->pool, cpu); + spin_lock_bh(&pool->lock); +- put_cpu(); ++ put_cpu_light(); + + /* peek cache of free slot */ + if (pool->left != FC_XID_UNKNOWN) { +diff -Nur linux-4.1.10.orig/drivers/scsi/libsas/sas_ata.c linux-4.1.10/drivers/scsi/libsas/sas_ata.c +--- linux-4.1.10.orig/drivers/scsi/libsas/sas_ata.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/scsi/libsas/sas_ata.c 2015-10-07 18:00:08.000000000 +0200 +@@ -190,7 +190,7 @@ + /* TODO: audit callers to ensure they are ready for qc_issue to + * unconditionally re-enable interrupts + */ +- local_irq_save(flags); ++ local_irq_save_nort(flags); + spin_unlock(ap->lock); + + /* If the device fell off, no sense in issuing commands */ +@@ -255,7 +255,7 @@ + + out: + spin_lock(ap->lock); +- local_irq_restore(flags); ++ local_irq_restore_nort(flags); + return ret; + } + +diff -Nur linux-4.1.10.orig/drivers/scsi/qla2xxx/qla_inline.h linux-4.1.10/drivers/scsi/qla2xxx/qla_inline.h +--- linux-4.1.10.orig/drivers/scsi/qla2xxx/qla_inline.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/scsi/qla2xxx/qla_inline.h 2015-10-07 18:00:08.000000000 +0200 +@@ -59,12 +59,12 @@ + { + unsigned long flags; + struct qla_hw_data *ha = rsp->hw; +- local_irq_save(flags); ++ local_irq_save_nort(flags); + if (IS_P3P_TYPE(ha)) + qla82xx_poll(0, rsp); + else + ha->isp_ops->intr_handler(0, rsp); +- local_irq_restore(flags); ++ local_irq_restore_nort(flags); + } + + static inline uint8_t * +diff -Nur linux-4.1.10.orig/drivers/thermal/x86_pkg_temp_thermal.c linux-4.1.10/drivers/thermal/x86_pkg_temp_thermal.c +--- linux-4.1.10.orig/drivers/thermal/x86_pkg_temp_thermal.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/thermal/x86_pkg_temp_thermal.c 2015-10-07 18:00:08.000000000 +0200 +@@ -29,6 +29,7 @@ + #include + #include + #include ++#include + #include + #include + +@@ -352,7 +353,7 @@ + } + } + +-static int pkg_temp_thermal_platform_thermal_notify(__u64 msr_val) ++static void platform_thermal_notify_work(struct swork_event *event) + { + unsigned long flags; + int cpu = smp_processor_id(); +@@ -369,7 +370,7 @@ + pkg_work_scheduled[phy_id]) { + disable_pkg_thres_interrupt(); + spin_unlock_irqrestore(&pkg_work_lock, flags); +- return -EINVAL; ++ return; + } + pkg_work_scheduled[phy_id] = 1; + spin_unlock_irqrestore(&pkg_work_lock, flags); +@@ -378,9 +379,48 @@ + schedule_delayed_work_on(cpu, + &per_cpu(pkg_temp_thermal_threshold_work, cpu), + msecs_to_jiffies(notify_delay_ms)); +} -+EXPORT_SYMBOL(__rt_spin_lock); + -+#ifdef CONFIG_DEBUG_LOCK_ALLOC -+void __lockfunc rt_spin_lock_nested(spinlock_t *lock, int subclass) -+{ -+ rt_spin_lock_fastlock(&lock->lock, rt_spin_lock_slowlock); -+ spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_); -+} -+EXPORT_SYMBOL(rt_spin_lock_nested); -+#endif ++#ifdef CONFIG_PREEMPT_RT_FULL ++static struct swork_event notify_work; + -+void __lockfunc rt_spin_unlock(spinlock_t *lock) ++static int thermal_notify_work_init(void) +{ -+ /* NOTE: we always pass in '1' for nested, for simplicity */ -+ spin_release(&lock->dep_map, 1, _RET_IP_); -+ rt_spin_lock_fastunlock(&lock->lock, rt_spin_lock_slowunlock); -+} -+EXPORT_SYMBOL(rt_spin_unlock); ++ int err; + -+void __lockfunc __rt_spin_unlock(struct rt_mutex *lock) -+{ -+ rt_spin_lock_fastunlock(lock, rt_spin_lock_slowunlock); -+} -+EXPORT_SYMBOL(__rt_spin_unlock); ++ err = swork_get(); ++ if (err) ++ return err; + -+/* -+ * Wait for the lock to get unlocked: instead of polling for an unlock -+ * (like raw spinlocks do), we lock and unlock, to force the kernel to -+ * schedule if there's contention: -+ */ -+void __lockfunc rt_spin_unlock_wait(spinlock_t *lock) ++ INIT_SWORK(¬ify_work, platform_thermal_notify_work); + return 0; + } + ++static void thermal_notify_work_cleanup(void) +{ -+ spin_lock(lock); -+ spin_unlock(lock); ++ swork_put(); +} -+EXPORT_SYMBOL(rt_spin_unlock_wait); + -+int __lockfunc __rt_spin_trylock(struct rt_mutex *lock) ++static int pkg_temp_thermal_platform_thermal_notify(__u64 msr_val) +{ -+ return rt_mutex_trylock(lock); ++ swork_queue(¬ify_work); ++ return 0; +} + -+int __lockfunc rt_spin_trylock(spinlock_t *lock) -+{ -+ int ret = rt_mutex_trylock(&lock->lock); -+ -+ if (ret) -+ spin_acquire(&lock->dep_map, 0, 1, _RET_IP_); -+ return ret; -+} -+EXPORT_SYMBOL(rt_spin_trylock); ++#else /* !CONFIG_PREEMPT_RT_FULL */ + -+int __lockfunc rt_spin_trylock_bh(spinlock_t *lock) -+{ -+ int ret; ++static int thermal_notify_work_init(void) { return 0; } + -+ local_bh_disable(); -+ ret = rt_mutex_trylock(&lock->lock); -+ if (ret) { -+ migrate_disable(); -+ spin_acquire(&lock->dep_map, 0, 1, _RET_IP_); -+ } else -+ local_bh_enable(); -+ return ret; -+} -+EXPORT_SYMBOL(rt_spin_trylock_bh); ++static void thermal_notify_work_cleanup(void) { } + -+int __lockfunc rt_spin_trylock_irqsave(spinlock_t *lock, unsigned long *flags) ++static int pkg_temp_thermal_platform_thermal_notify(__u64 msr_val) +{ -+ int ret; -+ -+ *flags = 0; -+ ret = rt_mutex_trylock(&lock->lock); -+ if (ret) { -+ migrate_disable(); -+ spin_acquire(&lock->dep_map, 0, 1, _RET_IP_); -+ } -+ return ret; -+} -+EXPORT_SYMBOL(rt_spin_trylock_irqsave); ++ platform_thermal_notify_work(NULL); + -+int atomic_dec_and_spin_lock(atomic_t *atomic, spinlock_t *lock) -+{ -+ /* Subtract 1 from counter unless that drops it to 0 (ie. it was 1) */ -+ if (atomic_add_unless(atomic, -1, 1)) -+ return 0; -+ migrate_disable(); -+ rt_spin_lock(lock); -+ if (atomic_dec_and_test(atomic)) -+ return 1; -+ rt_spin_unlock(lock); -+ migrate_enable(); + return 0; +} -+EXPORT_SYMBOL(atomic_dec_and_spin_lock); -+ -+ void -+__rt_spin_lock_init(spinlock_t *lock, char *name, struct lock_class_key *key) -+{ -+#ifdef CONFIG_DEBUG_LOCK_ALLOC -+ /* -+ * Make sure we are not reinitializing a held lock: -+ */ -+ debug_check_no_locks_freed((void *)lock, sizeof(*lock)); -+ lockdep_init_map(&lock->dep_map, name, key, 0); -+#endif -+} -+EXPORT_SYMBOL(__rt_spin_lock_init); ++#endif /* CONFIG_PREEMPT_RT_FULL */ + -+#endif /* PREEMPT_RT_FULL */ + static int find_siblings_cpu(int cpu) + { + int i; +@@ -584,6 +624,9 @@ + if (!x86_match_cpu(pkg_temp_thermal_ids)) + return -ENODEV; + ++ if (!thermal_notify_work_init()) ++ return -ENODEV; + + spin_lock_init(&pkg_work_lock); + platform_thermal_package_notify = + pkg_temp_thermal_platform_thermal_notify; +@@ -608,7 +651,7 @@ + kfree(pkg_work_scheduled); + platform_thermal_package_notify = NULL; + platform_thermal_package_rate_control = NULL; +- ++ thermal_notify_work_cleanup(); + return -ENODEV; + } + +@@ -633,6 +676,7 @@ + mutex_unlock(&phy_dev_list_mutex); + platform_thermal_package_notify = NULL; + platform_thermal_package_rate_control = NULL; ++ thermal_notify_work_cleanup(); + for_each_online_cpu(i) + cancel_delayed_work_sync( + &per_cpu(pkg_temp_thermal_threshold_work, i)); +diff -Nur linux-4.1.10.orig/drivers/tty/serial/8250/8250_core.c linux-4.1.10/drivers/tty/serial/8250/8250_core.c +--- linux-4.1.10.orig/drivers/tty/serial/8250/8250_core.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/tty/serial/8250/8250_core.c 2015-10-07 18:00:08.000000000 +0200 +@@ -36,6 +36,7 @@ + #include + #include + #include ++#include + #include + #include + #ifdef CONFIG_SPARC +@@ -80,7 +81,16 @@ + #define DEBUG_INTR(fmt...) do { } while (0) + #endif + +-#define PASS_LIMIT 512 ++/* ++ * On -rt we can have a more delays, and legitimately ++ * so - so don't drop work spuriously and spam the ++ * syslog: ++ */ +#ifdef CONFIG_PREEMPT_RT_FULL -+ static inline int __sched -+__mutex_lock_check_stamp(struct rt_mutex *lock, struct ww_acquire_ctx *ctx) -+{ -+ struct ww_mutex *ww = container_of(lock, struct ww_mutex, base.lock); -+ struct ww_acquire_ctx *hold_ctx = ACCESS_ONCE(ww->ctx); -+ -+ if (!hold_ctx) -+ return 0; -+ -+ if (unlikely(ctx == hold_ctx)) -+ return -EALREADY; -+ -+ if (ctx->stamp - hold_ctx->stamp <= LONG_MAX && -+ (ctx->stamp != hold_ctx->stamp || ctx > hold_ctx)) { -+#ifdef CONFIG_DEBUG_MUTEXES -+ DEBUG_LOCKS_WARN_ON(ctx->contending_lock); -+ ctx->contending_lock = ww; -+#endif -+ return -EDEADLK; -+ } -+ -+ return 0; -+} ++# define PASS_LIMIT 1000000 +#else -+ static inline int __sched -+__mutex_lock_check_stamp(struct rt_mutex *lock, struct ww_acquire_ctx *ctx) -+{ -+ BUG(); -+ return 0; -+} -+ ++# define PASS_LIMIT 512 +#endif -+ -+static inline int -+try_to_take_rt_mutex(struct rt_mutex *lock, struct task_struct *task, -+ struct rt_mutex_waiter *waiter) -+{ -+ return __try_to_take_rt_mutex(lock, task, waiter, STEAL_NORMAL); -+} -+ - /* - * Task blocks on lock. - * -@@ -896,6 +1279,23 @@ - return -EDEADLK; - raw_spin_lock_irqsave(&task->pi_lock, flags); -+ -+ /* -+ * In the case of futex requeue PI, this will be a proxy -+ * lock. The task will wake unaware that it is enqueueed on -+ * this lock. Avoid blocking on two locks and corrupting -+ * pi_blocked_on via the PI_WAKEUP_INPROGRESS -+ * flag. futex_wait_requeue_pi() sets this when it wakes up -+ * before requeue (due to a signal or timeout). Do not enqueue -+ * the task if PI_WAKEUP_INPROGRESS is set. -+ */ -+ if (task != current && task->pi_blocked_on == PI_WAKEUP_INPROGRESS) { -+ raw_spin_unlock_irqrestore(&task->pi_lock, flags); -+ return -EAGAIN; -+ } -+ -+ BUG_ON(rt_mutex_real_waiter(task->pi_blocked_on)); -+ - __rt_mutex_adjust_prio(task); - waiter->task = task; - waiter->lock = lock; -@@ -919,7 +1319,7 @@ - rt_mutex_enqueue_pi(owner, waiter); + #define BOTH_EMPTY (UART_LSR_TEMT | UART_LSR_THRE) - __rt_mutex_adjust_prio(owner); -- if (owner->pi_blocked_on) -+ if (rt_mutex_real_waiter(owner->pi_blocked_on)) - chain_walk = 1; - } else if (rt_mutex_cond_detect_deadlock(waiter, chwalk)) { - chain_walk = 1; -@@ -957,8 +1357,9 @@ - /* - * Wake up the next waiter on the lock. - * -- * Remove the top waiter from the current tasks pi waiter list and -- * wake it up. -+ * Remove the top waiter from the current tasks pi waiter list, -+ * wake it up and return whether the current task needs to undo -+ * a potential priority boosting. - * - * Called with lock->wait_lock held. - */ -@@ -996,7 +1397,7 @@ - * long as we hold lock->wait_lock. The waiter task needs to - * acquire it in order to dequeue the waiter. - */ -- wake_up_process(waiter->task); -+ rt_mutex_wake_waiter(waiter); - } +@@ -3364,7 +3374,7 @@ - /* -@@ -1010,7 +1411,7 @@ - { - bool is_top_waiter = (waiter == rt_mutex_top_waiter(lock)); - struct task_struct *owner = rt_mutex_owner(lock); -- struct rt_mutex *next_lock; -+ struct rt_mutex *next_lock = NULL; - unsigned long flags; + if (port->sysrq) + locked = 0; +- else if (oops_in_progress) ++ else if (oops_in_progress || in_kdb_printk()) + locked = spin_trylock_irqsave(&port->lock, flags); + else + spin_lock_irqsave(&port->lock, flags); +diff -Nur linux-4.1.10.orig/drivers/tty/serial/amba-pl011.c linux-4.1.10/drivers/tty/serial/amba-pl011.c +--- linux-4.1.10.orig/drivers/tty/serial/amba-pl011.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/tty/serial/amba-pl011.c 2015-10-07 18:00:08.000000000 +0200 +@@ -2000,13 +2000,19 @@ - raw_spin_lock_irqsave(¤t->pi_lock, flags); -@@ -1035,7 +1436,8 @@ - __rt_mutex_adjust_prio(owner); + clk_enable(uap->clk); - /* Store the lock on which owner is blocked or NULL */ -- next_lock = task_blocked_on_lock(owner); -+ if (rt_mutex_real_waiter(owner->pi_blocked_on)) -+ next_lock = task_blocked_on_lock(owner); +- local_irq_save(flags); ++ /* ++ * local_irq_save(flags); ++ * ++ * This local_irq_save() is nonsense. If we come in via sysrq ++ * handling then interrupts are already disabled. Aside of ++ * that the port.sysrq check is racy on SMP regardless. ++ */ + if (uap->port.sysrq) + locked = 0; + else if (oops_in_progress) +- locked = spin_trylock(&uap->port.lock); ++ locked = spin_trylock_irqsave(&uap->port.lock, flags); + else +- spin_lock(&uap->port.lock); ++ spin_lock_irqsave(&uap->port.lock, flags); - raw_spin_unlock_irqrestore(&owner->pi_lock, flags); + /* + * First save the CR then disable the interrupts +@@ -2028,8 +2034,7 @@ + writew(old_cr, uap->port.membase + UART011_CR); -@@ -1071,17 +1473,17 @@ - raw_spin_lock_irqsave(&task->pi_lock, flags); + if (locked) +- spin_unlock(&uap->port.lock); +- local_irq_restore(flags); ++ spin_unlock_irqrestore(&uap->port.lock, flags); - waiter = task->pi_blocked_on; -- if (!waiter || (waiter->prio == task->prio && -+ if (!rt_mutex_real_waiter(waiter) || (waiter->prio == task->prio && - !dl_prio(task->prio))) { - raw_spin_unlock_irqrestore(&task->pi_lock, flags); - return; - } - next_lock = waiter->lock; -- raw_spin_unlock_irqrestore(&task->pi_lock, flags); - - /* gets dropped in rt_mutex_adjust_prio_chain()! */ - get_task_struct(task); - -+ raw_spin_unlock_irqrestore(&task->pi_lock, flags); - rt_mutex_adjust_prio_chain(task, RT_MUTEX_MIN_CHAINWALK, NULL, - next_lock, NULL, task); + clk_disable(uap->clk); } -@@ -1099,7 +1501,8 @@ - static int __sched - __rt_mutex_slowlock(struct rt_mutex *lock, int state, - struct hrtimer_sleeper *timeout, -- struct rt_mutex_waiter *waiter) -+ struct rt_mutex_waiter *waiter, -+ struct ww_acquire_ctx *ww_ctx) - { - int ret = 0; +diff -Nur linux-4.1.10.orig/drivers/tty/serial/omap-serial.c linux-4.1.10/drivers/tty/serial/omap-serial.c +--- linux-4.1.10.orig/drivers/tty/serial/omap-serial.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/tty/serial/omap-serial.c 2015-10-07 18:00:08.000000000 +0200 +@@ -1282,13 +1282,10 @@ -@@ -1122,6 +1525,12 @@ - break; - } + pm_runtime_get_sync(up->dev); -+ if (ww_ctx && ww_ctx->acquired > 0) { -+ ret = __mutex_lock_check_stamp(lock, ww_ctx); -+ if (ret) -+ break; -+ } -+ - raw_spin_unlock(&lock->wait_lock); +- local_irq_save(flags); +- if (up->port.sysrq) +- locked = 0; +- else if (oops_in_progress) +- locked = spin_trylock(&up->port.lock); ++ if (up->port.sysrq || oops_in_progress) ++ locked = spin_trylock_irqsave(&up->port.lock, flags); + else +- spin_lock(&up->port.lock); ++ spin_lock_irqsave(&up->port.lock, flags); - debug_rt_mutex_print_deadlock(waiter); -@@ -1156,25 +1565,102 @@ - } + /* + * First save the IER then disable the interrupts +@@ -1317,8 +1314,7 @@ + pm_runtime_mark_last_busy(up->dev); + pm_runtime_put_autosuspend(up->dev); + if (locked) +- spin_unlock(&up->port.lock); +- local_irq_restore(flags); ++ spin_unlock_irqrestore(&up->port.lock, flags); } -+static __always_inline void ww_mutex_lock_acquired(struct ww_mutex *ww, -+ struct ww_acquire_ctx *ww_ctx) -+{ -+#ifdef CONFIG_DEBUG_MUTEXES -+ /* -+ * If this WARN_ON triggers, you used ww_mutex_lock to acquire, -+ * but released with a normal mutex_unlock in this call. -+ * -+ * This should never happen, always use ww_mutex_unlock. -+ */ -+ DEBUG_LOCKS_WARN_ON(ww->ctx); -+ -+ /* -+ * Not quite done after calling ww_acquire_done() ? -+ */ -+ DEBUG_LOCKS_WARN_ON(ww_ctx->done_acquire); -+ -+ if (ww_ctx->contending_lock) { -+ /* -+ * After -EDEADLK you tried to -+ * acquire a different ww_mutex? Bad! -+ */ -+ DEBUG_LOCKS_WARN_ON(ww_ctx->contending_lock != ww); -+ -+ /* -+ * You called ww_mutex_lock after receiving -EDEADLK, -+ * but 'forgot' to unlock everything else first? -+ */ -+ DEBUG_LOCKS_WARN_ON(ww_ctx->acquired > 0); -+ ww_ctx->contending_lock = NULL; -+ } -+ -+ /* -+ * Naughty, using a different class will lead to undefined behavior! -+ */ -+ DEBUG_LOCKS_WARN_ON(ww_ctx->ww_class != ww->ww_class); -+#endif -+ ww_ctx->acquired++; -+} -+ -+#ifdef CONFIG_PREEMPT_RT_FULL -+static void ww_mutex_account_lock(struct rt_mutex *lock, -+ struct ww_acquire_ctx *ww_ctx) -+{ -+ struct ww_mutex *ww = container_of(lock, struct ww_mutex, base.lock); -+ struct rt_mutex_waiter *waiter, *n; -+ -+ /* -+ * This branch gets optimized out for the common case, -+ * and is only important for ww_mutex_lock. -+ */ -+ ww_mutex_lock_acquired(ww, ww_ctx); -+ ww->ctx = ww_ctx; -+ -+ /* -+ * Give any possible sleeping processes the chance to wake up, -+ * so they can recheck if they have to back off. -+ */ -+ rbtree_postorder_for_each_entry_safe(waiter, n, &lock->waiters, -+ tree_entry) { -+ /* XXX debug rt mutex waiter wakeup */ -+ -+ BUG_ON(waiter->lock != lock); -+ rt_mutex_wake_waiter(waiter); -+ } -+} -+ -+#else -+ -+static void ww_mutex_account_lock(struct rt_mutex *lock, -+ struct ww_acquire_ctx *ww_ctx) -+{ -+ BUG(); -+} -+#endif -+ - /* - * Slow path lock function: - */ - static int __sched - rt_mutex_slowlock(struct rt_mutex *lock, int state, - struct hrtimer_sleeper *timeout, -- enum rtmutex_chainwalk chwalk) -+ enum rtmutex_chainwalk chwalk, -+ struct ww_acquire_ctx *ww_ctx) - { - struct rt_mutex_waiter waiter; - int ret = 0; + static int __init +diff -Nur linux-4.1.10.orig/drivers/usb/core/hcd.c linux-4.1.10/drivers/usb/core/hcd.c +--- linux-4.1.10.orig/drivers/usb/core/hcd.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/usb/core/hcd.c 2015-10-07 18:00:08.000000000 +0200 +@@ -1684,9 +1684,9 @@ + * and no one may trigger the above deadlock situation when + * running complete() in tasklet. + */ +- local_irq_save(flags); ++ local_irq_save_nort(flags); + urb->complete(urb); +- local_irq_restore(flags); ++ local_irq_restore_nort(flags); -- debug_rt_mutex_init_waiter(&waiter); -- RB_CLEAR_NODE(&waiter.pi_tree_entry); -- RB_CLEAR_NODE(&waiter.tree_entry); -+ rt_mutex_init_waiter(&waiter, false); + usb_anchor_resume_wakeups(anchor); + atomic_dec(&urb->use_count); +diff -Nur linux-4.1.10.orig/drivers/usb/gadget/function/f_fs.c linux-4.1.10/drivers/usb/gadget/function/f_fs.c +--- linux-4.1.10.orig/drivers/usb/gadget/function/f_fs.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/usb/gadget/function/f_fs.c 2015-10-07 18:00:08.000000000 +0200 +@@ -1405,7 +1405,7 @@ + pr_info("%s(): freeing\n", __func__); + ffs_data_clear(ffs); + BUG_ON(waitqueue_active(&ffs->ev.waitq) || +- waitqueue_active(&ffs->ep0req_completion.wait)); ++ swaitqueue_active(&ffs->ep0req_completion.wait)); + kfree(ffs->dev_name); + kfree(ffs); + } +diff -Nur linux-4.1.10.orig/drivers/usb/gadget/legacy/inode.c linux-4.1.10/drivers/usb/gadget/legacy/inode.c +--- linux-4.1.10.orig/drivers/usb/gadget/legacy/inode.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/drivers/usb/gadget/legacy/inode.c 2015-10-07 18:00:08.000000000 +0200 +@@ -345,7 +345,7 @@ + spin_unlock_irq (&epdata->dev->lock); - raw_spin_lock(&lock->wait_lock); + if (likely (value == 0)) { +- value = wait_event_interruptible (done.wait, done.done); ++ value = swait_event_interruptible (done.wait, done.done); + if (value != 0) { + spin_lock_irq (&epdata->dev->lock); + if (likely (epdata->ep != NULL)) { +@@ -354,7 +354,7 @@ + usb_ep_dequeue (epdata->ep, epdata->req); + spin_unlock_irq (&epdata->dev->lock); - /* Try to acquire the lock again: */ - if (try_to_take_rt_mutex(lock, current, NULL)) { -+ if (ww_ctx) -+ ww_mutex_account_lock(lock, ww_ctx); - raw_spin_unlock(&lock->wait_lock); - return 0; - } -@@ -1192,13 +1678,23 @@ +- wait_event (done.wait, done.done); ++ swait_event (done.wait, done.done); + if (epdata->status == -ECONNRESET) + epdata->status = -EINTR; + } else { +diff -Nur linux-4.1.10.orig/fs/aio.c linux-4.1.10/fs/aio.c +--- linux-4.1.10.orig/fs/aio.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/fs/aio.c 2015-10-07 18:00:08.000000000 +0200 +@@ -40,6 +40,7 @@ + #include + #include + #include ++#include - if (likely(!ret)) - /* sleep on the mutex */ -- ret = __rt_mutex_slowlock(lock, state, timeout, &waiter); -+ ret = __rt_mutex_slowlock(lock, state, timeout, &waiter, -+ ww_ctx); -+ else if (ww_ctx) { -+ /* ww_mutex received EDEADLK, let it become EALREADY */ -+ ret = __mutex_lock_check_stamp(lock, ww_ctx); -+ BUG_ON(!ret); -+ } + #include + #include +@@ -115,7 +116,7 @@ + struct page **ring_pages; + long nr_pages; - if (unlikely(ret)) { - __set_current_state(TASK_RUNNING); - if (rt_mutex_has_waiters(lock)) - remove_waiter(lock, &waiter); -- rt_mutex_handle_deadlock(ret, chwalk, &waiter); -+ /* ww_mutex want to report EDEADLK/EALREADY, let them */ -+ if (!ww_ctx) -+ rt_mutex_handle_deadlock(ret, chwalk, &waiter); -+ } else if (ww_ctx) { -+ ww_mutex_account_lock(lock, ww_ctx); - } +- struct work_struct free_work; ++ struct swork_event free_work; /* -@@ -1255,7 +1751,7 @@ - /* - * Slow path to release a rt-mutex: - */ --static void __sched -+static bool __sched - rt_mutex_slowunlock(struct rt_mutex *lock) + * signals when all in-flight requests are done +@@ -253,6 +254,7 @@ + .mount = aio_mount, + .kill_sb = kill_anon_super, + }; ++ BUG_ON(swork_get()); + aio_mnt = kern_mount(&aio_fs); + if (IS_ERR(aio_mnt)) + panic("Failed to create aio fs mount."); +@@ -559,9 +561,9 @@ + return cancel(&kiocb->common); + } + +-static void free_ioctx(struct work_struct *work) ++static void free_ioctx(struct swork_event *sev) { - raw_spin_lock(&lock->wait_lock); -@@ -1298,7 +1794,7 @@ - while (!rt_mutex_has_waiters(lock)) { - /* Drops lock->wait_lock ! */ - if (unlock_rt_mutex_safe(lock) == true) -- return; -+ return false; - /* Relock the rtmutex and try again */ - raw_spin_lock(&lock->wait_lock); - } -@@ -1311,8 +1807,7 @@ +- struct kioctx *ctx = container_of(work, struct kioctx, free_work); ++ struct kioctx *ctx = container_of(sev, struct kioctx, free_work); - raw_spin_unlock(&lock->wait_lock); + pr_debug("freeing %p\n", ctx); -- /* Undo pi boosting if necessary: */ -- rt_mutex_adjust_prio(current); -+ return true; +@@ -580,8 +582,8 @@ + if (ctx->rq_wait && atomic_dec_and_test(&ctx->rq_wait->count)) + complete(&ctx->rq_wait->comp); + +- INIT_WORK(&ctx->free_work, free_ioctx); +- schedule_work(&ctx->free_work); ++ INIT_SWORK(&ctx->free_work, free_ioctx); ++ swork_queue(&ctx->free_work); } /* -@@ -1323,31 +1818,36 @@ +@@ -589,9 +591,9 @@ + * and ctx->users has dropped to 0, so we know no more kiocbs can be submitted - + * now it's safe to cancel any that need to be. */ - static inline int - rt_mutex_fastlock(struct rt_mutex *lock, int state, -+ struct ww_acquire_ctx *ww_ctx, - int (*slowfn)(struct rt_mutex *lock, int state, - struct hrtimer_sleeper *timeout, -- enum rtmutex_chainwalk chwalk)) -+ enum rtmutex_chainwalk chwalk, -+ struct ww_acquire_ctx *ww_ctx)) +-static void free_ioctx_users(struct percpu_ref *ref) ++static void free_ioctx_users_work(struct swork_event *sev) { - if (likely(rt_mutex_cmpxchg(lock, NULL, current))) { - rt_mutex_deadlock_account_lock(lock, current); - return 0; - } else -- return slowfn(lock, state, NULL, RT_MUTEX_MIN_CHAINWALK); -+ return slowfn(lock, state, NULL, RT_MUTEX_MIN_CHAINWALK, -+ ww_ctx); +- struct kioctx *ctx = container_of(ref, struct kioctx, users); ++ struct kioctx *ctx = container_of(sev, struct kioctx, free_work); + struct aio_kiocb *req; + + spin_lock_irq(&ctx->ctx_lock); +@@ -610,6 +612,14 @@ + percpu_ref_put(&ctx->reqs); } - static inline int - rt_mutex_timed_fastlock(struct rt_mutex *lock, int state, - struct hrtimer_sleeper *timeout, - enum rtmutex_chainwalk chwalk, -+ struct ww_acquire_ctx *ww_ctx, - int (*slowfn)(struct rt_mutex *lock, int state, - struct hrtimer_sleeper *timeout, -- enum rtmutex_chainwalk chwalk)) -+ enum rtmutex_chainwalk chwalk, -+ struct ww_acquire_ctx *ww_ctx)) ++static void free_ioctx_users(struct percpu_ref *ref) ++{ ++ struct kioctx *ctx = container_of(ref, struct kioctx, users); ++ ++ INIT_SWORK(&ctx->free_work, free_ioctx_users_work); ++ swork_queue(&ctx->free_work); ++} ++ + static int ioctx_add_table(struct kioctx *ctx, struct mm_struct *mm) { - if (chwalk == RT_MUTEX_MIN_CHAINWALK && - likely(rt_mutex_cmpxchg(lock, NULL, current))) { - rt_mutex_deadlock_account_lock(lock, current); - return 0; - } else -- return slowfn(lock, state, timeout, chwalk); -+ return slowfn(lock, state, timeout, chwalk, ww_ctx); - } + unsigned i, new_nr; +diff -Nur linux-4.1.10.orig/fs/autofs4/autofs_i.h linux-4.1.10/fs/autofs4/autofs_i.h +--- linux-4.1.10.orig/fs/autofs4/autofs_i.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/fs/autofs4/autofs_i.h 2015-10-07 18:00:08.000000000 +0200 +@@ -34,6 +34,7 @@ + #include + #include + #include ++#include + #include + #include - static inline int -@@ -1363,12 +1863,14 @@ +diff -Nur linux-4.1.10.orig/fs/autofs4/expire.c linux-4.1.10/fs/autofs4/expire.c +--- linux-4.1.10.orig/fs/autofs4/expire.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/fs/autofs4/expire.c 2015-10-07 18:00:08.000000000 +0200 +@@ -150,7 +150,7 @@ + parent = p->d_parent; + if (!spin_trylock(&parent->d_lock)) { + spin_unlock(&p->d_lock); +- cpu_relax(); ++ cpu_chill(); + goto relock; + } + spin_unlock(&p->d_lock); +diff -Nur linux-4.1.10.orig/fs/buffer.c linux-4.1.10/fs/buffer.c +--- linux-4.1.10.orig/fs/buffer.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/fs/buffer.c 2015-10-07 18:00:08.000000000 +0200 +@@ -301,8 +301,7 @@ + * decide that the page is now completely done. + */ + first = page_buffers(page); +- local_irq_save(flags); +- bit_spin_lock(BH_Uptodate_Lock, &first->b_state); ++ flags = bh_uptodate_lock_irqsave(first); + clear_buffer_async_read(bh); + unlock_buffer(bh); + tmp = bh; +@@ -315,8 +314,7 @@ + } + tmp = tmp->b_this_page; + } while (tmp != bh); +- bit_spin_unlock(BH_Uptodate_Lock, &first->b_state); +- local_irq_restore(flags); ++ bh_uptodate_unlock_irqrestore(first, flags); - static inline void - rt_mutex_fastunlock(struct rt_mutex *lock, -- void (*slowfn)(struct rt_mutex *lock)) -+ bool (*slowfn)(struct rt_mutex *lock)) - { -- if (likely(rt_mutex_cmpxchg(lock, current, NULL))) -+ if (likely(rt_mutex_cmpxchg(lock, current, NULL))) { - rt_mutex_deadlock_account_unlock(current); -- else -- slowfn(lock); -+ } else if (slowfn(lock)) { -+ /* Undo pi boosting if necessary: */ -+ rt_mutex_adjust_prio(current); -+ } + /* + * If none of the buffers had errors and they are all +@@ -328,9 +326,7 @@ + return; + + still_busy: +- bit_spin_unlock(BH_Uptodate_Lock, &first->b_state); +- local_irq_restore(flags); +- return; ++ bh_uptodate_unlock_irqrestore(first, flags); } - /** -@@ -1380,7 +1882,7 @@ - { - might_sleep(); + /* +@@ -358,8 +354,7 @@ + } -- rt_mutex_fastlock(lock, TASK_UNINTERRUPTIBLE, rt_mutex_slowlock); -+ rt_mutex_fastlock(lock, TASK_UNINTERRUPTIBLE, NULL, rt_mutex_slowlock); - } - EXPORT_SYMBOL_GPL(rt_mutex_lock); + first = page_buffers(page); +- local_irq_save(flags); +- bit_spin_lock(BH_Uptodate_Lock, &first->b_state); ++ flags = bh_uptodate_lock_irqsave(first); -@@ -1397,7 +1899,7 @@ - { - might_sleep(); + clear_buffer_async_write(bh); + unlock_buffer(bh); +@@ -371,15 +366,12 @@ + } + tmp = tmp->b_this_page; + } +- bit_spin_unlock(BH_Uptodate_Lock, &first->b_state); +- local_irq_restore(flags); ++ bh_uptodate_unlock_irqrestore(first, flags); + end_page_writeback(page); + return; -- return rt_mutex_fastlock(lock, TASK_INTERRUPTIBLE, rt_mutex_slowlock); -+ return rt_mutex_fastlock(lock, TASK_INTERRUPTIBLE, NULL, rt_mutex_slowlock); + still_busy: +- bit_spin_unlock(BH_Uptodate_Lock, &first->b_state); +- local_irq_restore(flags); +- return; ++ bh_uptodate_unlock_irqrestore(first, flags); } - EXPORT_SYMBOL_GPL(rt_mutex_lock_interruptible); + EXPORT_SYMBOL(end_buffer_async_write); -@@ -1410,11 +1912,30 @@ - might_sleep(); +@@ -3325,6 +3317,7 @@ + struct buffer_head *ret = kmem_cache_zalloc(bh_cachep, gfp_flags); + if (ret) { + INIT_LIST_HEAD(&ret->b_assoc_buffers); ++ buffer_head_init_locks(ret); + preempt_disable(); + __this_cpu_inc(bh_accounting.nr); + recalc_bh_state(); +diff -Nur linux-4.1.10.orig/fs/dcache.c linux-4.1.10/fs/dcache.c +--- linux-4.1.10.orig/fs/dcache.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/fs/dcache.c 2015-10-07 18:00:08.000000000 +0200 +@@ -19,6 +19,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -589,7 +590,7 @@ - return rt_mutex_timed_fastlock(lock, TASK_INTERRUPTIBLE, timeout, -- RT_MUTEX_FULL_CHAINWALK, -+ RT_MUTEX_FULL_CHAINWALK, NULL, - rt_mutex_slowlock); + failed: + spin_unlock(&dentry->d_lock); +- cpu_relax(); ++ cpu_chill(); + return dentry; /* try again with same dentry */ } - /** -+ * rt_mutex_lock_killable - lock a rt_mutex killable -+ * -+ * @lock: the rt_mutex to be locked -+ * @detect_deadlock: deadlock detection on/off -+ * -+ * Returns: -+ * 0 on success -+ * -EINTR when interrupted by a signal -+ * -EDEADLK when the lock would deadlock (when deadlock detection is on) -+ */ -+int __sched rt_mutex_lock_killable(struct rt_mutex *lock) -+{ -+ might_sleep(); -+ -+ return rt_mutex_fastlock(lock, TASK_KILLABLE, NULL, rt_mutex_slowlock); -+} -+EXPORT_SYMBOL_GPL(rt_mutex_lock_killable); -+ -+/** - * rt_mutex_timed_lock - lock a rt_mutex interruptible - * the timeout structure is provided - * by the caller -@@ -1434,6 +1955,7 @@ +@@ -2395,7 +2396,7 @@ + if (dentry->d_lockref.count == 1) { + if (!spin_trylock(&inode->i_lock)) { + spin_unlock(&dentry->d_lock); +- cpu_relax(); ++ cpu_chill(); + goto again; + } + dentry->d_flags &= ~DCACHE_CANT_MOUNT; +diff -Nur linux-4.1.10.orig/fs/eventpoll.c linux-4.1.10/fs/eventpoll.c +--- linux-4.1.10.orig/fs/eventpoll.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/fs/eventpoll.c 2015-10-07 18:00:08.000000000 +0200 +@@ -505,12 +505,12 @@ + */ + static void ep_poll_safewake(wait_queue_head_t *wq) + { +- int this_cpu = get_cpu(); ++ int this_cpu = get_cpu_light(); - return rt_mutex_timed_fastlock(lock, TASK_INTERRUPTIBLE, timeout, - RT_MUTEX_MIN_CHAINWALK, -+ NULL, - rt_mutex_slowlock); + ep_call_nested(&poll_safewake_ncalls, EP_MAX_NESTS, + ep_poll_wakeup_proc, NULL, wq, (void *) (long) this_cpu); + +- put_cpu(); ++ put_cpu_light(); } - EXPORT_SYMBOL_GPL(rt_mutex_timed_lock); -@@ -1463,6 +1985,22 @@ - EXPORT_SYMBOL_GPL(rt_mutex_unlock); - /** -+ * rt_mutex_futex_unlock - Futex variant of rt_mutex_unlock -+ * @lock: the rt_mutex to be unlocked -+ * -+ * Returns: true/false indicating whether priority adjustment is -+ * required or not. -+ */ -+bool __sched rt_mutex_futex_unlock(struct rt_mutex *lock) -+{ -+ if (likely(rt_mutex_cmpxchg(lock, current, NULL))) { -+ rt_mutex_deadlock_account_unlock(current); -+ return false; -+ } -+ return rt_mutex_slowunlock(lock); -+} -+ -+/** - * rt_mutex_destroy - mark a mutex unusable - * @lock: the mutex to be destroyed - * -@@ -1492,13 +2030,12 @@ - void __rt_mutex_init(struct rt_mutex *lock, const char *name) - { - lock->owner = NULL; -- raw_spin_lock_init(&lock->wait_lock); - lock->waiters = RB_ROOT; - lock->waiters_leftmost = NULL; + static void ep_remove_wait_queue(struct eppoll_entry *pwq) +diff -Nur linux-4.1.10.orig/fs/exec.c linux-4.1.10/fs/exec.c +--- linux-4.1.10.orig/fs/exec.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/fs/exec.c 2015-10-07 18:00:08.000000000 +0200 +@@ -859,12 +859,14 @@ + } + } + task_lock(tsk); ++ preempt_disable_rt(); + active_mm = tsk->active_mm; + tsk->mm = mm; + tsk->active_mm = mm; + activate_mm(active_mm, mm); + tsk->mm->vmacache_seqnum = 0; + vmacache_flush(tsk); ++ preempt_enable_rt(); + task_unlock(tsk); + if (old_mm) { + up_read(&old_mm->mmap_sem); +diff -Nur linux-4.1.10.orig/fs/jbd/checkpoint.c linux-4.1.10/fs/jbd/checkpoint.c +--- linux-4.1.10.orig/fs/jbd/checkpoint.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/fs/jbd/checkpoint.c 2015-10-07 18:00:08.000000000 +0200 +@@ -129,6 +129,8 @@ + if (journal->j_flags & JFS_ABORT) + return; + spin_unlock(&journal->j_state_lock); ++ if (current->plug) ++ io_schedule(); + mutex_lock(&journal->j_checkpoint_mutex); - debug_rt_mutex_init(lock, name); + /* +diff -Nur linux-4.1.10.orig/fs/jbd2/checkpoint.c linux-4.1.10/fs/jbd2/checkpoint.c +--- linux-4.1.10.orig/fs/jbd2/checkpoint.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/fs/jbd2/checkpoint.c 2015-10-07 18:00:08.000000000 +0200 +@@ -116,6 +116,8 @@ + nblocks = jbd2_space_needed(journal); + while (jbd2_log_space_left(journal) < nblocks) { + write_unlock(&journal->j_state_lock); ++ if (current->plug) ++ io_schedule(); + mutex_lock(&journal->j_checkpoint_mutex); + + /* +diff -Nur linux-4.1.10.orig/fs/namespace.c linux-4.1.10/fs/namespace.c +--- linux-4.1.10.orig/fs/namespace.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/fs/namespace.c 2015-10-07 18:00:08.000000000 +0200 +@@ -14,6 +14,7 @@ + #include + #include + #include ++#include + #include + #include + #include /* init_rootfs */ +@@ -353,8 +354,11 @@ + * incremented count after it has set MNT_WRITE_HOLD. + */ + smp_mb(); +- while (ACCESS_ONCE(mnt->mnt.mnt_flags) & MNT_WRITE_HOLD) +- cpu_relax(); ++ while (ACCESS_ONCE(mnt->mnt.mnt_flags) & MNT_WRITE_HOLD) { ++ preempt_enable(); ++ cpu_chill(); ++ preempt_disable(); ++ } + /* + * After the slowpath clears MNT_WRITE_HOLD, mnt_is_readonly will + * be set to match its requirements. So we must not load that until +diff -Nur linux-4.1.10.orig/fs/ntfs/aops.c linux-4.1.10/fs/ntfs/aops.c +--- linux-4.1.10.orig/fs/ntfs/aops.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/fs/ntfs/aops.c 2015-10-07 18:00:08.000000000 +0200 +@@ -107,8 +107,7 @@ + "0x%llx.", (unsigned long long)bh->b_blocknr); + } + first = page_buffers(page); +- local_irq_save(flags); +- bit_spin_lock(BH_Uptodate_Lock, &first->b_state); ++ flags = bh_uptodate_lock_irqsave(first); + clear_buffer_async_read(bh); + unlock_buffer(bh); + tmp = bh; +@@ -123,8 +122,7 @@ + } + tmp = tmp->b_this_page; + } while (tmp != bh); +- bit_spin_unlock(BH_Uptodate_Lock, &first->b_state); +- local_irq_restore(flags); ++ bh_uptodate_unlock_irqrestore(first, flags); + /* + * If none of the buffers had errors then we can set the page uptodate, + * but we first have to perform the post read mst fixups, if the +@@ -145,13 +143,13 @@ + recs = PAGE_CACHE_SIZE / rec_size; + /* Should have been verified before we got here... */ + BUG_ON(!recs); +- local_irq_save(flags); ++ local_irq_save_nort(flags); + kaddr = kmap_atomic(page); + for (i = 0; i < recs; i++) + post_read_mst_fixup((NTFS_RECORD*)(kaddr + + i * rec_size), rec_size); + kunmap_atomic(kaddr); +- local_irq_restore(flags); ++ local_irq_restore_nort(flags); + flush_dcache_page(page); + if (likely(page_uptodate && !PageError(page))) + SetPageUptodate(page); +@@ -159,9 +157,7 @@ + unlock_page(page); + return; + still_busy: +- bit_spin_unlock(BH_Uptodate_Lock, &first->b_state); +- local_irq_restore(flags); +- return; ++ bh_uptodate_unlock_irqrestore(first, flags); } --EXPORT_SYMBOL_GPL(__rt_mutex_init); -+EXPORT_SYMBOL(__rt_mutex_init); /** - * rt_mutex_init_proxy_locked - initialize and lock a rt_mutex on behalf of a -@@ -1513,7 +2050,7 @@ - void rt_mutex_init_proxy_locked(struct rt_mutex *lock, - struct task_struct *proxy_owner) +diff -Nur linux-4.1.10.orig/fs/timerfd.c linux-4.1.10/fs/timerfd.c +--- linux-4.1.10.orig/fs/timerfd.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/fs/timerfd.c 2015-10-07 18:00:08.000000000 +0200 +@@ -450,7 +450,10 @@ + break; + } + spin_unlock_irq(&ctx->wqh.lock); +- cpu_relax(); ++ if (isalarm(ctx)) ++ hrtimer_wait_for_timer(&ctx->t.alarm.timer); ++ else ++ hrtimer_wait_for_timer(&ctx->t.tmr); + } + + /* +diff -Nur linux-4.1.10.orig/fs/xfs/xfs_inode.c linux-4.1.10/fs/xfs/xfs_inode.c +--- linux-4.1.10.orig/fs/xfs/xfs_inode.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/fs/xfs/xfs_inode.c 2015-10-07 18:00:08.000000000 +0200 +@@ -164,7 +164,7 @@ + (XFS_MMAPLOCK_SHARED | XFS_MMAPLOCK_EXCL)); + ASSERT((lock_flags & (XFS_ILOCK_SHARED | XFS_ILOCK_EXCL)) != + (XFS_ILOCK_SHARED | XFS_ILOCK_EXCL)); +- ASSERT((lock_flags & ~(XFS_LOCK_MASK | XFS_LOCK_DEP_MASK)) == 0); ++ ASSERT((lock_flags & ~(XFS_LOCK_MASK | XFS_LOCK_SUBCLASS_MASK)) == 0); + + if (lock_flags & XFS_IOLOCK_EXCL) + mrupdate_nested(&ip->i_iolock, XFS_IOLOCK_DEP(lock_flags)); +@@ -212,7 +212,7 @@ + (XFS_MMAPLOCK_SHARED | XFS_MMAPLOCK_EXCL)); + ASSERT((lock_flags & (XFS_ILOCK_SHARED | XFS_ILOCK_EXCL)) != + (XFS_ILOCK_SHARED | XFS_ILOCK_EXCL)); +- ASSERT((lock_flags & ~(XFS_LOCK_MASK | XFS_LOCK_DEP_MASK)) == 0); ++ ASSERT((lock_flags & ~(XFS_LOCK_MASK | XFS_LOCK_SUBCLASS_MASK)) == 0); + + if (lock_flags & XFS_IOLOCK_EXCL) { + if (!mrtryupdate(&ip->i_iolock)) +@@ -281,7 +281,7 @@ + (XFS_MMAPLOCK_SHARED | XFS_MMAPLOCK_EXCL)); + ASSERT((lock_flags & (XFS_ILOCK_SHARED | XFS_ILOCK_EXCL)) != + (XFS_ILOCK_SHARED | XFS_ILOCK_EXCL)); +- ASSERT((lock_flags & ~(XFS_LOCK_MASK | XFS_LOCK_DEP_MASK)) == 0); ++ ASSERT((lock_flags & ~(XFS_LOCK_MASK | XFS_LOCK_SUBCLASS_MASK)) == 0); + ASSERT(lock_flags != 0); + + if (lock_flags & XFS_IOLOCK_EXCL) +@@ -364,30 +364,38 @@ + + /* + * Bump the subclass so xfs_lock_inodes() acquires each lock with a different +- * value. This shouldn't be called for page fault locking, but we also need to +- * ensure we don't overrun the number of lockdep subclasses for the iolock or +- * mmaplock as that is limited to 12 by the mmap lock lockdep annotations. ++ * value. This can be called for any type of inode lock combination, including ++ * parent locking. Care must be taken to ensure we don't overrun the subclass ++ * storage fields in the class mask we build. + */ + static inline int + xfs_lock_inumorder(int lock_mode, int subclass) { -- __rt_mutex_init(lock, NULL); -+ rt_mutex_init(lock); - debug_rt_mutex_proxy_lock(lock, proxy_owner); - rt_mutex_set_owner(lock, proxy_owner); - rt_mutex_deadlock_account_lock(lock, proxy_owner); -@@ -1561,6 +2098,35 @@ - return 1; ++ int class = 0; ++ ++ ASSERT(!(lock_mode & (XFS_ILOCK_PARENT | XFS_ILOCK_RTBITMAP | ++ XFS_ILOCK_RTSUM))); ++ + if (lock_mode & (XFS_IOLOCK_SHARED|XFS_IOLOCK_EXCL)) { +- ASSERT(subclass + XFS_LOCK_INUMORDER < +- (1 << (XFS_MMAPLOCK_SHIFT - XFS_IOLOCK_SHIFT))); +- lock_mode |= (subclass + XFS_LOCK_INUMORDER) << XFS_IOLOCK_SHIFT; ++ ASSERT(subclass <= XFS_IOLOCK_MAX_SUBCLASS); ++ ASSERT(subclass + XFS_IOLOCK_PARENT_VAL < ++ MAX_LOCKDEP_SUBCLASSES); ++ class += subclass << XFS_IOLOCK_SHIFT; ++ if (lock_mode & XFS_IOLOCK_PARENT) ++ class += XFS_IOLOCK_PARENT_VAL << XFS_IOLOCK_SHIFT; } -+#ifdef CONFIG_PREEMPT_RT_FULL -+ /* -+ * In PREEMPT_RT there's an added race. -+ * If the task, that we are about to requeue, times out, -+ * it can set the PI_WAKEUP_INPROGRESS. This tells the requeue -+ * to skip this task. But right after the task sets -+ * its pi_blocked_on to PI_WAKEUP_INPROGRESS it can then -+ * block on the spin_lock(&hb->lock), which in RT is an rtmutex. -+ * This will replace the PI_WAKEUP_INPROGRESS with the actual -+ * lock that it blocks on. We *must not* place this task -+ * on this proxy lock in that case. -+ * -+ * To prevent this race, we first take the task's pi_lock -+ * and check if it has updated its pi_blocked_on. If it has, -+ * we assume that it woke up and we return -EAGAIN. -+ * Otherwise, we set the task's pi_blocked_on to -+ * PI_REQUEUE_INPROGRESS, so that if the task is waking up -+ * it will know that we are in the process of requeuing it. -+ */ -+ raw_spin_lock_irq(&task->pi_lock); -+ if (task->pi_blocked_on) { -+ raw_spin_unlock_irq(&task->pi_lock); -+ raw_spin_unlock(&lock->wait_lock); -+ return -EAGAIN; + if (lock_mode & (XFS_MMAPLOCK_SHARED|XFS_MMAPLOCK_EXCL)) { +- ASSERT(subclass + XFS_LOCK_INUMORDER < +- (1 << (XFS_ILOCK_SHIFT - XFS_MMAPLOCK_SHIFT))); +- lock_mode |= (subclass + XFS_LOCK_INUMORDER) << +- XFS_MMAPLOCK_SHIFT; ++ ASSERT(subclass <= XFS_MMAPLOCK_MAX_SUBCLASS); ++ class += subclass << XFS_MMAPLOCK_SHIFT; + } + +- if (lock_mode & (XFS_ILOCK_SHARED|XFS_ILOCK_EXCL)) +- lock_mode |= (subclass + XFS_LOCK_INUMORDER) << XFS_ILOCK_SHIFT; ++ if (lock_mode & (XFS_ILOCK_SHARED|XFS_ILOCK_EXCL)) { ++ ASSERT(subclass <= XFS_ILOCK_MAX_SUBCLASS); ++ class += subclass << XFS_ILOCK_SHIFT; + } -+ task->pi_blocked_on = PI_REQUEUE_INPROGRESS; -+ raw_spin_unlock_irq(&task->pi_lock); -+#endif -+ - /* We enforce deadlock detection for futexes */ - ret = task_blocks_on_rt_mutex(lock, waiter, task, - RT_MUTEX_FULL_CHAINWALK); -@@ -1631,7 +2197,7 @@ - set_current_state(TASK_INTERRUPTIBLE); - /* sleep on the mutex */ -- ret = __rt_mutex_slowlock(lock, TASK_INTERRUPTIBLE, to, waiter); -+ ret = __rt_mutex_slowlock(lock, TASK_INTERRUPTIBLE, to, waiter, NULL); +- return lock_mode; ++ return (lock_mode & ~XFS_LOCK_SUBCLASS_MASK) | class; + } - if (unlikely(ret)) - remove_waiter(lock, waiter); -@@ -1646,3 +2212,89 @@ + /* +@@ -399,6 +407,11 @@ + * transaction (such as truncate). This can result in deadlock since the long + * running trans might need to wait for the inode we just locked in order to + * push the tail and free space in the log. ++ * ++ * xfs_lock_inodes() can only be used to lock one type of lock at a time - ++ * the iolock, the mmaplock or the ilock, but not more than one at a time. If we ++ * lock more than one at a time, lockdep will report false positives saying we ++ * have violated locking orders. + */ + void + xfs_lock_inodes( +@@ -409,8 +422,29 @@ + int attempts = 0, i, j, try_lock; + xfs_log_item_t *lp; - return ret; - } -+ -+static inline int -+ww_mutex_deadlock_injection(struct ww_mutex *lock, struct ww_acquire_ctx *ctx) -+{ -+#ifdef CONFIG_DEBUG_WW_MUTEX_SLOWPATH -+ unsigned tmp; -+ -+ if (ctx->deadlock_inject_countdown-- == 0) { -+ tmp = ctx->deadlock_inject_interval; -+ if (tmp > UINT_MAX/4) -+ tmp = UINT_MAX; -+ else -+ tmp = tmp*2 + tmp + tmp/2; -+ -+ ctx->deadlock_inject_interval = tmp; -+ ctx->deadlock_inject_countdown = tmp; -+ ctx->contending_lock = lock; -+ -+ ww_mutex_unlock(lock); -+ -+ return -EDEADLK; -+ } -+#endif -+ -+ return 0; -+} -+ -+#ifdef CONFIG_PREEMPT_RT_FULL -+int __sched -+__ww_mutex_lock_interruptible(struct ww_mutex *lock, struct ww_acquire_ctx *ww_ctx) -+{ -+ int ret; -+ -+ might_sleep(); -+ -+ mutex_acquire_nest(&lock->base.dep_map, 0, 0, &ww_ctx->dep_map, _RET_IP_); -+ ret = rt_mutex_slowlock(&lock->base.lock, TASK_INTERRUPTIBLE, NULL, 0, ww_ctx); -+ if (ret) -+ mutex_release(&lock->base.dep_map, 1, _RET_IP_); -+ else if (!ret && ww_ctx->acquired > 1) -+ return ww_mutex_deadlock_injection(lock, ww_ctx); -+ -+ return ret; -+} -+EXPORT_SYMBOL_GPL(__ww_mutex_lock_interruptible); -+ -+int __sched -+__ww_mutex_lock(struct ww_mutex *lock, struct ww_acquire_ctx *ww_ctx) -+{ -+ int ret; -+ -+ might_sleep(); -+ -+ mutex_acquire_nest(&lock->base.dep_map, 0, 0, &ww_ctx->dep_map, _RET_IP_); -+ ret = rt_mutex_slowlock(&lock->base.lock, TASK_UNINTERRUPTIBLE, NULL, 0, ww_ctx); -+ if (ret) -+ mutex_release(&lock->base.dep_map, 1, _RET_IP_); -+ else if (!ret && ww_ctx->acquired > 1) -+ return ww_mutex_deadlock_injection(lock, ww_ctx); -+ -+ return ret; -+} -+EXPORT_SYMBOL_GPL(__ww_mutex_lock); -+ -+void __sched ww_mutex_unlock(struct ww_mutex *lock) -+{ -+ int nest = !!lock->ctx; -+ +- /* currently supports between 2 and 5 inodes */ + /* -+ * The unlocking fastpath is the 0->1 transition from 'locked' -+ * into 'unlocked' state: ++ * Currently supports between 2 and 5 inodes with exclusive locking. We ++ * support an arbitrary depth of locking here, but absolute limits on ++ * inodes depend on the the type of locking and the limits placed by ++ * lockdep annotations in xfs_lock_inumorder. These are all checked by ++ * the asserts. + */ -+ if (nest) { -+#ifdef CONFIG_DEBUG_MUTEXES -+ DEBUG_LOCKS_WARN_ON(!lock->ctx->acquired); -+#endif -+ if (lock->ctx->acquired > 0) -+ lock->ctx->acquired--; -+ lock->ctx = NULL; -+ } -+ -+ mutex_release(&lock->base.dep_map, nest, _RET_IP_); -+ rt_mutex_unlock(&lock->base.lock); -+} -+EXPORT_SYMBOL(ww_mutex_unlock); -+#endif -diff -Nur linux-4.1.6.orig/kernel/locking/rtmutex_common.h linux-4.1.6/kernel/locking/rtmutex_common.h ---- linux-4.1.6.orig/kernel/locking/rtmutex_common.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/locking/rtmutex_common.h 2015-09-08 23:49:08.101867828 +0200 -@@ -49,6 +49,7 @@ - struct rb_node pi_tree_entry; - struct task_struct *task; - struct rt_mutex *lock; -+ bool savestate; - #ifdef CONFIG_DEBUG_RT_MUTEXES - unsigned long ip; - struct pid *deadlock_task_pid; -@@ -119,6 +120,9 @@ - /* - * PI-futex support (proxy locking functions, etc.): + ASSERT(ips && inodes >= 2 && inodes <= 5); ++ ASSERT(lock_mode & (XFS_IOLOCK_EXCL | XFS_MMAPLOCK_EXCL | ++ XFS_ILOCK_EXCL)); ++ ASSERT(!(lock_mode & (XFS_IOLOCK_SHARED | XFS_MMAPLOCK_SHARED | ++ XFS_ILOCK_SHARED))); ++ ASSERT(!(lock_mode & XFS_IOLOCK_EXCL) || ++ inodes <= XFS_IOLOCK_MAX_SUBCLASS + 1); ++ ASSERT(!(lock_mode & XFS_MMAPLOCK_EXCL) || ++ inodes <= XFS_MMAPLOCK_MAX_SUBCLASS + 1); ++ ASSERT(!(lock_mode & XFS_ILOCK_EXCL) || ++ inodes <= XFS_ILOCK_MAX_SUBCLASS + 1); ++ ++ if (lock_mode & XFS_IOLOCK_EXCL) { ++ ASSERT(!(lock_mode & (XFS_MMAPLOCK_EXCL | XFS_ILOCK_EXCL))); ++ } else if (lock_mode & XFS_MMAPLOCK_EXCL) ++ ASSERT(!(lock_mode & XFS_ILOCK_EXCL)); + + try_lock = 0; + i = 0; +diff -Nur linux-4.1.10.orig/fs/xfs/xfs_inode.h linux-4.1.10/fs/xfs/xfs_inode.h +--- linux-4.1.10.orig/fs/xfs/xfs_inode.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/fs/xfs/xfs_inode.h 2015-10-07 18:00:08.000000000 +0200 +@@ -284,9 +284,9 @@ + * Flags for lockdep annotations. + * + * XFS_LOCK_PARENT - for directory operations that require locking a +- * parent directory inode and a child entry inode. The parent gets locked +- * with this flag so it gets a lockdep subclass of 1 and the child entry +- * lock will have a lockdep subclass of 0. ++ * parent directory inode and a child entry inode. IOLOCK requires nesting, ++ * MMAPLOCK does not support this class, ILOCK requires a single subclass ++ * to differentiate parent from child. + * + * XFS_LOCK_RTBITMAP/XFS_LOCK_RTSUM - the realtime device bitmap and summary + * inodes do not participate in the normal lock order, and thus have their +@@ -295,30 +295,63 @@ + * XFS_LOCK_INUMORDER - for locking several inodes at the some time + * with xfs_lock_inodes(). This flag is used as the starting subclass + * and each subsequent lock acquired will increment the subclass by one. +- * So the first lock acquired will have a lockdep subclass of 4, the +- * second lock will have a lockdep subclass of 5, and so on. It is +- * the responsibility of the class builder to shift this to the correct +- * portion of the lock_mode lockdep mask. ++ * However, MAX_LOCKDEP_SUBCLASSES == 8, which means we are greatly ++ * limited to the subclasses we can represent via nesting. We need at least ++ * 5 inodes nest depth for the ILOCK through rename, and we also have to support ++ * XFS_ILOCK_PARENT, which gives 6 subclasses. Then we have XFS_ILOCK_RTBITMAP ++ * and XFS_ILOCK_RTSUM, which are another 2 unique subclasses, so that's all ++ * 8 subclasses supported by lockdep. ++ * ++ * This also means we have to number the sub-classes in the lowest bits of ++ * the mask we keep, and we have to ensure we never exceed 3 bits of lockdep ++ * mask and we can't use bit-masking to build the subclasses. What a mess. ++ * ++ * Bit layout: ++ * ++ * Bit Lock Region ++ * 16-19 XFS_IOLOCK_SHIFT dependencies ++ * 20-23 XFS_MMAPLOCK_SHIFT dependencies ++ * 24-31 XFS_ILOCK_SHIFT dependencies ++ * ++ * IOLOCK values ++ * ++ * 0-3 subclass value ++ * 4-7 PARENT subclass values ++ * ++ * MMAPLOCK values ++ * ++ * 0-3 subclass value ++ * 4-7 unused ++ * ++ * ILOCK values ++ * 0-4 subclass values ++ * 5 PARENT subclass (not nestable) ++ * 6 RTBITMAP subclass (not nestable) ++ * 7 RTSUM subclass (not nestable) ++ * */ -+#define PI_WAKEUP_INPROGRESS ((struct rt_mutex_waiter *) 1) -+#define PI_REQUEUE_INPROGRESS ((struct rt_mutex_waiter *) 2) -+ - extern struct task_struct *rt_mutex_next_owner(struct rt_mutex *lock); - extern void rt_mutex_init_proxy_locked(struct rt_mutex *lock, - struct task_struct *proxy_owner); -@@ -132,10 +136,24 @@ - struct rt_mutex_waiter *waiter); - extern int rt_mutex_timed_futex_lock(struct rt_mutex *l, struct hrtimer_sleeper *to); +-#define XFS_LOCK_PARENT 1 +-#define XFS_LOCK_RTBITMAP 2 +-#define XFS_LOCK_RTSUM 3 +-#define XFS_LOCK_INUMORDER 4 +- +-#define XFS_IOLOCK_SHIFT 16 +-#define XFS_IOLOCK_PARENT (XFS_LOCK_PARENT << XFS_IOLOCK_SHIFT) ++#define XFS_IOLOCK_SHIFT 16 ++#define XFS_IOLOCK_PARENT_VAL 4 ++#define XFS_IOLOCK_MAX_SUBCLASS (XFS_IOLOCK_PARENT_VAL - 1) ++#define XFS_IOLOCK_DEP_MASK 0x000f0000 ++#define XFS_IOLOCK_PARENT (XFS_IOLOCK_PARENT_VAL << XFS_IOLOCK_SHIFT) + +-#define XFS_MMAPLOCK_SHIFT 20 ++#define XFS_MMAPLOCK_SHIFT 20 ++#define XFS_MMAPLOCK_NUMORDER 0 ++#define XFS_MMAPLOCK_MAX_SUBCLASS 3 ++#define XFS_MMAPLOCK_DEP_MASK 0x00f00000 + +-#define XFS_ILOCK_SHIFT 24 +-#define XFS_ILOCK_PARENT (XFS_LOCK_PARENT << XFS_ILOCK_SHIFT) +-#define XFS_ILOCK_RTBITMAP (XFS_LOCK_RTBITMAP << XFS_ILOCK_SHIFT) +-#define XFS_ILOCK_RTSUM (XFS_LOCK_RTSUM << XFS_ILOCK_SHIFT) ++#define XFS_ILOCK_SHIFT 24 ++#define XFS_ILOCK_PARENT_VAL 5 ++#define XFS_ILOCK_MAX_SUBCLASS (XFS_ILOCK_PARENT_VAL - 1) ++#define XFS_ILOCK_RTBITMAP_VAL 6 ++#define XFS_ILOCK_RTSUM_VAL 7 ++#define XFS_ILOCK_DEP_MASK 0xff000000 ++#define XFS_ILOCK_PARENT (XFS_ILOCK_PARENT_VAL << XFS_ILOCK_SHIFT) ++#define XFS_ILOCK_RTBITMAP (XFS_ILOCK_RTBITMAP_VAL << XFS_ILOCK_SHIFT) ++#define XFS_ILOCK_RTSUM (XFS_ILOCK_RTSUM_VAL << XFS_ILOCK_SHIFT) + +-#define XFS_IOLOCK_DEP_MASK 0x000f0000 +-#define XFS_MMAPLOCK_DEP_MASK 0x00f00000 +-#define XFS_ILOCK_DEP_MASK 0xff000000 +-#define XFS_LOCK_DEP_MASK (XFS_IOLOCK_DEP_MASK | \ ++#define XFS_LOCK_SUBCLASS_MASK (XFS_IOLOCK_DEP_MASK | \ + XFS_MMAPLOCK_DEP_MASK | \ + XFS_ILOCK_DEP_MASK) + +diff -Nur linux-4.1.10.orig/include/acpi/platform/aclinux.h linux-4.1.10/include/acpi/platform/aclinux.h +--- linux-4.1.10.orig/include/acpi/platform/aclinux.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/acpi/platform/aclinux.h 2015-10-07 18:00:08.000000000 +0200 +@@ -123,6 +123,7 @@ -+extern bool rt_mutex_futex_unlock(struct rt_mutex *lock); + #define acpi_cache_t struct kmem_cache + #define acpi_spinlock spinlock_t * ++#define acpi_raw_spinlock raw_spinlock_t * + #define acpi_cpu_flags unsigned long + + /* Use native linux version of acpi_os_allocate_zeroed */ +@@ -141,6 +142,20 @@ + #define ACPI_USE_ALTERNATE_PROTOTYPE_acpi_os_get_thread_id + #define ACPI_USE_ALTERNATE_PROTOTYPE_acpi_os_create_lock + ++#define acpi_os_create_raw_lock(__handle) \ ++({ \ ++ raw_spinlock_t *lock = ACPI_ALLOCATE(sizeof(*lock)); \ ++ \ ++ if (lock) { \ ++ *(__handle) = lock; \ ++ raw_spin_lock_init(*(__handle)); \ ++ } \ ++ lock ? AE_OK : AE_NO_MEMORY; \ ++ }) + -+extern void rt_mutex_adjust_prio(struct task_struct *task); ++#define acpi_os_delete_raw_lock(__handle) kfree(__handle) + - #ifdef CONFIG_DEBUG_RT_MUTEXES - # include "rtmutex-debug.h" - #else - # include "rtmutex.h" ++ + /* + * OSL interfaces used by debugger/disassembler + */ +diff -Nur linux-4.1.10.orig/include/asm-generic/bug.h linux-4.1.10/include/asm-generic/bug.h +--- linux-4.1.10.orig/include/asm-generic/bug.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/asm-generic/bug.h 2015-10-07 18:00:08.000000000 +0200 +@@ -206,6 +206,20 @@ + # define WARN_ON_SMP(x) ({0;}) #endif -+static inline void -+rt_mutex_init_waiter(struct rt_mutex_waiter *waiter, bool savestate) -+{ -+ debug_rt_mutex_init_waiter(waiter); -+ waiter->task = NULL; -+ waiter->savestate = savestate; -+ RB_CLEAR_NODE(&waiter->pi_tree_entry); -+ RB_CLEAR_NODE(&waiter->tree_entry); -+} ++#ifdef CONFIG_PREEMPT_RT_BASE ++# define BUG_ON_RT(c) BUG_ON(c) ++# define BUG_ON_NONRT(c) do { } while (0) ++# define WARN_ON_RT(condition) WARN_ON(condition) ++# define WARN_ON_NONRT(condition) do { } while (0) ++# define WARN_ON_ONCE_NONRT(condition) do { } while (0) ++#else ++# define BUG_ON_RT(c) do { } while (0) ++# define BUG_ON_NONRT(c) BUG_ON(c) ++# define WARN_ON_RT(condition) do { } while (0) ++# define WARN_ON_NONRT(condition) WARN_ON(condition) ++# define WARN_ON_ONCE_NONRT(condition) WARN_ON_ONCE(condition) ++#endif + + #endif /* __ASSEMBLY__ */ + #endif -diff -Nur linux-4.1.6.orig/kernel/locking/spinlock.c linux-4.1.6/kernel/locking/spinlock.c ---- linux-4.1.6.orig/kernel/locking/spinlock.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/locking/spinlock.c 2015-09-08 23:49:08.101867828 +0200 -@@ -124,8 +124,11 @@ - * __[spin|read|write]_lock_bh() +diff -Nur linux-4.1.10.orig/include/asm-generic/futex.h linux-4.1.10/include/asm-generic/futex.h +--- linux-4.1.10.orig/include/asm-generic/futex.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/asm-generic/futex.h 2015-10-07 18:00:08.000000000 +0200 +@@ -8,8 +8,7 @@ + #ifndef CONFIG_SMP + /* + * The following implementation only for uniprocessor machines. +- * For UP, it's relies on the fact that pagefault_disable() also disables +- * preemption to ensure mutual exclusion. ++ * It relies on preempt_disable() ensuring mutual exclusion. + * */ - BUILD_LOCK_OPS(spin, raw_spinlock); -+ -+#ifndef CONFIG_PREEMPT_RT_FULL - BUILD_LOCK_OPS(read, rwlock); - BUILD_LOCK_OPS(write, rwlock); -+#endif - #endif +@@ -38,6 +37,7 @@ + if (encoded_op & (FUTEX_OP_OPARG_SHIFT << 28)) + oparg = 1 << oparg; -@@ -209,6 +212,8 @@ - EXPORT_SYMBOL(_raw_spin_unlock_bh); - #endif ++ preempt_disable(); + pagefault_disable(); -+#ifndef CONFIG_PREEMPT_RT_FULL -+ - #ifndef CONFIG_INLINE_READ_TRYLOCK - int __lockfunc _raw_read_trylock(rwlock_t *lock) + ret = -EFAULT; +@@ -72,6 +72,7 @@ + + out_pagefault_enable: + pagefault_enable(); ++ preempt_enable(); + + if (ret == 0) { + switch (cmp) { +@@ -106,6 +107,7 @@ { -@@ -353,6 +358,8 @@ - EXPORT_SYMBOL(_raw_write_unlock_bh); - #endif + u32 val; -+#endif /* !PREEMPT_RT_FULL */ -+ - #ifdef CONFIG_DEBUG_LOCK_ALLOC ++ preempt_disable(); + if (unlikely(get_user(val, uaddr) != 0)) + return -EFAULT; - void __lockfunc _raw_spin_lock_nested(raw_spinlock_t *lock, int subclass) -diff -Nur linux-4.1.6.orig/kernel/locking/spinlock_debug.c linux-4.1.6/kernel/locking/spinlock_debug.c ---- linux-4.1.6.orig/kernel/locking/spinlock_debug.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/locking/spinlock_debug.c 2015-09-08 23:49:08.105867384 +0200 -@@ -31,6 +31,7 @@ +@@ -113,6 +115,7 @@ + return -EFAULT; - EXPORT_SYMBOL(__raw_spin_lock_init); + *uval = val; ++ preempt_enable(); -+#ifndef CONFIG_PREEMPT_RT_FULL - void __rwlock_init(rwlock_t *lock, const char *name, - struct lock_class_key *key) - { -@@ -48,6 +49,7 @@ + return 0; } +diff -Nur linux-4.1.10.orig/include/linux/blkdev.h linux-4.1.10/include/linux/blkdev.h +--- linux-4.1.10.orig/include/linux/blkdev.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/blkdev.h 2015-10-07 18:00:08.000000000 +0200 +@@ -101,6 +101,7 @@ + struct list_head queuelist; + union { + struct call_single_data csd; ++ struct work_struct work; + unsigned long fifo_time; + }; - EXPORT_SYMBOL(__rwlock_init); -+#endif +@@ -482,7 +483,7 @@ + struct throtl_data *td; + #endif + struct rcu_head rcu_head; +- wait_queue_head_t mq_freeze_wq; ++ struct swait_head mq_freeze_wq; + struct percpu_ref mq_usage_counter; + struct list_head all_q_node; - static void spin_dump(raw_spinlock_t *lock, const char *msg) - { -@@ -159,6 +161,7 @@ - arch_spin_unlock(&lock->raw_lock); - } +diff -Nur linux-4.1.10.orig/include/linux/blk-mq.h linux-4.1.10/include/linux/blk-mq.h +--- linux-4.1.10.orig/include/linux/blk-mq.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/blk-mq.h 2015-10-07 18:00:08.000000000 +0200 +@@ -202,6 +202,7 @@ -+#ifndef CONFIG_PREEMPT_RT_FULL - static void rwlock_bug(rwlock_t *lock, const char *msg) + struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *, const int ctx_index); + struct blk_mq_hw_ctx *blk_mq_alloc_single_hw_queue(struct blk_mq_tag_set *, unsigned int, int); ++void __blk_mq_complete_request_remote_work(struct work_struct *work); + + int blk_mq_request_started(struct request *rq); + void blk_mq_start_request(struct request *rq); +diff -Nur linux-4.1.10.orig/include/linux/bottom_half.h linux-4.1.10/include/linux/bottom_half.h +--- linux-4.1.10.orig/include/linux/bottom_half.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/bottom_half.h 2015-10-07 18:00:08.000000000 +0200 +@@ -4,6 +4,17 @@ + #include + #include + ++#ifdef CONFIG_PREEMPT_RT_FULL ++ ++extern void local_bh_disable(void); ++extern void _local_bh_enable(void); ++extern void local_bh_enable(void); ++extern void local_bh_enable_ip(unsigned long ip); ++extern void __local_bh_disable_ip(unsigned long ip, unsigned int cnt); ++extern void __local_bh_enable_ip(unsigned long ip, unsigned int cnt); ++ ++#else ++ + #ifdef CONFIG_TRACE_IRQFLAGS + extern void __local_bh_disable_ip(unsigned long ip, unsigned int cnt); + #else +@@ -31,5 +42,6 @@ { - if (!debug_locks_off()) -@@ -300,3 +303,5 @@ - debug_write_unlock(lock); - arch_write_unlock(&lock->raw_lock); + __local_bh_enable_ip(_THIS_IP_, SOFTIRQ_DISABLE_OFFSET); } -+ +#endif -diff -Nur linux-4.1.6.orig/kernel/panic.c linux-4.1.6/kernel/panic.c ---- linux-4.1.6.orig/kernel/panic.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/panic.c 2015-09-08 23:49:08.105867384 +0200 -@@ -387,9 +387,11 @@ - static int init_oops_id(void) - { -+#ifndef CONFIG_PREEMPT_RT_FULL - if (!oops_id) - get_random_bytes(&oops_id, sizeof(oops_id)); - else + #endif /* _LINUX_BH_H */ +diff -Nur linux-4.1.10.orig/include/linux/buffer_head.h linux-4.1.10/include/linux/buffer_head.h +--- linux-4.1.10.orig/include/linux/buffer_head.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/buffer_head.h 2015-10-07 18:00:08.000000000 +0200 +@@ -75,8 +75,52 @@ + struct address_space *b_assoc_map; /* mapping this buffer is + associated with */ + atomic_t b_count; /* users using this buffer_head */ ++#ifdef CONFIG_PREEMPT_RT_BASE ++ spinlock_t b_uptodate_lock; ++#if defined(CONFIG_JBD) || defined(CONFIG_JBD_MODULE) || \ ++ defined(CONFIG_JBD2) || defined(CONFIG_JBD2_MODULE) ++ spinlock_t b_state_lock; ++ spinlock_t b_journal_head_lock; +#endif - oops_id++; ++#endif + }; - return 0; -diff -Nur linux-4.1.6.orig/kernel/power/hibernate.c linux-4.1.6/kernel/power/hibernate.c ---- linux-4.1.6.orig/kernel/power/hibernate.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/power/hibernate.c 2015-09-08 23:49:08.105867384 +0200 -@@ -285,6 +285,8 @@ - - local_irq_disable(); - -+ system_state = SYSTEM_SUSPEND; -+ - error = syscore_suspend(); - if (error) { - printk(KERN_ERR "PM: Some system devices failed to power down, " -@@ -314,6 +316,7 @@ - syscore_resume(); - - Enable_irqs: -+ system_state = SYSTEM_RUNNING; - local_irq_enable(); - - Enable_cpus: -@@ -437,6 +440,7 @@ - goto Enable_cpus; - - local_irq_disable(); -+ system_state = SYSTEM_SUSPEND; - - error = syscore_suspend(); - if (error) -@@ -470,6 +474,7 @@ - syscore_resume(); - - Enable_irqs: -+ system_state = SYSTEM_RUNNING; - local_irq_enable(); - - Enable_cpus: -@@ -555,6 +560,7 @@ - goto Platform_finish; - - local_irq_disable(); -+ system_state = SYSTEM_SUSPEND; - syscore_suspend(); - if (pm_wakeup_pending()) { - error = -EAGAIN; -@@ -567,6 +573,7 @@ - - Power_up: - syscore_resume(); -+ system_state = SYSTEM_RUNNING; - local_irq_enable(); - enable_nonboot_cpus(); - -diff -Nur linux-4.1.6.orig/kernel/power/suspend.c linux-4.1.6/kernel/power/suspend.c ---- linux-4.1.6.orig/kernel/power/suspend.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/power/suspend.c 2015-09-08 23:49:08.105867384 +0200 -@@ -356,6 +356,8 @@ - arch_suspend_disable_irqs(); - BUG_ON(!irqs_disabled()); - -+ system_state = SYSTEM_SUSPEND; -+ - error = syscore_suspend(); - if (!error) { - *wakeup = pm_wakeup_pending(); -@@ -370,6 +372,8 @@ - syscore_resume(); - } - -+ system_state = SYSTEM_RUNNING; ++static inline unsigned long bh_uptodate_lock_irqsave(struct buffer_head *bh) ++{ ++ unsigned long flags; + - arch_suspend_enable_irqs(); - BUG_ON(irqs_disabled()); - -diff -Nur linux-4.1.6.orig/kernel/printk/printk.c linux-4.1.6/kernel/printk/printk.c ---- linux-4.1.6.orig/kernel/printk/printk.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/printk/printk.c 2015-09-08 23:49:08.105867384 +0200 -@@ -1163,6 +1163,7 @@ - { - char *text; - int len = 0; -+ int attempts = 0; - - text = kmalloc(LOG_LINE_MAX + PREFIX_MAX, GFP_KERNEL); - if (!text) -@@ -1174,7 +1175,14 @@ - u64 seq; - u32 idx; - enum log_flags prev; -- -+ int num_msg; -+try_again: -+ attempts++; -+ if (attempts > 10) { -+ len = -EBUSY; -+ goto out; -+ } -+ num_msg = 0; - if (clear_seq < log_first_seq) { - /* messages are gone, move to first available one */ - clear_seq = log_first_seq; -@@ -1195,6 +1203,14 @@ - prev = msg->flags; - idx = log_next(idx); - seq++; -+ num_msg++; -+ if (num_msg > 5) { -+ num_msg = 0; -+ raw_spin_unlock_irq(&logbuf_lock); -+ raw_spin_lock_irq(&logbuf_lock); -+ if (clear_seq < log_first_seq) -+ goto try_again; -+ } - } - - /* move first record forward until length fits into the buffer */ -@@ -1208,6 +1224,14 @@ - prev = msg->flags; - idx = log_next(idx); - seq++; -+ num_msg++; -+ if (num_msg > 5) { -+ num_msg = 0; -+ raw_spin_unlock_irq(&logbuf_lock); -+ raw_spin_lock_irq(&logbuf_lock); -+ if (clear_seq < log_first_seq) -+ goto try_again; -+ } - } - - /* last message fitting into this dump */ -@@ -1248,6 +1272,7 @@ - clear_seq = log_next_seq; - clear_idx = log_next_idx; - } -+out: - raw_spin_unlock_irq(&logbuf_lock); - - kfree(text); -@@ -1401,6 +1426,7 @@ - if (!console_drivers) - return; - -+ migrate_disable(); - for_each_console(con) { - if (exclusive_console && con != exclusive_console) - continue; -@@ -1413,6 +1439,7 @@ - continue; - con->write(con, text, len); - } -+ migrate_enable(); - } - - /* -@@ -1473,6 +1500,15 @@ - static int console_trylock_for_printk(void) - { - unsigned int cpu = smp_processor_id(); -+#ifdef CONFIG_PREEMPT_RT_FULL -+ int lock = !early_boot_irqs_disabled && (preempt_count() == 0) && -+ !irqs_disabled(); ++#ifndef CONFIG_PREEMPT_RT_BASE ++ local_irq_save(flags); ++ bit_spin_lock(BH_Uptodate_Lock, &bh->b_state); +#else -+ int lock = 1; ++ spin_lock_irqsave(&bh->b_uptodate_lock, flags); +#endif -+ -+ if (!lock) -+ return 0; - - if (!console_trylock()) - return 0; -@@ -1607,6 +1643,62 @@ - return textlen; - } - -+#ifdef CONFIG_EARLY_PRINTK -+struct console *early_console; -+ -+static void early_vprintk(const char *fmt, va_list ap) -+{ -+ if (early_console) { -+ char buf[512]; -+ int n = vscnprintf(buf, sizeof(buf), fmt, ap); -+ -+ early_console->write(early_console, buf, n); -+ } -+} -+ -+asmlinkage void early_printk(const char *fmt, ...) -+{ -+ va_list ap; -+ -+ va_start(ap, fmt); -+ early_vprintk(fmt, ap); -+ va_end(ap); -+} -+ -+/* -+ * This is independent of any log levels - a global -+ * kill switch that turns off all of printk. -+ * -+ * Used by the NMI watchdog if early-printk is enabled. -+ */ -+static bool __read_mostly printk_killswitch; -+ -+static int __init force_early_printk_setup(char *str) -+{ -+ printk_killswitch = true; -+ return 0; ++ return flags; +} -+early_param("force_early_printk", force_early_printk_setup); + -+void printk_kill(void) ++static inline void ++bh_uptodate_unlock_irqrestore(struct buffer_head *bh, unsigned long flags) +{ -+ printk_killswitch = true; ++#ifndef CONFIG_PREEMPT_RT_BASE ++ bit_spin_unlock(BH_Uptodate_Lock, &bh->b_state); ++ local_irq_restore(flags); ++#else ++ spin_unlock_irqrestore(&bh->b_uptodate_lock, flags); ++#endif +} + -+static int forced_early_printk(const char *fmt, va_list ap) -+{ -+ if (!printk_killswitch) -+ return 0; -+ early_vprintk(fmt, ap); -+ return 1; -+} -+#else -+static inline int forced_early_printk(const char *fmt, va_list ap) ++static inline void buffer_head_init_locks(struct buffer_head *bh) +{ -+ return 0; -+} ++#ifdef CONFIG_PREEMPT_RT_BASE ++ spin_lock_init(&bh->b_uptodate_lock); ++#if defined(CONFIG_JBD) || defined(CONFIG_JBD_MODULE) || \ ++ defined(CONFIG_JBD2) || defined(CONFIG_JBD2_MODULE) ++ spin_lock_init(&bh->b_state_lock); ++ spin_lock_init(&bh->b_journal_head_lock); +#endif ++#endif ++} + - asmlinkage int vprintk_emit(int facility, int level, - const char *dict, size_t dictlen, - const char *fmt, va_list args) -@@ -1623,6 +1715,13 @@ - /* cpu currently holding logbuf_lock in this function */ - static unsigned int logbuf_cpu = UINT_MAX; + /* + * macro tricks to expand the set_buffer_foo(), clear_buffer_foo() + * and buffer_foo() functions. +diff -Nur linux-4.1.10.orig/include/linux/cgroup.h linux-4.1.10/include/linux/cgroup.h +--- linux-4.1.10.orig/include/linux/cgroup.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/cgroup.h 2015-10-07 18:00:08.000000000 +0200 +@@ -22,6 +22,7 @@ + #include + #include + #include ++#include -+ /* -+ * Fall back to early_printk if a debugging subsystem has -+ * killed printk output -+ */ -+ if (unlikely(forced_early_printk(fmt, args))) -+ return 1; -+ - if (level == LOGLEVEL_SCHED) { - level = LOGLEVEL_DEFAULT; - in_sched = true; -@@ -1764,8 +1863,7 @@ - * console_sem which would prevent anyone from printing to - * console - */ -- preempt_disable(); + #ifdef CONFIG_CGROUPS + +@@ -91,6 +92,7 @@ + /* percpu_ref killing and RCU release */ + struct rcu_head rcu_head; + struct work_struct destroy_work; ++ struct swork_event destroy_swork; + }; + + /* bits in struct cgroup_subsys_state flags field */ +diff -Nur linux-4.1.10.orig/include/linux/completion.h linux-4.1.10/include/linux/completion.h +--- linux-4.1.10.orig/include/linux/completion.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/completion.h 2015-10-07 18:00:08.000000000 +0200 +@@ -7,8 +7,7 @@ + * Atomic wait-for-completion handler data structures. + * See kernel/sched/completion.c for details. + */ - -+ migrate_disable(); - /* - * Try to acquire and then immediately release the console - * semaphore. The release will print out buffers and wake up -@@ -1773,7 +1871,7 @@ - */ - if (console_trylock_for_printk()) - console_unlock(); -- preempt_enable(); -+ migrate_enable(); - lockdep_on(); - } +-#include ++#include -@@ -1902,26 +2000,6 @@ + /* + * struct completion - structure used to maintain state for a "completion" +@@ -24,11 +23,11 @@ + */ + struct completion { + unsigned int done; +- wait_queue_head_t wait; ++ struct swait_head wait; + }; - #endif /* CONFIG_PRINTK */ + #define COMPLETION_INITIALIZER(work) \ +- { 0, __WAIT_QUEUE_HEAD_INITIALIZER((work).wait) } ++ { 0, SWAIT_HEAD_INITIALIZER((work).wait) } --#ifdef CONFIG_EARLY_PRINTK --struct console *early_console; -- --asmlinkage __visible void early_printk(const char *fmt, ...) --{ -- va_list ap; -- char buf[512]; -- int n; -- -- if (!early_console) -- return; -- -- va_start(ap, fmt); -- n = vscnprintf(buf, sizeof(buf), fmt, ap); -- va_end(ap); -- -- early_console->write(early_console, buf, n); --} --#endif -- - static int __add_preferred_console(char *name, int idx, char *options, - char *brl_options) + #define COMPLETION_INITIALIZER_ONSTACK(work) \ + ({ init_completion(&work); work; }) +@@ -73,7 +72,7 @@ + static inline void init_completion(struct completion *x) { -@@ -2143,11 +2221,16 @@ - goto out; + x->done = 0; +- init_waitqueue_head(&x->wait); ++ init_swait_head(&x->wait); + } + + /** +diff -Nur linux-4.1.10.orig/include/linux/cpu.h linux-4.1.10/include/linux/cpu.h +--- linux-4.1.10.orig/include/linux/cpu.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/cpu.h 2015-10-07 18:00:08.000000000 +0200 +@@ -231,6 +231,8 @@ + extern void put_online_cpus(void); + extern void cpu_hotplug_disable(void); + extern void cpu_hotplug_enable(void); ++extern void pin_current_cpu(void); ++extern void unpin_current_cpu(void); + #define hotcpu_notifier(fn, pri) cpu_notifier(fn, pri) + #define __hotcpu_notifier(fn, pri) __cpu_notifier(fn, pri) + #define register_hotcpu_notifier(nb) register_cpu_notifier(nb) +@@ -249,6 +251,8 @@ + #define put_online_cpus() do { } while (0) + #define cpu_hotplug_disable() do { } while (0) + #define cpu_hotplug_enable() do { } while (0) ++static inline void pin_current_cpu(void) { } ++static inline void unpin_current_cpu(void) { } + #define hotcpu_notifier(fn, pri) do { (void)(fn); } while (0) + #define __hotcpu_notifier(fn, pri) do { (void)(fn); } while (0) + /* These aren't inline functions due to a GCC bug. */ +diff -Nur linux-4.1.10.orig/include/linux/delay.h linux-4.1.10/include/linux/delay.h +--- linux-4.1.10.orig/include/linux/delay.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/delay.h 2015-10-07 18:00:08.000000000 +0200 +@@ -52,4 +52,10 @@ + msleep(seconds * 1000); + } - len = cont_print_text(text, size); -+#ifndef CONFIG_PREEMPT_RT_FULL - raw_spin_unlock(&logbuf_lock); - stop_critical_timings(); - call_console_drivers(cont.level, text, len); - start_critical_timings(); - local_irq_restore(flags); -+#else -+ raw_spin_unlock_irqrestore(&logbuf_lock, flags); -+ call_console_drivers(cont.level, text, len); -+#endif - return; - out: - raw_spin_unlock_irqrestore(&logbuf_lock, flags); -@@ -2235,12 +2318,17 @@ - console_idx = log_next(console_idx); - console_seq++; - console_prev = msg->flags; +#ifdef CONFIG_PREEMPT_RT_FULL -+ raw_spin_unlock_irqrestore(&logbuf_lock, flags); -+ call_console_drivers(level, text, len); ++extern void cpu_chill(void); +#else - raw_spin_unlock(&logbuf_lock); - - stop_critical_timings(); /* don't trace print latency */ - call_console_drivers(level, text, len); - start_critical_timings(); - local_irq_restore(flags); ++# define cpu_chill() cpu_relax() +#endif - } - console_locked = 0; ++ + #endif /* defined(_LINUX_DELAY_H) */ +diff -Nur linux-4.1.10.orig/include/linux/ftrace_event.h linux-4.1.10/include/linux/ftrace_event.h +--- linux-4.1.10.orig/include/linux/ftrace_event.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/ftrace_event.h 2015-10-07 18:00:08.000000000 +0200 +@@ -66,6 +66,9 @@ + unsigned char flags; + unsigned char preempt_count; + int pid; ++ unsigned short migrate_disable; ++ unsigned short padding; ++ unsigned char preempt_lazy_count; + }; -diff -Nur linux-4.1.6.orig/kernel/ptrace.c linux-4.1.6/kernel/ptrace.c ---- linux-4.1.6.orig/kernel/ptrace.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/ptrace.c 2015-09-08 23:49:08.105867384 +0200 -@@ -129,7 +129,12 @@ + #define FTRACE_MAX_EVENT \ +diff -Nur linux-4.1.10.orig/include/linux/highmem.h linux-4.1.10/include/linux/highmem.h +--- linux-4.1.10.orig/include/linux/highmem.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/highmem.h 2015-10-07 18:00:08.000000000 +0200 +@@ -7,6 +7,7 @@ + #include + #include + #include ++#include - spin_lock_irq(&task->sighand->siglock); - if (task_is_traced(task) && !__fatal_signal_pending(task)) { -- task->state = __TASK_TRACED; -+ raw_spin_lock_irq(&task->pi_lock); -+ if (task->state & __TASK_TRACED) -+ task->state = __TASK_TRACED; -+ else -+ task->saved_state = __TASK_TRACED; -+ raw_spin_unlock_irq(&task->pi_lock); - ret = true; - } - spin_unlock_irq(&task->sighand->siglock); -diff -Nur linux-4.1.6.orig/kernel/rcu/tree.c linux-4.1.6/kernel/rcu/tree.c ---- linux-4.1.6.orig/kernel/rcu/tree.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/rcu/tree.c 2015-09-08 23:49:08.105867384 +0200 -@@ -56,6 +56,11 @@ - #include - #include - #include -+#include -+#include -+#include -+#include -+#include "../time/tick-internal.h" + #include - #include "tree.h" - #include "rcu.h" -@@ -220,6 +225,19 @@ - } - } +@@ -65,6 +66,7 @@ -+#ifdef CONFIG_PREEMPT_RT_FULL -+static void rcu_preempt_qs(void); -+ -+void rcu_bh_qs(void) -+{ -+ unsigned long flags; -+ -+ /* Callers to this function, rcu_preempt_qs(), must disable irqs. */ -+ local_irq_save(flags); -+ rcu_preempt_qs(); -+ local_irq_restore(flags); -+} -+#else - void rcu_bh_qs(void) + static inline void *kmap_atomic(struct page *page) { - if (!__this_cpu_read(rcu_bh_data.passed_quiesce)) { -@@ -229,6 +247,7 @@ - __this_cpu_write(rcu_bh_data.passed_quiesce, 1); - } ++ preempt_disable_nort(); + pagefault_disable(); + return page_address(page); + } +@@ -73,6 +75,7 @@ + static inline void __kunmap_atomic(void *addr) + { + pagefault_enable(); ++ preempt_enable_nort(); } -+#endif - static DEFINE_PER_CPU(int, rcu_sched_qs_mask); + #define kmap_atomic_pfn(pfn) kmap_atomic(pfn_to_page(pfn)) +@@ -85,32 +88,51 @@ -@@ -404,6 +423,7 @@ - } - EXPORT_SYMBOL_GPL(rcu_batches_completed_sched); + #if defined(CONFIG_HIGHMEM) || defined(CONFIG_X86_32) +#ifndef CONFIG_PREEMPT_RT_FULL - /* - * Return the number of RCU BH batches completed thus far for debug & stats. - */ -@@ -431,6 +451,13 @@ - } - EXPORT_SYMBOL_GPL(rcu_bh_force_quiescent_state); + DECLARE_PER_CPU(int, __kmap_atomic_idx); ++#endif + + static inline int kmap_atomic_idx_push(void) + { ++#ifndef CONFIG_PREEMPT_RT_FULL + int idx = __this_cpu_inc_return(__kmap_atomic_idx) - 1; +-#ifdef CONFIG_DEBUG_HIGHMEM ++# ifdef CONFIG_DEBUG_HIGHMEM + WARN_ON_ONCE(in_irq() && !irqs_disabled()); + BUG_ON(idx >= KM_TYPE_NR); +-#endif ++# endif + return idx; +#else -+void rcu_force_quiescent_state(void) -+{ -+} -+EXPORT_SYMBOL_GPL(rcu_force_quiescent_state); ++ current->kmap_idx++; ++ BUG_ON(current->kmap_idx > KM_TYPE_NR); ++ return current->kmap_idx - 1; +#endif -+ - /* - * Force a quiescent state for RCU-sched. - */ -@@ -1545,7 +1572,7 @@ - !ACCESS_ONCE(rsp->gp_flags) || - !rsp->gp_kthread) - return; -- wake_up(&rsp->gp_wq); -+ swait_wake(&rsp->gp_wq); } - /* -@@ -1986,7 +2013,7 @@ - ACCESS_ONCE(rsp->gpnum), - TPS("reqwait")); - rsp->gp_state = RCU_GP_WAIT_GPS; -- wait_event_interruptible(rsp->gp_wq, -+ swait_event_interruptible(rsp->gp_wq, - ACCESS_ONCE(rsp->gp_flags) & - RCU_GP_FLAG_INIT); - /* Locking provides needed memory barrier. */ -@@ -2015,7 +2042,7 @@ - ACCESS_ONCE(rsp->gpnum), - TPS("fqswait")); - rsp->gp_state = RCU_GP_WAIT_FQS; -- ret = wait_event_interruptible_timeout(rsp->gp_wq, -+ ret = swait_event_interruptible_timeout(rsp->gp_wq, - ((gf = ACCESS_ONCE(rsp->gp_flags)) & - RCU_GP_FLAG_FQS) || - (!ACCESS_ONCE(rnp->qsmask) && -@@ -2860,18 +2887,17 @@ - /* - * Do RCU core processing for the current CPU. - */ --static void rcu_process_callbacks(struct softirq_action *unused) -+static void rcu_process_callbacks(void) - { - struct rcu_state *rsp; - - if (cpu_is_offline(smp_processor_id())) - return; -- trace_rcu_utilization(TPS("Start RCU core")); - for_each_rcu_flavor(rsp) - __rcu_process_callbacks(rsp); -- trace_rcu_utilization(TPS("End RCU core")); - } - -+static DEFINE_PER_CPU(struct task_struct *, rcu_cpu_kthread_task); - /* - * Schedule RCU callback invocation. If the specified type of RCU - * does not support RCU priority boosting, just do a direct call, -@@ -2883,18 +2909,105 @@ + static inline int kmap_atomic_idx(void) { - if (unlikely(!ACCESS_ONCE(rcu_scheduler_fully_active))) - return; -- if (likely(!rsp->boost)) { -- rcu_do_batch(rsp, rdp); -+ rcu_do_batch(rsp, rdp); -+} -+ -+static void rcu_wake_cond(struct task_struct *t, int status) -+{ -+ /* -+ * If the thread is yielding, only wake it when this -+ * is invoked from idle -+ */ -+ if (t && (status != RCU_KTHREAD_YIELDING || is_idle_task(current))) -+ wake_up_process(t); -+} -+ -+/* -+ * Wake up this CPU's rcuc kthread to do RCU core processing. -+ */ -+static void invoke_rcu_core(void) -+{ -+ unsigned long flags; -+ struct task_struct *t; -+ -+ if (!cpu_online(smp_processor_id())) - return; -+ local_irq_save(flags); -+ __this_cpu_write(rcu_cpu_has_work, 1); -+ t = __this_cpu_read(rcu_cpu_kthread_task); -+ if (t != NULL && current != t) -+ rcu_wake_cond(t, __this_cpu_read(rcu_cpu_kthread_status)); -+ local_irq_restore(flags); -+} -+ -+static void rcu_cpu_kthread_park(unsigned int cpu) -+{ -+ per_cpu(rcu_cpu_kthread_status, cpu) = RCU_KTHREAD_OFFCPU; -+} -+ -+static int rcu_cpu_kthread_should_run(unsigned int cpu) -+{ -+ return __this_cpu_read(rcu_cpu_has_work); -+} -+ -+/* -+ * Per-CPU kernel thread that invokes RCU callbacks. This replaces the -+ * RCU softirq used in flavors and configurations of RCU that do not -+ * support RCU priority boosting. -+ */ -+static void rcu_cpu_kthread(unsigned int cpu) -+{ -+ unsigned int *statusp = this_cpu_ptr(&rcu_cpu_kthread_status); -+ char work, *workp = this_cpu_ptr(&rcu_cpu_has_work); -+ int spincnt; -+ -+ for (spincnt = 0; spincnt < 10; spincnt++) { -+ trace_rcu_utilization(TPS("Start CPU kthread@rcu_wait")); -+ local_bh_disable(); -+ *statusp = RCU_KTHREAD_RUNNING; -+ this_cpu_inc(rcu_cpu_kthread_loops); -+ local_irq_disable(); -+ work = *workp; -+ *workp = 0; -+ local_irq_enable(); -+ if (work) -+ rcu_process_callbacks(); -+ local_bh_enable(); -+ if (*workp == 0) { -+ trace_rcu_utilization(TPS("End CPU kthread@rcu_wait")); -+ *statusp = RCU_KTHREAD_WAITING; -+ return; -+ } - } -- invoke_rcu_callbacks_kthread(); -+ *statusp = RCU_KTHREAD_YIELDING; -+ trace_rcu_utilization(TPS("Start CPU kthread@rcu_yield")); -+ schedule_timeout_interruptible(2); -+ trace_rcu_utilization(TPS("End CPU kthread@rcu_yield")); -+ *statusp = RCU_KTHREAD_WAITING; ++#ifndef CONFIG_PREEMPT_RT_FULL + return __this_cpu_read(__kmap_atomic_idx) - 1; ++#else ++ return current->kmap_idx - 1; ++#endif } --static void invoke_rcu_core(void) -+static struct smp_hotplug_thread rcu_cpu_thread_spec = { -+ .store = &rcu_cpu_kthread_task, -+ .thread_should_run = rcu_cpu_kthread_should_run, -+ .thread_fn = rcu_cpu_kthread, -+ .thread_comm = "rcuc/%u", -+ .setup = rcu_cpu_kthread_setup, -+ .park = rcu_cpu_kthread_park, -+}; -+ -+/* -+ * Spawn per-CPU RCU core processing kthreads. -+ */ -+static int __init rcu_spawn_core_kthreads(void) + static inline void kmap_atomic_idx_pop(void) { -- if (cpu_online(smp_processor_id())) -- raise_softirq(RCU_SOFTIRQ); -+ int cpu; -+ -+ for_each_possible_cpu(cpu) -+ per_cpu(rcu_cpu_has_work, cpu) = 0; -+ BUG_ON(smpboot_register_percpu_thread(&rcu_cpu_thread_spec)); -+ return 0; - } -+early_initcall(rcu_spawn_core_kthreads); - - /* - * Handle any core-RCU processing required by a call_rcu() invocation. -@@ -3040,6 +3153,7 @@ - } - EXPORT_SYMBOL_GPL(call_rcu_sched); - +-#ifdef CONFIG_DEBUG_HIGHMEM +#ifndef CONFIG_PREEMPT_RT_FULL - /* - * Queue an RCU callback for invocation after a quicker grace period. - */ -@@ -3048,6 +3162,7 @@ - __call_rcu(head, func, &rcu_bh_state, -1, 0); - } - EXPORT_SYMBOL_GPL(call_rcu_bh); -+#endif ++# ifdef CONFIG_DEBUG_HIGHMEM + int idx = __this_cpu_dec_return(__kmap_atomic_idx); - /* - * Queue an RCU callback for lazy invocation after a grace period. -@@ -3139,6 +3254,7 @@ + BUG_ON(idx < 0); +-#else ++# else + __this_cpu_dec(__kmap_atomic_idx); ++# endif ++#else ++ current->kmap_idx--; ++# ifdef CONFIG_DEBUG_HIGHMEM ++ BUG_ON(current->kmap_idx < 0); ++# endif + #endif } - EXPORT_SYMBOL_GPL(synchronize_sched); -+#ifndef CONFIG_PREEMPT_RT_FULL - /** - * synchronize_rcu_bh - wait until an rcu_bh grace period has elapsed. - * -@@ -3165,6 +3281,7 @@ - wait_rcu_gp(call_rcu_bh); - } - EXPORT_SYMBOL_GPL(synchronize_rcu_bh); +diff -Nur linux-4.1.10.orig/include/linux/hrtimer.h linux-4.1.10/include/linux/hrtimer.h +--- linux-4.1.10.orig/include/linux/hrtimer.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/hrtimer.h 2015-10-07 18:00:08.000000000 +0200 +@@ -111,6 +111,11 @@ + enum hrtimer_restart (*function)(struct hrtimer *); + struct hrtimer_clock_base *base; + unsigned long state; ++ struct list_head cb_entry; ++ int irqsafe; ++#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST ++ ktime_t praecox; ++#endif + #ifdef CONFIG_TIMER_STATS + int start_pid; + void *start_site; +@@ -147,6 +152,7 @@ + int index; + clockid_t clockid; + struct timerqueue_head active; ++ struct list_head expired; + ktime_t resolution; + ktime_t (*get_time)(void); + ktime_t softirq_time; +@@ -194,6 +200,9 @@ + unsigned long nr_hangs; + ktime_t max_hang_time; + #endif ++#ifdef CONFIG_PREEMPT_RT_BASE ++ wait_queue_head_t wait; +#endif + struct hrtimer_clock_base clock_base[HRTIMER_MAX_CLOCK_BASES]; + }; - /** - * get_state_synchronize_rcu - Snapshot current RCU state -@@ -3677,6 +3794,7 @@ - mutex_unlock(&rsp->barrier_mutex); +@@ -381,6 +390,13 @@ + return hrtimer_start_expires(timer, HRTIMER_MODE_ABS); } -+#ifndef CONFIG_PREEMPT_RT_FULL - /** - * rcu_barrier_bh - Wait until all in-flight call_rcu_bh() callbacks complete. ++/* Softirq preemption could deadlock timer removal */ ++#ifdef CONFIG_PREEMPT_RT_BASE ++ extern void hrtimer_wait_for_timer(const struct hrtimer *timer); ++#else ++# define hrtimer_wait_for_timer(timer) do { cpu_relax(); } while (0) ++#endif ++ + /* Query timers: */ + extern ktime_t hrtimer_get_remaining(const struct hrtimer *timer); + extern int hrtimer_get_res(const clockid_t which_clock, struct timespec *tp); +diff -Nur linux-4.1.10.orig/include/linux/idr.h linux-4.1.10/include/linux/idr.h +--- linux-4.1.10.orig/include/linux/idr.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/idr.h 2015-10-07 18:00:08.000000000 +0200 +@@ -95,10 +95,14 @@ + * Each idr_preload() should be matched with an invocation of this + * function. See idr_preload() for details. */ -@@ -3685,6 +3803,7 @@ - _rcu_barrier(&rcu_bh_state); ++#ifdef CONFIG_PREEMPT_RT_FULL ++void idr_preload_end(void); ++#else + static inline void idr_preload_end(void) + { + preempt_enable(); } - EXPORT_SYMBOL_GPL(rcu_barrier_bh); +#endif /** - * rcu_barrier_sched - Wait for in-flight call_rcu_sched() callbacks. -@@ -4021,7 +4140,7 @@ - } - } + * idr_find - return pointer for given id +diff -Nur linux-4.1.10.orig/include/linux/init_task.h linux-4.1.10/include/linux/init_task.h +--- linux-4.1.10.orig/include/linux/init_task.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/init_task.h 2015-10-07 18:00:08.000000000 +0200 +@@ -147,9 +147,16 @@ + # define INIT_PERF_EVENTS(tsk) + #endif -- init_waitqueue_head(&rsp->gp_wq); -+ init_swait_head(&rsp->gp_wq); - rnp = rsp->level[rcu_num_lvls - 1]; - for_each_possible_cpu(i) { - while (i > rnp->grphi) -@@ -4120,7 +4239,6 @@ - rcu_init_one(&rcu_bh_state, &rcu_bh_data); - rcu_init_one(&rcu_sched_state, &rcu_sched_data); - __rcu_init_preempt(); -- open_softirq(RCU_SOFTIRQ, rcu_process_callbacks); ++#ifdef CONFIG_PREEMPT_RT_BASE ++# define INIT_TIMER_LIST .posix_timer_list = NULL, ++#else ++# define INIT_TIMER_LIST ++#endif ++ + #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN + # define INIT_VTIME(tsk) \ +- .vtime_seqlock = __SEQLOCK_UNLOCKED(tsk.vtime_seqlock), \ ++ .vtime_lock = __RAW_SPIN_LOCK_UNLOCKED(tsk.vtime_lock), \ ++ .vtime_seq = SEQCNT_ZERO(tsk.vtime_seq), \ + .vtime_snap = 0, \ + .vtime_snap_whence = VTIME_SYS, + #else +@@ -238,6 +245,7 @@ + .cpu_timers = INIT_CPU_TIMERS(tsk.cpu_timers), \ + .pi_lock = __RAW_SPIN_LOCK_UNLOCKED(tsk.pi_lock), \ + .timer_slack_ns = 50000, /* 50 usec default slack */ \ ++ INIT_TIMER_LIST \ + .pids = { \ + [PIDTYPE_PID] = INIT_PID_LINK(PIDTYPE_PID), \ + [PIDTYPE_PGID] = INIT_PID_LINK(PIDTYPE_PGID), \ +diff -Nur linux-4.1.10.orig/include/linux/interrupt.h linux-4.1.10/include/linux/interrupt.h +--- linux-4.1.10.orig/include/linux/interrupt.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/interrupt.h 2015-10-07 18:00:08.000000000 +0200 +@@ -61,6 +61,7 @@ + * interrupt handler after suspending interrupts. For system + * wakeup devices users need to implement wakeup detection in + * their interrupt handlers. ++ * IRQF_NO_SOFTIRQ_CALL - Do not process softirqs in the irq thread context (RT) + */ + #define IRQF_SHARED 0x00000080 + #define IRQF_PROBE_SHARED 0x00000100 +@@ -74,6 +75,7 @@ + #define IRQF_NO_THREAD 0x00010000 + #define IRQF_EARLY_RESUME 0x00020000 + #define IRQF_COND_SUSPEND 0x00040000 ++#define IRQF_NO_SOFTIRQ_CALL 0x00080000 - /* - * We don't need protection against CPU-hotplug here because -diff -Nur linux-4.1.6.orig/kernel/rcu/tree.h linux-4.1.6/kernel/rcu/tree.h ---- linux-4.1.6.orig/kernel/rcu/tree.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/rcu/tree.h 2015-09-08 23:49:08.105867384 +0200 -@@ -27,6 +27,7 @@ - #include - #include - #include -+#include + #define IRQF_TIMER (__IRQF_TIMER | IRQF_NO_SUSPEND | IRQF_NO_THREAD) - /* - * Define shape of hierarchy based on NR_CPUS, CONFIG_RCU_FANOUT, and -@@ -210,7 +211,7 @@ - /* This can happen due to race conditions. */ - #endif /* #ifdef CONFIG_RCU_BOOST */ - #ifdef CONFIG_RCU_NOCB_CPU -- wait_queue_head_t nocb_gp_wq[2]; -+ struct swait_head nocb_gp_wq[2]; - /* Place for rcu_nocb_kthread() to wait GP. */ - #endif /* #ifdef CONFIG_RCU_NOCB_CPU */ - int need_future_gp[2]; -@@ -349,7 +350,7 @@ - atomic_long_t nocb_q_count_lazy; /* invocation (all stages). */ - struct rcu_head *nocb_follower_head; /* CBs ready to invoke. */ - struct rcu_head **nocb_follower_tail; -- wait_queue_head_t nocb_wq; /* For nocb kthreads to sleep on. */ -+ struct swait_head nocb_wq; /* For nocb kthreads to sleep on. */ - struct task_struct *nocb_kthread; - int nocb_defer_wakeup; /* Defer wakeup of nocb_kthread. */ +@@ -102,6 +104,7 @@ + * @flags: flags (see IRQF_* above) + * @thread_fn: interrupt handler function for threaded interrupts + * @thread: thread pointer for threaded interrupts ++ * @secondary: pointer to secondary irqaction (force threading) + * @thread_flags: flags related to @thread + * @thread_mask: bitmask for keeping track of @thread activity + * @dir: pointer to the proc/irq/NN/name entry +@@ -113,6 +116,7 @@ + struct irqaction *next; + irq_handler_t thread_fn; + struct task_struct *thread; ++ struct irqaction *secondary; + unsigned int irq; + unsigned int flags; + unsigned long thread_flags; +@@ -184,7 +188,7 @@ + #ifdef CONFIG_LOCKDEP + # define local_irq_enable_in_hardirq() do { } while (0) + #else +-# define local_irq_enable_in_hardirq() local_irq_enable() ++# define local_irq_enable_in_hardirq() local_irq_enable_nort() + #endif -@@ -438,7 +439,7 @@ - unsigned long gpnum; /* Current gp number. */ - unsigned long completed; /* # of last completed gp. */ - struct task_struct *gp_kthread; /* Task for grace periods. */ -- wait_queue_head_t gp_wq; /* Where GP task waits. */ -+ struct swait_head gp_wq; /* Where GP task waits. */ - short gp_flags; /* Commands for GP task. */ - short gp_state; /* GP kthread sleep state. */ + extern void disable_irq_nosync(unsigned int irq); +@@ -215,6 +219,7 @@ + unsigned int irq; + struct kref kref; + struct work_struct work; ++ struct list_head list; + void (*notify)(struct irq_affinity_notify *, const cpumask_t *mask); + void (*release)(struct kref *ref); + }; +@@ -377,9 +382,13 @@ + bool state); -@@ -529,12 +530,10 @@ - DECLARE_PER_CPU(struct rcu_data, rcu_preempt_data); - #endif /* #ifdef CONFIG_PREEMPT_RCU */ + #ifdef CONFIG_IRQ_FORCED_THREADING ++# ifndef CONFIG_PREEMPT_RT_BASE + extern bool force_irqthreads; ++# else ++# define force_irqthreads (true) ++# endif + #else +-#define force_irqthreads (0) ++#define force_irqthreads (false) + #endif --#ifdef CONFIG_RCU_BOOST - DECLARE_PER_CPU(unsigned int, rcu_cpu_kthread_status); - DECLARE_PER_CPU(int, rcu_cpu_kthread_cpu); - DECLARE_PER_CPU(unsigned int, rcu_cpu_kthread_loops); - DECLARE_PER_CPU(char, rcu_cpu_has_work); --#endif /* #ifdef CONFIG_RCU_BOOST */ + #ifndef __ARCH_SET_SOFTIRQ_PENDING +@@ -435,9 +444,10 @@ + void (*action)(struct softirq_action *); + }; - #ifndef RCU_TREE_NONCORE ++#ifndef CONFIG_PREEMPT_RT_FULL + asmlinkage void do_softirq(void); + asmlinkage void __do_softirq(void); +- ++static inline void thread_do_softirq(void) { do_softirq(); } + #ifdef __ARCH_HAS_DO_SOFTIRQ + void do_softirq_own_stack(void); + #else +@@ -446,6 +456,9 @@ + __do_softirq(); + } + #endif ++#else ++extern void thread_do_softirq(void); ++#endif -@@ -553,10 +552,9 @@ - static void __init __rcu_init_preempt(void); - static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags); - static void rcu_preempt_boost_start_gp(struct rcu_node *rnp); --static void invoke_rcu_callbacks_kthread(void); - static bool rcu_is_callbacks_kthread(void); -+static void rcu_cpu_kthread_setup(unsigned int cpu); - #ifdef CONFIG_RCU_BOOST --static void rcu_preempt_do_callbacks(void); - static int rcu_spawn_one_boost_kthread(struct rcu_state *rsp, - struct rcu_node *rnp); - #endif /* #ifdef CONFIG_RCU_BOOST */ -diff -Nur linux-4.1.6.orig/kernel/rcu/tree_plugin.h linux-4.1.6/kernel/rcu/tree_plugin.h ---- linux-4.1.6.orig/kernel/rcu/tree_plugin.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/rcu/tree_plugin.h 2015-09-08 23:49:08.105867384 +0200 -@@ -24,27 +24,20 @@ - * Paul E. McKenney - */ + extern void open_softirq(int nr, void (*action)(struct softirq_action *)); + extern void softirq_init(void); +@@ -453,6 +466,7 @@ --#include --#include --#include --#include --#include "../time/tick-internal.h" -- - #ifdef CONFIG_RCU_BOOST + extern void raise_softirq_irqoff(unsigned int nr); + extern void raise_softirq(unsigned int nr); ++extern void softirq_check_pending_idle(void); - #include "../locking/rtmutex_common.h" + DECLARE_PER_CPU(struct task_struct *, ksoftirqd); -+#endif /* #ifdef CONFIG_RCU_BOOST */ +@@ -474,8 +488,9 @@ + to be executed on some cpu at least once after this. + * If the tasklet is already scheduled, but its execution is still not + started, it will be executed only once. +- * If this tasklet is already running on another CPU (or schedule is called +- from tasklet itself), it is rescheduled for later. ++ * If this tasklet is already running on another CPU, it is rescheduled ++ for later. ++ * Schedule must not be called from the tasklet itself (a lockup occurs) + * Tasklet is strictly serialized wrt itself, but not + wrt another tasklets. If client needs some intertask synchronization, + he makes it with spinlocks. +@@ -500,27 +515,36 @@ + enum + { + TASKLET_STATE_SCHED, /* Tasklet is scheduled for execution */ +- TASKLET_STATE_RUN /* Tasklet is running (SMP only) */ ++ TASKLET_STATE_RUN, /* Tasklet is running (SMP only) */ ++ TASKLET_STATE_PENDING /* Tasklet is pending */ + }; + +-#ifdef CONFIG_SMP ++#define TASKLET_STATEF_SCHED (1 << TASKLET_STATE_SCHED) ++#define TASKLET_STATEF_RUN (1 << TASKLET_STATE_RUN) ++#define TASKLET_STATEF_PENDING (1 << TASKLET_STATE_PENDING) + - /* - * Control variables for per-CPU and per-rcu_node kthreads. These - * handle all flavors of RCU. - */ --static DEFINE_PER_CPU(struct task_struct *, rcu_cpu_kthread_task); - DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_status); - DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_loops); - DEFINE_PER_CPU(char, rcu_cpu_has_work); ++#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT_FULL) + static inline int tasklet_trylock(struct tasklet_struct *t) + { + return !test_and_set_bit(TASKLET_STATE_RUN, &(t)->state); + } --#endif /* #ifdef CONFIG_RCU_BOOST */ -- - #ifdef CONFIG_RCU_NOCB_CPU - static cpumask_var_t rcu_nocb_mask; /* CPUs to have callbacks offloaded. */ - static bool have_rcu_nocb_mask; /* Was rcu_nocb_mask allocated? */ -@@ -291,7 +284,7 @@ - } ++static inline int tasklet_tryunlock(struct tasklet_struct *t) ++{ ++ return cmpxchg(&t->state, TASKLET_STATEF_RUN, 0) == TASKLET_STATEF_RUN; ++} ++ + static inline void tasklet_unlock(struct tasklet_struct *t) + { + smp_mb__before_atomic(); + clear_bit(TASKLET_STATE_RUN, &(t)->state); + } - /* Hardware IRQ handlers cannot block, complain if they get here. */ -- if (in_irq() || in_serving_softirq()) { -+ if (preempt_count() & (HARDIRQ_MASK | SOFTIRQ_OFFSET)) { - lockdep_rcu_suspicious(__FILE__, __LINE__, - "rcu_read_unlock() from irq or softirq with blocking in critical section!!!\n"); - pr_alert("->rcu_read_unlock_special: %#x (b: %d, nq: %d)\n", -@@ -496,15 +489,6 @@ - t->rcu_read_unlock_special.b.need_qs = true; +-static inline void tasklet_unlock_wait(struct tasklet_struct *t) +-{ +- while (test_bit(TASKLET_STATE_RUN, &(t)->state)) { barrier(); } +-} ++extern void tasklet_unlock_wait(struct tasklet_struct *t); ++ + #else + #define tasklet_trylock(t) 1 ++#define tasklet_tryunlock(t) 1 + #define tasklet_unlock_wait(t) do { } while (0) + #define tasklet_unlock(t) do { } while (0) + #endif +@@ -569,12 +593,7 @@ + smp_mb(); } --#ifdef CONFIG_RCU_BOOST -- --static void rcu_preempt_do_callbacks(void) +-static inline void tasklet_enable(struct tasklet_struct *t) -{ -- rcu_do_batch(&rcu_preempt_state, this_cpu_ptr(&rcu_preempt_data)); +- smp_mb__before_atomic(); +- atomic_dec(&t->count); -} - --#endif /* #ifdef CONFIG_RCU_BOOST */ -- - /* - * Queue a preemptible-RCU callback for invocation after a grace period. - */ -@@ -939,6 +923,19 @@ - - #endif /* #else #ifdef CONFIG_PREEMPT_RCU */ - -+/* -+ * If boosting, set rcuc kthreads to realtime priority. -+ */ -+static void rcu_cpu_kthread_setup(unsigned int cpu) -+{ -+#ifdef CONFIG_RCU_BOOST -+ struct sched_param sp; -+ -+ sp.sched_priority = kthread_prio; -+ sched_setscheduler_nocheck(current, SCHED_FIFO, &sp); -+#endif /* #ifdef CONFIG_RCU_BOOST */ -+} -+ - #ifdef CONFIG_RCU_BOOST - - #include "../locking/rtmutex_common.h" -@@ -970,16 +967,6 @@ - - #endif /* #else #ifdef CONFIG_RCU_TRACE */ - --static void rcu_wake_cond(struct task_struct *t, int status) --{ -- /* -- * If the thread is yielding, only wake it when this -- * is invoked from idle -- */ -- if (status != RCU_KTHREAD_YIELDING || is_idle_task(current)) -- wake_up_process(t); --} -- - /* - * Carry out RCU priority boosting on the task indicated by ->exp_tasks - * or ->boost_tasks, advancing the pointer to the next task in the -@@ -1125,23 +1112,6 @@ - } - - /* -- * Wake up the per-CPU kthread to invoke RCU callbacks. -- */ --static void invoke_rcu_callbacks_kthread(void) --{ -- unsigned long flags; -- -- local_irq_save(flags); -- __this_cpu_write(rcu_cpu_has_work, 1); -- if (__this_cpu_read(rcu_cpu_kthread_task) != NULL && -- current != __this_cpu_read(rcu_cpu_kthread_task)) { -- rcu_wake_cond(__this_cpu_read(rcu_cpu_kthread_task), -- __this_cpu_read(rcu_cpu_kthread_status)); -- } -- local_irq_restore(flags); --} -- --/* - * Is the current CPU running the RCU-callbacks kthread? - * Caller must have preemption disabled. - */ -@@ -1196,67 +1166,6 @@ - return 0; - } - --static void rcu_kthread_do_work(void) --{ -- rcu_do_batch(&rcu_sched_state, this_cpu_ptr(&rcu_sched_data)); -- rcu_do_batch(&rcu_bh_state, this_cpu_ptr(&rcu_bh_data)); -- rcu_preempt_do_callbacks(); --} -- --static void rcu_cpu_kthread_setup(unsigned int cpu) --{ -- struct sched_param sp; -- -- sp.sched_priority = kthread_prio; -- sched_setscheduler_nocheck(current, SCHED_FIFO, &sp); --} -- --static void rcu_cpu_kthread_park(unsigned int cpu) --{ -- per_cpu(rcu_cpu_kthread_status, cpu) = RCU_KTHREAD_OFFCPU; --} -- --static int rcu_cpu_kthread_should_run(unsigned int cpu) --{ -- return __this_cpu_read(rcu_cpu_has_work); --} -- --/* -- * Per-CPU kernel thread that invokes RCU callbacks. This replaces the -- * RCU softirq used in flavors and configurations of RCU that do not -- * support RCU priority boosting. -- */ --static void rcu_cpu_kthread(unsigned int cpu) --{ -- unsigned int *statusp = this_cpu_ptr(&rcu_cpu_kthread_status); -- char work, *workp = this_cpu_ptr(&rcu_cpu_has_work); -- int spincnt; -- -- for (spincnt = 0; spincnt < 10; spincnt++) { -- trace_rcu_utilization(TPS("Start CPU kthread@rcu_wait")); -- local_bh_disable(); -- *statusp = RCU_KTHREAD_RUNNING; -- this_cpu_inc(rcu_cpu_kthread_loops); -- local_irq_disable(); -- work = *workp; -- *workp = 0; -- local_irq_enable(); -- if (work) -- rcu_kthread_do_work(); -- local_bh_enable(); -- if (*workp == 0) { -- trace_rcu_utilization(TPS("End CPU kthread@rcu_wait")); -- *statusp = RCU_KTHREAD_WAITING; -- return; -- } -- } -- *statusp = RCU_KTHREAD_YIELDING; -- trace_rcu_utilization(TPS("Start CPU kthread@rcu_yield")); -- schedule_timeout_interruptible(2); -- trace_rcu_utilization(TPS("End CPU kthread@rcu_yield")); -- *statusp = RCU_KTHREAD_WAITING; --} -- - /* - * Set the per-rcu_node kthread's affinity to cover all CPUs that are - * served by the rcu_node in question. The CPU hotplug lock is still -@@ -1286,26 +1195,12 @@ - free_cpumask_var(cm); ++extern void tasklet_enable(struct tasklet_struct *t); + extern void tasklet_kill(struct tasklet_struct *t); + extern void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu); + extern void tasklet_init(struct tasklet_struct *t, +@@ -605,6 +624,12 @@ + tasklet_kill(&ttimer->tasklet); } --static struct smp_hotplug_thread rcu_cpu_thread_spec = { -- .store = &rcu_cpu_kthread_task, -- .thread_should_run = rcu_cpu_kthread_should_run, -- .thread_fn = rcu_cpu_kthread, -- .thread_comm = "rcuc/%u", -- .setup = rcu_cpu_kthread_setup, -- .park = rcu_cpu_kthread_park, --}; -- ++#ifdef CONFIG_PREEMPT_RT_FULL ++extern void softirq_early_init(void); ++#else ++static inline void softirq_early_init(void) { } ++#endif ++ /* - * Spawn boost kthreads -- called as soon as the scheduler is running. - */ - static void __init rcu_spawn_boost_kthreads(void) + * Autoprobing for irqs: + * +diff -Nur linux-4.1.10.orig/include/linux/io-mapping.h linux-4.1.10/include/linux/io-mapping.h +--- linux-4.1.10.orig/include/linux/io-mapping.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/io-mapping.h 2015-10-07 18:00:08.000000000 +0200 +@@ -141,6 +141,7 @@ + io_mapping_map_atomic_wc(struct io_mapping *mapping, + unsigned long offset) { - struct rcu_node *rnp; -- int cpu; -- -- for_each_possible_cpu(cpu) -- per_cpu(rcu_cpu_has_work, cpu) = 0; -- BUG_ON(smpboot_register_percpu_thread(&rcu_cpu_thread_spec)); - rcu_for_each_leaf_node(rcu_state_p, rnp) - (void)rcu_spawn_one_boost_kthread(rcu_state_p, rnp); - } -@@ -1328,11 +1223,6 @@ - raw_spin_unlock_irqrestore(&rnp->lock, flags); ++ preempt_disable(); + pagefault_disable(); + return ((char __force __iomem *) mapping) + offset; } - --static void invoke_rcu_callbacks_kthread(void) --{ -- WARN_ON_ONCE(1); --} -- - static bool rcu_is_callbacks_kthread(void) +@@ -149,6 +150,7 @@ + io_mapping_unmap_atomic(void __iomem *vaddr) { - return false; -@@ -1356,7 +1246,7 @@ - - #endif /* #else #ifdef CONFIG_RCU_BOOST */ + pagefault_enable(); ++ preempt_enable(); + } --#if !defined(CONFIG_RCU_FAST_NO_HZ) -+#if !defined(CONFIG_RCU_FAST_NO_HZ) || defined(CONFIG_PREEMPT_RT_FULL) + /* Non-atomic map/unmap */ +diff -Nur linux-4.1.10.orig/include/linux/irqdesc.h linux-4.1.10/include/linux/irqdesc.h +--- linux-4.1.10.orig/include/linux/irqdesc.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/irqdesc.h 2015-10-07 18:00:08.000000000 +0200 +@@ -63,6 +63,7 @@ + unsigned int irqs_unhandled; + atomic_t threads_handled; + int threads_handled_last; ++ u64 random_ip; + raw_spinlock_t lock; + struct cpumask *percpu_enabled; + #ifdef CONFIG_SMP +diff -Nur linux-4.1.10.orig/include/linux/irqflags.h linux-4.1.10/include/linux/irqflags.h +--- linux-4.1.10.orig/include/linux/irqflags.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/irqflags.h 2015-10-07 18:00:08.000000000 +0200 +@@ -25,8 +25,6 @@ + # define trace_softirqs_enabled(p) ((p)->softirqs_enabled) + # define trace_hardirq_enter() do { current->hardirq_context++; } while (0) + # define trace_hardirq_exit() do { current->hardirq_context--; } while (0) +-# define lockdep_softirq_enter() do { current->softirq_context++; } while (0) +-# define lockdep_softirq_exit() do { current->softirq_context--; } while (0) + # define INIT_TRACE_IRQFLAGS .softirqs_enabled = 1, + #else + # define trace_hardirqs_on() do { } while (0) +@@ -39,9 +37,15 @@ + # define trace_softirqs_enabled(p) 0 + # define trace_hardirq_enter() do { } while (0) + # define trace_hardirq_exit() do { } while (0) ++# define INIT_TRACE_IRQFLAGS ++#endif ++ ++#if defined(CONFIG_TRACE_IRQFLAGS) && !defined(CONFIG_PREEMPT_RT_FULL) ++# define lockdep_softirq_enter() do { current->softirq_context++; } while (0) ++# define lockdep_softirq_exit() do { current->softirq_context--; } while (0) ++#else + # define lockdep_softirq_enter() do { } while (0) + # define lockdep_softirq_exit() do { } while (0) +-# define INIT_TRACE_IRQFLAGS + #endif - /* - * Check to see if any future RCU-related work will need to be done -@@ -1374,7 +1264,9 @@ - return rcu_cpu_has_callbacks(NULL); - } - #endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */ -+#endif /* !defined(CONFIG_RCU_FAST_NO_HZ) || defined(CONFIG_PREEMPT_RT_FULL) */ + #if defined(CONFIG_IRQSOFF_TRACER) || \ +@@ -148,4 +152,23 @@ -+#if !defined(CONFIG_RCU_FAST_NO_HZ) - /* - * Because we do not have RCU_FAST_NO_HZ, don't bother cleaning up - * after it. -@@ -1472,6 +1364,8 @@ - return cbs_ready; - } + #define irqs_disabled_flags(flags) raw_irqs_disabled_flags(flags) -+#ifndef CONFIG_PREEMPT_RT_FULL ++/* ++ * local_irq* variants depending on RT/!RT ++ */ ++#ifdef CONFIG_PREEMPT_RT_FULL ++# define local_irq_disable_nort() do { } while (0) ++# define local_irq_enable_nort() do { } while (0) ++# define local_irq_save_nort(flags) local_save_flags(flags) ++# define local_irq_restore_nort(flags) (void)(flags) ++# define local_irq_disable_rt() local_irq_disable() ++# define local_irq_enable_rt() local_irq_enable() ++#else ++# define local_irq_disable_nort() local_irq_disable() ++# define local_irq_enable_nort() local_irq_enable() ++# define local_irq_save_nort(flags) local_irq_save(flags) ++# define local_irq_restore_nort(flags) local_irq_restore(flags) ++# define local_irq_disable_rt() do { } while (0) ++# define local_irq_enable_rt() do { } while (0) ++#endif + - /* - * Allow the CPU to enter dyntick-idle mode unless it has callbacks ready - * to invoke. If the CPU has callbacks, try to advance them. Tell the -@@ -1512,7 +1406,7 @@ - return 0; - } - #endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */ -- -+#endif /* #ifndef CONFIG_PREEMPT_RT_FULL */ - /* - * Prepare a CPU for idle from an RCU perspective. The first major task - * is to sense whether nohz mode has been enabled or disabled via sysfs. -@@ -1859,7 +1753,7 @@ + #endif +diff -Nur linux-4.1.10.orig/include/linux/irq.h linux-4.1.10/include/linux/irq.h +--- linux-4.1.10.orig/include/linux/irq.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/irq.h 2015-10-07 18:00:08.000000000 +0200 +@@ -72,6 +72,7 @@ + * IRQ_IS_POLLED - Always polled by another interrupt. Exclude + * it from the spurious interrupt detection + * mechanism and from core side polling. ++ * IRQ_NO_SOFTIRQ_CALL - No softirq processing in the irq thread context (RT) */ - static void rcu_nocb_gp_cleanup(struct rcu_state *rsp, struct rcu_node *rnp) - { -- wake_up_all(&rnp->nocb_gp_wq[rnp->completed & 0x1]); -+ swait_wake_all(&rnp->nocb_gp_wq[rnp->completed & 0x1]); - } + enum { + IRQ_TYPE_NONE = 0x00000000, +@@ -97,13 +98,14 @@ + IRQ_NOTHREAD = (1 << 16), + IRQ_PER_CPU_DEVID = (1 << 17), + IRQ_IS_POLLED = (1 << 18), ++ IRQ_NO_SOFTIRQ_CALL = (1 << 19), + }; - /* -@@ -1877,8 +1771,8 @@ + #define IRQF_MODIFY_MASK \ + (IRQ_TYPE_SENSE_MASK | IRQ_NOPROBE | IRQ_NOREQUEST | \ + IRQ_NOAUTOEN | IRQ_MOVE_PCNTXT | IRQ_LEVEL | IRQ_NO_BALANCING | \ + IRQ_PER_CPU | IRQ_NESTED_THREAD | IRQ_NOTHREAD | IRQ_PER_CPU_DEVID | \ +- IRQ_IS_POLLED) ++ IRQ_IS_POLLED | IRQ_NO_SOFTIRQ_CALL) - static void rcu_init_one_nocb(struct rcu_node *rnp) - { -- init_waitqueue_head(&rnp->nocb_gp_wq[0]); -- init_waitqueue_head(&rnp->nocb_gp_wq[1]); -+ init_swait_head(&rnp->nocb_gp_wq[0]); -+ init_swait_head(&rnp->nocb_gp_wq[1]); - } + #define IRQ_NO_BALANCING_MASK (IRQ_PER_CPU | IRQ_NO_BALANCING) - #ifndef CONFIG_RCU_NOCB_CPU_ALL -@@ -1903,7 +1797,7 @@ - if (ACCESS_ONCE(rdp_leader->nocb_leader_sleep) || force) { - /* Prior smp_mb__after_atomic() orders against prior enqueue. */ - ACCESS_ONCE(rdp_leader->nocb_leader_sleep) = false; -- wake_up(&rdp_leader->nocb_wq); -+ swait_wake(&rdp_leader->nocb_wq); - } - } +diff -Nur linux-4.1.10.orig/include/linux/irq_work.h linux-4.1.10/include/linux/irq_work.h +--- linux-4.1.10.orig/include/linux/irq_work.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/irq_work.h 2015-10-07 18:00:08.000000000 +0200 +@@ -16,6 +16,7 @@ + #define IRQ_WORK_BUSY 2UL + #define IRQ_WORK_FLAGS 3UL + #define IRQ_WORK_LAZY 4UL /* Doesn't want IPI, wait for tick */ ++#define IRQ_WORK_HARD_IRQ 8UL /* Run hard IRQ context, even on RT */ -@@ -2116,7 +2010,7 @@ - */ - trace_rcu_future_gp(rnp, rdp, c, TPS("StartWait")); - for (;;) { -- wait_event_interruptible( -+ swait_event_interruptible( - rnp->nocb_gp_wq[c & 0x1], - (d = ULONG_CMP_GE(ACCESS_ONCE(rnp->completed), c))); - if (likely(d)) -@@ -2144,7 +2038,7 @@ - /* Wait for callbacks to appear. */ - if (!rcu_nocb_poll) { - trace_rcu_nocb_wake(my_rdp->rsp->name, my_rdp->cpu, "Sleep"); -- wait_event_interruptible(my_rdp->nocb_wq, -+ swait_event_interruptible(my_rdp->nocb_wq, - !ACCESS_ONCE(my_rdp->nocb_leader_sleep)); - /* Memory barrier handled by smp_mb() calls below and repoll. */ - } else if (firsttime) { -@@ -2219,7 +2113,7 @@ - * List was empty, wake up the follower. - * Memory barriers supplied by atomic_long_add(). - */ -- wake_up(&rdp->nocb_wq); -+ swait_wake(&rdp->nocb_wq); - } - } + struct irq_work { + unsigned long flags; +diff -Nur linux-4.1.10.orig/include/linux/jbd_common.h linux-4.1.10/include/linux/jbd_common.h +--- linux-4.1.10.orig/include/linux/jbd_common.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/jbd_common.h 2015-10-07 18:00:08.000000000 +0200 +@@ -15,32 +15,56 @@ -@@ -2240,7 +2134,7 @@ - if (!rcu_nocb_poll) { - trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, - "FollowerSleep"); -- wait_event_interruptible(rdp->nocb_wq, -+ swait_event_interruptible(rdp->nocb_wq, - ACCESS_ONCE(rdp->nocb_follower_head)); - } else if (firsttime) { - /* Don't drown trace log with "Poll"! */ -@@ -2399,7 +2293,7 @@ - static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp) + static inline void jbd_lock_bh_state(struct buffer_head *bh) { - rdp->nocb_tail = &rdp->nocb_head; -- init_waitqueue_head(&rdp->nocb_wq); -+ init_swait_head(&rdp->nocb_wq); - rdp->nocb_follower_tail = &rdp->nocb_follower_head; - } - -diff -Nur linux-4.1.6.orig/kernel/rcu/update.c linux-4.1.6/kernel/rcu/update.c ---- linux-4.1.6.orig/kernel/rcu/update.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/rcu/update.c 2015-09-08 23:49:08.105867384 +0200 -@@ -227,6 +227,7 @@ - } - EXPORT_SYMBOL_GPL(rcu_read_lock_held); - -+#ifndef CONFIG_PREEMPT_RT_FULL - /** - * rcu_read_lock_bh_held() - might we be in RCU-bh read-side critical section? - * -@@ -253,6 +254,7 @@ - return in_softirq() || irqs_disabled(); - } - EXPORT_SYMBOL_GPL(rcu_read_lock_bh_held); ++#ifndef CONFIG_PREEMPT_RT_BASE + bit_spin_lock(BH_State, &bh->b_state); ++#else ++ spin_lock(&bh->b_state_lock); +#endif + } - #endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ - -diff -Nur linux-4.1.6.orig/kernel/relay.c linux-4.1.6/kernel/relay.c ---- linux-4.1.6.orig/kernel/relay.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/relay.c 2015-09-08 23:49:08.105867384 +0200 -@@ -339,6 +339,10 @@ + static inline int jbd_trylock_bh_state(struct buffer_head *bh) { - struct rchan_buf *buf = (struct rchan_buf *)data; - wake_up_interruptible(&buf->read_wait); -+ /* -+ * Stupid polling for now: -+ */ -+ mod_timer(&buf->timer, jiffies + 1); ++#ifndef CONFIG_PREEMPT_RT_BASE + return bit_spin_trylock(BH_State, &bh->b_state); ++#else ++ return spin_trylock(&bh->b_state_lock); ++#endif } - /** -@@ -356,6 +360,7 @@ - init_waitqueue_head(&buf->read_wait); - kref_init(&buf->kref); - setup_timer(&buf->timer, wakeup_readers, (unsigned long)buf); -+ mod_timer(&buf->timer, jiffies + 1); - } else - del_timer_sync(&buf->timer); - -@@ -739,15 +744,6 @@ - else - buf->early_bytes += buf->chan->subbuf_size - - buf->padding[old_subbuf]; -- smp_mb(); -- if (waitqueue_active(&buf->read_wait)) -- /* -- * Calling wake_up_interruptible() from here -- * will deadlock if we happen to be logging -- * from the scheduler (trying to re-grab -- * rq->lock), so defer it. -- */ -- mod_timer(&buf->timer, jiffies + 1); - } - - old = buf->data; -diff -Nur linux-4.1.6.orig/kernel/sched/completion.c linux-4.1.6/kernel/sched/completion.c ---- linux-4.1.6.orig/kernel/sched/completion.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/sched/completion.c 2015-09-08 23:49:08.105867384 +0200 -@@ -30,10 +30,10 @@ + static inline int jbd_is_locked_bh_state(struct buffer_head *bh) { - unsigned long flags; - -- spin_lock_irqsave(&x->wait.lock, flags); -+ raw_spin_lock_irqsave(&x->wait.lock, flags); - x->done++; -- __wake_up_locked(&x->wait, TASK_NORMAL, 1); -- spin_unlock_irqrestore(&x->wait.lock, flags); -+ __swait_wake_locked(&x->wait, TASK_NORMAL, 1); -+ raw_spin_unlock_irqrestore(&x->wait.lock, flags); ++#ifndef CONFIG_PREEMPT_RT_BASE + return bit_spin_is_locked(BH_State, &bh->b_state); ++#else ++ return spin_is_locked(&bh->b_state_lock); ++#endif } - EXPORT_SYMBOL(complete); -@@ -50,10 +50,10 @@ + static inline void jbd_unlock_bh_state(struct buffer_head *bh) { - unsigned long flags; - -- spin_lock_irqsave(&x->wait.lock, flags); -+ raw_spin_lock_irqsave(&x->wait.lock, flags); - x->done += UINT_MAX/2; -- __wake_up_locked(&x->wait, TASK_NORMAL, 0); -- spin_unlock_irqrestore(&x->wait.lock, flags); -+ __swait_wake_locked(&x->wait, TASK_NORMAL, 0); -+ raw_spin_unlock_irqrestore(&x->wait.lock, flags); ++#ifndef CONFIG_PREEMPT_RT_BASE + bit_spin_unlock(BH_State, &bh->b_state); ++#else ++ spin_unlock(&bh->b_state_lock); ++#endif } - EXPORT_SYMBOL(complete_all); -@@ -62,20 +62,20 @@ - long (*action)(long), long timeout, int state) + static inline void jbd_lock_bh_journal_head(struct buffer_head *bh) { - if (!x->done) { -- DECLARE_WAITQUEUE(wait, current); -+ DEFINE_SWAITER(wait); ++#ifndef CONFIG_PREEMPT_RT_BASE + bit_spin_lock(BH_JournalHead, &bh->b_state); ++#else ++ spin_lock(&bh->b_journal_head_lock); ++#endif + } -- __add_wait_queue_tail_exclusive(&x->wait, &wait); -+ swait_prepare_locked(&x->wait, &wait); - do { - if (signal_pending_state(state, current)) { - timeout = -ERESTARTSYS; - break; - } - __set_current_state(state); -- spin_unlock_irq(&x->wait.lock); -+ raw_spin_unlock_irq(&x->wait.lock); - timeout = action(timeout); -- spin_lock_irq(&x->wait.lock); -+ raw_spin_lock_irq(&x->wait.lock); - } while (!x->done && timeout); -- __remove_wait_queue(&x->wait, &wait); -+ swait_finish_locked(&x->wait, &wait); - if (!x->done) - return timeout; - } -@@ -89,9 +89,9 @@ + static inline void jbd_unlock_bh_journal_head(struct buffer_head *bh) { - might_sleep(); - -- spin_lock_irq(&x->wait.lock); -+ raw_spin_lock_irq(&x->wait.lock); - timeout = do_wait_for_common(x, action, timeout, state); -- spin_unlock_irq(&x->wait.lock); -+ raw_spin_unlock_irq(&x->wait.lock); - return timeout; ++#ifndef CONFIG_PREEMPT_RT_BASE + bit_spin_unlock(BH_JournalHead, &bh->b_state); ++#else ++ spin_unlock(&bh->b_journal_head_lock); ++#endif } -@@ -277,12 +277,12 @@ - if (!READ_ONCE(x->done)) - return 0; + #endif +diff -Nur linux-4.1.10.orig/include/linux/kdb.h linux-4.1.10/include/linux/kdb.h +--- linux-4.1.10.orig/include/linux/kdb.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/kdb.h 2015-10-07 18:00:08.000000000 +0200 +@@ -167,6 +167,7 @@ + extern __printf(1, 2) int kdb_printf(const char *, ...); + typedef __printf(1, 2) int (*kdb_printf_t)(const char *, ...); -- spin_lock_irqsave(&x->wait.lock, flags); -+ raw_spin_lock_irqsave(&x->wait.lock, flags); - if (!x->done) - ret = 0; - else - x->done--; -- spin_unlock_irqrestore(&x->wait.lock, flags); -+ raw_spin_unlock_irqrestore(&x->wait.lock, flags); - return ret; - } - EXPORT_SYMBOL(try_wait_for_completion); -@@ -311,7 +311,7 @@ - * after it's acquired the lock. - */ - smp_rmb(); -- spin_unlock_wait(&x->wait.lock); -+ raw_spin_unlock_wait(&x->wait.lock); - return true; - } - EXPORT_SYMBOL(completion_done); -diff -Nur linux-4.1.6.orig/kernel/sched/core.c linux-4.1.6/kernel/sched/core.c ---- linux-4.1.6.orig/kernel/sched/core.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/sched/core.c 2015-09-08 23:49:08.109866942 +0200 -@@ -282,7 +282,11 @@ - * Number of tasks to iterate in a single balance run. - * Limited because this is done with IRQs disabled. ++#define in_kdb_printk() (kdb_trap_printk) + extern void kdb_init(int level); + + /* Access to kdb specific polling devices */ +@@ -201,6 +202,7 @@ + extern int kdb_unregister(char *); + #else /* ! CONFIG_KGDB_KDB */ + static inline __printf(1, 2) int kdb_printf(const char *fmt, ...) { return 0; } ++#define in_kdb_printk() (0) + static inline void kdb_init(int level) {} + static inline int kdb_register(char *cmd, kdb_func_t func, char *usage, + char *help, short minlen) { return 0; } +diff -Nur linux-4.1.10.orig/include/linux/kernel.h linux-4.1.10/include/linux/kernel.h +--- linux-4.1.10.orig/include/linux/kernel.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/kernel.h 2015-10-07 18:00:08.000000000 +0200 +@@ -188,6 +188,9 @@ */ -+#ifndef CONFIG_PREEMPT_RT_FULL - const_debug unsigned int sysctl_sched_nr_migrate = 32; -+#else -+const_debug unsigned int sysctl_sched_nr_migrate = 8; -+#endif + # define might_sleep() \ + do { __might_sleep(__FILE__, __LINE__, 0); might_resched(); } while (0) ++ ++# define might_sleep_no_state_check() \ ++ do { ___might_sleep(__FILE__, __LINE__, 0); might_resched(); } while (0) + # define sched_annotate_sleep() (current->task_state_change = 0) + #else + static inline void ___might_sleep(const char *file, int line, +@@ -195,6 +198,7 @@ + static inline void __might_sleep(const char *file, int line, + int preempt_offset) { } + # define might_sleep() do { might_resched(); } while (0) ++# define might_sleep_no_state_check() do { might_resched(); } while (0) + # define sched_annotate_sleep() do { } while (0) + #endif - /* - * period over which we average the RT time consumption, measured -@@ -461,6 +465,7 @@ +@@ -244,7 +248,8 @@ - hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); - rq->hrtick_timer.function = hrtick; -+ rq->hrtick_timer.irqsafe = 1; + #if defined(CONFIG_MMU) && \ + (defined(CONFIG_PROVE_LOCKING) || defined(CONFIG_DEBUG_ATOMIC_SLEEP)) +-void might_fault(void); ++#define might_fault() __might_fault(__FILE__, __LINE__) ++void __might_fault(const char *file, int line); + #else + static inline void might_fault(void) { } + #endif +@@ -466,6 +471,7 @@ + SYSTEM_HALT, + SYSTEM_POWER_OFF, + SYSTEM_RESTART, ++ SYSTEM_SUSPEND, + } system_state; + + #define TAINT_PROPRIETARY_MODULE 0 +diff -Nur linux-4.1.10.orig/include/linux/kvm_host.h linux-4.1.10/include/linux/kvm_host.h +--- linux-4.1.10.orig/include/linux/kvm_host.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/kvm_host.h 2015-10-07 18:00:08.000000000 +0200 +@@ -230,7 +230,7 @@ + + int fpu_active; + int guest_fpu_loaded, guest_xcr0_loaded; +- wait_queue_head_t wq; ++ struct swait_head wq; + struct pid *pid; + int sigset_active; + sigset_t sigset; +@@ -690,7 +690,7 @@ } - #else /* CONFIG_SCHED_HRTICK */ - static inline void hrtick_clear(struct rq *rq) -@@ -541,6 +546,52 @@ #endif + +-static inline wait_queue_head_t *kvm_arch_vcpu_wq(struct kvm_vcpu *vcpu) ++static inline struct swait_head *kvm_arch_vcpu_wq(struct kvm_vcpu *vcpu) + { + #ifdef __KVM_HAVE_ARCH_WQP + return vcpu->arch.wqp; +diff -Nur linux-4.1.10.orig/include/linux/lglock.h linux-4.1.10/include/linux/lglock.h +--- linux-4.1.10.orig/include/linux/lglock.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/lglock.h 2015-10-07 18:00:08.000000000 +0200 +@@ -34,22 +34,39 @@ #endif -+void wake_q_add(struct wake_q_head *head, struct task_struct *task) -+{ -+ struct wake_q_node *node = &task->wake_q; -+ -+ /* -+ * Atomically grab the task, if ->wake_q is !nil already it means -+ * its already queued (either by us or someone else) and will get the -+ * wakeup due to that. -+ * -+ * This cmpxchg() implies a full barrier, which pairs with the write -+ * barrier implied by the wakeup in wake_up_list(). -+ */ -+ if (cmpxchg(&node->next, NULL, WAKE_Q_TAIL)) -+ return; -+ -+ get_task_struct(task); -+ -+ /* -+ * The head is context local, there can be no concurrency. -+ */ -+ *head->lastp = node; -+ head->lastp = &node->next; -+} -+ -+void wake_up_q(struct wake_q_head *head) -+{ -+ struct wake_q_node *node = head->first; -+ -+ while (node != WAKE_Q_TAIL) { -+ struct task_struct *task; -+ -+ task = container_of(node, struct task_struct, wake_q); -+ BUG_ON(!task); -+ /* task can safely be re-inserted now */ -+ node = node->next; -+ task->wake_q.next = NULL; -+ -+ /* -+ * wake_up_process() implies a wmb() to pair with the queueing -+ * in wake_q_add() so as not to miss wakeups. -+ */ -+ wake_up_process(task); -+ put_task_struct(task); -+ } -+} -+ - /* - * resched_curr - mark rq's current task 'to be rescheduled now'. - * -@@ -572,6 +623,38 @@ - trace_sched_wake_idle_without_ipi(cpu); - } + struct lglock { ++#ifndef CONFIG_PREEMPT_RT_FULL + arch_spinlock_t __percpu *lock; ++#else ++ struct rt_mutex __percpu *lock; ++#endif + #ifdef CONFIG_DEBUG_LOCK_ALLOC + struct lock_class_key lock_key; + struct lockdep_map lock_dep_map; + #endif + }; -+#ifdef CONFIG_PREEMPT_LAZY -+void resched_curr_lazy(struct rq *rq) -+{ -+ struct task_struct *curr = rq->curr; -+ int cpu; -+ -+ if (!sched_feat(PREEMPT_LAZY)) { -+ resched_curr(rq); -+ return; -+ } -+ -+ lockdep_assert_held(&rq->lock); -+ -+ if (test_tsk_need_resched(curr)) -+ return; -+ -+ if (test_tsk_need_resched_lazy(curr)) -+ return; -+ -+ set_tsk_need_resched_lazy(curr); +-#define DEFINE_LGLOCK(name) \ ++#ifndef CONFIG_PREEMPT_RT_FULL ++# define DEFINE_LGLOCK(name) \ + static DEFINE_PER_CPU(arch_spinlock_t, name ## _lock) \ + = __ARCH_SPIN_LOCK_UNLOCKED; \ + struct lglock name = { .lock = &name ## _lock } + +-#define DEFINE_STATIC_LGLOCK(name) \ ++# define DEFINE_STATIC_LGLOCK(name) \ + static DEFINE_PER_CPU(arch_spinlock_t, name ## _lock) \ + = __ARCH_SPIN_LOCK_UNLOCKED; \ + static struct lglock name = { .lock = &name ## _lock } ++#else + -+ cpu = cpu_of(rq); -+ if (cpu == smp_processor_id()) -+ return; ++# define DEFINE_LGLOCK(name) \ ++ static DEFINE_PER_CPU(struct rt_mutex, name ## _lock) \ ++ = __RT_MUTEX_INITIALIZER( name ## _lock); \ ++ struct lglock name = { .lock = &name ## _lock } + -+ /* NEED_RESCHED_LAZY must be visible before we test polling */ -+ smp_mb(); -+ if (!tsk_is_polling(curr)) -+ smp_send_reschedule(cpu); -+} ++# define DEFINE_STATIC_LGLOCK(name) \ ++ static DEFINE_PER_CPU(struct rt_mutex, name ## _lock) \ ++ = __RT_MUTEX_INITIALIZER( name ## _lock); \ ++ static struct lglock name = { .lock = &name ## _lock } ++#endif + + void lg_lock_init(struct lglock *lg, char *name); + void lg_local_lock(struct lglock *lg); +@@ -59,6 +76,12 @@ + void lg_global_lock(struct lglock *lg); + void lg_global_unlock(struct lglock *lg); + ++#ifndef CONFIG_PREEMPT_RT_FULL ++#define lg_global_trylock_relax(name) lg_global_lock(name) ++#else ++void lg_global_trylock_relax(struct lglock *lg); +#endif + - void resched_cpu(int cpu) - { - struct rq *rq = cpu_rq(cpu); -@@ -595,12 +678,14 @@ - */ - int get_nohz_timer_target(int pinned) - { -- int cpu = smp_processor_id(); -+ int cpu; - int i; - struct sched_domain *sd; + #else + /* When !CONFIG_SMP, map lglock to spinlock */ + #define lglock spinlock +diff -Nur linux-4.1.10.orig/include/linux/list_bl.h linux-4.1.10/include/linux/list_bl.h +--- linux-4.1.10.orig/include/linux/list_bl.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/list_bl.h 2015-10-07 18:00:08.000000000 +0200 +@@ -2,6 +2,7 @@ + #define _LINUX_LIST_BL_H -+ preempt_disable_rt(); -+ cpu = smp_processor_id(); - if (pinned || !get_sysctl_timer_migration() || !idle_cpu(cpu)) -- return cpu; -+ goto preempt_en_rt; + #include ++#include + #include - rcu_read_lock(); - for_each_domain(cpu, sd) { -@@ -613,6 +698,8 @@ - } - unlock: - rcu_read_unlock(); -+preempt_en_rt: -+ preempt_enable_rt(); - return cpu; - } /* -@@ -1164,6 +1251,18 @@ +@@ -32,13 +33,22 @@ - static int migration_cpu_stop(void *data); + struct hlist_bl_head { + struct hlist_bl_node *first; ++#ifdef CONFIG_PREEMPT_RT_BASE ++ raw_spinlock_t lock; ++#endif + }; -+static bool check_task_state(struct task_struct *p, long match_state) -+{ -+ bool match = false; -+ -+ raw_spin_lock_irq(&p->pi_lock); -+ if (p->state == match_state || p->saved_state == match_state) -+ match = true; -+ raw_spin_unlock_irq(&p->pi_lock); + struct hlist_bl_node { + struct hlist_bl_node *next, **pprev; + }; +-#define INIT_HLIST_BL_HEAD(ptr) \ +- ((ptr)->first = NULL) + -+ return match; ++static inline void INIT_HLIST_BL_HEAD(struct hlist_bl_head *h) ++{ ++ h->first = NULL; ++#ifdef CONFIG_PREEMPT_RT_BASE ++ raw_spin_lock_init(&h->lock); ++#endif +} -+ - /* - * wait_task_inactive - wait for a thread to unschedule. - * -@@ -1208,7 +1307,7 @@ - * is actually now running somewhere else! - */ - while (task_running(rq, p)) { -- if (match_state && unlikely(p->state != match_state)) -+ if (match_state && !check_task_state(p, match_state)) - return 0; - cpu_relax(); - } -@@ -1223,7 +1322,8 @@ - running = task_running(rq, p); - queued = task_on_rq_queued(p); - ncsw = 0; -- if (!match_state || p->state == match_state) -+ if (!match_state || p->state == match_state || -+ p->saved_state == match_state) - ncsw = p->nvcsw | LONG_MIN; /* sets MSB */ - task_rq_unlock(rq, p, &flags); -@@ -1449,10 +1549,6 @@ + static inline void INIT_HLIST_BL_NODE(struct hlist_bl_node *h) { - activate_task(rq, p, en_flags); - p->on_rq = TASK_ON_RQ_QUEUED; -- -- /* if a worker is waking up, notify workqueue */ -- if (p->flags & PF_WQ_WORKER) -- wq_worker_waking_up(p, cpu_of(rq)); - } - - /* -@@ -1666,8 +1762,27 @@ - */ - smp_mb__before_spinlock(); - raw_spin_lock_irqsave(&p->pi_lock, flags); -- if (!(p->state & state)) -+ if (!(p->state & state)) { -+ /* -+ * The task might be running due to a spinlock sleeper -+ * wakeup. Check the saved state and set it to running -+ * if the wakeup condition is true. -+ */ -+ if (!(wake_flags & WF_LOCK_SLEEPER)) { -+ if (p->saved_state & state) { -+ p->saved_state = TASK_RUNNING; -+ success = 1; -+ } -+ } - goto out; -+ } -+ -+ /* -+ * If this is a regular wakeup, then we can unconditionally -+ * clear the saved state of a "lock sleeper". -+ */ -+ if (!(wake_flags & WF_LOCK_SLEEPER)) -+ p->saved_state = TASK_RUNNING; - - success = 1; /* we're going to change ->state */ - cpu = task_cpu(p); -@@ -1710,42 +1825,6 @@ - } +@@ -117,12 +127,26 @@ - /** -- * try_to_wake_up_local - try to wake up a local task with rq lock held -- * @p: the thread to be awakened -- * -- * Put @p on the run-queue if it's not already there. The caller must -- * ensure that this_rq() is locked, @p is bound to this_rq() and not -- * the current task. -- */ --static void try_to_wake_up_local(struct task_struct *p) --{ -- struct rq *rq = task_rq(p); -- -- if (WARN_ON_ONCE(rq != this_rq()) || -- WARN_ON_ONCE(p == current)) -- return; -- -- lockdep_assert_held(&rq->lock); -- -- if (!raw_spin_trylock(&p->pi_lock)) { -- raw_spin_unlock(&rq->lock); -- raw_spin_lock(&p->pi_lock); -- raw_spin_lock(&rq->lock); -- } -- -- if (!(p->state & TASK_NORMAL)) -- goto out; -- -- if (!task_on_rq_queued(p)) -- ttwu_activate(rq, p, ENQUEUE_WAKEUP); -- -- ttwu_do_wakeup(rq, p, 0); -- ttwu_stat(p, smp_processor_id(), 0); --out: -- raw_spin_unlock(&p->pi_lock); --} -- --/** - * wake_up_process - Wake up a specific process - * @p: The process to be woken up. - * -@@ -1759,11 +1838,23 @@ - */ - int wake_up_process(struct task_struct *p) + static inline void hlist_bl_lock(struct hlist_bl_head *b) { -- WARN_ON(task_is_stopped_or_traced(p)); -+ WARN_ON(__task_is_stopped_or_traced(p)); - return try_to_wake_up(p, TASK_NORMAL, 0); ++#ifndef CONFIG_PREEMPT_RT_BASE + bit_spin_lock(0, (unsigned long *)b); ++#else ++ raw_spin_lock(&b->lock); ++#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK) ++ __set_bit(0, (unsigned long *)b); ++#endif ++#endif } - EXPORT_SYMBOL(wake_up_process); -+/** -+ * wake_up_lock_sleeper - Wake up a specific process blocked on a "sleeping lock" -+ * @p: The process to be woken up. -+ * -+ * Same as wake_up_process() above, but wake_flags=WF_LOCK_SLEEPER to indicate -+ * the nature of the wakeup. -+ */ -+int wake_up_lock_sleeper(struct task_struct *p) -+{ -+ return try_to_wake_up(p, TASK_ALL, WF_LOCK_SLEEPER); -+} -+ - int wake_up_state(struct task_struct *p, unsigned int state) + static inline void hlist_bl_unlock(struct hlist_bl_head *b) { - return try_to_wake_up(p, state, 0); -@@ -1959,6 +2050,9 @@ - p->on_cpu = 0; - #endif - init_task_preempt_count(p); -+#ifdef CONFIG_HAVE_PREEMPT_LAZY -+ task_thread_info(p)->preempt_lazy_count = 0; ++#ifndef CONFIG_PREEMPT_RT_BASE + __bit_spin_unlock(0, (unsigned long *)b); ++#else ++#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK) ++ __clear_bit(0, (unsigned long *)b); ++#endif ++ raw_spin_unlock(&b->lock); +#endif - #ifdef CONFIG_SMP - plist_node_init(&p->pushable_tasks, MAX_PRIO); - RB_CLEAR_NODE(&p->pushable_dl_tasks); -@@ -2231,8 +2325,12 @@ - finish_arch_post_lock_switch(); - - fire_sched_in_preempt_notifiers(current); -+ /* -+ * We use mmdrop_delayed() here so we don't have to do the -+ * full __mmdrop() when we are the last user. -+ */ - if (mm) -- mmdrop(mm); -+ mmdrop_delayed(mm); - if (unlikely(prev_state == TASK_DEAD)) { - if (prev->sched_class->task_dead) - prev->sched_class->task_dead(prev); -@@ -2650,6 +2748,133 @@ - schedstat_inc(this_rq(), sched_count); } -+#if defined(CONFIG_PREEMPT_RT_FULL) && defined(CONFIG_SMP) -+#define MIGRATE_DISABLE_SET_AFFIN (1<<30) /* Can't make a negative */ -+#define migrate_disabled_updated(p) ((p)->migrate_disable & MIGRATE_DISABLE_SET_AFFIN) -+#define migrate_disable_count(p) ((p)->migrate_disable & ~MIGRATE_DISABLE_SET_AFFIN) -+ -+static inline void update_migrate_disable(struct task_struct *p) -+{ -+ const struct cpumask *mask; + static inline bool hlist_bl_is_locked(struct hlist_bl_head *b) +diff -Nur linux-4.1.10.orig/include/linux/locallock.h linux-4.1.10/include/linux/locallock.h +--- linux-4.1.10.orig/include/linux/locallock.h 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/include/linux/locallock.h 2015-10-07 18:00:08.000000000 +0200 +@@ -0,0 +1,270 @@ ++#ifndef _LINUX_LOCALLOCK_H ++#define _LINUX_LOCALLOCK_H + -+ if (likely(!p->migrate_disable)) -+ return; ++#include ++#include + -+ /* Did we already update affinity? */ -+ if (unlikely(migrate_disabled_updated(p))) -+ return; ++#ifdef CONFIG_PREEMPT_RT_BASE + -+ /* -+ * Since this is always current we can get away with only locking -+ * rq->lock, the ->cpus_allowed value can normally only be changed -+ * while holding both p->pi_lock and rq->lock, but seeing that this -+ * is current, we cannot actually be waking up, so all code that -+ * relies on serialization against p->pi_lock is out of scope. -+ * -+ * Having rq->lock serializes us against things like -+ * set_cpus_allowed_ptr() that can still happen concurrently. -+ */ -+ mask = tsk_cpus_allowed(p); ++#ifdef CONFIG_DEBUG_SPINLOCK ++# define LL_WARN(cond) WARN_ON(cond) ++#else ++# define LL_WARN(cond) do { } while (0) ++#endif + -+ if (p->sched_class->set_cpus_allowed) -+ p->sched_class->set_cpus_allowed(p, mask); -+ /* mask==cpumask_of(task_cpu(p)) which has a cpumask_weight==1 */ -+ p->nr_cpus_allowed = 1; ++/* ++ * per cpu lock based substitute for local_irq_*() ++ */ ++struct local_irq_lock { ++ spinlock_t lock; ++ struct task_struct *owner; ++ int nestcnt; ++ unsigned long flags; ++}; + -+ /* Let migrate_enable know to fix things back up */ -+ p->migrate_disable |= MIGRATE_DISABLE_SET_AFFIN; -+} ++#define DEFINE_LOCAL_IRQ_LOCK(lvar) \ ++ DEFINE_PER_CPU(struct local_irq_lock, lvar) = { \ ++ .lock = __SPIN_LOCK_UNLOCKED((lvar).lock) } + -+void migrate_disable(void) -+{ -+ struct task_struct *p = current; ++#define DECLARE_LOCAL_IRQ_LOCK(lvar) \ ++ DECLARE_PER_CPU(struct local_irq_lock, lvar) + -+ if (in_atomic()) { -+#ifdef CONFIG_SCHED_DEBUG -+ p->migrate_disable_atomic++; -+#endif -+ return; -+ } ++#define local_irq_lock_init(lvar) \ ++ do { \ ++ int __cpu; \ ++ for_each_possible_cpu(__cpu) \ ++ spin_lock_init(&per_cpu(lvar, __cpu).lock); \ ++ } while (0) + -+#ifdef CONFIG_SCHED_DEBUG -+ if (unlikely(p->migrate_disable_atomic)) { -+ tracing_off(); -+ WARN_ON_ONCE(1); -+ } ++/* ++ * spin_lock|trylock|unlock_local flavour that does not migrate disable ++ * used for __local_lock|trylock|unlock where get_local_var/put_local_var ++ * already takes care of the migrate_disable/enable ++ * for CONFIG_PREEMPT_BASE map to the normal spin_* calls. ++ */ ++#ifdef CONFIG_PREEMPT_RT_FULL ++# define spin_lock_local(lock) rt_spin_lock(lock) ++# define spin_trylock_local(lock) rt_spin_trylock(lock) ++# define spin_unlock_local(lock) rt_spin_unlock(lock) ++#else ++# define spin_lock_local(lock) spin_lock(lock) ++# define spin_trylock_local(lock) spin_trylock(lock) ++# define spin_unlock_local(lock) spin_unlock(lock) +#endif + -+ if (p->migrate_disable) { -+ p->migrate_disable++; -+ return; ++static inline void __local_lock(struct local_irq_lock *lv) ++{ ++ if (lv->owner != current) { ++ spin_lock_local(&lv->lock); ++ LL_WARN(lv->owner); ++ LL_WARN(lv->nestcnt); ++ lv->owner = current; + } -+ -+ preempt_disable(); -+ preempt_lazy_disable(); -+ pin_current_cpu(); -+ p->migrate_disable = 1; -+ preempt_enable(); ++ lv->nestcnt++; +} -+EXPORT_SYMBOL(migrate_disable); + -+void migrate_enable(void) -+{ -+ struct task_struct *p = current; -+ const struct cpumask *mask; -+ unsigned long flags; -+ struct rq *rq; ++#define local_lock(lvar) \ ++ do { __local_lock(&get_local_var(lvar)); } while (0) + -+ if (in_atomic()) { -+#ifdef CONFIG_SCHED_DEBUG -+ p->migrate_disable_atomic--; -+#endif -+ return; ++static inline int __local_trylock(struct local_irq_lock *lv) ++{ ++ if (lv->owner != current && spin_trylock_local(&lv->lock)) { ++ LL_WARN(lv->owner); ++ LL_WARN(lv->nestcnt); ++ lv->owner = current; ++ lv->nestcnt = 1; ++ return 1; + } ++ return 0; ++} + -+#ifdef CONFIG_SCHED_DEBUG -+ if (unlikely(p->migrate_disable_atomic)) { -+ tracing_off(); -+ WARN_ON_ONCE(1); -+ } -+#endif -+ WARN_ON_ONCE(p->migrate_disable <= 0); ++#define local_trylock(lvar) \ ++ ({ \ ++ int __locked; \ ++ __locked = __local_trylock(&get_local_var(lvar)); \ ++ if (!__locked) \ ++ put_local_var(lvar); \ ++ __locked; \ ++ }) + -+ if (migrate_disable_count(p) > 1) { -+ p->migrate_disable--; ++static inline void __local_unlock(struct local_irq_lock *lv) ++{ ++ LL_WARN(lv->nestcnt == 0); ++ LL_WARN(lv->owner != current); ++ if (--lv->nestcnt) + return; -+ } -+ -+ preempt_disable(); -+ if (unlikely(migrate_disabled_updated(p))) { -+ /* -+ * Undo whatever update_migrate_disable() did, also see there -+ * about locking. -+ */ -+ rq = this_rq(); -+ raw_spin_lock_irqsave(&rq->lock, flags); -+ -+ /* -+ * Clearing migrate_disable causes tsk_cpus_allowed to -+ * show the tasks original cpu affinity. -+ */ -+ p->migrate_disable = 0; -+ mask = tsk_cpus_allowed(p); -+ if (p->sched_class->set_cpus_allowed) -+ p->sched_class->set_cpus_allowed(p, mask); -+ p->nr_cpus_allowed = cpumask_weight(mask); -+ raw_spin_unlock_irqrestore(&rq->lock, flags); -+ } else -+ p->migrate_disable = 0; + -+ unpin_current_cpu(); -+ preempt_enable(); -+ preempt_lazy_enable(); ++ lv->owner = NULL; ++ spin_unlock_local(&lv->lock); +} -+EXPORT_SYMBOL(migrate_enable); -+#else -+static inline void update_migrate_disable(struct task_struct *p) { } -+#define migrate_disabled_updated(p) 0 -+#endif + - /* - * Pick up the highest-prio task: - */ -@@ -2756,6 +2981,8 @@ - smp_mb__before_spinlock(); - raw_spin_lock_irq(&rq->lock); - -+ update_migrate_disable(prev); ++#define local_unlock(lvar) \ ++ do { \ ++ __local_unlock(this_cpu_ptr(&lvar)); \ ++ put_local_var(lvar); \ ++ } while (0) + - rq->clock_skip_update <<= 1; /* promote REQ to ACT */ - - switch_count = &prev->nivcsw; -@@ -2765,19 +2992,6 @@ - } else { - deactivate_task(rq, prev, DEQUEUE_SLEEP); - prev->on_rq = 0; -- -- /* -- * If a worker went to sleep, notify and ask workqueue -- * whether it wants to wake up a task to maintain -- * concurrency. -- */ -- if (prev->flags & PF_WQ_WORKER) { -- struct task_struct *to_wakeup; -- -- to_wakeup = wq_worker_sleeping(prev, cpu); -- if (to_wakeup) -- try_to_wake_up_local(to_wakeup); -- } - } - switch_count = &prev->nvcsw; - } -@@ -2787,6 +3001,7 @@ - - next = pick_next_task(rq, prev); - clear_tsk_need_resched(prev); -+ clear_tsk_need_resched_lazy(prev); - clear_preempt_need_resched(); - rq->clock_skip_update = 0; - -@@ -2807,8 +3022,19 @@ - - static inline void sched_submit_work(struct task_struct *tsk) - { -- if (!tsk->state || tsk_is_pi_blocked(tsk)) -+ if (!tsk->state) -+ return; -+ /* -+ * If a worker went to sleep, notify and ask workqueue whether -+ * it wants to wake up a task to maintain concurrency. -+ */ -+ if (tsk->flags & PF_WQ_WORKER) -+ wq_worker_sleeping(tsk); ++static inline void __local_lock_irq(struct local_irq_lock *lv) ++{ ++ spin_lock_irqsave(&lv->lock, lv->flags); ++ LL_WARN(lv->owner); ++ LL_WARN(lv->nestcnt); ++ lv->owner = current; ++ lv->nestcnt = 1; ++} + ++#define local_lock_irq(lvar) \ ++ do { __local_lock_irq(&get_local_var(lvar)); } while (0) + -+ if (tsk_is_pi_blocked(tsk)) - return; ++#define local_lock_irq_on(lvar, cpu) \ ++ do { __local_lock_irq(&per_cpu(lvar, cpu)); } while (0) + - /* - * If we are going to sleep and we have plugged IO queued, - * make sure to submit it to avoid deadlocks. -@@ -2817,6 +3043,12 @@ - blk_schedule_flush_plug(tsk); - } - -+static void sched_update_worker(struct task_struct *tsk) ++static inline void __local_unlock_irq(struct local_irq_lock *lv) +{ -+ if (tsk->flags & PF_WQ_WORKER) -+ wq_worker_running(tsk); ++ LL_WARN(!lv->nestcnt); ++ LL_WARN(lv->owner != current); ++ lv->owner = NULL; ++ lv->nestcnt = 0; ++ spin_unlock_irq(&lv->lock); +} + - asmlinkage __visible void __sched schedule(void) - { - struct task_struct *tsk = current; -@@ -2825,6 +3057,7 @@ - do { - __schedule(); - } while (need_resched()); -+ sched_update_worker(tsk); - } - EXPORT_SYMBOL(schedule); - -@@ -2916,6 +3149,14 @@ - if (likely(!preemptible())) - return; - -+#ifdef CONFIG_PREEMPT_LAZY -+ /* -+ * Check for lazy preemption -+ */ -+ if (current_thread_info()->preempt_lazy_count && -+ !test_thread_flag(TIF_NEED_RESCHED)) -+ return; -+#endif - do { - __preempt_count_add(PREEMPT_ACTIVE); - /* -@@ -2924,7 +3165,16 @@ - * an infinite recursion. - */ - prev_ctx = exception_enter(); -+ /* -+ * The add/subtract must not be traced by the function -+ * tracer. But we still want to account for the -+ * preempt off latency tracer. Since the _notrace versions -+ * of add/subtract skip the accounting for latency tracer -+ * we must force it manually. -+ */ -+ start_critical_timings(); - __schedule(); -+ stop_critical_timings(); - exception_exit(prev_ctx); - - __preempt_count_sub(PREEMPT_ACTIVE); -@@ -4261,6 +4511,7 @@ - } - EXPORT_SYMBOL(__cond_resched_lock); - -+#ifndef CONFIG_PREEMPT_RT_FULL - int __sched __cond_resched_softirq(void) - { - BUG_ON(!in_softirq()); -@@ -4274,6 +4525,7 @@ - return 0; - } - EXPORT_SYMBOL(__cond_resched_softirq); -+#endif - - /** - * yield - yield the current processor to other threads. -@@ -4628,7 +4880,9 @@ - - /* Set the preempt count _outside_ the spinlocks! */ - init_idle_preempt_count(idle, cpu); -- -+#ifdef CONFIG_HAVE_PREEMPT_LAZY -+ task_thread_info(idle)->preempt_lazy_count = 0; -+#endif - /* - * The idle tasks have their own, simple scheduling class: - */ -@@ -4748,11 +5002,91 @@ - - void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask) - { -- if (p->sched_class->set_cpus_allowed) -- p->sched_class->set_cpus_allowed(p, new_mask); -+ if (!migrate_disabled_updated(p)) { -+ if (p->sched_class->set_cpus_allowed) -+ p->sched_class->set_cpus_allowed(p, new_mask); -+ p->nr_cpus_allowed = cpumask_weight(new_mask); -+ } - - cpumask_copy(&p->cpus_allowed, new_mask); -- p->nr_cpus_allowed = cpumask_weight(new_mask); -+} ++#define local_unlock_irq(lvar) \ ++ do { \ ++ __local_unlock_irq(this_cpu_ptr(&lvar)); \ ++ put_local_var(lvar); \ ++ } while (0) + -+static DEFINE_PER_CPU(struct cpumask, sched_cpumasks); -+static DEFINE_MUTEX(sched_down_mutex); -+static cpumask_t sched_down_cpumask; ++#define local_unlock_irq_on(lvar, cpu) \ ++ do { \ ++ __local_unlock_irq(&per_cpu(lvar, cpu)); \ ++ } while (0) + -+void tell_sched_cpu_down_begin(int cpu) ++static inline int __local_lock_irqsave(struct local_irq_lock *lv) +{ -+ mutex_lock(&sched_down_mutex); -+ cpumask_set_cpu(cpu, &sched_down_cpumask); -+ mutex_unlock(&sched_down_mutex); ++ if (lv->owner != current) { ++ __local_lock_irq(lv); ++ return 0; ++ } else { ++ lv->nestcnt++; ++ return 1; ++ } +} + -+void tell_sched_cpu_down_done(int cpu) ++#define local_lock_irqsave(lvar, _flags) \ ++ do { \ ++ if (__local_lock_irqsave(&get_local_var(lvar))) \ ++ put_local_var(lvar); \ ++ _flags = __this_cpu_read(lvar.flags); \ ++ } while (0) ++ ++#define local_lock_irqsave_on(lvar, _flags, cpu) \ ++ do { \ ++ __local_lock_irqsave(&per_cpu(lvar, cpu)); \ ++ _flags = per_cpu(lvar, cpu).flags; \ ++ } while (0) ++ ++static inline int __local_unlock_irqrestore(struct local_irq_lock *lv, ++ unsigned long flags) +{ -+ mutex_lock(&sched_down_mutex); -+ cpumask_clear_cpu(cpu, &sched_down_cpumask); -+ mutex_unlock(&sched_down_mutex); ++ LL_WARN(!lv->nestcnt); ++ LL_WARN(lv->owner != current); ++ if (--lv->nestcnt) ++ return 0; ++ ++ lv->owner = NULL; ++ spin_unlock_irqrestore(&lv->lock, lv->flags); ++ return 1; +} + -+/** -+ * migrate_me - try to move the current task off this cpu -+ * -+ * Used by the pin_current_cpu() code to try to get tasks -+ * to move off the current CPU as it is going down. -+ * It will only move the task if the task isn't pinned to -+ * the CPU (with migrate_disable, affinity or NO_SETAFFINITY) -+ * and the task has to be in a RUNNING state. Otherwise the -+ * movement of the task will wake it up (change its state -+ * to running) when the task did not expect it. -+ * -+ * Returns 1 if it succeeded in moving the current task -+ * 0 otherwise. -+ */ -+int migrate_me(void) -+{ -+ struct task_struct *p = current; -+ struct migration_arg arg; -+ struct cpumask *cpumask; -+ struct cpumask *mask; -+ unsigned long flags; -+ unsigned int dest_cpu; -+ struct rq *rq; ++#define local_unlock_irqrestore(lvar, flags) \ ++ do { \ ++ if (__local_unlock_irqrestore(this_cpu_ptr(&lvar), flags)) \ ++ put_local_var(lvar); \ ++ } while (0) + -+ /* -+ * We can not migrate tasks bounded to a CPU or tasks not -+ * running. The movement of the task will wake it up. -+ */ -+ if (p->flags & PF_NO_SETAFFINITY || p->state) -+ return 0; ++#define local_unlock_irqrestore_on(lvar, flags, cpu) \ ++ do { \ ++ __local_unlock_irqrestore(&per_cpu(lvar, cpu), flags); \ ++ } while (0) + -+ mutex_lock(&sched_down_mutex); -+ rq = task_rq_lock(p, &flags); ++#define local_spin_trylock_irq(lvar, lock) \ ++ ({ \ ++ int __locked; \ ++ local_lock_irq(lvar); \ ++ __locked = spin_trylock(lock); \ ++ if (!__locked) \ ++ local_unlock_irq(lvar); \ ++ __locked; \ ++ }) + -+ cpumask = this_cpu_ptr(&sched_cpumasks); -+ mask = &p->cpus_allowed; ++#define local_spin_lock_irq(lvar, lock) \ ++ do { \ ++ local_lock_irq(lvar); \ ++ spin_lock(lock); \ ++ } while (0) + -+ cpumask_andnot(cpumask, mask, &sched_down_cpumask); ++#define local_spin_unlock_irq(lvar, lock) \ ++ do { \ ++ spin_unlock(lock); \ ++ local_unlock_irq(lvar); \ ++ } while (0) + -+ if (!cpumask_weight(cpumask)) { -+ /* It's only on this CPU? */ -+ task_rq_unlock(rq, p, &flags); -+ mutex_unlock(&sched_down_mutex); -+ return 0; -+ } ++#define local_spin_lock_irqsave(lvar, lock, flags) \ ++ do { \ ++ local_lock_irqsave(lvar, flags); \ ++ spin_lock(lock); \ ++ } while (0) + -+ dest_cpu = cpumask_any_and(cpu_active_mask, cpumask); ++#define local_spin_unlock_irqrestore(lvar, lock, flags) \ ++ do { \ ++ spin_unlock(lock); \ ++ local_unlock_irqrestore(lvar, flags); \ ++ } while (0) + -+ arg.task = p; -+ arg.dest_cpu = dest_cpu; ++#define get_locked_var(lvar, var) \ ++ (*({ \ ++ local_lock(lvar); \ ++ this_cpu_ptr(&var); \ ++ })) + -+ task_rq_unlock(rq, p, &flags); ++#define put_locked_var(lvar, var) local_unlock(lvar); + -+ stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg); -+ tlb_migrate_finish(p->mm); -+ mutex_unlock(&sched_down_mutex); ++#define local_lock_cpu(lvar) \ ++ ({ \ ++ local_lock(lvar); \ ++ smp_processor_id(); \ ++ }) + -+ return 1; - } - - /* -@@ -4798,7 +5132,7 @@ - do_set_cpus_allowed(p, new_mask); - - /* Can the task run on the task's current CPU? If so, we're done */ -- if (cpumask_test_cpu(task_cpu(p), new_mask)) -+ if (cpumask_test_cpu(task_cpu(p), new_mask) || __migrate_disabled(p)) - goto out; - - dest_cpu = cpumask_any_and(cpu_active_mask, new_mask); -@@ -4938,6 +5272,8 @@ - - #ifdef CONFIG_HOTPLUG_CPU - -+static DEFINE_PER_CPU(struct mm_struct *, idle_last_mm); ++#define local_unlock_cpu(lvar) local_unlock(lvar) ++ ++#else /* PREEMPT_RT_BASE */ ++ ++#define DEFINE_LOCAL_IRQ_LOCK(lvar) __typeof__(const int) lvar ++#define DECLARE_LOCAL_IRQ_LOCK(lvar) extern __typeof__(const int) lvar ++ ++static inline void local_irq_lock_init(int lvar) { } ++ ++#define local_lock(lvar) preempt_disable() ++#define local_unlock(lvar) preempt_enable() ++#define local_lock_irq(lvar) local_irq_disable() ++#define local_unlock_irq(lvar) local_irq_enable() ++#define local_lock_irqsave(lvar, flags) local_irq_save(flags) ++#define local_unlock_irqrestore(lvar, flags) local_irq_restore(flags) ++ ++#define local_spin_trylock_irq(lvar, lock) spin_trylock_irq(lock) ++#define local_spin_lock_irq(lvar, lock) spin_lock_irq(lock) ++#define local_spin_unlock_irq(lvar, lock) spin_unlock_irq(lock) ++#define local_spin_lock_irqsave(lvar, lock, flags) \ ++ spin_lock_irqsave(lock, flags) ++#define local_spin_unlock_irqrestore(lvar, lock, flags) \ ++ spin_unlock_irqrestore(lock, flags) ++ ++#define get_locked_var(lvar, var) get_cpu_var(var) ++#define put_locked_var(lvar, var) put_cpu_var(var) ++ ++#define local_lock_cpu(lvar) get_cpu() ++#define local_unlock_cpu(lvar) put_cpu() + - /* - * Ensures that the idle task is using init_mm right before its cpu goes - * offline. -@@ -4952,7 +5288,11 @@ - switch_mm(mm, &init_mm, current); - finish_arch_post_lock_switch(); - } -- mmdrop(mm); -+ /* -+ * Defer the cleanup to an alive cpu. On RT we can neither -+ * call mmdrop() nor mmdrop_delayed() from here. -+ */ -+ per_cpu(idle_last_mm, smp_processor_id()) = mm; - } - - /* -@@ -5295,6 +5635,10 @@ - - case CPU_DEAD: - calc_load_migrate(rq); -+ if (per_cpu(idle_last_mm, cpu)) { -+ mmdrop(per_cpu(idle_last_mm, cpu)); -+ per_cpu(idle_last_mm, cpu) = NULL; -+ } - break; - #endif - } -@@ -7266,7 +7610,8 @@ - #ifdef CONFIG_DEBUG_ATOMIC_SLEEP - static inline int preempt_count_equals(int preempt_offset) - { -- int nested = (preempt_count() & ~PREEMPT_ACTIVE) + rcu_preempt_depth(); -+ int nested = (preempt_count() & ~PREEMPT_ACTIVE) + -+ sched_rcu_preempt_depth(); - - return (nested == preempt_offset); - } -diff -Nur linux-4.1.6.orig/kernel/sched/cputime.c linux-4.1.6/kernel/sched/cputime.c ---- linux-4.1.6.orig/kernel/sched/cputime.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/sched/cputime.c 2015-09-08 23:49:08.109866942 +0200 -@@ -675,37 +675,45 @@ - - void vtime_account_system(struct task_struct *tsk) - { -- write_seqlock(&tsk->vtime_seqlock); -+ raw_spin_lock(&tsk->vtime_lock); -+ write_seqcount_begin(&tsk->vtime_seq); - __vtime_account_system(tsk); -- write_sequnlock(&tsk->vtime_seqlock); -+ write_seqcount_end(&tsk->vtime_seq); -+ raw_spin_unlock(&tsk->vtime_lock); - } - - void vtime_gen_account_irq_exit(struct task_struct *tsk) - { -- write_seqlock(&tsk->vtime_seqlock); -+ raw_spin_lock(&tsk->vtime_lock); -+ write_seqcount_begin(&tsk->vtime_seq); - __vtime_account_system(tsk); - if (context_tracking_in_user()) - tsk->vtime_snap_whence = VTIME_USER; -- write_sequnlock(&tsk->vtime_seqlock); -+ write_seqcount_end(&tsk->vtime_seq); -+ raw_spin_unlock(&tsk->vtime_lock); - } - - void vtime_account_user(struct task_struct *tsk) - { - cputime_t delta_cpu; - -- write_seqlock(&tsk->vtime_seqlock); -+ raw_spin_lock(&tsk->vtime_lock); -+ write_seqcount_begin(&tsk->vtime_seq); - delta_cpu = get_vtime_delta(tsk); - tsk->vtime_snap_whence = VTIME_SYS; - account_user_time(tsk, delta_cpu, cputime_to_scaled(delta_cpu)); -- write_sequnlock(&tsk->vtime_seqlock); -+ write_seqcount_end(&tsk->vtime_seq); -+ raw_spin_unlock(&tsk->vtime_lock); - } - - void vtime_user_enter(struct task_struct *tsk) - { -- write_seqlock(&tsk->vtime_seqlock); -+ raw_spin_lock(&tsk->vtime_lock); -+ write_seqcount_begin(&tsk->vtime_seq); - __vtime_account_system(tsk); - tsk->vtime_snap_whence = VTIME_USER; -- write_sequnlock(&tsk->vtime_seqlock); -+ write_seqcount_end(&tsk->vtime_seq); -+ raw_spin_unlock(&tsk->vtime_lock); - } - - void vtime_guest_enter(struct task_struct *tsk) -@@ -717,19 +725,23 @@ - * synchronization against the reader (task_gtime()) - * that can thus safely catch up with a tickless delta. - */ -- write_seqlock(&tsk->vtime_seqlock); -+ raw_spin_lock(&tsk->vtime_lock); -+ write_seqcount_begin(&tsk->vtime_seq); - __vtime_account_system(tsk); - current->flags |= PF_VCPU; -- write_sequnlock(&tsk->vtime_seqlock); -+ write_seqcount_end(&tsk->vtime_seq); -+ raw_spin_unlock(&tsk->vtime_lock); - } - EXPORT_SYMBOL_GPL(vtime_guest_enter); - - void vtime_guest_exit(struct task_struct *tsk) - { -- write_seqlock(&tsk->vtime_seqlock); -+ raw_spin_lock(&tsk->vtime_lock); -+ write_seqcount_begin(&tsk->vtime_seq); - __vtime_account_system(tsk); - current->flags &= ~PF_VCPU; -- write_sequnlock(&tsk->vtime_seqlock); -+ write_seqcount_end(&tsk->vtime_seq); -+ raw_spin_unlock(&tsk->vtime_lock); - } - EXPORT_SYMBOL_GPL(vtime_guest_exit); - -@@ -742,24 +754,30 @@ - - void arch_vtime_task_switch(struct task_struct *prev) - { -- write_seqlock(&prev->vtime_seqlock); -+ raw_spin_lock(&prev->vtime_lock); -+ write_seqcount_begin(&prev->vtime_seq); - prev->vtime_snap_whence = VTIME_SLEEPING; -- write_sequnlock(&prev->vtime_seqlock); -+ write_seqcount_end(&prev->vtime_seq); -+ raw_spin_unlock(&prev->vtime_lock); - -- write_seqlock(¤t->vtime_seqlock); -+ raw_spin_lock(¤t->vtime_lock); -+ write_seqcount_begin(¤t->vtime_seq); - current->vtime_snap_whence = VTIME_SYS; - current->vtime_snap = sched_clock_cpu(smp_processor_id()); -- write_sequnlock(¤t->vtime_seqlock); -+ write_seqcount_end(¤t->vtime_seq); -+ raw_spin_unlock(¤t->vtime_lock); - } - - void vtime_init_idle(struct task_struct *t, int cpu) - { - unsigned long flags; - -- write_seqlock_irqsave(&t->vtime_seqlock, flags); -+ raw_spin_lock_irqsave(&t->vtime_lock, flags); -+ write_seqcount_begin(&t->vtime_seq); - t->vtime_snap_whence = VTIME_SYS; - t->vtime_snap = sched_clock_cpu(cpu); -- write_sequnlock_irqrestore(&t->vtime_seqlock, flags); -+ write_seqcount_end(&t->vtime_seq); -+ raw_spin_unlock_irqrestore(&t->vtime_lock, flags); - } - - cputime_t task_gtime(struct task_struct *t) -@@ -768,13 +786,13 @@ - cputime_t gtime; - - do { -- seq = read_seqbegin(&t->vtime_seqlock); -+ seq = read_seqcount_begin(&t->vtime_seq); - - gtime = t->gtime; - if (t->flags & PF_VCPU) - gtime += vtime_delta(t); - -- } while (read_seqretry(&t->vtime_seqlock, seq)); -+ } while (read_seqcount_retry(&t->vtime_seq, seq)); - - return gtime; - } -@@ -797,7 +815,7 @@ - *udelta = 0; - *sdelta = 0; - -- seq = read_seqbegin(&t->vtime_seqlock); -+ seq = read_seqcount_begin(&t->vtime_seq); - - if (u_dst) - *u_dst = *u_src; -@@ -821,7 +839,7 @@ - if (t->vtime_snap_whence == VTIME_SYS) - *sdelta = delta; - } -- } while (read_seqretry(&t->vtime_seqlock, seq)); -+ } while (read_seqcount_retry(&t->vtime_seq, seq)); - } - - -diff -Nur linux-4.1.6.orig/kernel/sched/deadline.c linux-4.1.6/kernel/sched/deadline.c ---- linux-4.1.6.orig/kernel/sched/deadline.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/sched/deadline.c 2015-09-08 23:49:08.109866942 +0200 -@@ -637,6 +637,7 @@ - - hrtimer_init(timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); - timer->function = dl_task_timer; -+ timer->irqsafe = 1; - } - - static -diff -Nur linux-4.1.6.orig/kernel/sched/debug.c linux-4.1.6/kernel/sched/debug.c ---- linux-4.1.6.orig/kernel/sched/debug.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/sched/debug.c 2015-09-08 23:49:08.109866942 +0200 -@@ -260,6 +260,9 @@ - P(rt_throttled); - PN(rt_time); - PN(rt_runtime); -+#ifdef CONFIG_SMP -+ P(rt_nr_migratory); -+#endif - - #undef PN - #undef P -@@ -648,6 +651,10 @@ - #endif - P(policy); - P(prio); -+#ifdef CONFIG_PREEMPT_RT_FULL -+ P(migrate_disable); +#endif -+ P(nr_cpus_allowed); - #undef PN - #undef __PN - #undef P -diff -Nur linux-4.1.6.orig/kernel/sched/fair.c linux-4.1.6/kernel/sched/fair.c ---- linux-4.1.6.orig/kernel/sched/fair.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/sched/fair.c 2015-09-08 23:49:08.113866499 +0200 -@@ -3201,7 +3201,7 @@ - ideal_runtime = sched_slice(cfs_rq, curr); - delta_exec = curr->sum_exec_runtime - curr->prev_sum_exec_runtime; - if (delta_exec > ideal_runtime) { -- resched_curr(rq_of(cfs_rq)); -+ resched_curr_lazy(rq_of(cfs_rq)); - /* - * The current task ran long enough, ensure it doesn't get - * re-elected due to buddy favours. -@@ -3225,7 +3225,7 @@ - return; - - if (delta > ideal_runtime) -- resched_curr(rq_of(cfs_rq)); -+ resched_curr_lazy(rq_of(cfs_rq)); - } - - static void -@@ -3366,7 +3366,7 @@ - * validating it and just reschedule. - */ - if (queued) { -- resched_curr(rq_of(cfs_rq)); -+ resched_curr_lazy(rq_of(cfs_rq)); - return; - } - /* -@@ -3557,7 +3557,7 @@ - * hierarchy can be throttled - */ - if (!assign_cfs_rq_runtime(cfs_rq) && likely(cfs_rq->curr)) -- resched_curr(rq_of(cfs_rq)); -+ resched_curr_lazy(rq_of(cfs_rq)); - } - - static __always_inline -@@ -4180,7 +4180,7 @@ - - if (delta < 0) { - if (rq->curr == p) -- resched_curr(rq); -+ resched_curr_lazy(rq); - return; - } - hrtick_start(rq, delta); -@@ -5076,7 +5076,7 @@ - return; - - preempt: -- resched_curr(rq); -+ resched_curr_lazy(rq); - /* - * Only set the backward buddy when the current task is still - * on the rq. This can happen when a wakeup gets interleaved -@@ -7866,7 +7866,7 @@ - * 'current' within the tree based on its new key value. - */ - swap(curr->vruntime, se->vruntime); -- resched_curr(rq); -+ resched_curr_lazy(rq); - } - - se->vruntime -= cfs_rq->min_vruntime; -@@ -7891,7 +7891,7 @@ - */ - if (rq->curr == p) { - if (p->prio > oldprio) -- resched_curr(rq); -+ resched_curr_lazy(rq); - } else - check_preempt_curr(rq, p, 0); - } -diff -Nur linux-4.1.6.orig/kernel/sched/features.h linux-4.1.6/kernel/sched/features.h ---- linux-4.1.6.orig/kernel/sched/features.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/sched/features.h 2015-09-08 23:49:08.113866499 +0200 -@@ -50,11 +50,19 @@ - */ - SCHED_FEAT(NONTASK_CAPACITY, true) - -+#ifdef CONFIG_PREEMPT_RT_FULL -+SCHED_FEAT(TTWU_QUEUE, false) -+# ifdef CONFIG_PREEMPT_LAZY -+SCHED_FEAT(PREEMPT_LAZY, true) -+# endif -+#else + - /* - * Queue remote wakeups on the target CPU and process them - * using the scheduler IPI. Reduces rq->lock contention/bounces. - */ - SCHED_FEAT(TTWU_QUEUE, true) +#endif - - #ifdef HAVE_RT_PUSH_IPI - /* -diff -Nur linux-4.1.6.orig/kernel/sched/Makefile linux-4.1.6/kernel/sched/Makefile ---- linux-4.1.6.orig/kernel/sched/Makefile 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/sched/Makefile 2015-09-08 23:49:08.105867384 +0200 -@@ -13,7 +13,7 @@ - - obj-y += core.o proc.o clock.o cputime.o - obj-y += idle_task.o fair.o rt.o deadline.o stop_task.o --obj-y += wait.o completion.o idle.o -+obj-y += wait.o wait-simple.o work-simple.o completion.o idle.o - obj-$(CONFIG_SMP) += cpupri.o cpudeadline.o - obj-$(CONFIG_SCHED_AUTOGROUP) += auto_group.o - obj-$(CONFIG_SCHEDSTATS) += stats.o -diff -Nur linux-4.1.6.orig/kernel/sched/rt.c linux-4.1.6/kernel/sched/rt.c ---- linux-4.1.6.orig/kernel/sched/rt.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/sched/rt.c 2015-09-08 23:49:08.113866499 +0200 -@@ -44,6 +44,7 @@ - - hrtimer_init(&rt_b->rt_period_timer, - CLOCK_MONOTONIC, HRTIMER_MODE_REL); -+ rt_b->rt_period_timer.irqsafe = 1; - rt_b->rt_period_timer.function = sched_rt_period_timer; - } - -@@ -89,6 +90,7 @@ - rt_rq->push_cpu = nr_cpu_ids; - raw_spin_lock_init(&rt_rq->push_lock); - init_irq_work(&rt_rq->push_work, push_irq_work_func); -+ rt_rq->push_work.flags |= IRQ_WORK_HARD_IRQ; +diff -Nur linux-4.1.10.orig/include/linux/mm_types.h linux-4.1.10/include/linux/mm_types.h +--- linux-4.1.10.orig/include/linux/mm_types.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/mm_types.h 2015-10-07 18:00:08.000000000 +0200 +@@ -11,6 +11,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -453,6 +454,9 @@ + bool tlb_flush_pending; #endif - #endif /* CONFIG_SMP */ - /* We start is dequeued state, because no RT tasks are queued */ -diff -Nur linux-4.1.6.orig/kernel/sched/sched.h linux-4.1.6/kernel/sched/sched.h ---- linux-4.1.6.orig/kernel/sched/sched.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/sched/sched.h 2015-09-08 23:49:08.113866499 +0200 -@@ -1092,6 +1092,7 @@ - #define WF_SYNC 0x01 /* waker goes to sleep after wakeup */ - #define WF_FORK 0x02 /* child wakeup after fork */ - #define WF_MIGRATED 0x4 /* internal use, task got migrated */ -+#define WF_LOCK_SLEEPER 0x08 /* wakeup spinlock "sleeper" */ - - /* - * To aid in avoiding the subversion of "niceness" due to uneven distribution -@@ -1289,6 +1290,15 @@ - extern void resched_curr(struct rq *rq); - extern void resched_cpu(int cpu); - -+#ifdef CONFIG_PREEMPT_LAZY -+extern void resched_curr_lazy(struct rq *rq); -+#else -+static inline void resched_curr_lazy(struct rq *rq) -+{ -+ resched_curr(rq); -+} + struct uprobes_state uprobes_state; ++#ifdef CONFIG_PREEMPT_RT_BASE ++ struct rcu_head delayed_drop; ++#endif + #ifdef CONFIG_X86_INTEL_MPX + /* address of the bounds directory */ + void __user *bd_addr; +diff -Nur linux-4.1.10.orig/include/linux/mm_types.h.orig linux-4.1.10/include/linux/mm_types.h.orig +--- linux-4.1.10.orig/include/linux/mm_types.h.orig 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/include/linux/mm_types.h.orig 2015-10-03 13:49:38.000000000 +0200 +@@ -0,0 +1,539 @@ ++#ifndef _LINUX_MM_TYPES_H ++#define _LINUX_MM_TYPES_H ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#ifndef AT_VECTOR_SIZE_ARCH ++#define AT_VECTOR_SIZE_ARCH 0 +#endif ++#define AT_VECTOR_SIZE (2*(AT_VECTOR_SIZE_ARCH + AT_VECTOR_SIZE_BASE + 1)) ++ ++struct address_space; ++struct mem_cgroup; ++ ++#define USE_SPLIT_PTE_PTLOCKS (NR_CPUS >= CONFIG_SPLIT_PTLOCK_CPUS) ++#define USE_SPLIT_PMD_PTLOCKS (USE_SPLIT_PTE_PTLOCKS && \ ++ IS_ENABLED(CONFIG_ARCH_ENABLE_SPLIT_PMD_PTLOCK)) ++#define ALLOC_SPLIT_PTLOCKS (SPINLOCK_SIZE > BITS_PER_LONG/8) ++ ++typedef void compound_page_dtor(struct page *); + - extern struct rt_bandwidth def_rt_bandwidth; - extern void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime); - -diff -Nur linux-4.1.6.orig/kernel/sched/wait-simple.c linux-4.1.6/kernel/sched/wait-simple.c ---- linux-4.1.6.orig/kernel/sched/wait-simple.c 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.6/kernel/sched/wait-simple.c 2015-09-08 23:49:08.113866499 +0200 -@@ -0,0 +1,115 @@ +/* -+ * Simple waitqueues without fancy flags and callbacks -+ * -+ * (C) 2011 Thomas Gleixner -+ * -+ * Based on kernel/wait.c ++ * Each physical page in the system has a struct page associated with ++ * it to keep track of whatever it is we are using the page for at the ++ * moment. Note that we have no way to track which tasks are using ++ * a page, though if it is a pagecache page, rmap structures can tell us ++ * who is mapping it. + * -+ * For licencing details see kernel-base/COPYING ++ * The objects in struct page are organized in double word blocks in ++ * order to allows us to use atomic double word operations on portions ++ * of struct page. That is currently only used by slub but the arrangement ++ * allows the use of atomic double word operations on the flags/mapping ++ * and lru list pointers also. + */ -+#include -+#include -+#include -+#include ++struct page { ++ /* First double word block */ ++ unsigned long flags; /* Atomic flags, some possibly ++ * updated asynchronously */ ++ union { ++ struct address_space *mapping; /* If low bit clear, points to ++ * inode address_space, or NULL. ++ * If page mapped as anonymous ++ * memory, low bit is set, and ++ * it points to anon_vma object: ++ * see PAGE_MAPPING_ANON below. ++ */ ++ void *s_mem; /* slab first object */ ++ }; + -+/* Adds w to head->list. Must be called with head->lock locked. */ -+static inline void __swait_enqueue(struct swait_head *head, struct swaiter *w) -+{ -+ list_add(&w->node, &head->list); -+ /* We can't let the condition leak before the setting of head */ -+ smp_mb(); -+} ++ /* Second double word */ ++ struct { ++ union { ++ pgoff_t index; /* Our offset within mapping. */ ++ void *freelist; /* sl[aou]b first free object */ ++ }; + -+/* Removes w from head->list. Must be called with head->lock locked. */ -+static inline void __swait_dequeue(struct swaiter *w) -+{ -+ list_del_init(&w->node); -+} ++ union { ++#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \ ++ defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE) ++ /* Used for cmpxchg_double in slub */ ++ unsigned long counters; ++#else ++ /* ++ * Keep _count separate from slub cmpxchg_double data. ++ * As the rest of the double word is protected by ++ * slab_lock but _count is not. ++ */ ++ unsigned counters; ++#endif + -+void __init_swait_head(struct swait_head *head, struct lock_class_key *key) -+{ -+ raw_spin_lock_init(&head->lock); -+ lockdep_set_class(&head->lock, key); -+ INIT_LIST_HEAD(&head->list); -+} -+EXPORT_SYMBOL(__init_swait_head); ++ struct { ++ ++ union { ++ /* ++ * Count of ptes mapped in ++ * mms, to show when page is ++ * mapped & limit reverse map ++ * searches. ++ * ++ * Used also for tail pages ++ * refcounting instead of ++ * _count. Tail pages cannot ++ * be mapped and keeping the ++ * tail page _count zero at ++ * all times guarantees ++ * get_page_unless_zero() will ++ * never succeed on tail ++ * pages. ++ */ ++ atomic_t _mapcount; ++ ++ struct { /* SLUB */ ++ unsigned inuse:16; ++ unsigned objects:15; ++ unsigned frozen:1; ++ }; ++ int units; /* SLOB */ ++ }; ++ atomic_t _count; /* Usage count, see below. */ ++ }; ++ unsigned int active; /* SLAB */ ++ }; ++ }; + -+void swait_prepare_locked(struct swait_head *head, struct swaiter *w) -+{ -+ w->task = current; -+ if (list_empty(&w->node)) -+ __swait_enqueue(head, w); -+} ++ /* Third double word block */ ++ union { ++ struct list_head lru; /* Pageout list, eg. active_list ++ * protected by zone->lru_lock ! ++ * Can be used as a generic list ++ * by the page owner. ++ */ ++ struct { /* slub per cpu partial pages */ ++ struct page *next; /* Next partial slab */ ++#ifdef CONFIG_64BIT ++ int pages; /* Nr of partial slabs left */ ++ int pobjects; /* Approximate # of objects */ ++#else ++ short int pages; ++ short int pobjects; ++#endif ++ }; + -+void swait_prepare(struct swait_head *head, struct swaiter *w, int state) -+{ -+ unsigned long flags; ++ struct slab *slab_page; /* slab fields */ ++ struct rcu_head rcu_head; /* Used by SLAB ++ * when destroying via RCU ++ */ ++ /* First tail page of compound page */ ++ struct { ++ compound_page_dtor *compound_dtor; ++ unsigned long compound_order; ++ }; + -+ raw_spin_lock_irqsave(&head->lock, flags); -+ swait_prepare_locked(head, w); -+ __set_current_state(state); -+ raw_spin_unlock_irqrestore(&head->lock, flags); -+} -+EXPORT_SYMBOL(swait_prepare); ++#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && USE_SPLIT_PMD_PTLOCKS ++ pgtable_t pmd_huge_pte; /* protected by page->ptl */ ++#endif ++ }; + -+void swait_finish_locked(struct swait_head *head, struct swaiter *w) -+{ -+ __set_current_state(TASK_RUNNING); -+ if (w->task) -+ __swait_dequeue(w); -+} ++ /* Remainder is not double word aligned */ ++ union { ++ unsigned long private; /* Mapping-private opaque data: ++ * usually used for buffer_heads ++ * if PagePrivate set; used for ++ * swp_entry_t if PageSwapCache; ++ * indicates order in the buddy ++ * system if PG_buddy is set. ++ */ ++#if USE_SPLIT_PTE_PTLOCKS ++#if ALLOC_SPLIT_PTLOCKS ++ spinlock_t *ptl; ++#else ++ spinlock_t ptl; ++#endif ++#endif ++ struct kmem_cache *slab_cache; /* SL[AU]B: Pointer to slab */ ++ struct page *first_page; /* Compound tail pages */ ++ }; + -+void swait_finish(struct swait_head *head, struct swaiter *w) -+{ -+ unsigned long flags; ++#ifdef CONFIG_MEMCG ++ struct mem_cgroup *mem_cgroup; ++#endif + -+ __set_current_state(TASK_RUNNING); -+ if (w->task) { -+ raw_spin_lock_irqsave(&head->lock, flags); -+ __swait_dequeue(w); -+ raw_spin_unlock_irqrestore(&head->lock, flags); -+ } -+} -+EXPORT_SYMBOL(swait_finish); ++ /* ++ * On machines where all RAM is mapped into kernel address space, ++ * we can simply calculate the virtual address. On machines with ++ * highmem some memory is mapped into kernel virtual memory ++ * dynamically, so we need a place to store that address. ++ * Note that this field could be 16 bits on x86 ... ;) ++ * ++ * Architectures with slow multiplication can define ++ * WANT_PAGE_VIRTUAL in asm/page.h ++ */ ++#if defined(WANT_PAGE_VIRTUAL) ++ void *virtual; /* Kernel virtual address (NULL if ++ not kmapped, ie. highmem) */ ++#endif /* WANT_PAGE_VIRTUAL */ + -+unsigned int -+__swait_wake_locked(struct swait_head *head, unsigned int state, unsigned int num) -+{ -+ struct swaiter *curr, *next; -+ int woken = 0; ++#ifdef CONFIG_KMEMCHECK ++ /* ++ * kmemcheck wants to track the status of each byte in a page; this ++ * is a pointer to such a status block. NULL if not tracked. ++ */ ++ void *shadow; ++#endif + -+ list_for_each_entry_safe(curr, next, &head->list, node) { -+ if (wake_up_state(curr->task, state)) { -+ __swait_dequeue(curr); -+ /* -+ * The waiting task can free the waiter as -+ * soon as curr->task = NULL is written, -+ * without taking any locks. A memory barrier -+ * is required here to prevent the following -+ * store to curr->task from getting ahead of -+ * the dequeue operation. -+ */ -+ smp_wmb(); -+ curr->task = NULL; -+ if (++woken == num) -+ break; -+ } -+ } -+ return woken; ++#ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS ++ int _last_cpupid; ++#endif +} ++/* ++ * The struct page can be forced to be double word aligned so that atomic ops ++ * on double words work. The SLUB allocator can make use of such a feature. ++ */ ++#ifdef CONFIG_HAVE_ALIGNED_STRUCT_PAGE ++ __aligned(2 * sizeof(unsigned long)) ++#endif ++; + -+unsigned int -+__swait_wake(struct swait_head *head, unsigned int state, unsigned int num) -+{ -+ unsigned long flags; -+ int woken; ++struct page_frag { ++ struct page *page; ++#if (BITS_PER_LONG > 32) || (PAGE_SIZE >= 65536) ++ __u32 offset; ++ __u32 size; ++#else ++ __u16 offset; ++ __u16 size; ++#endif ++}; + -+ if (!swaitqueue_active(head)) -+ return 0; ++typedef unsigned long __nocast vm_flags_t; + -+ raw_spin_lock_irqsave(&head->lock, flags); -+ woken = __swait_wake_locked(head, state, num); -+ raw_spin_unlock_irqrestore(&head->lock, flags); -+ return woken; -+} -+EXPORT_SYMBOL(__swait_wake); -diff -Nur linux-4.1.6.orig/kernel/sched/work-simple.c linux-4.1.6/kernel/sched/work-simple.c ---- linux-4.1.6.orig/kernel/sched/work-simple.c 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.6/kernel/sched/work-simple.c 2015-09-08 23:49:08.113866499 +0200 -@@ -0,0 +1,172 @@ +/* -+ * Copyright (C) 2014 BMW Car IT GmbH, Daniel Wagner daniel.wagner@bmw-carit.de -+ * -+ * Provides a framework for enqueuing callbacks from irq context -+ * PREEMPT_RT_FULL safe. The callbacks are executed in kthread context. ++ * A region containing a mapping of a non-memory backed file under NOMMU ++ * conditions. These are held in a global tree and are pinned by the VMAs that ++ * map parts of them. + */ ++struct vm_region { ++ struct rb_node vm_rb; /* link in global region tree */ ++ vm_flags_t vm_flags; /* VMA vm_flags */ ++ unsigned long vm_start; /* start address of region */ ++ unsigned long vm_end; /* region initialised to here */ ++ unsigned long vm_top; /* region allocated to here */ ++ unsigned long vm_pgoff; /* the offset in vm_file corresponding to vm_start */ ++ struct file *vm_file; /* the backing file or NULL */ ++ ++ int vm_usage; /* region usage count (access under nommu_region_sem) */ ++ bool vm_icache_flushed : 1; /* true if the icache has been flushed for ++ * this region */ ++}; + -+#include -+#include -+#include -+#include -+#include ++/* ++ * This struct defines a memory VMM memory area. There is one of these ++ * per VM-area/task. A VM area is any part of the process virtual memory ++ * space that has a special rule for the page-fault handlers (ie a shared ++ * library, the executable area etc). ++ */ ++struct vm_area_struct { ++ /* The first cache line has the info for VMA tree walking. */ + -+#define SWORK_EVENT_PENDING (1 << 0) ++ unsigned long vm_start; /* Our start address within vm_mm. */ ++ unsigned long vm_end; /* The first byte after our end address ++ within vm_mm. */ + -+static DEFINE_MUTEX(worker_mutex); -+static struct sworker *glob_worker; ++ /* linked list of VM areas per task, sorted by address */ ++ struct vm_area_struct *vm_next, *vm_prev; + -+struct sworker { -+ struct list_head events; -+ struct swait_head wq; ++ struct rb_node vm_rb; + -+ raw_spinlock_t lock; ++ /* ++ * Largest free memory gap in bytes to the left of this VMA. ++ * Either between this VMA and vma->vm_prev, or between one of the ++ * VMAs below us in the VMA rbtree and its ->vm_prev. This helps ++ * get_unmapped_area find a free area of the right size. ++ */ ++ unsigned long rb_subtree_gap; + -+ struct task_struct *task; -+ int refs; -+}; ++ /* Second cache line starts here. */ + -+static bool swork_readable(struct sworker *worker) -+{ -+ bool r; ++ struct mm_struct *vm_mm; /* The address space we belong to. */ ++ pgprot_t vm_page_prot; /* Access permissions of this VMA. */ ++ unsigned long vm_flags; /* Flags, see mm.h. */ + -+ if (kthread_should_stop()) -+ return true; ++ /* ++ * For areas with an address space and backing store, ++ * linkage into the address_space->i_mmap interval tree. ++ */ ++ struct { ++ struct rb_node rb; ++ unsigned long rb_subtree_last; ++ } shared; + -+ raw_spin_lock_irq(&worker->lock); -+ r = !list_empty(&worker->events); -+ raw_spin_unlock_irq(&worker->lock); ++ /* ++ * A file's MAP_PRIVATE vma can be in both i_mmap tree and anon_vma ++ * list, after a COW of one of the file pages. A MAP_SHARED vma ++ * can only be in the i_mmap tree. An anonymous MAP_PRIVATE, stack ++ * or brk vma (with NULL file) can only be in an anon_vma list. ++ */ ++ struct list_head anon_vma_chain; /* Serialized by mmap_sem & ++ * page_table_lock */ ++ struct anon_vma *anon_vma; /* Serialized by page_table_lock */ + -+ return r; -+} ++ /* Function pointers to deal with this struct. */ ++ const struct vm_operations_struct *vm_ops; + -+static int swork_kthread(void *arg) -+{ -+ struct sworker *worker = arg; ++ /* Information about our backing store: */ ++ unsigned long vm_pgoff; /* Offset (within vm_file) in PAGE_SIZE ++ units, *not* PAGE_CACHE_SIZE */ ++ struct file * vm_file; /* File we map to (can be NULL). */ ++ void * vm_private_data; /* was vm_pte (shared mem) */ + -+ for (;;) { -+ swait_event_interruptible(worker->wq, -+ swork_readable(worker)); -+ if (kthread_should_stop()) -+ break; ++#ifndef CONFIG_MMU ++ struct vm_region *vm_region; /* NOMMU mapping region */ ++#endif ++#ifdef CONFIG_NUMA ++ struct mempolicy *vm_policy; /* NUMA policy for the VMA */ ++#endif ++}; + -+ raw_spin_lock_irq(&worker->lock); -+ while (!list_empty(&worker->events)) { -+ struct swork_event *sev; ++struct core_thread { ++ struct task_struct *task; ++ struct core_thread *next; ++}; + -+ sev = list_first_entry(&worker->events, -+ struct swork_event, item); -+ list_del(&sev->item); -+ raw_spin_unlock_irq(&worker->lock); ++struct core_state { ++ atomic_t nr_threads; ++ struct core_thread dumper; ++ struct completion startup; ++}; + -+ WARN_ON_ONCE(!test_and_clear_bit(SWORK_EVENT_PENDING, -+ &sev->flags)); -+ sev->func(sev); -+ raw_spin_lock_irq(&worker->lock); -+ } -+ raw_spin_unlock_irq(&worker->lock); -+ } -+ return 0; -+} ++enum { ++ MM_FILEPAGES, ++ MM_ANONPAGES, ++ MM_SWAPENTS, ++ NR_MM_COUNTERS ++}; + -+static struct sworker *swork_create(void) -+{ -+ struct sworker *worker; ++#if USE_SPLIT_PTE_PTLOCKS && defined(CONFIG_MMU) ++#define SPLIT_RSS_COUNTING ++/* per-thread cached information, */ ++struct task_rss_stat { ++ int events; /* for synchronization threshold */ ++ int count[NR_MM_COUNTERS]; ++}; ++#endif /* USE_SPLIT_PTE_PTLOCKS */ + -+ worker = kzalloc(sizeof(*worker), GFP_KERNEL); -+ if (!worker) -+ return ERR_PTR(-ENOMEM); ++struct mm_rss_stat { ++ atomic_long_t count[NR_MM_COUNTERS]; ++}; + -+ INIT_LIST_HEAD(&worker->events); -+ raw_spin_lock_init(&worker->lock); -+ init_swait_head(&worker->wq); ++struct kioctx_table; ++struct mm_struct { ++ struct vm_area_struct *mmap; /* list of VMAs */ ++ struct rb_root mm_rb; ++ u32 vmacache_seqnum; /* per-thread vmacache */ ++#ifdef CONFIG_MMU ++ unsigned long (*get_unmapped_area) (struct file *filp, ++ unsigned long addr, unsigned long len, ++ unsigned long pgoff, unsigned long flags); ++#endif ++ unsigned long mmap_base; /* base of mmap area */ ++ unsigned long mmap_legacy_base; /* base of mmap area in bottom-up allocations */ ++ unsigned long task_size; /* size of task vm space */ ++ unsigned long highest_vm_end; /* highest vma end address */ ++ pgd_t * pgd; ++ atomic_t mm_users; /* How many users with user space? */ ++ atomic_t mm_count; /* How many references to "struct mm_struct" (users count as 1) */ ++ atomic_long_t nr_ptes; /* PTE page table pages */ ++#if CONFIG_PGTABLE_LEVELS > 2 ++ atomic_long_t nr_pmds; /* PMD page table pages */ ++#endif ++ int map_count; /* number of VMAs */ + -+ worker->task = kthread_run(swork_kthread, worker, "kswork"); -+ if (IS_ERR(worker->task)) { -+ kfree(worker); -+ return ERR_PTR(-ENOMEM); -+ } ++ spinlock_t page_table_lock; /* Protects page tables and some counters */ ++ struct rw_semaphore mmap_sem; + -+ return worker; -+} ++ struct list_head mmlist; /* List of maybe swapped mm's. These are globally strung ++ * together off init_mm.mmlist, and are protected ++ * by mmlist_lock ++ */ + -+static void swork_destroy(struct sworker *worker) -+{ -+ kthread_stop(worker->task); + -+ WARN_ON(!list_empty(&worker->events)); -+ kfree(worker); -+} ++ unsigned long hiwater_rss; /* High-watermark of RSS usage */ ++ unsigned long hiwater_vm; /* High-water virtual memory usage */ + -+/** -+ * swork_queue - queue swork -+ * -+ * Returns %false if @work was already on a queue, %true otherwise. -+ * -+ * The work is queued and processed on a random CPU -+ */ -+bool swork_queue(struct swork_event *sev) -+{ -+ unsigned long flags; ++ unsigned long total_vm; /* Total pages mapped */ ++ unsigned long locked_vm; /* Pages that have PG_mlocked set */ ++ unsigned long pinned_vm; /* Refcount permanently increased */ ++ unsigned long shared_vm; /* Shared pages (files) */ ++ unsigned long exec_vm; /* VM_EXEC & ~VM_WRITE */ ++ unsigned long stack_vm; /* VM_GROWSUP/DOWN */ ++ unsigned long def_flags; ++ unsigned long start_code, end_code, start_data, end_data; ++ unsigned long start_brk, brk, start_stack; ++ unsigned long arg_start, arg_end, env_start, env_end; + -+ if (test_and_set_bit(SWORK_EVENT_PENDING, &sev->flags)) -+ return false; ++ unsigned long saved_auxv[AT_VECTOR_SIZE]; /* for /proc/PID/auxv */ + -+ raw_spin_lock_irqsave(&glob_worker->lock, flags); -+ list_add_tail(&sev->item, &glob_worker->events); -+ raw_spin_unlock_irqrestore(&glob_worker->lock, flags); ++ /* ++ * Special counters, in some configurations protected by the ++ * page_table_lock, in other configurations by being atomic. ++ */ ++ struct mm_rss_stat rss_stat; + -+ swait_wake(&glob_worker->wq); -+ return true; -+} -+EXPORT_SYMBOL_GPL(swork_queue); ++ struct linux_binfmt *binfmt; + -+/** -+ * swork_get - get an instance of the sworker -+ * -+ * Returns an negative error code if the initialization if the worker did not -+ * work, %0 otherwise. -+ * -+ */ -+int swork_get(void) -+{ -+ struct sworker *worker; ++ cpumask_var_t cpu_vm_mask_var; + -+ mutex_lock(&worker_mutex); -+ if (!glob_worker) { -+ worker = swork_create(); -+ if (IS_ERR(worker)) { -+ mutex_unlock(&worker_mutex); -+ return -ENOMEM; -+ } ++ /* Architecture-specific MM context */ ++ mm_context_t context; + -+ glob_worker = worker; -+ } ++ unsigned long flags; /* Must use atomic bitops to access the bits */ + -+ glob_worker->refs++; -+ mutex_unlock(&worker_mutex); ++ struct core_state *core_state; /* coredumping support */ ++#ifdef CONFIG_AIO ++ spinlock_t ioctx_lock; ++ struct kioctx_table __rcu *ioctx_table; ++#endif ++#ifdef CONFIG_MEMCG ++ /* ++ * "owner" points to a task that is regarded as the canonical ++ * user/owner of this mm. All of the following must be true in ++ * order for it to be changed: ++ * ++ * current == mm->owner ++ * current->mm != mm ++ * new_owner->mm == mm ++ * new_owner->alloc_lock is held ++ */ ++ struct task_struct __rcu *owner; ++#endif + -+ return 0; -+} -+EXPORT_SYMBOL_GPL(swork_get); ++ /* store ref to file /proc//exe symlink points to */ ++ struct file __rcu *exe_file; ++#ifdef CONFIG_MMU_NOTIFIER ++ struct mmu_notifier_mm *mmu_notifier_mm; ++#endif ++#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS ++ pgtable_t pmd_huge_pte; /* protected by page_table_lock */ ++#endif ++#ifdef CONFIG_CPUMASK_OFFSTACK ++ struct cpumask cpumask_allocation; ++#endif ++#ifdef CONFIG_NUMA_BALANCING ++ /* ++ * numa_next_scan is the next time that the PTEs will be marked ++ * pte_numa. NUMA hinting faults will gather statistics and migrate ++ * pages to new nodes if necessary. ++ */ ++ unsigned long numa_next_scan; + -+/** -+ * swork_put - puts an instance of the sworker -+ * -+ * Will destroy the sworker thread. This function must not be called until all -+ * queued events have been completed. -+ */ -+void swork_put(void) -+{ -+ mutex_lock(&worker_mutex); ++ /* Restart point for scanning and setting pte_numa */ ++ unsigned long numa_scan_offset; + -+ glob_worker->refs--; -+ if (glob_worker->refs > 0) -+ goto out; ++ /* numa_scan_seq prevents two threads setting pte_numa */ ++ int numa_scan_seq; ++#endif ++#if defined(CONFIG_NUMA_BALANCING) || defined(CONFIG_COMPACTION) ++ /* ++ * An operation with batched TLB flushing is going on. Anything that ++ * can move process memory needs to flush the TLB when moving a ++ * PROT_NONE or PROT_NUMA mapped page. ++ */ ++ bool tlb_flush_pending; ++#endif ++ struct uprobes_state uprobes_state; ++#ifdef CONFIG_X86_INTEL_MPX ++ /* address of the bounds directory */ ++ void __user *bd_addr; ++#endif ++}; + -+ swork_destroy(glob_worker); -+ glob_worker = NULL; -+out: -+ mutex_unlock(&worker_mutex); -+} -+EXPORT_SYMBOL_GPL(swork_put); -diff -Nur linux-4.1.6.orig/kernel/signal.c linux-4.1.6/kernel/signal.c ---- linux-4.1.6.orig/kernel/signal.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/signal.c 2015-09-08 23:49:08.113866499 +0200 -@@ -14,6 +14,7 @@ - #include - #include - #include -+#include - #include - #include - #include -@@ -352,13 +353,45 @@ - return false; - } - -+#ifdef __HAVE_ARCH_CMPXCHG -+static inline struct sigqueue *get_task_cache(struct task_struct *t) ++static inline void mm_init_cpumask(struct mm_struct *mm) +{ -+ struct sigqueue *q = t->sigqueue_cache; -+ -+ if (cmpxchg(&t->sigqueue_cache, q, NULL) != q) -+ return NULL; -+ return q; ++#ifdef CONFIG_CPUMASK_OFFSTACK ++ mm->cpu_vm_mask_var = &mm->cpumask_allocation; ++#endif ++ cpumask_clear(mm->cpu_vm_mask_var); +} + -+static inline int put_task_cache(struct task_struct *t, struct sigqueue *q) ++/* Future-safe accessor for struct mm_struct's cpu_vm_mask. */ ++static inline cpumask_t *mm_cpumask(struct mm_struct *mm) +{ -+ if (cmpxchg(&t->sigqueue_cache, NULL, q) == NULL) -+ return 0; -+ return 1; ++ return mm->cpu_vm_mask_var; +} + -+#else -+ -+static inline struct sigqueue *get_task_cache(struct task_struct *t) ++#if defined(CONFIG_NUMA_BALANCING) || defined(CONFIG_COMPACTION) ++/* ++ * Memory barriers to keep this state in sync are graciously provided by ++ * the page table locks, outside of which no page table modifications happen. ++ * The barriers below prevent the compiler from re-ordering the instructions ++ * around the memory barriers that are already present in the code. ++ */ ++static inline bool mm_tlb_flush_pending(struct mm_struct *mm) +{ -+ return NULL; ++ barrier(); ++ return mm->tlb_flush_pending; +} ++static inline void set_tlb_flush_pending(struct mm_struct *mm) ++{ ++ mm->tlb_flush_pending = true; + -+static inline int put_task_cache(struct task_struct *t, struct sigqueue *q) ++ /* ++ * Guarantee that the tlb_flush_pending store does not leak into the ++ * critical section updating the page tables ++ */ ++ smp_mb__before_spinlock(); ++} ++/* Clearing is done after a TLB flush, which also provides a barrier. */ ++static inline void clear_tlb_flush_pending(struct mm_struct *mm) +{ -+ return 1; ++ barrier(); ++ mm->tlb_flush_pending = false; +} -+ -+#endif -+ - /* - * allocate a new signal queue record - * - this may be called without locks if and only if t == current, otherwise an - * appropriate lock must be held to stop the target task from exiting - */ - static struct sigqueue * --__sigqueue_alloc(int sig, struct task_struct *t, gfp_t flags, int override_rlimit) -+__sigqueue_do_alloc(int sig, struct task_struct *t, gfp_t flags, -+ int override_rlimit, int fromslab) - { - struct sigqueue *q = NULL; - struct user_struct *user; -@@ -375,7 +408,10 @@ - if (override_rlimit || - atomic_read(&user->sigpending) <= - task_rlimit(t, RLIMIT_SIGPENDING)) { -- q = kmem_cache_alloc(sigqueue_cachep, flags); -+ if (!fromslab) -+ q = get_task_cache(t); -+ if (!q) -+ q = kmem_cache_alloc(sigqueue_cachep, flags); - } else { - print_dropped_signal(sig); - } -@@ -392,6 +428,13 @@ - return q; - } - -+static struct sigqueue * -+__sigqueue_alloc(int sig, struct task_struct *t, gfp_t flags, -+ int override_rlimit) ++#else ++static inline bool mm_tlb_flush_pending(struct mm_struct *mm) +{ -+ return __sigqueue_do_alloc(sig, t, flags, override_rlimit, 0); ++ return false; +} -+ - static void __sigqueue_free(struct sigqueue *q) - { - if (q->flags & SIGQUEUE_PREALLOC) -@@ -401,6 +444,21 @@ - kmem_cache_free(sigqueue_cachep, q); - } - -+static void sigqueue_free_current(struct sigqueue *q) ++static inline void set_tlb_flush_pending(struct mm_struct *mm) +{ -+ struct user_struct *up; -+ -+ if (q->flags & SIGQUEUE_PREALLOC) -+ return; -+ -+ up = q->user; -+ if (rt_prio(current->normal_prio) && !put_task_cache(current, q)) { -+ atomic_dec(&up->sigpending); -+ free_uid(up); -+ } else -+ __sigqueue_free(q); +} -+ - void flush_sigqueue(struct sigpending *queue) - { - struct sigqueue *q; -@@ -414,6 +472,21 @@ - } - - /* -+ * Called from __exit_signal. Flush tsk->pending and -+ * tsk->sigqueue_cache -+ */ -+void flush_task_sigqueue(struct task_struct *tsk) ++static inline void clear_tlb_flush_pending(struct mm_struct *mm) +{ -+ struct sigqueue *q; -+ -+ flush_sigqueue(&tsk->pending); -+ -+ q = get_task_cache(tsk); -+ if (q) -+ kmem_cache_free(sigqueue_cachep, q); +} ++#endif + -+/* - * Flush all pending signals for a task. - */ - void __flush_signals(struct task_struct *t) -@@ -565,7 +638,7 @@ - still_pending: - list_del_init(&first->list); - copy_siginfo(info, &first->info); -- __sigqueue_free(first); -+ sigqueue_free_current(first); - } else { - /* - * Ok, it wasn't in the queue. This must be -@@ -611,6 +684,8 @@ - { - int signr; - -+ WARN_ON_ONCE(tsk != current); -+ - /* We only dequeue private signals from ourselves, we don't let - * signalfd steal them - */ -@@ -1207,8 +1282,8 @@ - * We don't want to have recursive SIGSEGV's etc, for example, - * that is why we also clear SIGNAL_UNKILLABLE. - */ --int --force_sig_info(int sig, struct siginfo *info, struct task_struct *t) -+static int -+do_force_sig_info(int sig, struct siginfo *info, struct task_struct *t) - { - unsigned long int flags; - int ret, blocked, ignored; -@@ -1233,6 +1308,39 @@ - return ret; - } - -+int force_sig_info(int sig, struct siginfo *info, struct task_struct *t) ++struct vm_special_mapping +{ -+/* -+ * On some archs, PREEMPT_RT has to delay sending a signal from a trap -+ * since it can not enable preemption, and the signal code's spin_locks -+ * turn into mutexes. Instead, it must set TIF_NOTIFY_RESUME which will -+ * send the signal on exit of the trap. -+ */ -+#ifdef ARCH_RT_DELAYS_SIGNAL_SEND -+ if (in_atomic()) { -+ if (WARN_ON_ONCE(t != current)) -+ return 0; -+ if (WARN_ON_ONCE(t->forced_info.si_signo)) -+ return 0; ++ const char *name; ++ struct page **pages; ++}; + -+ if (is_si_special(info)) { -+ WARN_ON_ONCE(info != SEND_SIG_PRIV); -+ t->forced_info.si_signo = sig; -+ t->forced_info.si_errno = 0; -+ t->forced_info.si_code = SI_KERNEL; -+ t->forced_info.si_pid = 0; -+ t->forced_info.si_uid = 0; -+ } else { -+ t->forced_info = *info; -+ } ++enum tlb_flush_reason { ++ TLB_FLUSH_ON_TASK_SWITCH, ++ TLB_REMOTE_SHOOTDOWN, ++ TLB_LOCAL_SHOOTDOWN, ++ TLB_LOCAL_MM_SHOOTDOWN, ++ NR_TLB_FLUSH_REASONS, ++}; + -+ set_tsk_thread_flag(t, TIF_NOTIFY_RESUME); -+ return 0; -+ } ++ /* ++ * A swap entry has to fit into a "unsigned long", as the entry is hidden ++ * in the "index" field of the swapper address space. ++ */ ++typedef struct { ++ unsigned long val; ++} swp_entry_t; ++ ++#endif /* _LINUX_MM_TYPES_H */ +diff -Nur linux-4.1.10.orig/include/linux/mutex.h linux-4.1.10/include/linux/mutex.h +--- linux-4.1.10.orig/include/linux/mutex.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/mutex.h 2015-10-07 18:00:08.000000000 +0200 +@@ -19,6 +19,17 @@ + #include + #include + ++#ifdef CONFIG_DEBUG_LOCK_ALLOC ++# define __DEP_MAP_MUTEX_INITIALIZER(lockname) \ ++ , .dep_map = { .name = #lockname } ++#else ++# define __DEP_MAP_MUTEX_INITIALIZER(lockname) +#endif -+ return do_force_sig_info(sig, info, t); -+} ++ ++#ifdef CONFIG_PREEMPT_RT_FULL ++# include ++#else + /* - * Nuke all other threads in the group. - */ -@@ -1267,12 +1375,12 @@ - * Disable interrupts early to avoid deadlocks. - * See rcu_read_unlock() comment header for details. - */ -- local_irq_save(*flags); -+ local_irq_save_nort(*flags); - rcu_read_lock(); - sighand = rcu_dereference(tsk->sighand); - if (unlikely(sighand == NULL)) { - rcu_read_unlock(); -- local_irq_restore(*flags); -+ local_irq_restore_nort(*flags); - break; - } - /* -@@ -1293,7 +1401,7 @@ - } - spin_unlock(&sighand->siglock); - rcu_read_unlock(); -- local_irq_restore(*flags); -+ local_irq_restore_nort(*flags); - } - - return sighand; -@@ -1536,7 +1644,8 @@ - */ - struct sigqueue *sigqueue_alloc(void) - { -- struct sigqueue *q = __sigqueue_alloc(-1, current, GFP_KERNEL, 0); -+ /* Preallocated sigqueue objects always from the slabcache ! */ -+ struct sigqueue *q = __sigqueue_do_alloc(-1, current, GFP_KERNEL, 0, 1); - - if (q) - q->flags |= SIGQUEUE_PREALLOC; -@@ -1897,15 +2006,7 @@ - if (gstop_done && ptrace_reparented(current)) - do_notify_parent_cldstop(current, false, why); + * Simple, straightforward mutexes with strict semantics: + * +@@ -99,13 +110,6 @@ + static inline void mutex_destroy(struct mutex *lock) {} + #endif -- /* -- * Don't want to allow preemption here, because -- * sys_ptrace() needs this task to be inactive. -- * -- * XXX: implement read_unlock_no_resched(). -- */ -- preempt_disable(); - read_unlock(&tasklist_lock); -- preempt_enable_no_resched(); - freezable_schedule(); - } else { - /* -diff -Nur linux-4.1.6.orig/kernel/softirq.c linux-4.1.6/kernel/softirq.c ---- linux-4.1.6.orig/kernel/softirq.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/softirq.c 2015-09-08 23:49:08.117866054 +0200 -@@ -21,10 +21,12 @@ - #include - #include - #include -+#include - #include - #include - #include - #include -+#include - #include +-#ifdef CONFIG_DEBUG_LOCK_ALLOC +-# define __DEP_MAP_MUTEX_INITIALIZER(lockname) \ +- , .dep_map = { .name = #lockname } +-#else +-# define __DEP_MAP_MUTEX_INITIALIZER(lockname) +-#endif +- + #define __MUTEX_INITIALIZER(lockname) \ + { .count = ATOMIC_INIT(1) \ + , .wait_lock = __SPIN_LOCK_UNLOCKED(lockname.wait_lock) \ +@@ -173,6 +177,8 @@ + extern int mutex_trylock(struct mutex *lock); + extern void mutex_unlock(struct mutex *lock); - #define CREATE_TRACE_POINTS -@@ -62,6 +64,98 @@ - "TASKLET", "SCHED", "HRTIMER", "RCU" - }; ++#endif /* !PREEMPT_RT_FULL */ ++ + extern int atomic_dec_and_mutex_lock(atomic_t *cnt, struct mutex *lock); -+#ifdef CONFIG_NO_HZ_COMMON -+# ifdef CONFIG_PREEMPT_RT_FULL + #endif /* __LINUX_MUTEX_H */ +diff -Nur linux-4.1.10.orig/include/linux/mutex_rt.h linux-4.1.10/include/linux/mutex_rt.h +--- linux-4.1.10.orig/include/linux/mutex_rt.h 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/include/linux/mutex_rt.h 2015-10-07 18:00:08.000000000 +0200 +@@ -0,0 +1,84 @@ ++#ifndef __LINUX_MUTEX_RT_H ++#define __LINUX_MUTEX_RT_H + -+struct softirq_runner { -+ struct task_struct *runner[NR_SOFTIRQS]; -+}; ++#ifndef __LINUX_MUTEX_H ++#error "Please include mutex.h" ++#endif + -+static DEFINE_PER_CPU(struct softirq_runner, softirq_runners); ++#include + -+static inline void softirq_set_runner(unsigned int sirq) -+{ -+ struct softirq_runner *sr = this_cpu_ptr(&softirq_runners); ++/* FIXME: Just for __lockfunc */ ++#include + -+ sr->runner[sirq] = current; -+} ++struct mutex { ++ struct rt_mutex lock; ++#ifdef CONFIG_DEBUG_LOCK_ALLOC ++ struct lockdep_map dep_map; ++#endif ++}; + -+static inline void softirq_clr_runner(unsigned int sirq) -+{ -+ struct softirq_runner *sr = this_cpu_ptr(&softirq_runners); ++#define __MUTEX_INITIALIZER(mutexname) \ ++ { \ ++ .lock = __RT_MUTEX_INITIALIZER(mutexname.lock) \ ++ __DEP_MAP_MUTEX_INITIALIZER(mutexname) \ ++ } + -+ sr->runner[sirq] = NULL; -+} ++#define DEFINE_MUTEX(mutexname) \ ++ struct mutex mutexname = __MUTEX_INITIALIZER(mutexname) + -+/* -+ * On preempt-rt a softirq running context might be blocked on a -+ * lock. There might be no other runnable task on this CPU because the -+ * lock owner runs on some other CPU. So we have to go into idle with -+ * the pending bit set. Therefor we need to check this otherwise we -+ * warn about false positives which confuses users and defeats the -+ * whole purpose of this test. -+ * -+ * This code is called with interrupts disabled. -+ */ -+void softirq_check_pending_idle(void) -+{ -+ static int rate_limit; -+ struct softirq_runner *sr = this_cpu_ptr(&softirq_runners); -+ u32 warnpending; -+ int i; ++extern void __mutex_do_init(struct mutex *lock, const char *name, struct lock_class_key *key); ++extern void __lockfunc _mutex_lock(struct mutex *lock); ++extern int __lockfunc _mutex_lock_interruptible(struct mutex *lock); ++extern int __lockfunc _mutex_lock_killable(struct mutex *lock); ++extern void __lockfunc _mutex_lock_nested(struct mutex *lock, int subclass); ++extern void __lockfunc _mutex_lock_nest_lock(struct mutex *lock, struct lockdep_map *nest_lock); ++extern int __lockfunc _mutex_lock_interruptible_nested(struct mutex *lock, int subclass); ++extern int __lockfunc _mutex_lock_killable_nested(struct mutex *lock, int subclass); ++extern int __lockfunc _mutex_trylock(struct mutex *lock); ++extern void __lockfunc _mutex_unlock(struct mutex *lock); + -+ if (rate_limit >= 10) -+ return; ++#define mutex_is_locked(l) rt_mutex_is_locked(&(l)->lock) ++#define mutex_lock(l) _mutex_lock(l) ++#define mutex_lock_interruptible(l) _mutex_lock_interruptible(l) ++#define mutex_lock_killable(l) _mutex_lock_killable(l) ++#define mutex_trylock(l) _mutex_trylock(l) ++#define mutex_unlock(l) _mutex_unlock(l) ++#define mutex_destroy(l) rt_mutex_destroy(&(l)->lock) + -+ warnpending = local_softirq_pending() & SOFTIRQ_STOP_IDLE_MASK; -+ for (i = 0; i < NR_SOFTIRQS; i++) { -+ struct task_struct *tsk = sr->runner[i]; -+ -+ /* -+ * The wakeup code in rtmutex.c wakes up the task -+ * _before_ it sets pi_blocked_on to NULL under -+ * tsk->pi_lock. So we need to check for both: state -+ * and pi_blocked_on. -+ */ -+ if (tsk) { -+ raw_spin_lock(&tsk->pi_lock); -+ if (tsk->pi_blocked_on || tsk->state == TASK_RUNNING) { -+ /* Clear all bits pending in that task */ -+ warnpending &= ~(tsk->softirqs_raised); -+ warnpending &= ~(1 << i); -+ } -+ raw_spin_unlock(&tsk->pi_lock); -+ } -+ } -+ -+ if (warnpending) { -+ printk(KERN_ERR "NOHZ: local_softirq_pending %02x\n", -+ warnpending); -+ rate_limit++; -+ } -+} -+# else -+/* -+ * On !PREEMPT_RT we just printk rate limited: -+ */ -+void softirq_check_pending_idle(void) -+{ -+ static int rate_limit; ++#ifdef CONFIG_DEBUG_LOCK_ALLOC ++# define mutex_lock_nested(l, s) _mutex_lock_nested(l, s) ++# define mutex_lock_interruptible_nested(l, s) \ ++ _mutex_lock_interruptible_nested(l, s) ++# define mutex_lock_killable_nested(l, s) \ ++ _mutex_lock_killable_nested(l, s) + -+ if (rate_limit < 10 && -+ (local_softirq_pending() & SOFTIRQ_STOP_IDLE_MASK)) { -+ printk(KERN_ERR "NOHZ: local_softirq_pending %02x\n", -+ local_softirq_pending()); -+ rate_limit++; -+ } -+} -+# endif ++# define mutex_lock_nest_lock(lock, nest_lock) \ ++do { \ ++ typecheck(struct lockdep_map *, &(nest_lock)->dep_map); \ ++ _mutex_lock_nest_lock(lock, &(nest_lock)->dep_map); \ ++} while (0) + -+#else /* !CONFIG_NO_HZ_COMMON */ -+static inline void softirq_set_runner(unsigned int sirq) { } -+static inline void softirq_clr_runner(unsigned int sirq) { } ++#else ++# define mutex_lock_nested(l, s) _mutex_lock(l) ++# define mutex_lock_interruptible_nested(l, s) \ ++ _mutex_lock_interruptible(l) ++# define mutex_lock_killable_nested(l, s) \ ++ _mutex_lock_killable(l) ++# define mutex_lock_nest_lock(lock, nest_lock) mutex_lock(lock) +#endif + - /* - * we cannot loop indefinitely here to avoid userspace starvation, - * but we also don't want to introduce a worst case 1/HZ latency -@@ -77,6 +171,68 @@ - wake_up_process(tsk); - } - -+static void handle_softirq(unsigned int vec_nr) -+{ -+ struct softirq_action *h = softirq_vec + vec_nr; -+ int prev_count; -+ -+ prev_count = preempt_count(); -+ -+ kstat_incr_softirqs_this_cpu(vec_nr); -+ -+ trace_softirq_entry(vec_nr); -+ h->action(h); -+ trace_softirq_exit(vec_nr); -+ if (unlikely(prev_count != preempt_count())) { -+ pr_err("huh, entered softirq %u %s %p with preempt_count %08x, exited with %08x?\n", -+ vec_nr, softirq_to_name[vec_nr], h->action, -+ prev_count, preempt_count()); -+ preempt_count_set(prev_count); -+ } -+} -+ -+#ifndef CONFIG_PREEMPT_RT_FULL -+static inline int ksoftirqd_softirq_pending(void) -+{ -+ return local_softirq_pending(); -+} ++# define mutex_init(mutex) \ ++do { \ ++ static struct lock_class_key __key; \ ++ \ ++ rt_mutex_init(&(mutex)->lock); \ ++ __mutex_do_init((mutex), #mutex, &__key); \ ++} while (0) + -+static void handle_pending_softirqs(u32 pending) -+{ -+ struct softirq_action *h = softirq_vec; -+ int softirq_bit; ++# define __mutex_init(mutex, name, key) \ ++do { \ ++ rt_mutex_init(&(mutex)->lock); \ ++ __mutex_do_init((mutex), name, key); \ ++} while (0) + -+ local_irq_enable(); ++#endif +diff -Nur linux-4.1.10.orig/include/linux/netdevice.h linux-4.1.10/include/linux/netdevice.h +--- linux-4.1.10.orig/include/linux/netdevice.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/netdevice.h 2015-10-07 18:00:08.000000000 +0200 +@@ -2469,6 +2469,7 @@ + unsigned int dropped; + struct sk_buff_head input_pkt_queue; + struct napi_struct backlog; ++ struct sk_buff_head tofree_queue; + + }; + +diff -Nur linux-4.1.10.orig/include/linux/netfilter/x_tables.h linux-4.1.10/include/linux/netfilter/x_tables.h +--- linux-4.1.10.orig/include/linux/netfilter/x_tables.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/netfilter/x_tables.h 2015-10-07 18:00:08.000000000 +0200 +@@ -3,6 +3,7 @@ + + + #include ++#include + #include + + /** +@@ -282,6 +283,8 @@ + */ + DECLARE_PER_CPU(seqcount_t, xt_recseq); + ++DECLARE_LOCAL_IRQ_LOCK(xt_write_lock); + -+ h = softirq_vec; + /** + * xt_write_recseq_begin - start of a write section + * +@@ -296,6 +299,9 @@ + { + unsigned int addend; + ++ /* RT protection */ ++ local_lock(xt_write_lock); + -+ while ((softirq_bit = ffs(pending))) { -+ unsigned int vec_nr; + /* + * Low order bit of sequence is set if we already + * called xt_write_recseq_begin(). +@@ -326,6 +332,7 @@ + /* this is kind of a write_seqcount_end(), but addend is 0 or 1 */ + smp_wmb(); + __this_cpu_add(xt_recseq.sequence, addend); ++ local_unlock(xt_write_lock); + } + + /* +diff -Nur linux-4.1.10.orig/include/linux/notifier.h linux-4.1.10/include/linux/notifier.h +--- linux-4.1.10.orig/include/linux/notifier.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/notifier.h 2015-10-07 18:00:08.000000000 +0200 +@@ -6,7 +6,7 @@ + * + * Alan Cox + */ +- + -+ h += softirq_bit - 1; -+ vec_nr = h - softirq_vec; -+ handle_softirq(vec_nr); + #ifndef _LINUX_NOTIFIER_H + #define _LINUX_NOTIFIER_H + #include +@@ -42,9 +42,7 @@ + * in srcu_notifier_call_chain(): no cache bounces and no memory barriers. + * As compensation, srcu_notifier_chain_unregister() is rather expensive. + * SRCU notifier chains should be used when the chain will be called very +- * often but notifier_blocks will seldom be removed. Also, SRCU notifier +- * chains are slightly more difficult to use because they require special +- * runtime initialization. ++ * often but notifier_blocks will seldom be removed. + */ + + typedef int (*notifier_fn_t)(struct notifier_block *nb, +@@ -88,7 +86,7 @@ + (name)->head = NULL; \ + } while (0) + +-/* srcu_notifier_heads must be initialized and cleaned up dynamically */ ++/* srcu_notifier_heads must be cleaned up dynamically */ + extern void srcu_init_notifier_head(struct srcu_notifier_head *nh); + #define srcu_cleanup_notifier_head(name) \ + cleanup_srcu_struct(&(name)->srcu); +@@ -101,7 +99,13 @@ + .head = NULL } + #define RAW_NOTIFIER_INIT(name) { \ + .head = NULL } +-/* srcu_notifier_heads cannot be initialized statically */ + -+ h++; -+ pending >>= softirq_bit; ++#define SRCU_NOTIFIER_INIT(name, pcpu) \ ++ { \ ++ .mutex = __MUTEX_INITIALIZER(name.mutex), \ ++ .head = NULL, \ ++ .srcu = __SRCU_STRUCT_INIT(name.srcu, pcpu), \ + } + + #define ATOMIC_NOTIFIER_HEAD(name) \ + struct atomic_notifier_head name = \ +@@ -113,6 +117,18 @@ + struct raw_notifier_head name = \ + RAW_NOTIFIER_INIT(name) + ++#define _SRCU_NOTIFIER_HEAD(name, mod) \ ++ static DEFINE_PER_CPU(struct srcu_struct_array, \ ++ name##_head_srcu_array); \ ++ mod struct srcu_notifier_head name = \ ++ SRCU_NOTIFIER_INIT(name, name##_head_srcu_array) + -+ rcu_bh_qs(); -+ local_irq_disable(); -+} ++#define SRCU_NOTIFIER_HEAD(name) \ ++ _SRCU_NOTIFIER_HEAD(name, ) + -+static void run_ksoftirqd(unsigned int cpu) -+{ -+ local_irq_disable(); -+ if (ksoftirqd_softirq_pending()) { -+ __do_softirq(); -+ local_irq_enable(); -+ cond_resched_rcu_qs(); -+ return; -+ } -+ local_irq_enable(); -+} ++#define SRCU_NOTIFIER_HEAD_STATIC(name) \ ++ _SRCU_NOTIFIER_HEAD(name, static) + - /* - * preempt_count and SOFTIRQ_OFFSET usage: - * - preempt_count is changed by SOFTIRQ_OFFSET on entering or leaving -@@ -232,10 +388,8 @@ - unsigned long end = jiffies + MAX_SOFTIRQ_TIME; - unsigned long old_flags = current->flags; - int max_restart = MAX_SOFTIRQ_RESTART; -- struct softirq_action *h; - bool in_hardirq; - __u32 pending; -- int softirq_bit; - - /* - * Mask out PF_MEMALLOC s current task context is borrowed for the -@@ -254,36 +408,7 @@ - /* Reset the pending bitmask before enabling irqs */ - set_softirq_pending(0); - -- local_irq_enable(); -- -- h = softirq_vec; -- -- while ((softirq_bit = ffs(pending))) { -- unsigned int vec_nr; -- int prev_count; -- -- h += softirq_bit - 1; -- -- vec_nr = h - softirq_vec; -- prev_count = preempt_count(); -- -- kstat_incr_softirqs_this_cpu(vec_nr); -- -- trace_softirq_entry(vec_nr); -- h->action(h); -- trace_softirq_exit(vec_nr); -- if (unlikely(prev_count != preempt_count())) { -- pr_err("huh, entered softirq %u %s %p with preempt_count %08x, exited with %08x?\n", -- vec_nr, softirq_to_name[vec_nr], h->action, -- prev_count, preempt_count()); -- preempt_count_set(prev_count); -- } -- h++; -- pending >>= softirq_bit; -- } -- -- rcu_bh_qs(); -- local_irq_disable(); -+ handle_pending_softirqs(pending); + #ifdef __KERNEL__ - pending = local_softirq_pending(); - if (pending) { -@@ -320,6 +445,285 @@ - } + extern int atomic_notifier_chain_register(struct atomic_notifier_head *nh, +@@ -182,12 +198,12 @@ /* -+ * This function must run with irqs disabled! -+ */ -+void raise_softirq_irqoff(unsigned int nr) -+{ -+ __raise_softirq_irqoff(nr); -+ -+ /* -+ * If we're in an interrupt or softirq, we're done -+ * (this also catches softirq-disabled code). We will -+ * actually run the softirq once we return from -+ * the irq or softirq. -+ * -+ * Otherwise we wake up ksoftirqd to make sure we -+ * schedule the softirq soon. -+ */ -+ if (!in_interrupt()) -+ wakeup_softirqd(); -+} + * Declared notifiers so far. I can imagine quite a few more chains +- * over time (eg laptop power reset chains, reboot chain (to clean ++ * over time (eg laptop power reset chains, reboot chain (to clean + * device units up), device [un]mount chain, module load/unload chain, +- * low memory chain, screenblank chain (for plug in modular screenblankers) ++ * low memory chain, screenblank chain (for plug in modular screenblankers) + * VC switch chains (for loadable kernel svgalib VC switch helpers) etc... + */ +- + -+void __raise_softirq_irqoff(unsigned int nr) -+{ -+ trace_softirq_raise(nr); -+ or_softirq_pending(1UL << nr); -+} + /* CPU notfiers are defined in include/linux/cpu.h. */ + + /* netdevice notifiers are defined in include/linux/netdevice.h */ +diff -Nur linux-4.1.10.orig/include/linux/percpu.h linux-4.1.10/include/linux/percpu.h +--- linux-4.1.10.orig/include/linux/percpu.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/percpu.h 2015-10-07 18:00:08.000000000 +0200 +@@ -24,6 +24,35 @@ + PERCPU_MODULE_RESERVE) + #endif + ++#ifdef CONFIG_PREEMPT_RT_FULL + -+static inline void local_bh_disable_nort(void) { local_bh_disable(); } -+static inline void _local_bh_enable_nort(void) { _local_bh_enable(); } -+static void ksoftirqd_set_sched_params(unsigned int cpu) { } -+static void ksoftirqd_clr_sched_params(unsigned int cpu, bool online) { } ++#define get_local_var(var) (*({ \ ++ migrate_disable(); \ ++ this_cpu_ptr(&var); })) + -+#else /* !PREEMPT_RT_FULL */ ++#define put_local_var(var) do { \ ++ (void)&(var); \ ++ migrate_enable(); \ ++} while (0) + -+/* -+ * On RT we serialize softirq execution with a cpu local lock per softirq -+ */ -+static DEFINE_PER_CPU(struct local_irq_lock [NR_SOFTIRQS], local_softirq_locks); ++# define get_local_ptr(var) ({ \ ++ migrate_disable(); \ ++ this_cpu_ptr(var); }) + -+void __init softirq_early_init(void) -+{ -+ int i; ++# define put_local_ptr(var) do { \ ++ (void)(var); \ ++ migrate_enable(); \ ++} while (0) + -+ for (i = 0; i < NR_SOFTIRQS; i++) -+ local_irq_lock_init(local_softirq_locks[i]); -+} ++#else + -+static void lock_softirq(int which) -+{ -+ local_lock(local_softirq_locks[which]); -+} ++#define get_local_var(var) get_cpu_var(var) ++#define put_local_var(var) put_cpu_var(var) ++#define get_local_ptr(var) get_cpu_ptr(var) ++#define put_local_ptr(var) put_cpu_ptr(var) + -+static void unlock_softirq(int which) -+{ -+ local_unlock(local_softirq_locks[which]); -+} ++#endif + -+static void do_single_softirq(int which) -+{ -+ unsigned long old_flags = current->flags; + /* minimum unit size, also is the maximum supported allocation size */ + #define PCPU_MIN_UNIT_SIZE PFN_ALIGN(32 << 10) + +diff -Nur linux-4.1.10.orig/include/linux/pid.h linux-4.1.10/include/linux/pid.h +--- linux-4.1.10.orig/include/linux/pid.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/pid.h 2015-10-07 18:00:08.000000000 +0200 +@@ -2,6 +2,7 @@ + #define _LINUX_PID_H + + #include ++#include + + enum pid_type + { +diff -Nur linux-4.1.10.orig/include/linux/preempt.h linux-4.1.10/include/linux/preempt.h +--- linux-4.1.10.orig/include/linux/preempt.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/preempt.h 2015-10-07 18:00:08.000000000 +0200 +@@ -33,6 +33,20 @@ + #define preempt_count_inc() preempt_count_add(1) + #define preempt_count_dec() preempt_count_sub(1) + ++#ifdef CONFIG_PREEMPT_LAZY ++#define add_preempt_lazy_count(val) do { preempt_lazy_count() += (val); } while (0) ++#define sub_preempt_lazy_count(val) do { preempt_lazy_count() -= (val); } while (0) ++#define inc_preempt_lazy_count() add_preempt_lazy_count(1) ++#define dec_preempt_lazy_count() sub_preempt_lazy_count(1) ++#define preempt_lazy_count() (current_thread_info()->preempt_lazy_count) ++#else ++#define add_preempt_lazy_count(val) do { } while (0) ++#define sub_preempt_lazy_count(val) do { } while (0) ++#define inc_preempt_lazy_count() do { } while (0) ++#define dec_preempt_lazy_count() do { } while (0) ++#define preempt_lazy_count() (0) ++#endif + -+ current->flags &= ~PF_MEMALLOC; -+ vtime_account_irq_enter(current); -+ current->flags |= PF_IN_SOFTIRQ; -+ lockdep_softirq_enter(); -+ local_irq_enable(); -+ handle_softirq(which); -+ local_irq_disable(); -+ lockdep_softirq_exit(); -+ current->flags &= ~PF_IN_SOFTIRQ; -+ vtime_account_irq_enter(current); -+ tsk_restore_flags(current, old_flags, PF_MEMALLOC); -+} + #ifdef CONFIG_PREEMPT_COUNT + + #define preempt_disable() \ +@@ -41,13 +55,25 @@ + barrier(); \ + } while (0) + ++#define preempt_lazy_disable() \ ++do { \ ++ inc_preempt_lazy_count(); \ ++ barrier(); \ ++} while (0) + -+/* -+ * Called with interrupts disabled. Process softirqs which were raised -+ * in current context (or on behalf of ksoftirqd). -+ */ -+static void do_current_softirqs(void) -+{ -+ while (current->softirqs_raised) { -+ int i = __ffs(current->softirqs_raised); -+ unsigned int pending, mask = (1U << i); + #define sched_preempt_enable_no_resched() \ + do { \ + barrier(); \ + preempt_count_dec(); \ + } while (0) + +-#define preempt_enable_no_resched() sched_preempt_enable_no_resched() ++#ifdef CONFIG_PREEMPT_RT_BASE ++# define preempt_enable_no_resched() sched_preempt_enable_no_resched() ++# define preempt_check_resched_rt() preempt_check_resched() ++#else ++# define preempt_enable_no_resched() preempt_enable() ++# define preempt_check_resched_rt() barrier(); ++#endif + + #ifdef CONFIG_PREEMPT + #define preempt_enable() \ +@@ -63,6 +89,13 @@ + __preempt_schedule(); \ + } while (0) + ++#define preempt_lazy_enable() \ ++do { \ ++ dec_preempt_lazy_count(); \ ++ barrier(); \ ++ preempt_check_resched(); \ ++} while (0) + -+ current->softirqs_raised &= ~mask; -+ local_irq_enable(); + #else + #define preempt_enable() \ + do { \ +@@ -121,6 +154,7 @@ + #define preempt_disable_notrace() barrier() + #define preempt_enable_no_resched_notrace() barrier() + #define preempt_enable_notrace() barrier() ++#define preempt_check_resched_rt() barrier() + + #endif /* CONFIG_PREEMPT_COUNT */ + +@@ -140,10 +174,31 @@ + } while (0) + #define preempt_fold_need_resched() \ + do { \ +- if (tif_need_resched()) \ ++ if (tif_need_resched_now()) \ + set_preempt_need_resched(); \ + } while (0) + ++#ifdef CONFIG_PREEMPT_RT_FULL ++# define preempt_disable_rt() preempt_disable() ++# define preempt_enable_rt() preempt_enable() ++# define preempt_disable_nort() barrier() ++# define preempt_enable_nort() barrier() ++# ifdef CONFIG_SMP ++ extern void migrate_disable(void); ++ extern void migrate_enable(void); ++# else /* CONFIG_SMP */ ++# define migrate_disable() barrier() ++# define migrate_enable() barrier() ++# endif /* CONFIG_SMP */ ++#else ++# define preempt_disable_rt() barrier() ++# define preempt_enable_rt() barrier() ++# define preempt_disable_nort() preempt_disable() ++# define preempt_enable_nort() preempt_enable() ++# define migrate_disable() preempt_disable() ++# define migrate_enable() preempt_enable() ++#endif + -+ /* -+ * If the lock is contended, we boost the owner to -+ * process the softirq or leave the critical section -+ * now. -+ */ -+ lock_softirq(i); -+ local_irq_disable(); -+ softirq_set_runner(i); -+ /* -+ * Check with the local_softirq_pending() bits, -+ * whether we need to process this still or if someone -+ * else took care of it. -+ */ -+ pending = local_softirq_pending(); -+ if (pending & mask) { -+ set_softirq_pending(pending & ~mask); -+ do_single_softirq(i); -+ } -+ softirq_clr_runner(i); -+ unlock_softirq(i); -+ WARN_ON(current->softirq_nestcnt != 1); -+ } -+} + #ifdef CONFIG_PREEMPT_NOTIFIERS + + struct preempt_notifier; +diff -Nur linux-4.1.10.orig/include/linux/preempt_mask.h linux-4.1.10/include/linux/preempt_mask.h +--- linux-4.1.10.orig/include/linux/preempt_mask.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/preempt_mask.h 2015-10-07 18:00:08.000000000 +0200 +@@ -44,16 +44,26 @@ + #define HARDIRQ_OFFSET (1UL << HARDIRQ_SHIFT) + #define NMI_OFFSET (1UL << NMI_SHIFT) + +-#define SOFTIRQ_DISABLE_OFFSET (2 * SOFTIRQ_OFFSET) ++#ifndef CONFIG_PREEMPT_RT_FULL ++# define SOFTIRQ_DISABLE_OFFSET (2 * SOFTIRQ_OFFSET) ++#else ++# define SOFTIRQ_DISABLE_OFFSET (0) ++#endif + + #define PREEMPT_ACTIVE_BITS 1 + #define PREEMPT_ACTIVE_SHIFT (NMI_SHIFT + NMI_BITS) + #define PREEMPT_ACTIVE (__IRQ_MASK(PREEMPT_ACTIVE_BITS) << PREEMPT_ACTIVE_SHIFT) + + #define hardirq_count() (preempt_count() & HARDIRQ_MASK) +-#define softirq_count() (preempt_count() & SOFTIRQ_MASK) + #define irq_count() (preempt_count() & (HARDIRQ_MASK | SOFTIRQ_MASK \ + | NMI_MASK)) ++#ifndef CONFIG_PREEMPT_RT_FULL ++# define softirq_count() (preempt_count() & SOFTIRQ_MASK) ++# define in_serving_softirq() (softirq_count() & SOFTIRQ_OFFSET) ++#else ++# define softirq_count() (0UL) ++extern int in_serving_softirq(void); ++#endif + + /* + * Are we doing bottom half or hardware interrupt processing? +@@ -64,7 +74,6 @@ + #define in_irq() (hardirq_count()) + #define in_softirq() (softirq_count()) + #define in_interrupt() (irq_count()) +-#define in_serving_softirq() (softirq_count() & SOFTIRQ_OFFSET) + + /* + * Are we in NMI context? +diff -Nur linux-4.1.10.orig/include/linux/printk.h linux-4.1.10/include/linux/printk.h +--- linux-4.1.10.orig/include/linux/printk.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/printk.h 2015-10-07 18:00:08.000000000 +0200 +@@ -115,9 +115,11 @@ + #ifdef CONFIG_EARLY_PRINTK + extern asmlinkage __printf(1, 2) + void early_printk(const char *fmt, ...); ++extern void printk_kill(void); + #else + static inline __printf(1, 2) __cold + void early_printk(const char *s, ...) { } ++static inline void printk_kill(void) { } + #endif + + typedef int(*printk_func_t)(const char *fmt, va_list args); +diff -Nur linux-4.1.10.orig/include/linux/radix-tree.h linux-4.1.10/include/linux/radix-tree.h +--- linux-4.1.10.orig/include/linux/radix-tree.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/radix-tree.h 2015-10-07 18:00:08.000000000 +0200 +@@ -277,8 +277,13 @@ + unsigned int radix_tree_gang_lookup_slot(struct radix_tree_root *root, + void ***results, unsigned long *indices, + unsigned long first_index, unsigned int max_items); ++#ifndef CONFIG_PREEMPT_RT_FULL + int radix_tree_preload(gfp_t gfp_mask); + int radix_tree_maybe_preload(gfp_t gfp_mask); ++#else ++static inline int radix_tree_preload(gfp_t gm) { return 0; } ++static inline int radix_tree_maybe_preload(gfp_t gfp_mask) { return 0; } ++#endif + void radix_tree_init(void); + void *radix_tree_tag_set(struct radix_tree_root *root, + unsigned long index, unsigned int tag); +@@ -303,7 +308,7 @@ + + static inline void radix_tree_preload_end(void) + { +- preempt_enable(); ++ preempt_enable_nort(); + } + + /** +diff -Nur linux-4.1.10.orig/include/linux/random.h linux-4.1.10/include/linux/random.h +--- linux-4.1.10.orig/include/linux/random.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/random.h 2015-10-07 18:00:08.000000000 +0200 +@@ -11,7 +11,7 @@ + extern void add_device_randomness(const void *, unsigned int); + extern void add_input_randomness(unsigned int type, unsigned int code, + unsigned int value); +-extern void add_interrupt_randomness(int irq, int irq_flags); ++extern void add_interrupt_randomness(int irq, int irq_flags, __u64 ip); + + extern void get_random_bytes(void *buf, int nbytes); + extern void get_random_bytes_arch(void *buf, int nbytes); +diff -Nur linux-4.1.10.orig/include/linux/rcupdate.h linux-4.1.10/include/linux/rcupdate.h +--- linux-4.1.10.orig/include/linux/rcupdate.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/rcupdate.h 2015-10-07 18:00:08.000000000 +0200 +@@ -167,6 +167,9 @@ + + #endif /* #else #ifdef CONFIG_PREEMPT_RCU */ + ++#ifdef CONFIG_PREEMPT_RT_FULL ++#define call_rcu_bh call_rcu ++#else + /** + * call_rcu_bh() - Queue an RCU for invocation after a quicker grace period. + * @head: structure to be used for queueing the RCU updates. +@@ -190,6 +193,7 @@ + */ + void call_rcu_bh(struct rcu_head *head, + void (*func)(struct rcu_head *head)); ++#endif + + /** + * call_rcu_sched() - Queue an RCU for invocation after sched grace period. +@@ -260,6 +264,11 @@ + * types of kernel builds, the rcu_read_lock() nesting depth is unknowable. + */ + #define rcu_preempt_depth() (current->rcu_read_lock_nesting) ++#ifndef CONFIG_PREEMPT_RT_FULL ++#define sched_rcu_preempt_depth() rcu_preempt_depth() ++#else ++static inline int sched_rcu_preempt_depth(void) { return 0; } ++#endif + + #else /* #ifdef CONFIG_PREEMPT_RCU */ + +@@ -283,6 +292,8 @@ + return 0; + } + ++#define sched_rcu_preempt_depth() rcu_preempt_depth() + -+static void __local_bh_disable(void) + #endif /* #else #ifdef CONFIG_PREEMPT_RCU */ + + /* Internal to kernel */ +@@ -463,7 +474,14 @@ + int debug_lockdep_rcu_enabled(void); + + int rcu_read_lock_held(void); ++#ifdef CONFIG_PREEMPT_RT_FULL ++static inline int rcu_read_lock_bh_held(void) +{ -+ if (++current->softirq_nestcnt == 1) -+ migrate_disable(); ++ return rcu_read_lock_held(); +} ++#else + int rcu_read_lock_bh_held(void); ++#endif + + /** + * rcu_read_lock_sched_held() - might we be in RCU-sched read-side critical section? +@@ -990,10 +1008,14 @@ + static inline void rcu_read_lock_bh(void) + { + local_bh_disable(); ++#ifdef CONFIG_PREEMPT_RT_FULL ++ rcu_read_lock(); ++#else + __acquire(RCU_BH); + rcu_lock_acquire(&rcu_bh_lock_map); + rcu_lockdep_assert(rcu_is_watching(), + "rcu_read_lock_bh() used illegally while idle"); ++#endif + } + + /* +@@ -1003,10 +1025,14 @@ + */ + static inline void rcu_read_unlock_bh(void) + { ++#ifdef CONFIG_PREEMPT_RT_FULL ++ rcu_read_unlock(); ++#else + rcu_lockdep_assert(rcu_is_watching(), + "rcu_read_unlock_bh() used illegally while idle"); + rcu_lock_release(&rcu_bh_lock_map); + __release(RCU_BH); ++#endif + local_bh_enable(); + } + +diff -Nur linux-4.1.10.orig/include/linux/rcutree.h linux-4.1.10/include/linux/rcutree.h +--- linux-4.1.10.orig/include/linux/rcutree.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/rcutree.h 2015-10-07 18:00:08.000000000 +0200 +@@ -46,7 +46,11 @@ + rcu_note_context_switch(); + } + ++#ifdef CONFIG_PREEMPT_RT_FULL ++# define synchronize_rcu_bh synchronize_rcu ++#else + void synchronize_rcu_bh(void); ++#endif + void synchronize_sched_expedited(void); + void synchronize_rcu_expedited(void); + +@@ -74,7 +78,11 @@ + } + + void rcu_barrier(void); ++#ifdef CONFIG_PREEMPT_RT_FULL ++# define rcu_barrier_bh rcu_barrier ++#else + void rcu_barrier_bh(void); ++#endif + void rcu_barrier_sched(void); + unsigned long get_state_synchronize_rcu(void); + void cond_synchronize_rcu(unsigned long oldstate); +@@ -85,12 +93,10 @@ + unsigned long rcu_batches_started_bh(void); + unsigned long rcu_batches_started_sched(void); + unsigned long rcu_batches_completed(void); +-unsigned long rcu_batches_completed_bh(void); + unsigned long rcu_batches_completed_sched(void); + void show_rcu_gp_kthreads(void); + + void rcu_force_quiescent_state(void); +-void rcu_bh_force_quiescent_state(void); + void rcu_sched_force_quiescent_state(void); + + void exit_rcu(void); +@@ -100,6 +106,14 @@ + + bool rcu_is_watching(void); + ++#ifndef CONFIG_PREEMPT_RT_FULL ++void rcu_bh_force_quiescent_state(void); ++unsigned long rcu_batches_completed_bh(void); ++#else ++# define rcu_bh_force_quiescent_state rcu_force_quiescent_state ++# define rcu_batches_completed_bh rcu_batches_completed ++#endif + -+void local_bh_disable(void) -+{ -+ __local_bh_disable(); -+} -+EXPORT_SYMBOL(local_bh_disable); + void rcu_all_qs(void); + + #endif /* __LINUX_RCUTREE_H */ +diff -Nur linux-4.1.10.orig/include/linux/rtmutex.h linux-4.1.10/include/linux/rtmutex.h +--- linux-4.1.10.orig/include/linux/rtmutex.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/rtmutex.h 2015-10-07 18:00:08.000000000 +0200 +@@ -14,10 +14,14 @@ + + #include + #include +-#include ++#include + + extern int max_lock_depth; /* for sysctl */ + ++#ifdef CONFIG_DEBUG_MUTEXES ++#include ++#endif + -+void __local_bh_disable_ip(unsigned long ip, unsigned int cnt) -+{ -+ __local_bh_disable(); -+ if (cnt & PREEMPT_CHECK_OFFSET) -+ preempt_disable(); -+} + /** + * The rt_mutex structure + * +@@ -31,8 +35,8 @@ + struct rb_root waiters; + struct rb_node *waiters_leftmost; + struct task_struct *owner; +-#ifdef CONFIG_DEBUG_RT_MUTEXES + int save_state; ++#ifdef CONFIG_DEBUG_RT_MUTEXES + const char *name, *file; + int line; + void *magic; +@@ -55,22 +59,33 @@ + # define rt_mutex_debug_check_no_locks_held(task) do { } while (0) + #endif + ++# define rt_mutex_init(mutex) \ ++ do { \ ++ raw_spin_lock_init(&(mutex)->wait_lock); \ ++ __rt_mutex_init(mutex, #mutex); \ ++ } while (0) + -+static void __local_bh_enable(void) -+{ -+ if (WARN_ON(current->softirq_nestcnt == 0)) -+ return; + #ifdef CONFIG_DEBUG_RT_MUTEXES + # define __DEBUG_RT_MUTEX_INITIALIZER(mutexname) \ + , .name = #mutexname, .file = __FILE__, .line = __LINE__ +-# define rt_mutex_init(mutex) __rt_mutex_init(mutex, __func__) + extern void rt_mutex_debug_task_free(struct task_struct *tsk); + #else + # define __DEBUG_RT_MUTEX_INITIALIZER(mutexname) +-# define rt_mutex_init(mutex) __rt_mutex_init(mutex, NULL) + # define rt_mutex_debug_task_free(t) do { } while (0) + #endif + +-#define __RT_MUTEX_INITIALIZER(mutexname) \ +- { .wait_lock = __RAW_SPIN_LOCK_UNLOCKED(mutexname.wait_lock) \ ++#define __RT_MUTEX_INITIALIZER_PLAIN(mutexname) \ ++ .wait_lock = __RAW_SPIN_LOCK_UNLOCKED(mutexname.wait_lock) \ + , .waiters = RB_ROOT \ + , .owner = NULL \ +- __DEBUG_RT_MUTEX_INITIALIZER(mutexname)} ++ __DEBUG_RT_MUTEX_INITIALIZER(mutexname) + -+ local_irq_disable(); -+ if (current->softirq_nestcnt == 1 && current->softirqs_raised) -+ do_current_softirqs(); -+ local_irq_enable(); ++#define __RT_MUTEX_INITIALIZER(mutexname) \ ++ { __RT_MUTEX_INITIALIZER_PLAIN(mutexname) } + -+ if (--current->softirq_nestcnt == 0) -+ migrate_enable(); -+} ++#define __RT_MUTEX_INITIALIZER_SAVE_STATE(mutexname) \ ++ { __RT_MUTEX_INITIALIZER_PLAIN(mutexname) \ ++ , .save_state = 1 } + + #define DEFINE_RT_MUTEX(mutexname) \ + struct rt_mutex mutexname = __RT_MUTEX_INITIALIZER(mutexname) +@@ -91,6 +106,7 @@ + + extern void rt_mutex_lock(struct rt_mutex *lock); + extern int rt_mutex_lock_interruptible(struct rt_mutex *lock); ++extern int rt_mutex_lock_killable(struct rt_mutex *lock); + extern int rt_mutex_timed_lock(struct rt_mutex *lock, + struct hrtimer_sleeper *timeout); + +diff -Nur linux-4.1.10.orig/include/linux/rwlock_rt.h linux-4.1.10/include/linux/rwlock_rt.h +--- linux-4.1.10.orig/include/linux/rwlock_rt.h 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/include/linux/rwlock_rt.h 2015-10-07 18:00:08.000000000 +0200 +@@ -0,0 +1,99 @@ ++#ifndef __LINUX_RWLOCK_RT_H ++#define __LINUX_RWLOCK_RT_H + -+void local_bh_enable(void) -+{ -+ __local_bh_enable(); -+} -+EXPORT_SYMBOL(local_bh_enable); ++#ifndef __LINUX_SPINLOCK_H ++#error Do not include directly. Use spinlock.h ++#endif + -+extern void __local_bh_enable_ip(unsigned long ip, unsigned int cnt) -+{ -+ __local_bh_enable(); -+ if (cnt & PREEMPT_CHECK_OFFSET) -+ preempt_enable(); -+} ++#define rwlock_init(rwl) \ ++do { \ ++ static struct lock_class_key __key; \ ++ \ ++ rt_mutex_init(&(rwl)->lock); \ ++ __rt_rwlock_init(rwl, #rwl, &__key); \ ++} while (0) + -+void local_bh_enable_ip(unsigned long ip) -+{ -+ local_bh_enable(); -+} -+EXPORT_SYMBOL(local_bh_enable_ip); ++extern void __lockfunc rt_write_lock(rwlock_t *rwlock); ++extern void __lockfunc rt_read_lock(rwlock_t *rwlock); ++extern int __lockfunc rt_write_trylock(rwlock_t *rwlock); ++extern int __lockfunc rt_write_trylock_irqsave(rwlock_t *trylock, unsigned long *flags); ++extern int __lockfunc rt_read_trylock(rwlock_t *rwlock); ++extern void __lockfunc rt_write_unlock(rwlock_t *rwlock); ++extern void __lockfunc rt_read_unlock(rwlock_t *rwlock); ++extern unsigned long __lockfunc rt_write_lock_irqsave(rwlock_t *rwlock); ++extern unsigned long __lockfunc rt_read_lock_irqsave(rwlock_t *rwlock); ++extern void __rt_rwlock_init(rwlock_t *rwlock, char *name, struct lock_class_key *key); + -+void _local_bh_enable(void) -+{ -+ if (WARN_ON(current->softirq_nestcnt == 0)) -+ return; -+ if (--current->softirq_nestcnt == 0) -+ migrate_enable(); -+} -+EXPORT_SYMBOL(_local_bh_enable); ++#define read_trylock(lock) __cond_lock(lock, rt_read_trylock(lock)) ++#define write_trylock(lock) __cond_lock(lock, rt_write_trylock(lock)) + -+int in_serving_softirq(void) -+{ -+ return current->flags & PF_IN_SOFTIRQ; -+} -+EXPORT_SYMBOL(in_serving_softirq); ++#define write_trylock_irqsave(lock, flags) \ ++ __cond_lock(lock, rt_write_trylock_irqsave(lock, &flags)) + -+/* Called with preemption disabled */ -+static void run_ksoftirqd(unsigned int cpu) -+{ -+ local_irq_disable(); -+ current->softirq_nestcnt++; ++#define read_lock_irqsave(lock, flags) \ ++ do { \ ++ typecheck(unsigned long, flags); \ ++ flags = rt_read_lock_irqsave(lock); \ ++ } while (0) + -+ do_current_softirqs(); -+ current->softirq_nestcnt--; -+ rcu_note_context_switch(); -+ local_irq_enable(); -+} ++#define write_lock_irqsave(lock, flags) \ ++ do { \ ++ typecheck(unsigned long, flags); \ ++ flags = rt_write_lock_irqsave(lock); \ ++ } while (0) + -+/* -+ * Called from netif_rx_ni(). Preemption enabled, but migration -+ * disabled. So the cpu can't go away under us. -+ */ -+void thread_do_softirq(void) -+{ -+ if (!in_serving_softirq() && current->softirqs_raised) { -+ current->softirq_nestcnt++; -+ do_current_softirqs(); -+ current->softirq_nestcnt--; -+ } -+} ++#define read_lock(lock) rt_read_lock(lock) + -+static void do_raise_softirq_irqoff(unsigned int nr) -+{ -+ trace_softirq_raise(nr); -+ or_softirq_pending(1UL << nr); ++#define read_lock_bh(lock) \ ++ do { \ ++ local_bh_disable(); \ ++ rt_read_lock(lock); \ ++ } while (0) + -+ /* -+ * If we are not in a hard interrupt and inside a bh disabled -+ * region, we simply raise the flag on current. local_bh_enable() -+ * will make sure that the softirq is executed. Otherwise we -+ * delegate it to ksoftirqd. -+ */ -+ if (!in_irq() && current->softirq_nestcnt) -+ current->softirqs_raised |= (1U << nr); -+ else if (__this_cpu_read(ksoftirqd)) -+ __this_cpu_read(ksoftirqd)->softirqs_raised |= (1U << nr); -+} ++#define read_lock_irq(lock) read_lock(lock) + -+void __raise_softirq_irqoff(unsigned int nr) -+{ -+ do_raise_softirq_irqoff(nr); -+ if (!in_irq() && !current->softirq_nestcnt) -+ wakeup_softirqd(); -+} ++#define write_lock(lock) rt_write_lock(lock) + -+/* -+ * This function must run with irqs disabled! -+ */ -+void raise_softirq_irqoff(unsigned int nr) -+{ -+ do_raise_softirq_irqoff(nr); ++#define write_lock_bh(lock) \ ++ do { \ ++ local_bh_disable(); \ ++ rt_write_lock(lock); \ ++ } while (0) + -+ /* -+ * If we're in an hard interrupt we let irq return code deal -+ * with the wakeup of ksoftirqd. -+ */ -+ if (in_irq()) -+ return; -+ /* -+ * If we are in thread context but outside of a bh disabled -+ * region, we need to wake ksoftirqd as well. -+ * -+ * CHECKME: Some of the places which do that could be wrapped -+ * into local_bh_disable/enable pairs. Though it's unclear -+ * whether this is worth the effort. To find those places just -+ * raise a WARN() if the condition is met. -+ */ -+ if (!current->softirq_nestcnt) -+ wakeup_softirqd(); -+} ++#define write_lock_irq(lock) write_lock(lock) + -+static inline int ksoftirqd_softirq_pending(void) -+{ -+ return current->softirqs_raised; -+} ++#define read_unlock(lock) rt_read_unlock(lock) + -+static inline void local_bh_disable_nort(void) { } -+static inline void _local_bh_enable_nort(void) { } ++#define read_unlock_bh(lock) \ ++ do { \ ++ rt_read_unlock(lock); \ ++ local_bh_enable(); \ ++ } while (0) + -+static inline void ksoftirqd_set_sched_params(unsigned int cpu) -+{ -+ struct sched_param param = { .sched_priority = 1 }; ++#define read_unlock_irq(lock) read_unlock(lock) + -+ sched_setscheduler(current, SCHED_FIFO, ¶m); -+ /* Take over all pending softirqs when starting */ -+ local_irq_disable(); -+ current->softirqs_raised = local_softirq_pending(); -+ local_irq_enable(); -+} ++#define write_unlock(lock) rt_write_unlock(lock) + -+static inline void ksoftirqd_clr_sched_params(unsigned int cpu, bool online) -+{ -+ struct sched_param param = { .sched_priority = 0 }; ++#define write_unlock_bh(lock) \ ++ do { \ ++ rt_write_unlock(lock); \ ++ local_bh_enable(); \ ++ } while (0) + -+ sched_setscheduler(current, SCHED_NORMAL, ¶m); -+} ++#define write_unlock_irq(lock) write_unlock(lock) + -+#endif /* PREEMPT_RT_FULL */ -+/* - * Enter an interrupt context. - */ - void irq_enter(void) -@@ -330,9 +734,9 @@ - * Prevent raise_softirq from needlessly waking up ksoftirqd - * here, as softirq will be serviced on return from interrupt. - */ -- local_bh_disable(); -+ local_bh_disable_nort(); - tick_irq_enter(); -- _local_bh_enable(); -+ _local_bh_enable_nort(); - } ++#define read_unlock_irqrestore(lock, flags) \ ++ do { \ ++ typecheck(unsigned long, flags); \ ++ (void) flags; \ ++ rt_read_unlock(lock); \ ++ } while (0) ++ ++#define write_unlock_irqrestore(lock, flags) \ ++ do { \ ++ typecheck(unsigned long, flags); \ ++ (void) flags; \ ++ rt_write_unlock(lock); \ ++ } while (0) ++ ++#endif +diff -Nur linux-4.1.10.orig/include/linux/rwlock_types.h linux-4.1.10/include/linux/rwlock_types.h +--- linux-4.1.10.orig/include/linux/rwlock_types.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/rwlock_types.h 2015-10-07 18:00:08.000000000 +0200 +@@ -1,6 +1,10 @@ + #ifndef __LINUX_RWLOCK_TYPES_H + #define __LINUX_RWLOCK_TYPES_H - __irq_enter(); -@@ -340,6 +744,7 @@ ++#if !defined(__LINUX_SPINLOCK_TYPES_H) ++# error "Do not include directly, include spinlock_types.h" ++#endif ++ + /* + * include/linux/rwlock_types.h - generic rwlock type definitions + * and initializers +@@ -43,6 +47,7 @@ + RW_DEP_MAP_INIT(lockname) } + #endif - static inline void invoke_softirq(void) - { -+#ifndef CONFIG_PREEMPT_RT_FULL - if (!force_irqthreads) { - #ifdef CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK - /* -@@ -359,6 +764,15 @@ - } else { - wakeup_softirqd(); - } -+#else /* PREEMPT_RT_FULL */ -+ unsigned long flags; +-#define DEFINE_RWLOCK(x) rwlock_t x = __RW_LOCK_UNLOCKED(x) ++#define DEFINE_RWLOCK(name) \ ++ rwlock_t name __cacheline_aligned_in_smp = __RW_LOCK_UNLOCKED(name) + + #endif /* __LINUX_RWLOCK_TYPES_H */ +diff -Nur linux-4.1.10.orig/include/linux/rwlock_types_rt.h linux-4.1.10/include/linux/rwlock_types_rt.h +--- linux-4.1.10.orig/include/linux/rwlock_types_rt.h 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/include/linux/rwlock_types_rt.h 2015-10-07 18:00:08.000000000 +0200 +@@ -0,0 +1,33 @@ ++#ifndef __LINUX_RWLOCK_TYPES_RT_H ++#define __LINUX_RWLOCK_TYPES_RT_H + -+ local_irq_save(flags); -+ if (__this_cpu_read(ksoftirqd) && -+ __this_cpu_read(ksoftirqd)->softirqs_raised) -+ wakeup_softirqd(); -+ local_irq_restore(flags); ++#ifndef __LINUX_SPINLOCK_TYPES_H ++#error "Do not include directly. Include spinlock_types.h instead" +#endif - } - - static inline void tick_irq_exit(void) -@@ -395,26 +809,6 @@ - trace_hardirq_exit(); /* must be last! */ - } ++ ++/* ++ * rwlocks - rtmutex which allows single reader recursion ++ */ ++typedef struct { ++ struct rt_mutex lock; ++ int read_depth; ++ unsigned int break_lock; ++#ifdef CONFIG_DEBUG_LOCK_ALLOC ++ struct lockdep_map dep_map; ++#endif ++} rwlock_t; ++ ++#ifdef CONFIG_DEBUG_LOCK_ALLOC ++# define RW_DEP_MAP_INIT(lockname) .dep_map = { .name = #lockname } ++#else ++# define RW_DEP_MAP_INIT(lockname) ++#endif ++ ++#define __RW_LOCK_UNLOCKED(name) \ ++ { .lock = __RT_MUTEX_INITIALIZER_SAVE_STATE(name.lock), \ ++ RW_DEP_MAP_INIT(name) } ++ ++#define DEFINE_RWLOCK(name) \ ++ rwlock_t name __cacheline_aligned_in_smp = __RW_LOCK_UNLOCKED(name) ++ ++#endif +diff -Nur linux-4.1.10.orig/include/linux/rwsem.h linux-4.1.10/include/linux/rwsem.h +--- linux-4.1.10.orig/include/linux/rwsem.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/rwsem.h 2015-10-07 18:00:08.000000000 +0200 +@@ -18,6 +18,10 @@ + #include + #endif --/* -- * This function must run with irqs disabled! -- */ --inline void raise_softirq_irqoff(unsigned int nr) --{ -- __raise_softirq_irqoff(nr); -- -- /* -- * If we're in an interrupt or softirq, we're done -- * (this also catches softirq-disabled code). We will -- * actually run the softirq once we return from -- * the irq or softirq. -- * -- * Otherwise we wake up ksoftirqd to make sure we -- * schedule the softirq soon. -- */ -- if (!in_interrupt()) -- wakeup_softirqd(); --} -- - void raise_softirq(unsigned int nr) - { - unsigned long flags; -@@ -424,12 +818,6 @@ - local_irq_restore(flags); - } ++#ifdef CONFIG_PREEMPT_RT_FULL ++#include ++#else /* PREEMPT_RT_FULL */ ++ + struct rw_semaphore; --void __raise_softirq_irqoff(unsigned int nr) --{ -- trace_softirq_raise(nr); -- or_softirq_pending(1UL << nr); --} -- - void open_softirq(int nr, void (*action)(struct softirq_action *)) - { - softirq_vec[nr].action = action; -@@ -446,15 +834,45 @@ - static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec); - static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec); + #ifdef CONFIG_RWSEM_GENERIC_SPINLOCK +@@ -177,4 +181,6 @@ + # define up_read_non_owner(sem) up_read(sem) + #endif -+static void inline -+__tasklet_common_schedule(struct tasklet_struct *t, struct tasklet_head *head, unsigned int nr) ++#endif /* !PREEMPT_RT_FULL */ ++ + #endif /* _LINUX_RWSEM_H */ +diff -Nur linux-4.1.10.orig/include/linux/rwsem_rt.h linux-4.1.10/include/linux/rwsem_rt.h +--- linux-4.1.10.orig/include/linux/rwsem_rt.h 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/include/linux/rwsem_rt.h 2015-10-07 18:00:08.000000000 +0200 +@@ -0,0 +1,140 @@ ++#ifndef _LINUX_RWSEM_RT_H ++#define _LINUX_RWSEM_RT_H ++ ++#ifndef _LINUX_RWSEM_H ++#error "Include rwsem.h" ++#endif ++ ++/* ++ * RW-semaphores are a spinlock plus a reader-depth count. ++ * ++ * Note that the semantics are different from the usual ++ * Linux rw-sems, in PREEMPT_RT mode we do not allow ++ * multiple readers to hold the lock at once, we only allow ++ * a read-lock owner to read-lock recursively. This is ++ * better for latency, makes the implementation inherently ++ * fair and makes it simpler as well. ++ */ ++ ++#include ++ ++struct rw_semaphore { ++ struct rt_mutex lock; ++ int read_depth; ++#ifdef CONFIG_DEBUG_LOCK_ALLOC ++ struct lockdep_map dep_map; ++#endif ++}; ++ ++#define __RWSEM_INITIALIZER(name) \ ++ { .lock = __RT_MUTEX_INITIALIZER(name.lock), \ ++ RW_DEP_MAP_INIT(name) } ++ ++#define DECLARE_RWSEM(lockname) \ ++ struct rw_semaphore lockname = __RWSEM_INITIALIZER(lockname) ++ ++extern void __rt_rwsem_init(struct rw_semaphore *rwsem, const char *name, ++ struct lock_class_key *key); ++ ++#define __rt_init_rwsem(sem, name, key) \ ++ do { \ ++ rt_mutex_init(&(sem)->lock); \ ++ __rt_rwsem_init((sem), (name), (key));\ ++ } while (0) ++ ++#define __init_rwsem(sem, name, key) __rt_init_rwsem(sem, name, key) ++ ++# define rt_init_rwsem(sem) \ ++do { \ ++ static struct lock_class_key __key; \ ++ \ ++ __rt_init_rwsem((sem), #sem, &__key); \ ++} while (0) ++ ++extern void rt_down_write(struct rw_semaphore *rwsem); ++extern void rt_down_read_nested(struct rw_semaphore *rwsem, int subclass); ++extern void rt_down_write_nested(struct rw_semaphore *rwsem, int subclass); ++extern void rt_down_write_nested_lock(struct rw_semaphore *rwsem, ++ struct lockdep_map *nest); ++extern void rt_down_read(struct rw_semaphore *rwsem); ++extern int rt_down_write_trylock(struct rw_semaphore *rwsem); ++extern int rt_down_read_trylock(struct rw_semaphore *rwsem); ++extern void __rt_up_read(struct rw_semaphore *rwsem); ++extern void rt_up_read(struct rw_semaphore *rwsem); ++extern void rt_up_write(struct rw_semaphore *rwsem); ++extern void rt_downgrade_write(struct rw_semaphore *rwsem); ++ ++#define init_rwsem(sem) rt_init_rwsem(sem) ++#define rwsem_is_locked(s) rt_mutex_is_locked(&(s)->lock) ++ ++static inline int rwsem_is_contended(struct rw_semaphore *sem) +{ -+ if (tasklet_trylock(t)) { -+again: -+ /* We may have been preempted before tasklet_trylock -+ * and __tasklet_action may have already run. -+ * So double check the sched bit while the takslet -+ * is locked before adding it to the list. -+ */ -+ if (test_bit(TASKLET_STATE_SCHED, &t->state)) { -+ t->next = NULL; -+ *head->tail = t; -+ head->tail = &(t->next); -+ raise_softirq_irqoff(nr); -+ tasklet_unlock(t); -+ } else { -+ /* This is subtle. If we hit the corner case above -+ * It is possible that we get preempted right here, -+ * and another task has successfully called -+ * tasklet_schedule(), then this function, and -+ * failed on the trylock. Thus we must be sure -+ * before releasing the tasklet lock, that the -+ * SCHED_BIT is clear. Otherwise the tasklet -+ * may get its SCHED_BIT set, but not added to the -+ * list -+ */ -+ if (!tasklet_tryunlock(t)) -+ goto again; -+ } -+ } ++ /* rt_mutex_has_waiters() */ ++ return !RB_EMPTY_ROOT(&sem->lock.waiters); +} + - void __tasklet_schedule(struct tasklet_struct *t) - { - unsigned long flags; - - local_irq_save(flags); -- t->next = NULL; -- *__this_cpu_read(tasklet_vec.tail) = t; -- __this_cpu_write(tasklet_vec.tail, &(t->next)); -- raise_softirq_irqoff(TASKLET_SOFTIRQ); -+ __tasklet_common_schedule(t, this_cpu_ptr(&tasklet_vec), TASKLET_SOFTIRQ); - local_irq_restore(flags); - } - EXPORT_SYMBOL(__tasklet_schedule); -@@ -464,10 +882,7 @@ - unsigned long flags; - - local_irq_save(flags); -- t->next = NULL; -- *__this_cpu_read(tasklet_hi_vec.tail) = t; -- __this_cpu_write(tasklet_hi_vec.tail, &(t->next)); -- raise_softirq_irqoff(HI_SOFTIRQ); -+ __tasklet_common_schedule(t, this_cpu_ptr(&tasklet_hi_vec), HI_SOFTIRQ); - local_irq_restore(flags); - } - EXPORT_SYMBOL(__tasklet_hi_schedule); -@@ -476,82 +891,122 @@ - { - BUG_ON(!irqs_disabled()); - -- t->next = __this_cpu_read(tasklet_hi_vec.head); -- __this_cpu_write(tasklet_hi_vec.head, t); -- __raise_softirq_irqoff(HI_SOFTIRQ); -+ __tasklet_hi_schedule(t); - } - EXPORT_SYMBOL(__tasklet_hi_schedule_first); - --static void tasklet_action(struct softirq_action *a) -+void tasklet_enable(struct tasklet_struct *t) - { -- struct tasklet_struct *list; -+ if (!atomic_dec_and_test(&t->count)) -+ return; -+ if (test_and_clear_bit(TASKLET_STATE_PENDING, &t->state)) -+ tasklet_schedule(t); ++static inline void down_read(struct rw_semaphore *sem) ++{ ++ rt_down_read(sem); +} -+EXPORT_SYMBOL(tasklet_enable); - -- local_irq_disable(); -- list = __this_cpu_read(tasklet_vec.head); -- __this_cpu_write(tasklet_vec.head, NULL); -- __this_cpu_write(tasklet_vec.tail, this_cpu_ptr(&tasklet_vec.head)); -- local_irq_enable(); -+static void __tasklet_action(struct softirq_action *a, -+ struct tasklet_struct *list) ++ ++static inline int down_read_trylock(struct rw_semaphore *sem) +{ -+ int loops = 1000000; - - while (list) { - struct tasklet_struct *t = list; - - list = list->next; - -- if (tasklet_trylock(t)) { -- if (!atomic_read(&t->count)) { -- if (!test_and_clear_bit(TASKLET_STATE_SCHED, -- &t->state)) -- BUG(); -- t->func(t->data); -- tasklet_unlock(t); -- continue; -- } -- tasklet_unlock(t); -+ /* -+ * Should always succeed - after a tasklist got on the -+ * list (after getting the SCHED bit set from 0 to 1), -+ * nothing but the tasklet softirq it got queued to can -+ * lock it: -+ */ -+ if (!tasklet_trylock(t)) { -+ WARN_ON(1); -+ continue; - } - -- local_irq_disable(); - t->next = NULL; -- *__this_cpu_read(tasklet_vec.tail) = t; -- __this_cpu_write(tasklet_vec.tail, &(t->next)); -- __raise_softirq_irqoff(TASKLET_SOFTIRQ); -- local_irq_enable(); ++ return rt_down_read_trylock(sem); ++} + -+ /* -+ * If we cannot handle the tasklet because it's disabled, -+ * mark it as pending. tasklet_enable() will later -+ * re-schedule the tasklet. -+ */ -+ if (unlikely(atomic_read(&t->count))) { -+out_disabled: -+ /* implicit unlock: */ -+ wmb(); -+ t->state = TASKLET_STATEF_PENDING; -+ continue; -+ } ++static inline void down_write(struct rw_semaphore *sem) ++{ ++ rt_down_write(sem); ++} + -+ /* -+ * After this point on the tasklet might be rescheduled -+ * on another CPU, but it can only be added to another -+ * CPU's tasklet list if we unlock the tasklet (which we -+ * dont do yet). -+ */ -+ if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state)) -+ WARN_ON(1); ++static inline int down_write_trylock(struct rw_semaphore *sem) ++{ ++ return rt_down_write_trylock(sem); ++} + -+again: -+ t->func(t->data); ++static inline void __up_read(struct rw_semaphore *sem) ++{ ++ __rt_up_read(sem); ++} + -+ /* -+ * Try to unlock the tasklet. We must use cmpxchg, because -+ * another CPU might have scheduled or disabled the tasklet. -+ * We only allow the STATE_RUN -> 0 transition here. -+ */ -+ while (!tasklet_tryunlock(t)) { -+ /* -+ * If it got disabled meanwhile, bail out: -+ */ -+ if (atomic_read(&t->count)) -+ goto out_disabled; -+ /* -+ * If it got scheduled meanwhile, re-execute -+ * the tasklet function: -+ */ -+ if (test_and_clear_bit(TASKLET_STATE_SCHED, &t->state)) -+ goto again; -+ if (!--loops) { -+ printk("hm, tasklet state: %08lx\n", t->state); -+ WARN_ON(1); -+ tasklet_unlock(t); -+ break; -+ } -+ } - } - } - -+static void tasklet_action(struct softirq_action *a) ++static inline void up_read(struct rw_semaphore *sem) +{ -+ struct tasklet_struct *list; ++ rt_up_read(sem); ++} + -+ local_irq_disable(); ++static inline void up_write(struct rw_semaphore *sem) ++{ ++ rt_up_write(sem); ++} + -+ list = __this_cpu_read(tasklet_vec.head); -+ __this_cpu_write(tasklet_vec.head, NULL); -+ __this_cpu_write(tasklet_vec.tail, this_cpu_ptr(&tasklet_vec.head)); ++static inline void downgrade_write(struct rw_semaphore *sem) ++{ ++ rt_downgrade_write(sem); ++} + -+ local_irq_enable(); ++static inline void down_read_nested(struct rw_semaphore *sem, int subclass) ++{ ++ return rt_down_read_nested(sem, subclass); ++} + -+ __tasklet_action(a, list); ++static inline void down_write_nested(struct rw_semaphore *sem, int subclass) ++{ ++ rt_down_write_nested(sem, subclass); ++} ++#ifdef CONFIG_DEBUG_LOCK_ALLOC ++static inline void down_write_nest_lock(struct rw_semaphore *sem, ++ struct rw_semaphore *nest_lock) ++{ ++ rt_down_write_nested_lock(sem, &nest_lock->dep_map); +} + - static void tasklet_hi_action(struct softirq_action *a) - { - struct tasklet_struct *list; - - local_irq_disable(); ++#else + - list = __this_cpu_read(tasklet_hi_vec.head); - __this_cpu_write(tasklet_hi_vec.head, NULL); - __this_cpu_write(tasklet_hi_vec.tail, this_cpu_ptr(&tasklet_hi_vec.head)); -- local_irq_enable(); -- -- while (list) { -- struct tasklet_struct *t = list; -- -- list = list->next; - -- if (tasklet_trylock(t)) { -- if (!atomic_read(&t->count)) { -- if (!test_and_clear_bit(TASKLET_STATE_SCHED, -- &t->state)) -- BUG(); -- t->func(t->data); -- tasklet_unlock(t); -- continue; -- } -- tasklet_unlock(t); -- } -+ local_irq_enable(); ++static inline void down_write_nest_lock(struct rw_semaphore *sem, ++ struct rw_semaphore *nest_lock) ++{ ++ rt_down_write_nested_lock(sem, NULL); ++} ++#endif ++#endif +diff -Nur linux-4.1.10.orig/include/linux/sched.h linux-4.1.10/include/linux/sched.h +--- linux-4.1.10.orig/include/linux/sched.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/sched.h 2015-10-07 18:00:08.000000000 +0200 +@@ -26,6 +26,7 @@ + #include + #include + #include ++#include -- local_irq_disable(); -- t->next = NULL; -- *__this_cpu_read(tasklet_hi_vec.tail) = t; -- __this_cpu_write(tasklet_hi_vec.tail, &(t->next)); -- __raise_softirq_irqoff(HI_SOFTIRQ); -- local_irq_enable(); -- } -+ __tasklet_action(a, list); - } + #include + #include +@@ -234,10 +235,7 @@ + TASK_UNINTERRUPTIBLE | __TASK_STOPPED | \ + __TASK_TRACED | EXIT_ZOMBIE | EXIT_DEAD) - void tasklet_init(struct tasklet_struct *t, -@@ -572,7 +1027,7 @@ +-#define task_is_traced(task) ((task->state & __TASK_TRACED) != 0) + #define task_is_stopped(task) ((task->state & __TASK_STOPPED) != 0) +-#define task_is_stopped_or_traced(task) \ +- ((task->state & (__TASK_STOPPED | __TASK_TRACED)) != 0) + #define task_contributes_to_load(task) \ + ((task->state & TASK_UNINTERRUPTIBLE) != 0 && \ + (task->flags & PF_FROZEN) == 0) +@@ -302,6 +300,11 @@ - while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) { - do { -- yield(); -+ msleep(1); - } while (test_bit(TASKLET_STATE_SCHED, &t->state)); - } - tasklet_unlock_wait(t); -@@ -646,25 +1101,26 @@ - open_softirq(HI_SOFTIRQ, tasklet_hi_action); - } + #endif --static int ksoftirqd_should_run(unsigned int cpu) --{ -- return local_softirq_pending(); --} -- --static void run_ksoftirqd(unsigned int cpu) -+#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT_FULL) -+void tasklet_unlock_wait(struct tasklet_struct *t) - { -- local_irq_disable(); -- if (local_softirq_pending()) { -+ while (test_bit(TASKLET_STATE_RUN, &(t)->state)) { - /* -- * We can safely run softirq on inline stack, as we are not deep -- * in the task stack here. -+ * Hack for now to avoid this busy-loop: - */ -- __do_softirq(); -- local_irq_enable(); -- cond_resched_rcu_qs(); -- return; ++#define __set_current_state_no_track(state_value) \ ++ do { current->state = (state_value); } while (0) ++#define set_current_state_no_track(state_value) \ ++ set_mb(current->state, (state_value)) ++ + /* Task command name length */ + #define TASK_COMM_LEN 16 + +@@ -900,6 +903,50 @@ + #define SCHED_CAPACITY_SCALE (1L << SCHED_CAPACITY_SHIFT) + + /* ++ * Wake-queues are lists of tasks with a pending wakeup, whose ++ * callers have already marked the task as woken internally, ++ * and can thus carry on. A common use case is being able to ++ * do the wakeups once the corresponding user lock as been ++ * released. ++ * ++ * We hold reference to each task in the list across the wakeup, ++ * thus guaranteeing that the memory is still valid by the time ++ * the actual wakeups are performed in wake_up_q(). ++ * ++ * One per task suffices, because there's never a need for a task to be ++ * in two wake queues simultaneously; it is forbidden to abandon a task ++ * in a wake queue (a call to wake_up_q() _must_ follow), so if a task is ++ * already in a wake queue, the wakeup will happen soon and the second ++ * waker can just skip it. ++ * ++ * The WAKE_Q macro declares and initializes the list head. ++ * wake_up_q() does NOT reinitialize the list; it's expected to be ++ * called near the end of a function, where the fact that the queue is ++ * not used again will be easy to see by inspection. ++ * ++ * Note that this can cause spurious wakeups. schedule() callers ++ * must ensure the call is done inside a loop, confirming that the ++ * wakeup condition has in fact occurred. ++ */ ++struct wake_q_node { ++ struct wake_q_node *next; ++}; ++ ++struct wake_q_head { ++ struct wake_q_node *first; ++ struct wake_q_node **lastp; ++}; ++ ++#define WAKE_Q_TAIL ((struct wake_q_node *) 0x01) ++ ++#define WAKE_Q(name) \ ++ struct wake_q_head name = { WAKE_Q_TAIL, &name.first } ++ ++extern void wake_q_add(struct wake_q_head *head, ++ struct task_struct *task); ++extern void wake_up_q(struct wake_q_head *head); ++ ++/* + * sched-domains (multiprocessor balancing) declarations: + */ + #ifdef CONFIG_SMP +@@ -1291,6 +1338,7 @@ + + struct task_struct { + volatile long state; /* -1 unrunnable, 0 runnable, >0 stopped */ ++ volatile long saved_state; /* saved state for "spinlock sleepers" */ + void *stack; + atomic_t usage; + unsigned int flags; /* per process flags, defined below */ +@@ -1327,6 +1375,12 @@ + #endif + + unsigned int policy; +#ifdef CONFIG_PREEMPT_RT_FULL -+ msleep(1); -+#else -+ barrier(); ++ int migrate_disable; ++# ifdef CONFIG_SCHED_DEBUG ++ int migrate_disable_atomic; ++# endif +#endif - } -- local_irq_enable(); -+} -+EXPORT_SYMBOL(tasklet_unlock_wait); + int nr_cpus_allowed; + cpumask_t cpus_allowed; + +@@ -1434,7 +1488,8 @@ + struct cputime prev_cputime; + #endif + #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN +- seqlock_t vtime_seqlock; ++ raw_spinlock_t vtime_lock; ++ seqcount_t vtime_seq; + unsigned long long vtime_snap; + enum { + VTIME_SLEEPING = 0, +@@ -1450,6 +1505,9 @@ + + struct task_cputime cputime_expires; + struct list_head cpu_timers[3]; ++#ifdef CONFIG_PREEMPT_RT_BASE ++ struct task_struct *posix_timer_list; +#endif -+ -+static int ksoftirqd_should_run(unsigned int cpu) -+{ -+ return ksoftirqd_softirq_pending(); - } - #ifdef CONFIG_HOTPLUG_CPU -@@ -746,6 +1202,8 @@ + /* process credentials */ + const struct cred __rcu *real_cred; /* objective and real subjective task +@@ -1482,10 +1540,15 @@ + /* signal handlers */ + struct signal_struct *signal; + struct sighand_struct *sighand; ++ struct sigqueue *sigqueue_cache; - static struct smp_hotplug_thread softirq_threads = { - .store = &ksoftirqd, -+ .setup = ksoftirqd_set_sched_params, -+ .cleanup = ksoftirqd_clr_sched_params, - .thread_should_run = ksoftirqd_should_run, - .thread_fn = run_ksoftirqd, - .thread_comm = "ksoftirqd/%u", -diff -Nur linux-4.1.6.orig/kernel/stop_machine.c linux-4.1.6/kernel/stop_machine.c ---- linux-4.1.6.orig/kernel/stop_machine.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/stop_machine.c 2015-09-08 23:49:08.117866054 +0200 -@@ -30,12 +30,12 @@ - atomic_t nr_todo; /* nr left to execute */ - bool executed; /* actually executed? */ - int ret; /* collected return value */ -- struct completion completion; /* fired if nr_todo reaches 0 */ -+ struct task_struct *waiter; /* woken when nr_todo reaches 0 */ - }; + sigset_t blocked, real_blocked; + sigset_t saved_sigmask; /* restored if set_restore_sigmask() was used */ + struct sigpending pending; ++#ifdef CONFIG_PREEMPT_RT_FULL ++ /* TODO: move me into ->restart_block ? */ ++ struct siginfo forced_info; ++#endif - /* the actual stopper, one per every possible cpu, enabled on online cpus */ - struct cpu_stopper { -- spinlock_t lock; -+ raw_spinlock_t lock; - bool enabled; /* is this stopper enabled? */ - struct list_head works; /* list of pending works */ + unsigned long sas_ss_sp; + size_t sas_ss_size; +@@ -1511,6 +1574,8 @@ + /* Protection of the PI data structures: */ + raw_spinlock_t pi_lock; + ++ struct wake_q_node wake_q; ++ + #ifdef CONFIG_RT_MUTEXES + /* PI waiters blocked on a rt_mutex held by this task */ + struct rb_root pi_waiters; +@@ -1705,6 +1770,12 @@ + unsigned long trace; + /* bitmask and counter of trace recursion */ + unsigned long trace_recursion; ++#ifdef CONFIG_WAKEUP_LATENCY_HIST ++ u64 preempt_timestamp_hist; ++#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST ++ long timer_offset; ++#endif ++#endif + #endif /* CONFIG_TRACING */ + #ifdef CONFIG_MEMCG + struct memcg_oom_info { +@@ -1721,14 +1792,23 @@ + unsigned int sequential_io; + unsigned int sequential_io_avg; + #endif ++#ifdef CONFIG_PREEMPT_RT_BASE ++ struct rcu_head put_rcu; ++ int softirq_nestcnt; ++ unsigned int softirqs_raised; ++#endif ++#ifdef CONFIG_PREEMPT_RT_FULL ++# if defined CONFIG_HIGHMEM || defined CONFIG_X86_32 ++ int kmap_idx; ++ pte_t kmap_pte[KM_TYPE_NR]; ++# endif ++#endif + #ifdef CONFIG_DEBUG_ATOMIC_SLEEP + unsigned long task_state_change; + #endif ++ int pagefault_disabled; }; -@@ -56,7 +56,7 @@ - { - memset(done, 0, sizeof(*done)); - atomic_set(&done->nr_todo, nr_todo); -- init_completion(&done->completion); -+ done->waiter = current; - } - /* signal completion unless @done is NULL */ -@@ -65,8 +65,10 @@ - if (done) { - if (executed) - done->executed = true; -- if (atomic_dec_and_test(&done->nr_todo)) -- complete(&done->completion); -+ if (atomic_dec_and_test(&done->nr_todo)) { -+ wake_up_process(done->waiter); -+ done->waiter = NULL; -+ } - } +-/* Future-safe accessor for struct task_struct's cpus_allowed. */ +-#define tsk_cpus_allowed(tsk) (&(tsk)->cpus_allowed) +- + #define TNF_MIGRATED 0x01 + #define TNF_NO_GROUP 0x02 + #define TNF_SHARED 0x04 +@@ -1917,6 +1997,15 @@ + extern void free_task(struct task_struct *tsk); + #define get_task_struct(tsk) do { atomic_inc(&(tsk)->usage); } while(0) + ++#ifdef CONFIG_PREEMPT_RT_BASE ++extern void __put_task_struct_cb(struct rcu_head *rhp); ++ ++static inline void put_task_struct(struct task_struct *t) ++{ ++ if (atomic_dec_and_test(&t->usage)) ++ call_rcu(&t->put_rcu, __put_task_struct_cb); ++} ++#else + extern void __put_task_struct(struct task_struct *t); + + static inline void put_task_struct(struct task_struct *t) +@@ -1924,6 +2013,7 @@ + if (atomic_dec_and_test(&t->usage)) + __put_task_struct(t); } ++#endif -@@ -78,7 +80,7 @@ + #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN + extern void task_cputime(struct task_struct *t, +@@ -1962,6 +2052,7 @@ + /* + * Per process flags + */ ++#define PF_IN_SOFTIRQ 0x00000001 /* Task is serving softirq */ + #define PF_EXITING 0x00000004 /* getting shut down */ + #define PF_EXITPIDONE 0x00000008 /* pi exit done on shut down */ + #define PF_VCPU 0x00000010 /* I'm a virtual CPU */ +@@ -2126,6 +2217,10 @@ - unsigned long flags; + extern int set_cpus_allowed_ptr(struct task_struct *p, + const struct cpumask *new_mask); ++int migrate_me(void); ++void tell_sched_cpu_down_begin(int cpu); ++void tell_sched_cpu_down_done(int cpu); ++ + #else + static inline void do_set_cpus_allowed(struct task_struct *p, + const struct cpumask *new_mask) +@@ -2138,6 +2233,9 @@ + return -EINVAL; + return 0; + } ++static inline int migrate_me(void) { return 0; } ++static inline void tell_sched_cpu_down_begin(int cpu) { } ++static inline void tell_sched_cpu_down_done(int cpu) { } + #endif -- spin_lock_irqsave(&stopper->lock, flags); -+ raw_spin_lock_irqsave(&stopper->lock, flags); + #ifdef CONFIG_NO_HZ_COMMON +@@ -2354,6 +2452,7 @@ - if (stopper->enabled) { - list_add_tail(&work->list, &stopper->works); -@@ -86,7 +88,23 @@ - } else - cpu_stop_signal_done(work->done, false); + extern int wake_up_state(struct task_struct *tsk, unsigned int state); + extern int wake_up_process(struct task_struct *tsk); ++extern int wake_up_lock_sleeper(struct task_struct * tsk); + extern void wake_up_new_task(struct task_struct *tsk); + #ifdef CONFIG_SMP + extern void kick_process(struct task_struct *tsk); +@@ -2470,12 +2569,24 @@ -- spin_unlock_irqrestore(&stopper->lock, flags); -+ raw_spin_unlock_irqrestore(&stopper->lock, flags); -+} + /* mmdrop drops the mm and the page tables */ + extern void __mmdrop(struct mm_struct *); + -+static void wait_for_stop_done(struct cpu_stop_done *done) -+{ -+ set_current_state(TASK_UNINTERRUPTIBLE); -+ while (atomic_read(&done->nr_todo)) { -+ schedule(); -+ set_current_state(TASK_UNINTERRUPTIBLE); -+ } -+ /* -+ * We need to wait until cpu_stop_signal_done() has cleared -+ * done->waiter. -+ */ -+ while (done->waiter) -+ cpu_relax(); -+ set_current_state(TASK_RUNNING); + static inline void mmdrop(struct mm_struct * mm) + { + if (unlikely(atomic_dec_and_test(&mm->mm_count))) + __mmdrop(mm); } - /** -@@ -120,7 +138,7 @@ ++#ifdef CONFIG_PREEMPT_RT_BASE ++extern void __mmdrop_delayed(struct rcu_head *rhp); ++static inline void mmdrop_delayed(struct mm_struct *mm) ++{ ++ if (atomic_dec_and_test(&mm->mm_count)) ++ call_rcu(&mm->delayed_drop, __mmdrop_delayed); ++} ++#else ++# define mmdrop_delayed(mm) mmdrop(mm) ++#endif ++ + /* mmput gets rid of the mappings and all user-space */ + extern void mmput(struct mm_struct *); + /* Grab a reference to a task's mm, if it is not already going away */ +@@ -2787,6 +2898,43 @@ + return unlikely(test_tsk_thread_flag(tsk,TIF_NEED_RESCHED)); + } - cpu_stop_init_done(&done, 1); - cpu_stop_queue_work(cpu, &work); -- wait_for_completion(&done.completion); -+ wait_for_stop_done(&done); - return done.executed ? done.ret : -ENOENT; ++#ifdef CONFIG_PREEMPT_LAZY ++static inline void set_tsk_need_resched_lazy(struct task_struct *tsk) ++{ ++ set_tsk_thread_flag(tsk,TIF_NEED_RESCHED_LAZY); ++} ++ ++static inline void clear_tsk_need_resched_lazy(struct task_struct *tsk) ++{ ++ clear_tsk_thread_flag(tsk,TIF_NEED_RESCHED_LAZY); ++} ++ ++static inline int test_tsk_need_resched_lazy(struct task_struct *tsk) ++{ ++ return unlikely(test_tsk_thread_flag(tsk,TIF_NEED_RESCHED_LAZY)); ++} ++ ++static inline int need_resched_lazy(void) ++{ ++ return test_thread_flag(TIF_NEED_RESCHED_LAZY); ++} ++ ++static inline int need_resched_now(void) ++{ ++ return test_thread_flag(TIF_NEED_RESCHED); ++} ++ ++#else ++static inline void clear_tsk_need_resched_lazy(struct task_struct *tsk) { } ++static inline int need_resched_lazy(void) { return 0; } ++ ++static inline int need_resched_now(void) ++{ ++ return test_thread_flag(TIF_NEED_RESCHED); ++} ++ ++#endif ++ + static inline int restart_syscall(void) + { + set_tsk_thread_flag(current, TIF_SIGPENDING); +@@ -2818,6 +2966,51 @@ + return (state & TASK_INTERRUPTIBLE) || __fatal_signal_pending(p); } -@@ -248,7 +266,7 @@ - struct irq_cpu_stop_queue_work_info call_args; - struct multi_stop_data msdata; ++static inline bool __task_is_stopped_or_traced(struct task_struct *task) ++{ ++ if (task->state & (__TASK_STOPPED | __TASK_TRACED)) ++ return true; ++#ifdef CONFIG_PREEMPT_RT_FULL ++ if (task->saved_state & (__TASK_STOPPED | __TASK_TRACED)) ++ return true; ++#endif ++ return false; ++} ++ ++static inline bool task_is_stopped_or_traced(struct task_struct *task) ++{ ++ bool traced_stopped; ++ ++#ifdef CONFIG_PREEMPT_RT_FULL ++ unsigned long flags; ++ ++ raw_spin_lock_irqsave(&task->pi_lock, flags); ++ traced_stopped = __task_is_stopped_or_traced(task); ++ raw_spin_unlock_irqrestore(&task->pi_lock, flags); ++#else ++ traced_stopped = __task_is_stopped_or_traced(task); ++#endif ++ return traced_stopped; ++} ++ ++static inline bool task_is_traced(struct task_struct *task) ++{ ++ bool traced = false; ++ ++ if (task->state & __TASK_TRACED) ++ return true; ++#ifdef CONFIG_PREEMPT_RT_FULL ++ /* in case the task is sleeping on tasklist_lock */ ++ raw_spin_lock_irq(&task->pi_lock); ++ if (task->state & __TASK_TRACED) ++ traced = true; ++ else if (task->saved_state & __TASK_TRACED) ++ traced = true; ++ raw_spin_unlock_irq(&task->pi_lock); ++#endif ++ return traced; ++} ++ + /* + * cond_resched() and cond_resched_lock(): latency reduction via + * explicit rescheduling in places that are safe. The return +@@ -2834,7 +3027,7 @@ -- preempt_disable(); -+ preempt_disable_nort(); - msdata = (struct multi_stop_data){ - .fn = fn, - .data = arg, -@@ -281,7 +299,7 @@ - * This relies on the stopper workqueues to be FIFO. - */ - if (!cpu_active(cpu1) || !cpu_active(cpu2)) { -- preempt_enable(); -+ preempt_enable_nort(); - return -ENOENT; - } + extern int __cond_resched_lock(spinlock_t *lock); -@@ -295,9 +313,9 @@ - &irq_cpu_stop_queue_work, - &call_args, 1); - lg_local_unlock(&stop_cpus_lock); -- preempt_enable(); -+ preempt_enable_nort(); +-#ifdef CONFIG_PREEMPT_COUNT ++#if defined(CONFIG_PREEMPT_COUNT) && !defined(CONFIG_PREEMPT_RT_FULL) + #define PREEMPT_LOCK_OFFSET PREEMPT_OFFSET + #else + #define PREEMPT_LOCK_OFFSET 0 +@@ -2845,12 +3038,16 @@ + __cond_resched_lock(lock); \ + }) -- wait_for_completion(&done.completion); -+ wait_for_stop_done(&done); ++#ifndef CONFIG_PREEMPT_RT_FULL + extern int __cond_resched_softirq(void); - return done.executed ? done.ret : -ENOENT; - } -@@ -329,7 +347,7 @@ + #define cond_resched_softirq() ({ \ + ___might_sleep(__FILE__, __LINE__, SOFTIRQ_DISABLE_OFFSET); \ + __cond_resched_softirq(); \ + }) ++#else ++# define cond_resched_softirq() cond_resched() ++#endif - static void queue_stop_cpus_work(const struct cpumask *cpumask, - cpu_stop_fn_t fn, void *arg, -- struct cpu_stop_done *done) -+ struct cpu_stop_done *done, bool inactive) + static inline void cond_resched_rcu(void) { - struct cpu_stop_work *work; - unsigned int cpu; -@@ -343,11 +361,13 @@ - } +@@ -3017,6 +3214,26 @@ - /* -- * Disable preemption while queueing to avoid getting -- * preempted by a stopper which might wait for other stoppers -- * to enter @fn which can lead to deadlock. -+ * Make sure that all work is queued on all cpus before -+ * any of the cpus can execute it. - */ -- lg_global_lock(&stop_cpus_lock); -+ if (!inactive) -+ lg_global_lock(&stop_cpus_lock); -+ else -+ lg_global_trylock_relax(&stop_cpus_lock); - for_each_cpu(cpu, cpumask) - cpu_stop_queue_work(cpu, &per_cpu(stop_cpus_work, cpu)); - lg_global_unlock(&stop_cpus_lock); -@@ -359,8 +379,8 @@ - struct cpu_stop_done done; + #endif /* CONFIG_SMP */ - cpu_stop_init_done(&done, cpumask_weight(cpumask)); -- queue_stop_cpus_work(cpumask, fn, arg, &done); -- wait_for_completion(&done.completion); -+ queue_stop_cpus_work(cpumask, fn, arg, &done, false); -+ wait_for_stop_done(&done); - return done.executed ? done.ret : -ENOENT; - } - -@@ -439,9 +459,9 @@ - unsigned long flags; - int run; ++static inline int __migrate_disabled(struct task_struct *p) ++{ ++#ifdef CONFIG_PREEMPT_RT_FULL ++ return p->migrate_disable; ++#else ++ return 0; ++#endif ++} ++ ++/* Future-safe accessor for struct task_struct's cpus_allowed. */ ++static inline const struct cpumask *tsk_cpus_allowed(struct task_struct *p) ++{ ++#ifdef CONFIG_PREEMPT_RT_FULL ++ if (p->migrate_disable) ++ return cpumask_of(task_cpu(p)); ++#endif ++ ++ return &p->cpus_allowed; ++} ++ + extern long sched_setaffinity(pid_t pid, const struct cpumask *new_mask); + extern long sched_getaffinity(pid_t pid, struct cpumask *mask); -- spin_lock_irqsave(&stopper->lock, flags); -+ raw_spin_lock_irqsave(&stopper->lock, flags); - run = !list_empty(&stopper->works); -- spin_unlock_irqrestore(&stopper->lock, flags); -+ raw_spin_unlock_irqrestore(&stopper->lock, flags); - return run; +diff -Nur linux-4.1.10.orig/include/linux/seqlock.h linux-4.1.10/include/linux/seqlock.h +--- linux-4.1.10.orig/include/linux/seqlock.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/seqlock.h 2015-10-07 18:00:08.000000000 +0200 +@@ -219,20 +219,30 @@ + return __read_seqcount_retry(s, start); } -@@ -453,13 +473,13 @@ - - repeat: - work = NULL; -- spin_lock_irq(&stopper->lock); -+ raw_spin_lock_irq(&stopper->lock); - if (!list_empty(&stopper->works)) { - work = list_first_entry(&stopper->works, - struct cpu_stop_work, list); - list_del_init(&work->list); - } -- spin_unlock_irq(&stopper->lock); -+ raw_spin_unlock_irq(&stopper->lock); - - if (work) { - cpu_stop_fn_t fn = work->fn; -@@ -467,6 +487,16 @@ - struct cpu_stop_done *done = work->done; - char ksym_buf[KSYM_NAME_LEN] __maybe_unused; +- +- +-static inline void raw_write_seqcount_begin(seqcount_t *s) ++static inline void __raw_write_seqcount_begin(seqcount_t *s) + { + s->sequence++; + smp_wmb(); + } -+ /* -+ * Wait until the stopper finished scheduling on all -+ * cpus -+ */ -+ lg_global_lock(&stop_cpus_lock); -+ /* -+ * Let other cpu threads continue as well -+ */ -+ lg_global_unlock(&stop_cpus_lock); +-static inline void raw_write_seqcount_end(seqcount_t *s) ++static inline void raw_write_seqcount_begin(seqcount_t *s) ++{ ++ preempt_disable_rt(); ++ __raw_write_seqcount_begin(s); ++} + - /* cpu stop callbacks are not allowed to sleep */ - preempt_disable(); - -@@ -481,7 +511,13 @@ - kallsyms_lookup((unsigned long)fn, NULL, NULL, NULL, - ksym_buf), arg); ++static inline void __raw_write_seqcount_end(seqcount_t *s) + { + smp_wmb(); + s->sequence++; + } ++static inline void raw_write_seqcount_end(seqcount_t *s) ++{ ++ __raw_write_seqcount_end(s); ++ preempt_enable_rt(); ++} ++ + /* + * raw_write_seqcount_latch - redirect readers to even/odd copy + * @s: pointer to seqcount_t +@@ -305,10 +315,32 @@ + /* + * Read side functions for starting and finalizing a read side section. + */ ++#ifndef CONFIG_PREEMPT_RT_FULL + static inline unsigned read_seqbegin(const seqlock_t *sl) + { + return read_seqcount_begin(&sl->seqcount); + } ++#else ++/* ++ * Starvation safe read side for RT ++ */ ++static inline unsigned read_seqbegin(seqlock_t *sl) ++{ ++ unsigned ret; ++ ++repeat: ++ ret = ACCESS_ONCE(sl->seqcount.sequence); ++ if (unlikely(ret & 1)) { + /* -+ * Make sure that the wakeup and setting done->waiter -+ * to NULL is atomic. ++ * Take the lock and let the writer proceed (i.e. evtl ++ * boost it), otherwise we could loop here forever. + */ -+ local_irq_disable(); - cpu_stop_signal_done(done, true); -+ local_irq_enable(); - goto repeat; - } ++ spin_unlock_wait(&sl->lock); ++ goto repeat; ++ } ++ return ret; ++} ++#endif + + static inline unsigned read_seqretry(const seqlock_t *sl, unsigned start) + { +@@ -323,36 +355,36 @@ + static inline void write_seqlock(seqlock_t *sl) + { + spin_lock(&sl->lock); +- write_seqcount_begin(&sl->seqcount); ++ __raw_write_seqcount_begin(&sl->seqcount); } -@@ -500,20 +536,20 @@ - unsigned long flags; - /* drain remaining works */ -- spin_lock_irqsave(&stopper->lock, flags); -+ raw_spin_lock_irqsave(&stopper->lock, flags); - list_for_each_entry(work, &stopper->works, list) - cpu_stop_signal_done(work->done, false); - stopper->enabled = false; -- spin_unlock_irqrestore(&stopper->lock, flags); -+ raw_spin_unlock_irqrestore(&stopper->lock, flags); + static inline void write_sequnlock(seqlock_t *sl) + { +- write_seqcount_end(&sl->seqcount); ++ __raw_write_seqcount_end(&sl->seqcount); + spin_unlock(&sl->lock); } - static void cpu_stop_unpark(unsigned int cpu) + static inline void write_seqlock_bh(seqlock_t *sl) { - struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu); + spin_lock_bh(&sl->lock); +- write_seqcount_begin(&sl->seqcount); ++ __raw_write_seqcount_begin(&sl->seqcount); + } -- spin_lock_irq(&stopper->lock); -+ raw_spin_lock_irq(&stopper->lock); - stopper->enabled = true; -- spin_unlock_irq(&stopper->lock); -+ raw_spin_unlock_irq(&stopper->lock); + static inline void write_sequnlock_bh(seqlock_t *sl) + { +- write_seqcount_end(&sl->seqcount); ++ __raw_write_seqcount_end(&sl->seqcount); + spin_unlock_bh(&sl->lock); } - static struct smp_hotplug_thread cpu_stop_threads = { -@@ -535,10 +571,12 @@ - for_each_possible_cpu(cpu) { - struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu); + static inline void write_seqlock_irq(seqlock_t *sl) + { + spin_lock_irq(&sl->lock); +- write_seqcount_begin(&sl->seqcount); ++ __raw_write_seqcount_begin(&sl->seqcount); + } -- spin_lock_init(&stopper->lock); -+ raw_spin_lock_init(&stopper->lock); - INIT_LIST_HEAD(&stopper->works); - } + static inline void write_sequnlock_irq(seqlock_t *sl) + { +- write_seqcount_end(&sl->seqcount); ++ __raw_write_seqcount_end(&sl->seqcount); + spin_unlock_irq(&sl->lock); + } -+ lg_lock_init(&stop_cpus_lock, "stop_cpus_lock"); -+ - BUG_ON(smpboot_register_percpu_thread(&cpu_stop_threads)); - stop_machine_initialized = true; - return 0; -@@ -634,11 +672,11 @@ - set_state(&msdata, MULTI_STOP_PREPARE); - cpu_stop_init_done(&done, num_active_cpus()); - queue_stop_cpus_work(cpu_active_mask, multi_cpu_stop, &msdata, -- &done); -+ &done, true); - ret = multi_cpu_stop(&msdata); +@@ -361,7 +393,7 @@ + unsigned long flags; - /* Busy wait for completion. */ -- while (!completion_done(&done.completion)) -+ while (atomic_read(&done.nr_todo)) - cpu_relax(); + spin_lock_irqsave(&sl->lock, flags); +- write_seqcount_begin(&sl->seqcount); ++ __raw_write_seqcount_begin(&sl->seqcount); + return flags; + } - mutex_unlock(&stop_cpus_mutex); -diff -Nur linux-4.1.6.orig/kernel/time/hrtimer.c linux-4.1.6/kernel/time/hrtimer.c ---- linux-4.1.6.orig/kernel/time/hrtimer.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/time/hrtimer.c 2015-09-08 23:49:08.117866054 +0200 -@@ -48,11 +48,13 @@ - #include - #include - #include -+#include - #include +@@ -371,7 +403,7 @@ + static inline void + write_sequnlock_irqrestore(seqlock_t *sl, unsigned long flags) + { +- write_seqcount_end(&sl->seqcount); ++ __raw_write_seqcount_end(&sl->seqcount); + spin_unlock_irqrestore(&sl->lock, flags); + } - #include +diff -Nur linux-4.1.10.orig/include/linux/signal.h linux-4.1.10/include/linux/signal.h +--- linux-4.1.10.orig/include/linux/signal.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/signal.h 2015-10-07 18:00:08.000000000 +0200 +@@ -218,6 +218,7 @@ + } - #include -+#include + extern void flush_sigqueue(struct sigpending *queue); ++extern void flush_task_sigqueue(struct task_struct *tsk); - #include "tick-internal.h" + /* Test if 'sig' is valid signal. Use this instead of testing _NSIG directly */ + static inline int valid_signal(unsigned long sig) +diff -Nur linux-4.1.10.orig/include/linux/skbuff.h linux-4.1.10/include/linux/skbuff.h +--- linux-4.1.10.orig/include/linux/skbuff.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/skbuff.h 2015-10-07 18:00:08.000000000 +0200 +@@ -187,6 +187,7 @@ -@@ -576,8 +578,7 @@ - * When the callback is running, we do not reprogram the clock event - * device. The timer callback is either running on a different CPU or - * the callback is executed in the hrtimer_interrupt context. The -- * reprogramming is handled either by the softirq, which called the -- * callback or at the end of the hrtimer_interrupt. -+ * reprogramming is handled at the end of the hrtimer_interrupt. - */ - if (hrtimer_callback_running(timer)) - return 0; -@@ -621,6 +622,9 @@ - return res; - } + __u32 qlen; + spinlock_t lock; ++ raw_spinlock_t raw_lock; + }; -+static void __run_hrtimer(struct hrtimer *timer, ktime_t *now); -+static int hrtimer_rt_defer(struct hrtimer *timer); -+ - /* - * Initialize the high resolution related parts of cpu_base - */ -@@ -630,6 +634,21 @@ - base->hres_active = 0; + struct sk_buff; +@@ -1336,6 +1337,12 @@ + __skb_queue_head_init(list); } -+static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer, -+ struct hrtimer_clock_base *base, -+ int wakeup) ++static inline void skb_queue_head_init_raw(struct sk_buff_head *list) +{ -+ if (!hrtimer_reprogram(timer, base)) -+ return 0; -+ if (!wakeup) -+ return -ETIME; -+#ifdef CONFIG_PREEMPT_RT_BASE -+ if (!hrtimer_rt_defer(timer)) -+ return -ETIME; -+#endif -+ return 1; ++ raw_spin_lock_init(&list->raw_lock); ++ __skb_queue_head_init(list); +} + - static inline ktime_t hrtimer_update_base(struct hrtimer_cpu_base *base) + static inline void skb_queue_head_init_class(struct sk_buff_head *list, + struct lock_class_key *class) { - ktime_t *offs_real = &base->clock_base[HRTIMER_BASE_REALTIME].offset; -@@ -695,6 +714,44 @@ - - static DECLARE_WORK(hrtimer_work, clock_was_set_work); +diff -Nur linux-4.1.10.orig/include/linux/smp.h linux-4.1.10/include/linux/smp.h +--- linux-4.1.10.orig/include/linux/smp.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/smp.h 2015-10-07 18:00:08.000000000 +0200 +@@ -185,6 +185,9 @@ + #define get_cpu() ({ preempt_disable(); smp_processor_id(); }) + #define put_cpu() preempt_enable() -+#ifdef CONFIG_PREEMPT_RT_FULL -+/* -+ * RT can not call schedule_work from real interrupt context. -+ * Need to make a thread to do the real work. -+ */ -+static struct task_struct *clock_set_delay_thread; -+static bool do_clock_set_delay; -+ -+static int run_clock_set_delay(void *ignore) -+{ -+ while (!kthread_should_stop()) { -+ set_current_state(TASK_INTERRUPTIBLE); -+ if (do_clock_set_delay) { -+ do_clock_set_delay = false; -+ schedule_work(&hrtimer_work); -+ } -+ schedule(); -+ } -+ __set_current_state(TASK_RUNNING); -+ return 0; -+} -+ -+void clock_was_set_delayed(void) -+{ -+ do_clock_set_delay = true; -+ /* Make visible before waking up process */ -+ smp_wmb(); -+ wake_up_process(clock_set_delay_thread); -+} ++#define get_cpu_light() ({ migrate_disable(); smp_processor_id(); }) ++#define put_cpu_light() migrate_enable() + -+static __init int create_clock_set_delay_thread(void) -+{ -+ clock_set_delay_thread = kthread_run(run_clock_set_delay, NULL, "kclksetdelayd"); -+ BUG_ON(!clock_set_delay_thread); -+ return 0; -+} -+early_initcall(create_clock_set_delay_thread); -+#else /* PREEMPT_RT_FULL */ /* - * Called from timekeeping and resume code to reprogramm the hrtimer - * interrupt device on all cpus. -@@ -703,6 +760,7 @@ - { - schedule_work(&hrtimer_work); + * Callback to arch code if there's nosmp or maxcpus=0 on the + * boot command line: +diff -Nur linux-4.1.10.orig/include/linux/spinlock_api_smp.h linux-4.1.10/include/linux/spinlock_api_smp.h +--- linux-4.1.10.orig/include/linux/spinlock_api_smp.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/spinlock_api_smp.h 2015-10-07 18:00:08.000000000 +0200 +@@ -189,6 +189,8 @@ + return 0; } + +-#include ++#ifndef CONFIG_PREEMPT_RT_FULL ++# include +#endif - #else + #endif /* __LINUX_SPINLOCK_API_SMP_H */ +diff -Nur linux-4.1.10.orig/include/linux/spinlock.h linux-4.1.10/include/linux/spinlock.h +--- linux-4.1.10.orig/include/linux/spinlock.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/spinlock.h 2015-10-07 18:00:08.000000000 +0200 +@@ -281,7 +281,11 @@ + #define raw_spin_can_lock(lock) (!raw_spin_is_locked(lock)) -@@ -711,6 +769,13 @@ - static inline int hrtimer_switch_to_hres(void) { return 0; } - static inline void - hrtimer_force_reprogram(struct hrtimer_cpu_base *base, int skip_equal) { } -+static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer, -+ struct hrtimer_clock_base *base, -+ int wakeup) -+{ -+ return 0; -+} -+ - static inline int hrtimer_reprogram(struct hrtimer *timer, - struct hrtimer_clock_base *base) - { -@@ -718,7 +783,6 @@ - } - static inline void hrtimer_init_hres(struct hrtimer_cpu_base *base) { } - static inline void retrigger_next_event(void *arg) { } -- - #endif /* CONFIG_HIGH_RES_TIMERS */ + /* Include rwlock functions */ +-#include ++#ifdef CONFIG_PREEMPT_RT_FULL ++# include ++#else ++# include ++#endif /* -@@ -836,6 +900,32 @@ - } - EXPORT_SYMBOL_GPL(hrtimer_forward); + * Pull the _spin_*()/_read_*()/_write_*() functions/declarations: +@@ -292,6 +296,10 @@ + # include + #endif -+#ifdef CONFIG_PREEMPT_RT_BASE -+# define wake_up_timer_waiters(b) wake_up(&(b)->wait) ++#ifdef CONFIG_PREEMPT_RT_FULL ++# include ++#else /* PREEMPT_RT_FULL */ + -+/** -+ * hrtimer_wait_for_timer - Wait for a running timer -+ * -+ * @timer: timer to wait for -+ * -+ * The function waits in case the timers callback function is -+ * currently executed on the waitqueue of the timer base. The -+ * waitqueue is woken up after the timer callback function has -+ * finished execution. -+ */ -+void hrtimer_wait_for_timer(const struct hrtimer *timer) -+{ -+ struct hrtimer_clock_base *base = timer->base; + /* + * Map the spin_lock functions to the raw variants for PREEMPT_RT=n + */ +@@ -426,4 +434,6 @@ + #define atomic_dec_and_lock(atomic, lock) \ + __cond_lock(lock, _atomic_dec_and_lock(atomic, lock)) + ++#endif /* !PREEMPT_RT_FULL */ + -+ if (base && base->cpu_base && !timer->irqsafe) -+ wait_event(base->cpu_base->wait, -+ !(timer->state & HRTIMER_STATE_CALLBACK)); -+} + #endif /* __LINUX_SPINLOCK_H */ +diff -Nur linux-4.1.10.orig/include/linux/spinlock_rt.h linux-4.1.10/include/linux/spinlock_rt.h +--- linux-4.1.10.orig/include/linux/spinlock_rt.h 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/include/linux/spinlock_rt.h 2015-10-07 18:00:08.000000000 +0200 +@@ -0,0 +1,174 @@ ++#ifndef __LINUX_SPINLOCK_RT_H ++#define __LINUX_SPINLOCK_RT_H + -+#else -+# define wake_up_timer_waiters(b) do { } while (0) ++#ifndef __LINUX_SPINLOCK_H ++#error Do not include directly. Use spinlock.h +#endif + - /* - * enqueue_hrtimer - internal function to (re)start a timer - * -@@ -879,6 +969,11 @@ - if (!(timer->state & HRTIMER_STATE_ENQUEUED)) - goto out; - -+ if (unlikely(!list_empty(&timer->cb_entry))) { -+ list_del_init(&timer->cb_entry); -+ goto out; -+ } ++#include + - next_timer = timerqueue_getnext(&base->active); - timerqueue_del(&base->active, &timer->node); - if (&timer->node == next_timer) { -@@ -966,7 +1061,16 @@ - new_base = switch_hrtimer_base(timer, base, mode & HRTIMER_MODE_PINNED); - - timer_stats_hrtimer_set_start_info(timer); -+#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST -+ { -+ ktime_t now = new_base->get_time(); - -+ if (ktime_to_ns(tim) < ktime_to_ns(now)) -+ timer->praecox = now; -+ else -+ timer->praecox = ktime_set(0, 0); -+ } -+#endif - leftmost = enqueue_hrtimer(timer, new_base); - - if (!leftmost) { -@@ -980,15 +1084,26 @@ - * on dynticks target. - */ - wake_up_nohz_cpu(new_base->cpu_base->cpu); -- } else if (new_base->cpu_base == this_cpu_ptr(&hrtimer_bases) && -- hrtimer_reprogram(timer, new_base)) { -+ } else if (new_base->cpu_base == this_cpu_ptr(&hrtimer_bases)) { ++extern void ++__rt_spin_lock_init(spinlock_t *lock, char *name, struct lock_class_key *key); + -+ ret = hrtimer_enqueue_reprogram(timer, new_base, wakeup); -+ if (ret < 0) { -+ /* -+ * In case we failed to reprogram the timer (mostly -+ * because out current timer is already elapsed), -+ * remove it again and report a failure. This avoids -+ * stale base->first entries. -+ */ -+ debug_deactivate(timer); -+ __remove_hrtimer(timer, new_base, -+ timer->state & HRTIMER_STATE_CALLBACK, 0); -+ } else if (ret > 0) { - /* - * Only allow reprogramming if the new base is on this CPU. - * (it might still be on another CPU if the timer was pending) - * - * XXX send_remote_softirq() ? - */ -- if (wakeup) { - /* - * We need to drop cpu_base->lock to avoid a - * lock ordering issue vs. rq->lock. -@@ -996,9 +1111,7 @@ - raw_spin_unlock(&new_base->cpu_base->lock); - raise_softirq_irqoff(HRTIMER_SOFTIRQ); - local_irq_restore(flags); -- return ret; -- } else { -- __raise_softirq_irqoff(HRTIMER_SOFTIRQ); -+ return 0; - } - } - -@@ -1089,7 +1202,7 @@ - - if (ret >= 0) - return ret; -- cpu_relax(); -+ hrtimer_wait_for_timer(timer); - } - } - EXPORT_SYMBOL_GPL(hrtimer_cancel); -@@ -1153,6 +1266,7 @@ - - base = hrtimer_clockid_to_base(clock_id); - timer->base = &cpu_base->clock_base[base]; -+ INIT_LIST_HEAD(&timer->cb_entry); - timerqueue_init(&timer->node); - - #ifdef CONFIG_TIMER_STATS -@@ -1236,6 +1350,126 @@ - timer->state &= ~HRTIMER_STATE_CALLBACK; - } - -+static enum hrtimer_restart hrtimer_wakeup(struct hrtimer *timer); ++#define spin_lock_init(slock) \ ++do { \ ++ static struct lock_class_key __key; \ ++ \ ++ rt_mutex_init(&(slock)->lock); \ ++ __rt_spin_lock_init(slock, #slock, &__key); \ ++} while (0) + -+#ifdef CONFIG_PREEMPT_RT_BASE -+static void hrtimer_rt_reprogram(int restart, struct hrtimer *timer, -+ struct hrtimer_clock_base *base) -+{ -+ /* -+ * Note, we clear the callback flag before we requeue the -+ * timer otherwise we trigger the callback_running() check -+ * in hrtimer_reprogram(). -+ */ -+ timer->state &= ~HRTIMER_STATE_CALLBACK; ++extern void __lockfunc rt_spin_lock(spinlock_t *lock); ++extern unsigned long __lockfunc rt_spin_lock_trace_flags(spinlock_t *lock); ++extern void __lockfunc rt_spin_lock_nested(spinlock_t *lock, int subclass); ++extern void __lockfunc rt_spin_unlock(spinlock_t *lock); ++extern void __lockfunc rt_spin_unlock_wait(spinlock_t *lock); ++extern int __lockfunc rt_spin_trylock_irqsave(spinlock_t *lock, unsigned long *flags); ++extern int __lockfunc rt_spin_trylock_bh(spinlock_t *lock); ++extern int __lockfunc rt_spin_trylock(spinlock_t *lock); ++extern int atomic_dec_and_spin_lock(atomic_t *atomic, spinlock_t *lock); + -+ if (restart != HRTIMER_NORESTART) { -+ BUG_ON(hrtimer_active(timer)); -+ /* -+ * Enqueue the timer, if it's the leftmost timer then -+ * we need to reprogram it. -+ */ -+ if (!enqueue_hrtimer(timer, base)) -+ return; ++/* ++ * lockdep-less calls, for derived types like rwlock: ++ * (for trylock they can use rt_mutex_trylock() directly. ++ */ ++extern void __lockfunc __rt_spin_lock(struct rt_mutex *lock); ++extern void __lockfunc __rt_spin_unlock(struct rt_mutex *lock); ++extern int __lockfunc __rt_spin_trylock(struct rt_mutex *lock); + -+#ifndef CONFIG_HIGH_RES_TIMERS -+ } ++#define spin_lock(lock) \ ++ do { \ ++ migrate_disable(); \ ++ rt_spin_lock(lock); \ ++ } while (0) ++ ++#define spin_lock_bh(lock) \ ++ do { \ ++ local_bh_disable(); \ ++ migrate_disable(); \ ++ rt_spin_lock(lock); \ ++ } while (0) ++ ++#define spin_lock_irq(lock) spin_lock(lock) ++ ++#define spin_do_trylock(lock) __cond_lock(lock, rt_spin_trylock(lock)) ++ ++#define spin_trylock(lock) \ ++({ \ ++ int __locked; \ ++ migrate_disable(); \ ++ __locked = spin_do_trylock(lock); \ ++ if (!__locked) \ ++ migrate_enable(); \ ++ __locked; \ ++}) ++ ++#ifdef CONFIG_LOCKDEP ++# define spin_lock_nested(lock, subclass) \ ++ do { \ ++ migrate_disable(); \ ++ rt_spin_lock_nested(lock, subclass); \ ++ } while (0) ++ ++#define spin_lock_bh_nested(lock, subclass) \ ++ do { \ ++ local_bh_disable(); \ ++ migrate_disable(); \ ++ rt_spin_lock_nested(lock, subclass); \ ++ } while (0) ++ ++# define spin_lock_irqsave_nested(lock, flags, subclass) \ ++ do { \ ++ typecheck(unsigned long, flags); \ ++ flags = 0; \ ++ migrate_disable(); \ ++ rt_spin_lock_nested(lock, subclass); \ ++ } while (0) +#else -+ if (base->cpu_base->hres_active && -+ hrtimer_reprogram(timer, base)) -+ goto requeue; ++# define spin_lock_nested(lock, subclass) spin_lock(lock) ++# define spin_lock_bh_nested(lock, subclass) spin_lock_bh(lock) + -+ } else if (hrtimer_active(timer)) { -+ /* -+ * If the timer was rearmed on another CPU, reprogram -+ * the event device. -+ */ -+ if (&timer->node == base->active.next && -+ base->cpu_base->hres_active && -+ hrtimer_reprogram(timer, base)) -+ goto requeue; -+ } -+ return; ++# define spin_lock_irqsave_nested(lock, flags, subclass) \ ++ do { \ ++ typecheck(unsigned long, flags); \ ++ flags = 0; \ ++ spin_lock(lock); \ ++ } while (0) ++#endif + -+requeue: -+ /* -+ * Timer is expired. Thus move it from tree to pending list -+ * again. -+ */ -+ __remove_hrtimer(timer, base, timer->state, 0); -+ list_add_tail(&timer->cb_entry, &base->expired); ++#define spin_lock_irqsave(lock, flags) \ ++ do { \ ++ typecheck(unsigned long, flags); \ ++ flags = 0; \ ++ spin_lock(lock); \ ++ } while (0) ++ ++static inline unsigned long spin_lock_trace_flags(spinlock_t *lock) ++{ ++ unsigned long flags = 0; ++#ifdef CONFIG_TRACE_IRQFLAGS ++ flags = rt_spin_lock_trace_flags(lock); ++#else ++ spin_lock(lock); /* lock_local */ +#endif ++ return flags; +} + -+/* -+ * The changes in mainline which removed the callback modes from -+ * hrtimer are not yet working with -rt. The non wakeup_process() -+ * based callbacks which involve sleeping locks need to be treated -+ * seperately. -+ */ -+static void hrtimer_rt_run_pending(void) -+{ -+ enum hrtimer_restart (*fn)(struct hrtimer *); -+ struct hrtimer_cpu_base *cpu_base; -+ struct hrtimer_clock_base *base; -+ struct hrtimer *timer; -+ int index, restart; ++/* FIXME: we need rt_spin_lock_nest_lock */ ++#define spin_lock_nest_lock(lock, nest_lock) spin_lock_nested(lock, 0) + -+ local_irq_disable(); -+ cpu_base = &per_cpu(hrtimer_bases, smp_processor_id()); ++#define spin_unlock(lock) \ ++ do { \ ++ rt_spin_unlock(lock); \ ++ migrate_enable(); \ ++ } while (0) + -+ raw_spin_lock(&cpu_base->lock); ++#define spin_unlock_bh(lock) \ ++ do { \ ++ rt_spin_unlock(lock); \ ++ migrate_enable(); \ ++ local_bh_enable(); \ ++ } while (0) + -+ for (index = 0; index < HRTIMER_MAX_CLOCK_BASES; index++) { -+ base = &cpu_base->clock_base[index]; ++#define spin_unlock_irq(lock) spin_unlock(lock) + -+ while (!list_empty(&base->expired)) { -+ timer = list_first_entry(&base->expired, -+ struct hrtimer, cb_entry); ++#define spin_unlock_irqrestore(lock, flags) \ ++ do { \ ++ typecheck(unsigned long, flags); \ ++ (void) flags; \ ++ spin_unlock(lock); \ ++ } while (0) + -+ /* -+ * Same as the above __run_hrtimer function -+ * just we run with interrupts enabled. -+ */ -+ debug_hrtimer_deactivate(timer); -+ __remove_hrtimer(timer, base, HRTIMER_STATE_CALLBACK, 0); -+ timer_stats_account_hrtimer(timer); -+ fn = timer->function; ++#define spin_trylock_bh(lock) __cond_lock(lock, rt_spin_trylock_bh(lock)) ++#define spin_trylock_irq(lock) spin_trylock(lock) + -+ raw_spin_unlock_irq(&cpu_base->lock); -+ restart = fn(timer); -+ raw_spin_lock_irq(&cpu_base->lock); ++#define spin_trylock_irqsave(lock, flags) \ ++ rt_spin_trylock_irqsave(lock, &(flags)) + -+ hrtimer_rt_reprogram(restart, timer, base); -+ } -+ } ++#define spin_unlock_wait(lock) rt_spin_unlock_wait(lock) + -+ raw_spin_unlock_irq(&cpu_base->lock); ++#ifdef CONFIG_GENERIC_LOCKBREAK ++# define spin_is_contended(lock) ((lock)->break_lock) ++#else ++# define spin_is_contended(lock) (((void)(lock), 0)) ++#endif + -+ wake_up_timer_waiters(cpu_base); ++static inline int spin_can_lock(spinlock_t *lock) ++{ ++ return !rt_mutex_is_locked(&lock->lock); +} + -+static int hrtimer_rt_defer(struct hrtimer *timer) ++static inline int spin_is_locked(spinlock_t *lock) +{ -+ if (timer->irqsafe) -+ return 0; -+ -+ __remove_hrtimer(timer, timer->base, timer->state, 0); -+ list_add_tail(&timer->cb_entry, &timer->base->expired); -+ return 1; ++ return rt_mutex_is_locked(&lock->lock); +} + -+#else -+ -+static inline void hrtimer_rt_run_pending(void) ++static inline void assert_spin_locked(spinlock_t *lock) +{ -+ hrtimer_peek_ahead_timers(); ++ BUG_ON(!spin_is_locked(lock)); +} + -+static inline int hrtimer_rt_defer(struct hrtimer *timer) { return 0; } ++#define atomic_dec_and_lock(atomic, lock) \ ++ atomic_dec_and_spin_lock(atomic, lock) + +#endif -+ - #ifdef CONFIG_HIGH_RES_TIMERS - - /* -@@ -1246,7 +1480,7 @@ - { - struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases); - ktime_t expires_next, now, entry_time, delta; -- int i, retries = 0; -+ int i, retries = 0, raise = 0; - - BUG_ON(!cpu_base->hres_active); - cpu_base->nr_events++; -@@ -1281,6 +1515,15 @@ - - timer = container_of(node, struct hrtimer, node); - -+ trace_hrtimer_interrupt(raw_smp_processor_id(), -+ ktime_to_ns(ktime_sub(ktime_to_ns(timer->praecox) ? -+ timer->praecox : hrtimer_get_expires(timer), -+ basenow)), -+ current, -+ timer->function == hrtimer_wakeup ? -+ container_of(timer, struct hrtimer_sleeper, -+ timer)->task : NULL); -+ - /* - * The immediate goal for using the softexpires is - * minimizing wakeups, not running timers at the -@@ -1296,7 +1539,10 @@ - if (basenow.tv64 < hrtimer_get_softexpires_tv64(timer)) - break; - -- __run_hrtimer(timer, &basenow); -+ if (!hrtimer_rt_defer(timer)) -+ __run_hrtimer(timer, &basenow); -+ else -+ raise = 1; - } - } - /* Reevaluate the clock bases for the next expiry */ -@@ -1313,7 +1559,7 @@ - if (expires_next.tv64 == KTIME_MAX || - !tick_program_event(expires_next, 0)) { - cpu_base->hang_detected = 0; -- return; -+ goto out; - } - - /* -@@ -1357,6 +1603,9 @@ - tick_program_event(expires_next, 1); - printk_once(KERN_WARNING "hrtimer: interrupt took %llu ns\n", - ktime_to_ns(delta)); -+out: -+ if (raise) -+ raise_softirq_irqoff(HRTIMER_SOFTIRQ); - } +diff -Nur linux-4.1.10.orig/include/linux/spinlock_types.h linux-4.1.10/include/linux/spinlock_types.h +--- linux-4.1.10.orig/include/linux/spinlock_types.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/spinlock_types.h 2015-10-07 18:00:08.000000000 +0200 +@@ -9,80 +9,15 @@ + * Released under the General Public License (GPL). + */ - /* -@@ -1392,18 +1641,18 @@ - __hrtimer_peek_ahead_timers(); - local_irq_restore(flags); - } +-#if defined(CONFIG_SMP) +-# include +-#else +-# include +-#endif - --static void run_hrtimer_softirq(struct softirq_action *h) --{ -- hrtimer_peek_ahead_timers(); --} +-#include - - #else /* CONFIG_HIGH_RES_TIMERS */ - - static inline void __hrtimer_peek_ahead_timers(void) { } - - #endif /* !CONFIG_HIGH_RES_TIMERS */ - -+ -+static void run_hrtimer_softirq(struct softirq_action *h) -+{ -+ hrtimer_rt_run_pending(); -+} -+ - /* - * Called from timer softirq every jiffy, expire hrtimers: - * -@@ -1436,7 +1685,7 @@ - struct timerqueue_node *node; - struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases); - struct hrtimer_clock_base *base; -- int index, gettime = 1; -+ int index, gettime = 1, raise = 0; +-typedef struct raw_spinlock { +- arch_spinlock_t raw_lock; +-#ifdef CONFIG_GENERIC_LOCKBREAK +- unsigned int break_lock; +-#endif +-#ifdef CONFIG_DEBUG_SPINLOCK +- unsigned int magic, owner_cpu; +- void *owner; +-#endif +-#ifdef CONFIG_DEBUG_LOCK_ALLOC +- struct lockdep_map dep_map; +-#endif +-} raw_spinlock_t; +- +-#define SPINLOCK_MAGIC 0xdead4ead +- +-#define SPINLOCK_OWNER_INIT ((void *)-1L) +- +-#ifdef CONFIG_DEBUG_LOCK_ALLOC +-# define SPIN_DEP_MAP_INIT(lockname) .dep_map = { .name = #lockname } +-#else +-# define SPIN_DEP_MAP_INIT(lockname) +-#endif ++#include - if (hrtimer_hres_active()) - return; -@@ -1461,10 +1710,16 @@ - hrtimer_get_expires_tv64(timer)) - break; +-#ifdef CONFIG_DEBUG_SPINLOCK +-# define SPIN_DEBUG_INIT(lockname) \ +- .magic = SPINLOCK_MAGIC, \ +- .owner_cpu = -1, \ +- .owner = SPINLOCK_OWNER_INIT, ++#ifndef CONFIG_PREEMPT_RT_FULL ++# include ++# include + #else +-# define SPIN_DEBUG_INIT(lockname) ++# include ++# include ++# include + #endif -- __run_hrtimer(timer, &base->softirq_time); -+ if (!hrtimer_rt_defer(timer)) -+ __run_hrtimer(timer, &base->softirq_time); -+ else -+ raise = 1; - } - raw_spin_unlock(&cpu_base->lock); - } +-#define __RAW_SPIN_LOCK_INITIALIZER(lockname) \ +- { \ +- .raw_lock = __ARCH_SPIN_LOCK_UNLOCKED, \ +- SPIN_DEBUG_INIT(lockname) \ +- SPIN_DEP_MAP_INIT(lockname) } +- +-#define __RAW_SPIN_LOCK_UNLOCKED(lockname) \ +- (raw_spinlock_t) __RAW_SPIN_LOCK_INITIALIZER(lockname) +- +-#define DEFINE_RAW_SPINLOCK(x) raw_spinlock_t x = __RAW_SPIN_LOCK_UNLOCKED(x) +- +-typedef struct spinlock { +- union { +- struct raw_spinlock rlock; +- +-#ifdef CONFIG_DEBUG_LOCK_ALLOC +-# define LOCK_PADSIZE (offsetof(struct raw_spinlock, dep_map)) +- struct { +- u8 __padding[LOCK_PADSIZE]; +- struct lockdep_map dep_map; +- }; +-#endif +- }; +-} spinlock_t; +- +-#define __SPIN_LOCK_INITIALIZER(lockname) \ +- { { .rlock = __RAW_SPIN_LOCK_INITIALIZER(lockname) } } +- +-#define __SPIN_LOCK_UNLOCKED(lockname) \ +- (spinlock_t ) __SPIN_LOCK_INITIALIZER(lockname) +- +-#define DEFINE_SPINLOCK(x) spinlock_t x = __SPIN_LOCK_UNLOCKED(x) +- +-#include +- + #endif /* __LINUX_SPINLOCK_TYPES_H */ +diff -Nur linux-4.1.10.orig/include/linux/spinlock_types_nort.h linux-4.1.10/include/linux/spinlock_types_nort.h +--- linux-4.1.10.orig/include/linux/spinlock_types_nort.h 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/include/linux/spinlock_types_nort.h 2015-10-07 18:00:08.000000000 +0200 +@@ -0,0 +1,33 @@ ++#ifndef __LINUX_SPINLOCK_TYPES_NORT_H ++#define __LINUX_SPINLOCK_TYPES_NORT_H + -+ if (raise) -+ raise_softirq_irqoff(HRTIMER_SOFTIRQ); - } - - /* -@@ -1486,16 +1741,18 @@ - void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, struct task_struct *task) - { - sl->timer.function = hrtimer_wakeup; -+ sl->timer.irqsafe = 1; - sl->task = task; - } - EXPORT_SYMBOL_GPL(hrtimer_init_sleeper); - --static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mode) -+static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mode, -+ unsigned long state) - { - hrtimer_init_sleeper(t, current); - - do { -- set_current_state(TASK_INTERRUPTIBLE); -+ set_current_state(state); - hrtimer_start_expires(&t->timer, mode); - if (!hrtimer_active(&t->timer)) - t->task = NULL; -@@ -1539,7 +1796,8 @@ - HRTIMER_MODE_ABS); - hrtimer_set_expires_tv64(&t.timer, restart->nanosleep.expires); - -- if (do_nanosleep(&t, HRTIMER_MODE_ABS)) -+ /* cpu_chill() does not care about restart state. */ -+ if (do_nanosleep(&t, HRTIMER_MODE_ABS, TASK_INTERRUPTIBLE)) - goto out; - - rmtp = restart->nanosleep.rmtp; -@@ -1556,8 +1814,10 @@ - return ret; - } - --long hrtimer_nanosleep(struct timespec *rqtp, struct timespec __user *rmtp, -- const enum hrtimer_mode mode, const clockid_t clockid) -+static long -+__hrtimer_nanosleep(struct timespec *rqtp, struct timespec __user *rmtp, -+ const enum hrtimer_mode mode, const clockid_t clockid, -+ unsigned long state) - { - struct restart_block *restart; - struct hrtimer_sleeper t; -@@ -1570,7 +1830,7 @@ - - hrtimer_init_on_stack(&t.timer, clockid, mode); - hrtimer_set_expires_range_ns(&t.timer, timespec_to_ktime(*rqtp), slack); -- if (do_nanosleep(&t, mode)) -+ if (do_nanosleep(&t, mode, state)) - goto out; - - /* Absolute timers do not update the rmtp value and restart: */ -@@ -1597,6 +1857,12 @@ - return ret; - } - -+long hrtimer_nanosleep(struct timespec *rqtp, struct timespec __user *rmtp, -+ const enum hrtimer_mode mode, const clockid_t clockid) -+{ -+ return __hrtimer_nanosleep(rqtp, rmtp, mode, clockid, TASK_INTERRUPTIBLE); -+} ++#ifndef __LINUX_SPINLOCK_TYPES_H ++#error "Do not include directly. Include spinlock_types.h instead" ++#endif + - SYSCALL_DEFINE2(nanosleep, struct timespec __user *, rqtp, - struct timespec __user *, rmtp) - { -@@ -1611,6 +1877,26 @@ - return hrtimer_nanosleep(&tu, rmtp, HRTIMER_MODE_REL, CLOCK_MONOTONIC); - } - -+#ifdef CONFIG_PREEMPT_RT_FULL +/* -+ * Sleep for 1 ms in hope whoever holds what we want will let it go. ++ * The non RT version maps spinlocks to raw_spinlocks + */ -+void cpu_chill(void) -+{ -+ struct timespec tu = { -+ .tv_nsec = NSEC_PER_MSEC, -+ }; -+ unsigned int freeze_flag = current->flags & PF_NOFREEZE; ++typedef struct spinlock { ++ union { ++ struct raw_spinlock rlock; + -+ current->flags |= PF_NOFREEZE; -+ __hrtimer_nanosleep(&tu, NULL, HRTIMER_MODE_REL, CLOCK_MONOTONIC, -+ TASK_UNINTERRUPTIBLE); -+ if (!freeze_flag) -+ current->flags &= ~PF_NOFREEZE; -+} -+EXPORT_SYMBOL(cpu_chill); ++#ifdef CONFIG_DEBUG_LOCK_ALLOC ++# define LOCK_PADSIZE (offsetof(struct raw_spinlock, dep_map)) ++ struct { ++ u8 __padding[LOCK_PADSIZE]; ++ struct lockdep_map dep_map; ++ }; +#endif ++ }; ++} spinlock_t; ++ ++#define __SPIN_LOCK_INITIALIZER(lockname) \ ++ { { .rlock = __RAW_SPIN_LOCK_INITIALIZER(lockname) } } ++ ++#define __SPIN_LOCK_UNLOCKED(lockname) \ ++ (spinlock_t ) __SPIN_LOCK_INITIALIZER(lockname) ++ ++#define DEFINE_SPINLOCK(x) spinlock_t x = __SPIN_LOCK_UNLOCKED(x) + - /* - * Functions related to boot-time initialization: - */ -@@ -1622,10 +1908,14 @@ - for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) { - cpu_base->clock_base[i].cpu_base = cpu_base; - timerqueue_init_head(&cpu_base->clock_base[i].active); -+ INIT_LIST_HEAD(&cpu_base->clock_base[i].expired); - } - - cpu_base->cpu = cpu; - hrtimer_init_hres(cpu_base); -+#ifdef CONFIG_PREEMPT_RT_BASE -+ init_waitqueue_head(&cpu_base->wait); +#endif - } +diff -Nur linux-4.1.10.orig/include/linux/spinlock_types_raw.h linux-4.1.10/include/linux/spinlock_types_raw.h +--- linux-4.1.10.orig/include/linux/spinlock_types_raw.h 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/include/linux/spinlock_types_raw.h 2015-10-07 18:00:08.000000000 +0200 +@@ -0,0 +1,56 @@ ++#ifndef __LINUX_SPINLOCK_TYPES_RAW_H ++#define __LINUX_SPINLOCK_TYPES_RAW_H ++ ++#if defined(CONFIG_SMP) ++# include ++#else ++# include ++#endif ++ ++#include ++ ++typedef struct raw_spinlock { ++ arch_spinlock_t raw_lock; ++#ifdef CONFIG_GENERIC_LOCKBREAK ++ unsigned int break_lock; ++#endif ++#ifdef CONFIG_DEBUG_SPINLOCK ++ unsigned int magic, owner_cpu; ++ void *owner; ++#endif ++#ifdef CONFIG_DEBUG_LOCK_ALLOC ++ struct lockdep_map dep_map; ++#endif ++} raw_spinlock_t; ++ ++#define SPINLOCK_MAGIC 0xdead4ead ++ ++#define SPINLOCK_OWNER_INIT ((void *)-1L) ++ ++#ifdef CONFIG_DEBUG_LOCK_ALLOC ++# define SPIN_DEP_MAP_INIT(lockname) .dep_map = { .name = #lockname } ++#else ++# define SPIN_DEP_MAP_INIT(lockname) ++#endif ++ ++#ifdef CONFIG_DEBUG_SPINLOCK ++# define SPIN_DEBUG_INIT(lockname) \ ++ .magic = SPINLOCK_MAGIC, \ ++ .owner_cpu = -1, \ ++ .owner = SPINLOCK_OWNER_INIT, ++#else ++# define SPIN_DEBUG_INIT(lockname) ++#endif ++ ++#define __RAW_SPIN_LOCK_INITIALIZER(lockname) \ ++ { \ ++ .raw_lock = __ARCH_SPIN_LOCK_UNLOCKED, \ ++ SPIN_DEBUG_INIT(lockname) \ ++ SPIN_DEP_MAP_INIT(lockname) } ++ ++#define __RAW_SPIN_LOCK_UNLOCKED(lockname) \ ++ (raw_spinlock_t) __RAW_SPIN_LOCK_INITIALIZER(lockname) ++ ++#define DEFINE_RAW_SPINLOCK(x) raw_spinlock_t x = __RAW_SPIN_LOCK_UNLOCKED(x) ++ ++#endif +diff -Nur linux-4.1.10.orig/include/linux/spinlock_types_rt.h linux-4.1.10/include/linux/spinlock_types_rt.h +--- linux-4.1.10.orig/include/linux/spinlock_types_rt.h 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/include/linux/spinlock_types_rt.h 2015-10-07 18:00:08.000000000 +0200 +@@ -0,0 +1,51 @@ ++#ifndef __LINUX_SPINLOCK_TYPES_RT_H ++#define __LINUX_SPINLOCK_TYPES_RT_H ++ ++#ifndef __LINUX_SPINLOCK_TYPES_H ++#error "Do not include directly. Include spinlock_types.h instead" ++#endif ++ ++#include ++ ++/* ++ * PREEMPT_RT: spinlocks - an RT mutex plus lock-break field: ++ */ ++typedef struct spinlock { ++ struct rt_mutex lock; ++ unsigned int break_lock; ++#ifdef CONFIG_DEBUG_LOCK_ALLOC ++ struct lockdep_map dep_map; ++#endif ++} spinlock_t; ++ ++#ifdef CONFIG_DEBUG_RT_MUTEXES ++# define __RT_SPIN_INITIALIZER(name) \ ++ { \ ++ .wait_lock = __RAW_SPIN_LOCK_UNLOCKED(name.wait_lock), \ ++ .save_state = 1, \ ++ .file = __FILE__, \ ++ .line = __LINE__ , \ ++ } ++#else ++# define __RT_SPIN_INITIALIZER(name) \ ++ { \ ++ .wait_lock = __RAW_SPIN_LOCK_UNLOCKED(name.wait_lock), \ ++ .save_state = 1, \ ++ } ++#endif ++ ++/* ++.wait_list = PLIST_HEAD_INIT_RAW((name).lock.wait_list, (name).lock.wait_lock) ++*/ ++ ++#define __SPIN_LOCK_UNLOCKED(name) \ ++ { .lock = __RT_SPIN_INITIALIZER(name.lock), \ ++ SPIN_DEP_MAP_INIT(name) } ++ ++#define __DEFINE_SPINLOCK(name) \ ++ spinlock_t name = __SPIN_LOCK_UNLOCKED(name) ++ ++#define DEFINE_SPINLOCK(name) \ ++ spinlock_t name __cacheline_aligned_in_smp = __SPIN_LOCK_UNLOCKED(name) ++ ++#endif +diff -Nur linux-4.1.10.orig/include/linux/srcu.h linux-4.1.10/include/linux/srcu.h +--- linux-4.1.10.orig/include/linux/srcu.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/srcu.h 2015-10-07 18:00:08.000000000 +0200 +@@ -84,10 +84,10 @@ - #ifdef CONFIG_HOTPLUG_CPU -@@ -1731,9 +2021,7 @@ - hrtimer_cpu_notify(&hrtimers_nb, (unsigned long)CPU_UP_PREPARE, - (void *)(long)smp_processor_id()); - register_cpu_notifier(&hrtimers_nb); --#ifdef CONFIG_HIGH_RES_TIMERS - open_softirq(HRTIMER_SOFTIRQ, run_hrtimer_softirq); --#endif - } + void process_srcu(struct work_struct *work); - /** -diff -Nur linux-4.1.6.orig/kernel/time/itimer.c linux-4.1.6/kernel/time/itimer.c ---- linux-4.1.6.orig/kernel/time/itimer.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/time/itimer.c 2015-09-08 23:49:08.117866054 +0200 -@@ -213,6 +213,7 @@ - /* We are sharing ->siglock with it_real_fn() */ - if (hrtimer_try_to_cancel(timer) < 0) { - spin_unlock_irq(&tsk->sighand->siglock); -+ hrtimer_wait_for_timer(&tsk->signal->real_timer); - goto again; - } - expires = timeval_to_ktime(value->it_value); -diff -Nur linux-4.1.6.orig/kernel/time/jiffies.c linux-4.1.6/kernel/time/jiffies.c ---- linux-4.1.6.orig/kernel/time/jiffies.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/time/jiffies.c 2015-09-08 23:49:08.117866054 +0200 -@@ -74,7 +74,8 @@ - .max_cycles = 10, - }; +-#define __SRCU_STRUCT_INIT(name) \ ++#define __SRCU_STRUCT_INIT(name, pcpu_name) \ + { \ + .completed = -300, \ +- .per_cpu_ref = &name##_srcu_array, \ ++ .per_cpu_ref = &pcpu_name, \ + .queue_lock = __SPIN_LOCK_UNLOCKED(name.queue_lock), \ + .running = false, \ + .batch_queue = RCU_BATCH_INIT(name.batch_queue), \ +@@ -104,7 +104,7 @@ + */ + #define __DEFINE_SRCU(name, is_static) \ + static DEFINE_PER_CPU(struct srcu_struct_array, name##_srcu_array);\ +- is_static struct srcu_struct name = __SRCU_STRUCT_INIT(name) ++ is_static struct srcu_struct name = __SRCU_STRUCT_INIT(name, name##_srcu_array) + #define DEFINE_SRCU(name) __DEFINE_SRCU(name, /* not static */) + #define DEFINE_STATIC_SRCU(name) __DEFINE_SRCU(name, static) --__cacheline_aligned_in_smp DEFINE_SEQLOCK(jiffies_lock); -+__cacheline_aligned_in_smp DEFINE_RAW_SPINLOCK(jiffies_lock); -+__cacheline_aligned_in_smp seqcount_t jiffies_seq; +diff -Nur linux-4.1.10.orig/include/linux/swap.h linux-4.1.10/include/linux/swap.h +--- linux-4.1.10.orig/include/linux/swap.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/swap.h 2015-10-07 18:00:08.000000000 +0200 +@@ -11,6 +11,7 @@ + #include + #include + #include ++#include + #include - #if (BITS_PER_LONG < 64) - u64 get_jiffies_64(void) -@@ -83,9 +84,9 @@ - u64 ret; + struct notifier_block; +@@ -252,7 +253,8 @@ + void *workingset_eviction(struct address_space *mapping, struct page *page); + bool workingset_refault(void *shadow); + void workingset_activation(struct page *page); +-extern struct list_lru workingset_shadow_nodes; ++extern struct list_lru __workingset_shadow_nodes; ++DECLARE_LOCAL_IRQ_LOCK(workingset_shadow_lock); - do { -- seq = read_seqbegin(&jiffies_lock); -+ seq = read_seqcount_begin(&jiffies_seq); - ret = jiffies_64; -- } while (read_seqretry(&jiffies_lock, seq)); -+ } while (read_seqcount_retry(&jiffies_seq, seq)); - return ret; - } - EXPORT_SYMBOL(get_jiffies_64); -diff -Nur linux-4.1.6.orig/kernel/time/ntp.c linux-4.1.6/kernel/time/ntp.c ---- linux-4.1.6.orig/kernel/time/ntp.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/time/ntp.c 2015-09-08 23:49:08.117866054 +0200 -@@ -10,6 +10,7 @@ - #include - #include - #include -+#include - #include - #include - #include -@@ -529,10 +530,52 @@ - &sync_cmos_work, timespec_to_jiffies(&next)); - } + static inline unsigned int workingset_node_pages(struct radix_tree_node *node) + { +@@ -296,6 +298,7 @@ -+#ifdef CONFIG_PREEMPT_RT_FULL -+/* -+ * RT can not call schedule_delayed_work from real interrupt context. -+ * Need to make a thread to do the real work. -+ */ -+static struct task_struct *cmos_delay_thread; -+static bool do_cmos_delay; -+ -+static int run_cmos_delay(void *ignore) -+{ -+ while (!kthread_should_stop()) { -+ set_current_state(TASK_INTERRUPTIBLE); -+ if (do_cmos_delay) { -+ do_cmos_delay = false; -+ queue_delayed_work(system_power_efficient_wq, -+ &sync_cmos_work, 0); -+ } -+ schedule(); -+ } -+ __set_current_state(TASK_RUNNING); -+ return 0; -+} + + /* linux/mm/swap.c */ ++DECLARE_LOCAL_IRQ_LOCK(swapvec_lock); + extern void lru_cache_add(struct page *); + extern void lru_cache_add_anon(struct page *page); + extern void lru_cache_add_file(struct page *page); +diff -Nur linux-4.1.10.orig/include/linux/thread_info.h linux-4.1.10/include/linux/thread_info.h +--- linux-4.1.10.orig/include/linux/thread_info.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/thread_info.h 2015-10-07 18:00:08.000000000 +0200 +@@ -102,7 +102,17 @@ + #define test_thread_flag(flag) \ + test_ti_thread_flag(current_thread_info(), flag) + +-#define tif_need_resched() test_thread_flag(TIF_NEED_RESCHED) ++#ifdef CONFIG_PREEMPT_LAZY ++#define tif_need_resched() (test_thread_flag(TIF_NEED_RESCHED) || \ ++ test_thread_flag(TIF_NEED_RESCHED_LAZY)) ++#define tif_need_resched_now() (test_thread_flag(TIF_NEED_RESCHED)) ++#define tif_need_resched_lazy() test_thread_flag(TIF_NEED_RESCHED_LAZY)) + -+void ntp_notify_cmos_timer(void) ++#else ++#define tif_need_resched() test_thread_flag(TIF_NEED_RESCHED) ++#define tif_need_resched_now() test_thread_flag(TIF_NEED_RESCHED) ++#define tif_need_resched_lazy() 0 ++#endif + + #if defined TIF_RESTORE_SIGMASK && !defined HAVE_SET_RESTORE_SIGMASK + /* +diff -Nur linux-4.1.10.orig/include/linux/timer.h linux-4.1.10/include/linux/timer.h +--- linux-4.1.10.orig/include/linux/timer.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/timer.h 2015-10-07 18:00:08.000000000 +0200 +@@ -241,7 +241,7 @@ + + extern int try_to_del_timer_sync(struct timer_list *timer); + +-#ifdef CONFIG_SMP ++#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT_FULL) + extern int del_timer_sync(struct timer_list *timer); + #else + # define del_timer_sync(t) del_timer(t) +diff -Nur linux-4.1.10.orig/include/linux/uaccess.h linux-4.1.10/include/linux/uaccess.h +--- linux-4.1.10.orig/include/linux/uaccess.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/uaccess.h 2015-10-07 18:00:08.000000000 +0200 +@@ -1,21 +1,31 @@ + #ifndef __LINUX_UACCESS_H__ + #define __LINUX_UACCESS_H__ + +-#include ++#include + #include + ++static __always_inline void pagefault_disabled_inc(void) +{ -+ do_cmos_delay = true; -+ /* Make visible before waking up process */ -+ smp_wmb(); -+ wake_up_process(cmos_delay_thread); ++ current->pagefault_disabled++; +} + -+static __init int create_cmos_delay_thread(void) ++static __always_inline void pagefault_disabled_dec(void) +{ -+ cmos_delay_thread = kthread_run(run_cmos_delay, NULL, "kcmosdelayd"); -+ BUG_ON(!cmos_delay_thread); -+ return 0; ++ current->pagefault_disabled--; ++ WARN_ON(current->pagefault_disabled < 0); +} -+early_initcall(create_cmos_delay_thread); -+ -+#else + - void ntp_notify_cmos_timer(void) - { - queue_delayed_work(system_power_efficient_wq, &sync_cmos_work, 0); - } -+#endif /* CONFIG_PREEMPT_RT_FULL */ - - #else - void ntp_notify_cmos_timer(void) { } -diff -Nur linux-4.1.6.orig/kernel/time/posix-cpu-timers.c linux-4.1.6/kernel/time/posix-cpu-timers.c ---- linux-4.1.6.orig/kernel/time/posix-cpu-timers.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/time/posix-cpu-timers.c 2015-09-08 23:49:08.117866054 +0200 -@@ -3,6 +3,7 @@ + /* +- * These routines enable/disable the pagefault handler in that +- * it will not take any locks and go straight to the fixup table. ++ * These routines enable/disable the pagefault handler. If disabled, it will ++ * not take any locks and go straight to the fixup table. + * +- * They have great resemblance to the preempt_disable/enable calls +- * and in fact they are identical; this is because currently there is +- * no other way to make the pagefault handlers do this. So we do +- * disable preemption but we don't necessarily care about that. ++ * User access methods will not sleep when called from a pagefault_disabled() ++ * environment. */ - - #include -+#include - #include - #include - #include -@@ -626,7 +627,7 @@ + static inline void pagefault_disable(void) + { +- preempt_count_inc(); ++ migrate_disable(); ++ pagefault_disabled_inc(); /* - * Disarm any old timer after extracting its expiry time. - */ -- WARN_ON_ONCE(!irqs_disabled()); -+ WARN_ON_ONCE_NONRT(!irqs_disabled()); + * make sure to have issued the store before a pagefault + * can hit. +@@ -25,18 +35,32 @@ - ret = 0; - old_incr = timer->it.cpu.incr; -@@ -1047,7 +1048,7 @@ + static inline void pagefault_enable(void) + { +-#ifndef CONFIG_PREEMPT /* - * Now re-arm for the new expiry time. + * make sure to issue those last loads/stores before enabling + * the pagefault handler again. */ -- WARN_ON_ONCE(!irqs_disabled()); -+ WARN_ON_ONCE_NONRT(!irqs_disabled()); - arm_timer(timer); - unlock_task_sighand(p, &flags); + barrier(); +- preempt_count_dec(); +-#else +- preempt_enable(); +-#endif ++ pagefault_disabled_dec(); ++ migrate_enable(); + } -@@ -1113,10 +1114,11 @@ - sig = tsk->signal; - if (sig->cputimer.running) { - struct task_cputime group_sample; -+ unsigned long flags; ++/* ++ * Is the pagefault handler disabled? If so, user access methods will not sleep. ++ */ ++#define pagefault_disabled() (current->pagefault_disabled != 0) ++ ++/* ++ * The pagefault handler is in general disabled by pagefault_disable() or ++ * when in irq context (via in_atomic()). ++ * ++ * This function should only be used by the fault handlers. Other users should ++ * stick to pagefault_disabled(). ++ * Please NEVER use preempt_disable() to disable the fault handler. With ++ * !CONFIG_PREEMPT_COUNT, this is like a NOP. So the handler won't be disabled. ++ * in_atomic() will report different values based on !CONFIG_PREEMPT_COUNT. ++ */ ++#define faulthandler_disabled() (pagefault_disabled() || in_atomic()) ++ + #ifndef ARCH_HAS_NOCACHE_UACCESS -- raw_spin_lock(&sig->cputimer.lock); -+ raw_spin_lock_irqsave(&sig->cputimer.lock, flags); - group_sample = sig->cputimer.cputime; -- raw_spin_unlock(&sig->cputimer.lock); -+ raw_spin_unlock_irqrestore(&sig->cputimer.lock, flags); + static inline unsigned long __copy_from_user_inatomic_nocache(void *to, +diff -Nur linux-4.1.10.orig/include/linux/uprobes.h linux-4.1.10/include/linux/uprobes.h +--- linux-4.1.10.orig/include/linux/uprobes.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/uprobes.h 2015-10-07 18:00:08.000000000 +0200 +@@ -27,6 +27,7 @@ + #include + #include + #include ++#include - if (task_cputime_expired(&group_sample, &sig->cputime_expires)) - return 1; -@@ -1130,13 +1132,13 @@ - * already updated our counts. We need to check if any timers fire now. - * Interrupts are disabled. + struct vm_area_struct; + struct mm_struct; +diff -Nur linux-4.1.10.orig/include/linux/vmstat.h linux-4.1.10/include/linux/vmstat.h +--- linux-4.1.10.orig/include/linux/vmstat.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/vmstat.h 2015-10-07 18:00:08.000000000 +0200 +@@ -33,7 +33,9 @@ */ --void run_posix_cpu_timers(struct task_struct *tsk) -+static void __run_posix_cpu_timers(struct task_struct *tsk) + static inline void __count_vm_event(enum vm_event_item item) { - LIST_HEAD(firing); - struct k_itimer *timer, *next; - unsigned long flags; ++ preempt_disable_rt(); + raw_cpu_inc(vm_event_states.event[item]); ++ preempt_enable_rt(); + } -- WARN_ON_ONCE(!irqs_disabled()); -+ WARN_ON_ONCE_NONRT(!irqs_disabled()); + static inline void count_vm_event(enum vm_event_item item) +@@ -43,7 +45,9 @@ - /* - * The fast path checks that there are no expired thread or thread -@@ -1194,6 +1196,190 @@ - } + static inline void __count_vm_events(enum vm_event_item item, long delta) + { ++ preempt_disable_rt(); + raw_cpu_add(vm_event_states.event[item], delta); ++ preempt_enable_rt(); } -+#ifdef CONFIG_PREEMPT_RT_BASE -+#include -+#include -+DEFINE_PER_CPU(struct task_struct *, posix_timer_task); -+DEFINE_PER_CPU(struct task_struct *, posix_timer_tasklist); + static inline void count_vm_events(enum vm_event_item item, long delta) +diff -Nur linux-4.1.10.orig/include/linux/wait.h linux-4.1.10/include/linux/wait.h +--- linux-4.1.10.orig/include/linux/wait.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/linux/wait.h 2015-10-07 18:00:08.000000000 +0200 +@@ -8,6 +8,7 @@ + #include + #include + #include ++#include + + typedef struct __wait_queue wait_queue_t; + typedef int (*wait_queue_func_t)(wait_queue_t *wait, unsigned mode, int flags, void *key); +diff -Nur linux-4.1.10.orig/include/linux/wait-simple.h linux-4.1.10/include/linux/wait-simple.h +--- linux-4.1.10.orig/include/linux/wait-simple.h 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/include/linux/wait-simple.h 2015-10-07 18:00:08.000000000 +0200 +@@ -0,0 +1,207 @@ ++#ifndef _LINUX_WAIT_SIMPLE_H ++#define _LINUX_WAIT_SIMPLE_H + -+static int posix_cpu_timers_thread(void *data) -+{ -+ int cpu = (long)data; ++#include ++#include + -+ BUG_ON(per_cpu(posix_timer_task,cpu) != current); ++#include + -+ while (!kthread_should_stop()) { -+ struct task_struct *tsk = NULL; -+ struct task_struct *next = NULL; ++struct swaiter { ++ struct task_struct *task; ++ struct list_head node; ++}; + -+ if (cpu_is_offline(cpu)) -+ goto wait_to_die; ++#define DEFINE_SWAITER(name) \ ++ struct swaiter name = { \ ++ .task = current, \ ++ .node = LIST_HEAD_INIT((name).node), \ ++ } + -+ /* grab task list */ -+ raw_local_irq_disable(); -+ tsk = per_cpu(posix_timer_tasklist, cpu); -+ per_cpu(posix_timer_tasklist, cpu) = NULL; -+ raw_local_irq_enable(); ++struct swait_head { ++ raw_spinlock_t lock; ++ struct list_head list; ++}; + -+ /* its possible the list is empty, just return */ -+ if (!tsk) { -+ set_current_state(TASK_INTERRUPTIBLE); -+ schedule(); -+ __set_current_state(TASK_RUNNING); -+ continue; -+ } ++#define SWAIT_HEAD_INITIALIZER(name) { \ ++ .lock = __RAW_SPIN_LOCK_UNLOCKED(name.lock), \ ++ .list = LIST_HEAD_INIT((name).list), \ ++ } + -+ /* Process task list */ -+ while (1) { -+ /* save next */ -+ next = tsk->posix_timer_list; ++#define DEFINE_SWAIT_HEAD(name) \ ++ struct swait_head name = SWAIT_HEAD_INITIALIZER(name) + -+ /* run the task timers, clear its ptr and -+ * unreference it -+ */ -+ __run_posix_cpu_timers(tsk); -+ tsk->posix_timer_list = NULL; -+ put_task_struct(tsk); ++extern void __init_swait_head(struct swait_head *h, struct lock_class_key *key); + -+ /* check if this is the last on the list */ -+ if (next == tsk) -+ break; -+ tsk = next; -+ } -+ } -+ return 0; ++#define init_swait_head(swh) \ ++ do { \ ++ static struct lock_class_key __key; \ ++ \ ++ __init_swait_head((swh), &__key); \ ++ } while (0) + -+wait_to_die: -+ /* Wait for kthread_stop */ -+ set_current_state(TASK_INTERRUPTIBLE); -+ while (!kthread_should_stop()) { -+ schedule(); -+ set_current_state(TASK_INTERRUPTIBLE); -+ } -+ __set_current_state(TASK_RUNNING); -+ return 0; -+} ++/* ++ * Waiter functions ++ */ ++extern void swait_prepare_locked(struct swait_head *head, struct swaiter *w); ++extern void swait_prepare(struct swait_head *head, struct swaiter *w, int state); ++extern void swait_finish_locked(struct swait_head *head, struct swaiter *w); ++extern void swait_finish(struct swait_head *head, struct swaiter *w); + -+static inline int __fastpath_timer_check(struct task_struct *tsk) ++/* Check whether a head has waiters enqueued */ ++static inline bool swaitqueue_active(struct swait_head *h) +{ -+ /* tsk == current, ensure it is safe to use ->signal/sighand */ -+ if (unlikely(tsk->exit_state)) -+ return 0; ++ /* Make sure the condition is visible before checking list_empty() */ ++ smp_mb(); ++ return !list_empty(&h->list); ++} + -+ if (!task_cputime_zero(&tsk->cputime_expires)) -+ return 1; ++/* ++ * Wakeup functions ++ */ ++extern unsigned int __swait_wake(struct swait_head *head, unsigned int state, unsigned int num); ++extern unsigned int __swait_wake_locked(struct swait_head *head, unsigned int state, unsigned int num); + -+ if (!task_cputime_zero(&tsk->signal->cputime_expires)) -+ return 1; ++#define swait_wake(head) __swait_wake(head, TASK_NORMAL, 1) ++#define swait_wake_interruptible(head) __swait_wake(head, TASK_INTERRUPTIBLE, 1) ++#define swait_wake_all(head) __swait_wake(head, TASK_NORMAL, 0) ++#define swait_wake_all_interruptible(head) __swait_wake(head, TASK_INTERRUPTIBLE, 0) + -+ return 0; -+} ++/* ++ * Event API ++ */ ++#define __swait_event(wq, condition) \ ++do { \ ++ DEFINE_SWAITER(__wait); \ ++ \ ++ for (;;) { \ ++ swait_prepare(&wq, &__wait, TASK_UNINTERRUPTIBLE); \ ++ if (condition) \ ++ break; \ ++ schedule(); \ ++ } \ ++ swait_finish(&wq, &__wait); \ ++} while (0) + -+void run_posix_cpu_timers(struct task_struct *tsk) -+{ -+ unsigned long cpu = smp_processor_id(); -+ struct task_struct *tasklist; ++/** ++ * swait_event - sleep until a condition gets true ++ * @wq: the waitqueue to wait on ++ * @condition: a C expression for the event to wait for ++ * ++ * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the ++ * @condition evaluates to true. The @condition is checked each time ++ * the waitqueue @wq is woken up. ++ * ++ * wake_up() has to be called after changing any variable that could ++ * change the result of the wait condition. ++ */ ++#define swait_event(wq, condition) \ ++do { \ ++ if (condition) \ ++ break; \ ++ __swait_event(wq, condition); \ ++} while (0) + -+ BUG_ON(!irqs_disabled()); -+ if(!per_cpu(posix_timer_task, cpu)) -+ return; -+ /* get per-cpu references */ -+ tasklist = per_cpu(posix_timer_tasklist, cpu); ++#define __swait_event_interruptible(wq, condition, ret) \ ++do { \ ++ DEFINE_SWAITER(__wait); \ ++ \ ++ for (;;) { \ ++ swait_prepare(&wq, &__wait, TASK_INTERRUPTIBLE); \ ++ if (condition) \ ++ break; \ ++ if (signal_pending(current)) { \ ++ ret = -ERESTARTSYS; \ ++ break; \ ++ } \ ++ schedule(); \ ++ } \ ++ swait_finish(&wq, &__wait); \ ++} while (0) + -+ /* check to see if we're already queued */ -+ if (!tsk->posix_timer_list && __fastpath_timer_check(tsk)) { -+ get_task_struct(tsk); -+ if (tasklist) { -+ tsk->posix_timer_list = tasklist; -+ } else { -+ /* -+ * The list is terminated by a self-pointing -+ * task_struct -+ */ -+ tsk->posix_timer_list = tsk; -+ } -+ per_cpu(posix_timer_tasklist, cpu) = tsk; ++#define __swait_event_interruptible_timeout(wq, condition, ret) \ ++do { \ ++ DEFINE_SWAITER(__wait); \ ++ \ ++ for (;;) { \ ++ swait_prepare(&wq, &__wait, TASK_INTERRUPTIBLE); \ ++ if (condition) \ ++ break; \ ++ if (signal_pending(current)) { \ ++ ret = -ERESTARTSYS; \ ++ break; \ ++ } \ ++ ret = schedule_timeout(ret); \ ++ if (!ret) \ ++ break; \ ++ } \ ++ swait_finish(&wq, &__wait); \ ++} while (0) + -+ wake_up_process(per_cpu(posix_timer_task, cpu)); -+ } -+} ++/** ++ * swait_event_interruptible - sleep until a condition gets true ++ * @wq: the waitqueue to wait on ++ * @condition: a C expression for the event to wait for ++ * ++ * The process is put to sleep (TASK_INTERRUPTIBLE) until the ++ * @condition evaluates to true. The @condition is checked each time ++ * the waitqueue @wq is woken up. ++ * ++ * wake_up() has to be called after changing any variable that could ++ * change the result of the wait condition. ++ */ ++#define swait_event_interruptible(wq, condition) \ ++({ \ ++ int __ret = 0; \ ++ if (!(condition)) \ ++ __swait_event_interruptible(wq, condition, __ret); \ ++ __ret; \ ++}) + -+/* -+ * posix_cpu_thread_call - callback that gets triggered when a CPU is added. -+ * Here we can start up the necessary migration thread for the new CPU. ++#define swait_event_interruptible_timeout(wq, condition, timeout) \ ++({ \ ++ int __ret = timeout; \ ++ if (!(condition)) \ ++ __swait_event_interruptible_timeout(wq, condition, __ret); \ ++ __ret; \ ++}) ++ ++#define __swait_event_timeout(wq, condition, ret) \ ++do { \ ++ DEFINE_SWAITER(__wait); \ ++ \ ++ for (;;) { \ ++ swait_prepare(&wq, &__wait, TASK_UNINTERRUPTIBLE); \ ++ if (condition) \ ++ break; \ ++ ret = schedule_timeout(ret); \ ++ if (!ret) \ ++ break; \ ++ } \ ++ swait_finish(&wq, &__wait); \ ++} while (0) ++ ++/** ++ * swait_event_timeout - sleep until a condition gets true or a timeout elapses ++ * @wq: the waitqueue to wait on ++ * @condition: a C expression for the event to wait for ++ * @timeout: timeout, in jiffies ++ * ++ * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the ++ * @condition evaluates to true. The @condition is checked each time ++ * the waitqueue @wq is woken up. ++ * ++ * wake_up() has to be called after changing any variable that could ++ * change the result of the wait condition. ++ * ++ * The function returns 0 if the @timeout elapsed, and the remaining ++ * jiffies if the condition evaluated to true before the timeout elapsed. + */ -+static int posix_cpu_thread_call(struct notifier_block *nfb, -+ unsigned long action, void *hcpu) -+{ -+ int cpu = (long)hcpu; -+ struct task_struct *p; -+ struct sched_param param; ++#define swait_event_timeout(wq, condition, timeout) \ ++({ \ ++ long __ret = timeout; \ ++ if (!(condition)) \ ++ __swait_event_timeout(wq, condition, __ret); \ ++ __ret; \ ++}) + -+ switch (action) { -+ case CPU_UP_PREPARE: -+ p = kthread_create(posix_cpu_timers_thread, hcpu, -+ "posixcputmr/%d",cpu); -+ if (IS_ERR(p)) -+ return NOTIFY_BAD; -+ p->flags |= PF_NOFREEZE; -+ kthread_bind(p, cpu); -+ /* Must be high prio to avoid getting starved */ -+ param.sched_priority = MAX_RT_PRIO-1; -+ sched_setscheduler(p, SCHED_FIFO, ¶m); -+ per_cpu(posix_timer_task,cpu) = p; -+ break; -+ case CPU_ONLINE: -+ /* Strictly unneccessary, as first user will wake it. */ -+ wake_up_process(per_cpu(posix_timer_task,cpu)); -+ break; -+#ifdef CONFIG_HOTPLUG_CPU -+ case CPU_UP_CANCELED: -+ /* Unbind it from offline cpu so it can run. Fall thru. */ -+ kthread_bind(per_cpu(posix_timer_task, cpu), -+ cpumask_any(cpu_online_mask)); -+ kthread_stop(per_cpu(posix_timer_task,cpu)); -+ per_cpu(posix_timer_task,cpu) = NULL; -+ break; -+ case CPU_DEAD: -+ kthread_stop(per_cpu(posix_timer_task,cpu)); -+ per_cpu(posix_timer_task,cpu) = NULL; -+ break; +#endif -+ } -+ return NOTIFY_OK; -+} +diff -Nur linux-4.1.10.orig/include/linux/work-simple.h linux-4.1.10/include/linux/work-simple.h +--- linux-4.1.10.orig/include/linux/work-simple.h 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/include/linux/work-simple.h 2015-10-07 18:00:08.000000000 +0200 +@@ -0,0 +1,24 @@ ++#ifndef _LINUX_SWORK_H ++#define _LINUX_SWORK_H + -+/* Register at highest priority so that task migration (migrate_all_tasks) -+ * happens before everything else. -+ */ -+static struct notifier_block posix_cpu_thread_notifier = { -+ .notifier_call = posix_cpu_thread_call, -+ .priority = 10 ++#include ++ ++struct swork_event { ++ struct list_head item; ++ unsigned long flags; ++ void (*func)(struct swork_event *); +}; + -+static int __init posix_cpu_thread_init(void) ++static inline void INIT_SWORK(struct swork_event *event, ++ void (*func)(struct swork_event *)) +{ -+ void *hcpu = (void *)(long)smp_processor_id(); -+ /* Start one for boot CPU. */ -+ unsigned long cpu; ++ event->flags = 0; ++ event->func = func; ++} + -+ /* init the per-cpu posix_timer_tasklets */ -+ for_each_possible_cpu(cpu) -+ per_cpu(posix_timer_tasklist, cpu) = NULL; ++bool swork_queue(struct swork_event *sev); + -+ posix_cpu_thread_call(&posix_cpu_thread_notifier, CPU_UP_PREPARE, hcpu); -+ posix_cpu_thread_call(&posix_cpu_thread_notifier, CPU_ONLINE, hcpu); -+ register_cpu_notifier(&posix_cpu_thread_notifier); -+ return 0; -+} -+early_initcall(posix_cpu_thread_init); -+#else /* CONFIG_PREEMPT_RT_BASE */ -+void run_posix_cpu_timers(struct task_struct *tsk) -+{ -+ __run_posix_cpu_timers(tsk); -+} -+#endif /* CONFIG_PREEMPT_RT_BASE */ ++int swork_get(void); ++void swork_put(void); + - /* - * Set one of the process-wide special case CPU timers or RLIMIT_CPU. - * The tsk->sighand->siglock must be held by the caller. -diff -Nur linux-4.1.6.orig/kernel/time/posix-timers.c linux-4.1.6/kernel/time/posix-timers.c ---- linux-4.1.6.orig/kernel/time/posix-timers.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/time/posix-timers.c 2015-09-08 23:49:08.117866054 +0200 -@@ -499,6 +499,7 @@ - static struct pid *good_sigevent(sigevent_t * event) ++#endif /* _LINUX_SWORK_H */ +diff -Nur linux-4.1.10.orig/include/net/dst.h linux-4.1.10/include/net/dst.h +--- linux-4.1.10.orig/include/net/dst.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/net/dst.h 2015-10-07 18:00:08.000000000 +0200 +@@ -403,7 +403,7 @@ + static inline int dst_neigh_output(struct dst_entry *dst, struct neighbour *n, + struct sk_buff *skb) { - struct task_struct *rtn = current->group_leader; -+ int sig = event->sigev_signo; +- const struct hh_cache *hh; ++ struct hh_cache *hh; - if ((event->sigev_notify & SIGEV_THREAD_ID ) && - (!(rtn = find_task_by_vpid(event->sigev_notify_thread_id)) || -@@ -507,7 +508,8 @@ - return NULL; + if (dst->pending_confirm) { + unsigned long now = jiffies; +diff -Nur linux-4.1.10.orig/include/net/neighbour.h linux-4.1.10/include/net/neighbour.h +--- linux-4.1.10.orig/include/net/neighbour.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/net/neighbour.h 2015-10-07 18:00:08.000000000 +0200 +@@ -445,7 +445,7 @@ + } + #endif - if (((event->sigev_notify & ~SIGEV_THREAD_ID) != SIGEV_NONE) && -- ((event->sigev_signo <= 0) || (event->sigev_signo > SIGRTMAX))) -+ (sig <= 0 || sig > SIGRTMAX || sig_kernel_only(sig) || -+ sig_kernel_coredump(sig))) - return NULL; +-static inline int neigh_hh_output(const struct hh_cache *hh, struct sk_buff *skb) ++static inline int neigh_hh_output(struct hh_cache *hh, struct sk_buff *skb) + { + unsigned int seq; + int hh_len; +@@ -500,7 +500,7 @@ - return task_pid(rtn); -@@ -819,6 +821,20 @@ - return overrun; - } + #define NEIGH_CB(skb) ((struct neighbour_cb *)(skb)->cb) -+/* -+ * Protected by RCU! -+ */ -+static void timer_wait_for_callback(struct k_clock *kc, struct k_itimer *timr) -+{ -+#ifdef CONFIG_PREEMPT_RT_FULL -+ if (kc->timer_set == common_timer_set) -+ hrtimer_wait_for_timer(&timr->it.real.timer); -+ else -+ /* FIXME: Whacky hack for posix-cpu-timers */ -+ schedule_timeout(1); +-static inline void neigh_ha_snapshot(char *dst, const struct neighbour *n, ++static inline void neigh_ha_snapshot(char *dst, struct neighbour *n, + const struct net_device *dev) + { + unsigned int seq; +diff -Nur linux-4.1.10.orig/include/net/netns/ipv4.h linux-4.1.10/include/net/netns/ipv4.h +--- linux-4.1.10.orig/include/net/netns/ipv4.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/include/net/netns/ipv4.h 2015-10-07 18:00:08.000000000 +0200 +@@ -69,6 +69,7 @@ + + int sysctl_icmp_echo_ignore_all; + int sysctl_icmp_echo_ignore_broadcasts; ++ int sysctl_icmp_echo_sysrq; + int sysctl_icmp_ignore_bogus_error_responses; + int sysctl_icmp_ratelimit; + int sysctl_icmp_ratemask; +diff -Nur linux-4.1.10.orig/include/trace/events/hist.h linux-4.1.10/include/trace/events/hist.h +--- linux-4.1.10.orig/include/trace/events/hist.h 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/include/trace/events/hist.h 2015-10-07 18:00:08.000000000 +0200 +@@ -0,0 +1,72 @@ ++#undef TRACE_SYSTEM ++#define TRACE_SYSTEM hist ++ ++#if !defined(_TRACE_HIST_H) || defined(TRACE_HEADER_MULTI_READ) ++#define _TRACE_HIST_H ++ ++#include "latency_hist.h" ++#include ++ ++#if !defined(CONFIG_PREEMPT_OFF_HIST) && !defined(CONFIG_INTERRUPT_OFF_HIST) ++#define trace_preemptirqsoff_hist(a, b) ++#else ++TRACE_EVENT(preemptirqsoff_hist, ++ ++ TP_PROTO(int reason, int starthist), ++ ++ TP_ARGS(reason, starthist), ++ ++ TP_STRUCT__entry( ++ __field(int, reason) ++ __field(int, starthist) ++ ), ++ ++ TP_fast_assign( ++ __entry->reason = reason; ++ __entry->starthist = starthist; ++ ), ++ ++ TP_printk("reason=%s starthist=%s", getaction(__entry->reason), ++ __entry->starthist ? "start" : "stop") ++); ++#endif ++ ++#ifndef CONFIG_MISSED_TIMER_OFFSETS_HIST ++#define trace_hrtimer_interrupt(a, b, c, d) ++#else ++TRACE_EVENT(hrtimer_interrupt, ++ ++ TP_PROTO(int cpu, long long offset, struct task_struct *curr, ++ struct task_struct *task), ++ ++ TP_ARGS(cpu, offset, curr, task), ++ ++ TP_STRUCT__entry( ++ __field(int, cpu) ++ __field(long long, offset) ++ __array(char, ccomm, TASK_COMM_LEN) ++ __field(int, cprio) ++ __array(char, tcomm, TASK_COMM_LEN) ++ __field(int, tprio) ++ ), ++ ++ TP_fast_assign( ++ __entry->cpu = cpu; ++ __entry->offset = offset; ++ memcpy(__entry->ccomm, curr->comm, TASK_COMM_LEN); ++ __entry->cprio = curr->prio; ++ memcpy(__entry->tcomm, task != NULL ? task->comm : "", ++ task != NULL ? TASK_COMM_LEN : 7); ++ __entry->tprio = task != NULL ? task->prio : -1; ++ ), ++ ++ TP_printk("cpu=%d offset=%lld curr=%s[%d] thread=%s[%d]", ++ __entry->cpu, __entry->offset, __entry->ccomm, ++ __entry->cprio, __entry->tcomm, __entry->tprio) ++); +#endif ++ ++#endif /* _TRACE_HIST_H */ ++ ++/* This part must be outside protection */ ++#include +diff -Nur linux-4.1.10.orig/include/trace/events/latency_hist.h linux-4.1.10/include/trace/events/latency_hist.h +--- linux-4.1.10.orig/include/trace/events/latency_hist.h 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/include/trace/events/latency_hist.h 2015-10-07 18:00:08.000000000 +0200 +@@ -0,0 +1,29 @@ ++#ifndef _LATENCY_HIST_H ++#define _LATENCY_HIST_H ++ ++enum hist_action { ++ IRQS_ON, ++ PREEMPT_ON, ++ TRACE_STOP, ++ IRQS_OFF, ++ PREEMPT_OFF, ++ TRACE_START, ++}; ++ ++static char *actions[] = { ++ "IRQS_ON", ++ "PREEMPT_ON", ++ "TRACE_STOP", ++ "IRQS_OFF", ++ "PREEMPT_OFF", ++ "TRACE_START", ++}; ++ ++static inline char *getaction(int action) ++{ ++ if (action >= 0 && action <= sizeof(actions)/sizeof(actions[0])) ++ return actions[action]; ++ return "unknown"; +} + - /* Set a POSIX.1b interval timer. */ - /* timr->it_lock is taken. */ - static int -@@ -896,6 +912,7 @@ - if (!timr) - return -EINVAL; ++#endif /* _LATENCY_HIST_H */ +diff -Nur linux-4.1.10.orig/init/Kconfig linux-4.1.10/init/Kconfig +--- linux-4.1.10.orig/init/Kconfig 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/init/Kconfig 2015-10-07 18:00:08.000000000 +0200 +@@ -637,7 +637,7 @@ -+ rcu_read_lock(); - kc = clockid_to_kclock(timr->it_clock); - if (WARN_ON_ONCE(!kc || !kc->timer_set)) - error = -EINVAL; -@@ -904,9 +921,12 @@ + config RCU_FAST_NO_HZ + bool "Accelerate last non-dyntick-idle CPU's grace periods" +- depends on NO_HZ_COMMON && SMP ++ depends on NO_HZ_COMMON && SMP && !PREEMPT_RT_FULL + default n + help + This option permits CPUs to enter dynticks-idle state even if +@@ -664,7 +664,7 @@ + config RCU_BOOST + bool "Enable RCU priority boosting" + depends on RT_MUTEXES && PREEMPT_RCU +- default n ++ default y if PREEMPT_RT_FULL + help + This option boosts the priority of preempted RCU readers that + block the current preemptible RCU grace period for too long. +@@ -1101,6 +1101,7 @@ + config RT_GROUP_SCHED + bool "Group scheduling for SCHED_RR/FIFO" + depends on CGROUP_SCHED ++ depends on !PREEMPT_RT_FULL + default n + help + This feature lets you explicitly allocate real CPU bandwidth +@@ -1688,6 +1689,7 @@ - unlock_timer(timr, flag); - if (error == TIMER_RETRY) { -+ timer_wait_for_callback(kc, timr); - rtn = NULL; // We already got the old time... -+ rcu_read_unlock(); - goto retry; - } -+ rcu_read_unlock(); + config SLAB + bool "SLAB" ++ depends on !PREEMPT_RT_FULL + help + The regular slab allocator that is established and known to work + well in all environments. It organizes cache hot objects in +@@ -1706,6 +1708,7 @@ + config SLOB + depends on EXPERT + bool "SLOB (Simple Allocator)" ++ depends on !PREEMPT_RT_FULL + help + SLOB replaces the stock allocator with a drastically simpler + allocator. SLOB is generally more space efficient but +@@ -1715,7 +1718,7 @@ - if (old_setting && !error && - copy_to_user(old_setting, &old_spec, sizeof (old_spec))) -@@ -944,10 +964,15 @@ - if (!timer) - return -EINVAL; - -+ rcu_read_lock(); - if (timer_delete_hook(timer) == TIMER_RETRY) { - unlock_timer(timer, flags); -+ timer_wait_for_callback(clockid_to_kclock(timer->it_clock), -+ timer); -+ rcu_read_unlock(); - goto retry_delete; - } -+ rcu_read_unlock(); + config SLUB_CPU_PARTIAL + default y +- depends on SLUB && SMP ++ depends on SLUB && SMP && !PREEMPT_RT_FULL + bool "SLUB per cpu partial cache" + help + Per cpu partial caches accellerate objects allocation and freeing +diff -Nur linux-4.1.10.orig/init/main.c linux-4.1.10/init/main.c +--- linux-4.1.10.orig/init/main.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/init/main.c 2015-10-07 18:00:08.000000000 +0200 +@@ -525,6 +525,7 @@ + setup_command_line(command_line); + setup_nr_cpu_ids(); + setup_per_cpu_areas(); ++ softirq_early_init(); + smp_prepare_boot_cpu(); /* arch-specific boot-cpu hooks */ - spin_lock(¤t->sighand->siglock); - list_del(&timer->list); -@@ -973,8 +998,18 @@ - retry_delete: - spin_lock_irqsave(&timer->it_lock, flags); + build_all_zonelists(NULL, NULL); +diff -Nur linux-4.1.10.orig/init/Makefile linux-4.1.10/init/Makefile +--- linux-4.1.10.orig/init/Makefile 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/init/Makefile 2015-10-07 18:00:08.000000000 +0200 +@@ -33,4 +33,4 @@ + include/generated/compile.h: FORCE + @$($(quiet)chk_compile.h) + $(Q)$(CONFIG_SHELL) $(srctree)/scripts/mkcompile_h $@ \ +- "$(UTS_MACHINE)" "$(CONFIG_SMP)" "$(CONFIG_PREEMPT)" "$(CC) $(KBUILD_CFLAGS)" ++ "$(UTS_MACHINE)" "$(CONFIG_SMP)" "$(CONFIG_PREEMPT)" "$(CONFIG_PREEMPT_RT_FULL)" "$(CC) $(KBUILD_CFLAGS)" +diff -Nur linux-4.1.10.orig/ipc/mqueue.c linux-4.1.10/ipc/mqueue.c +--- linux-4.1.10.orig/ipc/mqueue.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/ipc/mqueue.c 2015-10-07 18:00:08.000000000 +0200 +@@ -47,8 +47,7 @@ + #define RECV 1 -+ /* On RT we can race with a deletion */ -+ if (!timer->it_signal) { -+ unlock_timer(timer, flags); -+ return; -+ } -+ - if (timer_delete_hook(timer) == TIMER_RETRY) { -+ rcu_read_lock(); - unlock_timer(timer, flags); -+ timer_wait_for_callback(clockid_to_kclock(timer->it_clock), -+ timer); -+ rcu_read_unlock(); - goto retry_delete; - } - list_del(&timer->list); -diff -Nur linux-4.1.6.orig/kernel/time/tick-common.c linux-4.1.6/kernel/time/tick-common.c ---- linux-4.1.6.orig/kernel/time/tick-common.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/time/tick-common.c 2015-09-08 23:49:08.117866054 +0200 -@@ -78,13 +78,15 @@ - static void tick_periodic(int cpu) - { - if (tick_do_timer_cpu == cpu) { -- write_seqlock(&jiffies_lock); -+ raw_spin_lock(&jiffies_lock); -+ write_seqcount_begin(&jiffies_seq); + #define STATE_NONE 0 +-#define STATE_PENDING 1 +-#define STATE_READY 2 ++#define STATE_READY 1 - /* Keep track of the next tick event */ - tick_next_period = ktime_add(tick_next_period, tick_period); + struct posix_msg_tree_node { + struct rb_node rb_node; +@@ -568,15 +567,12 @@ + wq_add(info, sr, ewp); - do_timer(1); -- write_sequnlock(&jiffies_lock); -+ write_seqcount_end(&jiffies_seq); -+ raw_spin_unlock(&jiffies_lock); - update_wall_time(); - } + for (;;) { +- set_current_state(TASK_INTERRUPTIBLE); ++ __set_current_state(TASK_INTERRUPTIBLE); -@@ -146,9 +148,9 @@ - ktime_t next; + spin_unlock(&info->lock); + time = schedule_hrtimeout_range_clock(timeout, 0, + HRTIMER_MODE_ABS, CLOCK_REALTIME); - do { -- seq = read_seqbegin(&jiffies_lock); -+ seq = read_seqcount_begin(&jiffies_seq); - next = tick_next_period; -- } while (read_seqretry(&jiffies_lock, seq)); -+ } while (read_seqcount_retry(&jiffies_seq, seq)); +- while (ewp->state == STATE_PENDING) +- cpu_relax(); +- + if (ewp->state == STATE_READY) { + retval = 0; + goto out; +@@ -904,11 +900,15 @@ + * list of waiting receivers. A sender checks that list before adding the new + * message into the message array. If there is a waiting receiver, then it + * bypasses the message array and directly hands the message over to the +- * receiver. +- * The receiver accepts the message and returns without grabbing the queue +- * spinlock. Therefore an intermediate STATE_PENDING state and memory barriers +- * are necessary. The same algorithm is used for sysv semaphores, see +- * ipc/sem.c for more details. ++ * receiver. The receiver accepts the message and returns without grabbing the ++ * queue spinlock: ++ * ++ * - Set pointer to message. ++ * - Queue the receiver task for later wakeup (without the info->lock). ++ * - Update its state to STATE_READY. Now the receiver can continue. ++ * - Wake up the process after the lock is dropped. Should the process wake up ++ * before this wakeup (due to a timeout or a signal) it will either see ++ * STATE_READY and continue or acquire the lock to check the state again. + * + * The same algorithm is used for senders. + */ +@@ -916,21 +916,29 @@ + /* pipelined_send() - send a message directly to the task waiting in + * sys_mq_timedreceive() (without inserting message into a queue). + */ +-static inline void pipelined_send(struct mqueue_inode_info *info, ++static inline void pipelined_send(struct wake_q_head *wake_q, ++ struct mqueue_inode_info *info, + struct msg_msg *message, + struct ext_wait_queue *receiver) + { + receiver->msg = message; + list_del(&receiver->list); +- receiver->state = STATE_PENDING; +- wake_up_process(receiver->task); +- smp_wmb(); ++ wake_q_add(wake_q, receiver->task); ++ /* ++ * Rely on the implicit cmpxchg barrier from wake_q_add such ++ * that we can ensure that updating receiver->state is the last ++ * write operation: As once set, the receiver can continue, ++ * and if we don't have the reference count from the wake_q, ++ * yet, at that point we can later have a use-after-free ++ * condition and bogus wakeup. ++ */ + receiver->state = STATE_READY; + } - clockevents_set_state(dev, CLOCK_EVT_STATE_ONESHOT); + /* pipelined_receive() - if there is task waiting in sys_mq_timedsend() + * gets its message and put to the queue (we have one free place for sure). */ +-static inline void pipelined_receive(struct mqueue_inode_info *info) ++static inline void pipelined_receive(struct wake_q_head *wake_q, ++ struct mqueue_inode_info *info) + { + struct ext_wait_queue *sender = wq_get_first_waiter(info, SEND); -diff -Nur linux-4.1.6.orig/kernel/time/tick-sched.c linux-4.1.6/kernel/time/tick-sched.c ---- linux-4.1.6.orig/kernel/time/tick-sched.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/time/tick-sched.c 2015-09-08 23:49:08.117866054 +0200 -@@ -62,7 +62,8 @@ +@@ -941,10 +949,9 @@ + } + if (msg_insert(sender->msg, info)) return; ++ + list_del(&sender->list); +- sender->state = STATE_PENDING; +- wake_up_process(sender->task); +- smp_wmb(); ++ wake_q_add(wake_q, sender->task); + sender->state = STATE_READY; + } - /* Reevalute with jiffies_lock held */ -- write_seqlock(&jiffies_lock); -+ raw_spin_lock(&jiffies_lock); -+ write_seqcount_begin(&jiffies_seq); +@@ -962,6 +969,7 @@ + struct timespec ts; + struct posix_msg_tree_node *new_leaf = NULL; + int ret = 0; ++ WAKE_Q(wake_q); - delta = ktime_sub(now, last_jiffies_update); - if (delta.tv64 >= tick_period.tv64) { -@@ -85,10 +86,12 @@ - /* Keep the tick_next_period variable up to date */ - tick_next_period = ktime_add(last_jiffies_update, tick_period); + if (u_abs_timeout) { + int res = prepare_timeout(u_abs_timeout, &expires, &ts); +@@ -1045,7 +1053,7 @@ } else { -- write_sequnlock(&jiffies_lock); -+ write_seqcount_end(&jiffies_seq); -+ raw_spin_unlock(&jiffies_lock); - return; + receiver = wq_get_first_waiter(info, RECV); + if (receiver) { +- pipelined_send(info, msg_ptr, receiver); ++ pipelined_send(&wake_q, info, msg_ptr, receiver); + } else { + /* adds message to the queue */ + ret = msg_insert(msg_ptr, info); +@@ -1058,6 +1066,7 @@ } -- write_sequnlock(&jiffies_lock); -+ write_seqcount_end(&jiffies_seq); -+ raw_spin_unlock(&jiffies_lock); - update_wall_time(); - } - -@@ -99,12 +102,14 @@ - { - ktime_t period; + out_unlock: + spin_unlock(&info->lock); ++ wake_up_q(&wake_q); + out_free: + if (ret) + free_msg(msg_ptr); +@@ -1144,14 +1153,17 @@ + msg_ptr = wait.msg; + } + } else { ++ WAKE_Q(wake_q); ++ + msg_ptr = msg_get(info); -- write_seqlock(&jiffies_lock); -+ raw_spin_lock(&jiffies_lock); -+ write_seqcount_begin(&jiffies_seq); - /* Did we start the jiffies update yet ? */ - if (last_jiffies_update.tv64 == 0) - last_jiffies_update = tick_next_period; - period = last_jiffies_update; -- write_sequnlock(&jiffies_lock); -+ write_seqcount_end(&jiffies_seq); -+ raw_spin_unlock(&jiffies_lock); - return period; - } + inode->i_atime = inode->i_mtime = inode->i_ctime = + CURRENT_TIME; -@@ -176,6 +181,11 @@ - return false; + /* There is now free space in queue. */ +- pipelined_receive(info); ++ pipelined_receive(&wake_q, info); + spin_unlock(&info->lock); ++ wake_up_q(&wake_q); + ret = 0; } + if (ret == 0) { +diff -Nur linux-4.1.10.orig/ipc/msg.c linux-4.1.10/ipc/msg.c +--- linux-4.1.10.orig/ipc/msg.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/ipc/msg.c 2015-10-07 18:00:08.000000000 +0200 +@@ -188,6 +188,12 @@ + struct msg_receiver *msr, *t; -+ if (!arch_irq_work_has_interrupt()) { -+ trace_tick_stop(0, "missing irq work interrupt\n"); -+ return false; -+ } + list_for_each_entry_safe(msr, t, &msq->q_receivers, r_list) { ++ /* ++ * Make sure that the wakeup doesnt preempt ++ * this CPU prematurely. (on PREEMPT_RT) ++ */ ++ preempt_disable_rt(); + - /* sched_clock_tick() needs us? */ - #ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK - /* -@@ -222,6 +232,7 @@ - - static DEFINE_PER_CPU(struct irq_work, nohz_full_kick_work) = { - .func = nohz_full_kick_work_func, -+ .flags = IRQ_WORK_HARD_IRQ, - }; - - /* -@@ -578,10 +589,10 @@ + msr->r_msg = NULL; /* initialize expunge ordering */ + wake_up_process(msr->r_tsk); + /* +@@ -198,6 +204,8 @@ + */ + smp_mb(); + msr->r_msg = ERR_PTR(res); ++ ++ preempt_enable_rt(); + } + } - /* Read jiffies and the time when jiffies were updated last */ - do { -- seq = read_seqbegin(&jiffies_lock); -+ seq = read_seqcount_begin(&jiffies_seq); - last_update = last_jiffies_update; - last_jiffies = jiffies; -- } while (read_seqretry(&jiffies_lock, seq)); -+ } while (read_seqcount_retry(&jiffies_seq, seq)); +@@ -574,6 +582,11 @@ + if (testmsg(msg, msr->r_msgtype, msr->r_mode) && + !security_msg_queue_msgrcv(msq, msg, msr->r_tsk, + msr->r_msgtype, msr->r_mode)) { ++ /* ++ * Make sure that the wakeup doesnt preempt ++ * this CPU prematurely. (on PREEMPT_RT) ++ */ ++ preempt_disable_rt(); - if (rcu_needs_cpu(&rcu_delta_jiffies) || - arch_needs_cpu() || irq_work_needs_cpu()) { -@@ -759,14 +770,7 @@ - return false; + list_del(&msr->r_list); + if (msr->r_maxsize < msg->m_ts) { +@@ -595,12 +608,13 @@ + */ + smp_mb(); + msr->r_msg = msg; ++ preempt_enable_rt(); - if (unlikely(local_softirq_pending() && cpu_online(cpu))) { -- static int ratelimit; -- -- if (ratelimit < 10 && -- (local_softirq_pending() & SOFTIRQ_STOP_IDLE_MASK)) { -- pr_warn("NOHZ: local_softirq_pending %02x\n", -- (unsigned int) local_softirq_pending()); -- ratelimit++; -- } -+ softirq_check_pending_idle(); - return false; + return 1; + } ++ preempt_enable_rt(); + } } +- + return 0; + } -@@ -1154,6 +1158,7 @@ - * Emulate tick processing via per-CPU hrtimers: - */ - hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); -+ ts->sched_timer.irqsafe = 1; - ts->sched_timer.function = tick_sched_timer; +diff -Nur linux-4.1.10.orig/ipc/sem.c linux-4.1.10/ipc/sem.c +--- linux-4.1.10.orig/ipc/sem.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/ipc/sem.c 2015-10-07 18:00:08.000000000 +0200 +@@ -690,6 +690,13 @@ + static void wake_up_sem_queue_prepare(struct list_head *pt, + struct sem_queue *q, int error) + { ++#ifdef CONFIG_PREEMPT_RT_BASE ++ struct task_struct *p = q->sleeper; ++ get_task_struct(p); ++ q->status = error; ++ wake_up_process(p); ++ put_task_struct(p); ++#else + if (list_empty(pt)) { + /* + * Hold preempt off so that we don't get preempted and have the +@@ -701,6 +708,7 @@ + q->pid = error; - /* Get the next period (per cpu) */ -diff -Nur linux-4.1.6.orig/kernel/time/timekeeping.c linux-4.1.6/kernel/time/timekeeping.c ---- linux-4.1.6.orig/kernel/time/timekeeping.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/time/timekeeping.c 2015-09-08 23:49:08.125865168 +0200 -@@ -2065,8 +2065,10 @@ + list_add_tail(&q->list, pt); ++#endif + } + + /** +@@ -714,6 +722,7 @@ */ - void xtime_update(unsigned long ticks) + static void wake_up_sem_queue_do(struct list_head *pt) { -- write_seqlock(&jiffies_lock); -+ raw_spin_lock(&jiffies_lock); -+ write_seqcount_begin(&jiffies_seq); - do_timer(ticks); -- write_sequnlock(&jiffies_lock); -+ write_seqcount_end(&jiffies_seq); -+ raw_spin_unlock(&jiffies_lock); - update_wall_time(); ++#ifndef CONFIG_PREEMPT_RT_BASE + struct sem_queue *q, *t; + int did_something; + +@@ -726,6 +735,7 @@ + } + if (did_something) + preempt_enable(); ++#endif } -diff -Nur linux-4.1.6.orig/kernel/time/timekeeping.h linux-4.1.6/kernel/time/timekeeping.h ---- linux-4.1.6.orig/kernel/time/timekeeping.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/time/timekeeping.h 2015-09-08 23:49:08.125865168 +0200 -@@ -22,7 +22,8 @@ - extern void do_timer(unsigned long ticks); - extern void update_wall_time(void); --extern seqlock_t jiffies_lock; -+extern raw_spinlock_t jiffies_lock; -+extern seqcount_t jiffies_seq; + static void unlink_queue(struct sem_array *sma, struct sem_queue *q) +diff -Nur linux-4.1.10.orig/kernel/cgroup.c linux-4.1.10/kernel/cgroup.c +--- linux-4.1.10.orig/kernel/cgroup.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/cgroup.c 2015-10-07 18:00:08.000000000 +0200 +@@ -4422,10 +4422,10 @@ + queue_work(cgroup_destroy_wq, &css->destroy_work); + } - #define CS_NAME_LEN 32 +-static void css_release_work_fn(struct work_struct *work) ++static void css_release_work_fn(struct swork_event *sev) + { + struct cgroup_subsys_state *css = +- container_of(work, struct cgroup_subsys_state, destroy_work); ++ container_of(sev, struct cgroup_subsys_state, destroy_swork); + struct cgroup_subsys *ss = css->ss; + struct cgroup *cgrp = css->cgroup; -diff -Nur linux-4.1.6.orig/kernel/time/timer.c linux-4.1.6/kernel/time/timer.c ---- linux-4.1.6.orig/kernel/time/timer.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/time/timer.c 2015-09-08 23:49:08.125865168 +0200 -@@ -78,6 +78,9 @@ - struct tvec_base { - spinlock_t lock; - struct timer_list *running_timer; -+#ifdef CONFIG_PREEMPT_RT_FULL -+ wait_queue_head_t wait_for_running_timer; -+#endif - unsigned long timer_jiffies; - unsigned long next_timer; - unsigned long active_timers; -@@ -768,6 +771,36 @@ - } +@@ -4464,8 +4464,8 @@ + struct cgroup_subsys_state *css = + container_of(ref, struct cgroup_subsys_state, refcnt); + +- INIT_WORK(&css->destroy_work, css_release_work_fn); +- queue_work(cgroup_destroy_wq, &css->destroy_work); ++ INIT_SWORK(&css->destroy_swork, css_release_work_fn); ++ swork_queue(&css->destroy_swork); } -+#ifndef CONFIG_PREEMPT_RT_FULL -+static inline struct tvec_base *switch_timer_base(struct timer_list *timer, -+ struct tvec_base *old, -+ struct tvec_base *new) -+{ -+ /* See the comment in lock_timer_base() */ -+ timer_set_base(timer, NULL); -+ spin_unlock(&old->lock); -+ spin_lock(&new->lock); -+ timer_set_base(timer, new); -+ return new; -+} + static void init_and_link_css(struct cgroup_subsys_state *css, +@@ -5069,6 +5069,7 @@ + */ + cgroup_destroy_wq = alloc_workqueue("cgroup_destroy", 0, 1); + BUG_ON(!cgroup_destroy_wq); ++ BUG_ON(swork_get()); + + /* + * Used to destroy pidlists and separate to serve as flush domain. +diff -Nur linux-4.1.10.orig/kernel/cgroup.c.orig linux-4.1.10/kernel/cgroup.c.orig +--- linux-4.1.10.orig/kernel/cgroup.c.orig 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/kernel/cgroup.c.orig 2015-10-03 13:49:38.000000000 +0200 +@@ -0,0 +1,5602 @@ ++/* ++ * Generic process-grouping system. ++ * ++ * Based originally on the cpuset system, extracted by Paul Menage ++ * Copyright (C) 2006 Google, Inc ++ * ++ * Notifications support ++ * Copyright (C) 2009 Nokia Corporation ++ * Author: Kirill A. Shutemov ++ * ++ * Copyright notices from the original cpuset code: ++ * -------------------------------------------------- ++ * Copyright (C) 2003 BULL SA. ++ * Copyright (C) 2004-2006 Silicon Graphics, Inc. ++ * ++ * Portions derived from Patrick Mochel's sysfs code. ++ * sysfs is Copyright (c) 2001-3 Patrick Mochel ++ * ++ * 2003-10-10 Written by Simon Derr. ++ * 2003-10-22 Updates by Stephen Hemminger. ++ * 2004 May-July Rework by Paul Jackson. ++ * --------------------------------------------------- ++ * ++ * This file is subject to the terms and conditions of the GNU General Public ++ * License. See the file COPYING in the main directory of the Linux ++ * distribution for more details. ++ */ ++ ++#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include /* TODO: replace with more sophisticated array */ ++#include ++#include ++ ++#include ++ ++/* ++ * pidlists linger the following amount before being destroyed. The goal ++ * is avoiding frequent destruction in the middle of consecutive read calls ++ * Expiring in the middle is a performance problem not a correctness one. ++ * 1 sec should be enough. ++ */ ++#define CGROUP_PIDLIST_DESTROY_DELAY HZ ++ ++#define CGROUP_FILE_NAME_MAX (MAX_CGROUP_TYPE_NAMELEN + \ ++ MAX_CFTYPE_NAME + 2) ++ ++/* ++ * cgroup_mutex is the master lock. Any modification to cgroup or its ++ * hierarchy must be performed while holding it. ++ * ++ * css_set_rwsem protects task->cgroups pointer, the list of css_set ++ * objects, and the chain of tasks off each css_set. ++ * ++ * These locks are exported if CONFIG_PROVE_RCU so that accessors in ++ * cgroup.h can use them for lockdep annotations. ++ */ ++#ifdef CONFIG_PROVE_RCU ++DEFINE_MUTEX(cgroup_mutex); ++DECLARE_RWSEM(css_set_rwsem); ++EXPORT_SYMBOL_GPL(cgroup_mutex); ++EXPORT_SYMBOL_GPL(css_set_rwsem); +#else -+static inline struct tvec_base *switch_timer_base(struct timer_list *timer, -+ struct tvec_base *old, -+ struct tvec_base *new) -+{ -+ /* -+ * We cannot do the above because we might be preempted and -+ * then the preempter would see NULL and loop forever. -+ */ -+ if (spin_trylock(&new->lock)) { -+ timer_set_base(timer, new); -+ spin_unlock(&old->lock); -+ return new; -+ } -+ return old; -+} ++static DEFINE_MUTEX(cgroup_mutex); ++static DECLARE_RWSEM(css_set_rwsem); +#endif + - static inline int - __mod_timer(struct timer_list *timer, unsigned long expires, - bool pending_only, int pinned) -@@ -798,14 +831,8 @@ - * handler yet has not finished. This also guarantees that - * the timer is serialized wrt itself. - */ -- if (likely(base->running_timer != timer)) { -- /* See the comment in lock_timer_base() */ -- timer_set_base(timer, NULL); -- spin_unlock(&base->lock); -- base = new_base; -- spin_lock(&base->lock); -- timer_set_base(timer, base); -- } -+ if (likely(base->running_timer != timer)) -+ base = switch_timer_base(timer, base, new_base); - } - - timer->expires = expires; -@@ -979,6 +1006,29 @@ - } - EXPORT_SYMBOL_GPL(add_timer_on); - -+#ifdef CONFIG_PREEMPT_RT_FULL +/* -+ * Wait for a running timer ++ * Protects cgroup_idr and css_idr so that IDs can be released without ++ * grabbing cgroup_mutex. + */ -+static void wait_for_running_timer(struct timer_list *timer) -+{ -+ struct tvec_base *base = timer->base; ++static DEFINE_SPINLOCK(cgroup_idr_lock); + -+ if (base->running_timer == timer) -+ wait_event(base->wait_for_running_timer, -+ base->running_timer != timer); -+} ++/* ++ * Protects cgroup_subsys->release_agent_path. Modifying it also requires ++ * cgroup_mutex. Reading requires either cgroup_mutex or this spinlock. ++ */ ++static DEFINE_SPINLOCK(release_agent_path_lock); + -+# define wakeup_timer_waiters(b) wake_up(&(b)->wait_for_running_timer) -+#else -+static inline void wait_for_running_timer(struct timer_list *timer) -+{ -+ cpu_relax(); -+} ++#define cgroup_assert_mutex_or_rcu_locked() \ ++ rcu_lockdep_assert(rcu_read_lock_held() || \ ++ lockdep_is_held(&cgroup_mutex), \ ++ "cgroup_mutex or RCU read lock required"); + -+# define wakeup_timer_waiters(b) do { } while (0) -+#endif ++/* ++ * cgroup destruction makes heavy use of work items and there can be a lot ++ * of concurrent destructions. Use a separate workqueue so that cgroup ++ * destruction work items don't end up filling up max_active of system_wq ++ * which may lead to deadlock. ++ */ ++static struct workqueue_struct *cgroup_destroy_wq; + - /** - * del_timer - deactive a timer. - * @timer: the timer to be deactivated -@@ -1036,7 +1086,7 @@ - } - EXPORT_SYMBOL(try_to_del_timer_sync); - --#ifdef CONFIG_SMP -+#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT_FULL) - static DEFINE_PER_CPU(struct tvec_base, __tvec_bases); - - /** -@@ -1098,7 +1148,7 @@ - int ret = try_to_del_timer_sync(timer); - if (ret >= 0) - return ret; -- cpu_relax(); -+ wait_for_running_timer(timer); - } - } - EXPORT_SYMBOL(del_timer_sync); -@@ -1219,15 +1269,17 @@ - if (irqsafe) { - spin_unlock(&base->lock); - call_timer_fn(timer, fn, data); -+ base->running_timer = NULL; - spin_lock(&base->lock); - } else { - spin_unlock_irq(&base->lock); - call_timer_fn(timer, fn, data); -+ base->running_timer = NULL; - spin_lock_irq(&base->lock); - } - } - } -- base->running_timer = NULL; -+ wakeup_timer_waiters(base); - spin_unlock_irq(&base->lock); - } - -@@ -1367,6 +1419,14 @@ - if (cpu_is_offline(smp_processor_id())) - return expires; - -+#ifdef CONFIG_PREEMPT_RT_FULL -+ /* -+ * On PREEMPT_RT we cannot sleep here. As a result we can't take -+ * the base lock to check when the next timer is pending and so -+ * we assume the next jiffy. -+ */ -+ return now + 1; -+#endif - spin_lock(&base->lock); - if (base->active_timers) { - if (time_before_eq(base->next_timer, base->timer_jiffies)) -@@ -1392,13 +1452,13 @@ - - /* Note: this timer irq context must be accounted for as well. */ - account_process_tick(p, user_tick); -+ scheduler_tick(); - run_local_timers(); - rcu_check_callbacks(user_tick); --#ifdef CONFIG_IRQ_WORK -+#if defined(CONFIG_IRQ_WORK) && !defined(CONFIG_PREEMPT_RT_FULL) - if (in_irq()) - irq_work_tick(); - #endif -- scheduler_tick(); - run_posix_cpu_timers(p); - } - -@@ -1411,6 +1471,10 @@ - - hrtimer_run_pending(); - -+#if defined(CONFIG_IRQ_WORK) && defined(CONFIG_PREEMPT_RT_FULL) -+ irq_work_tick(); -+#endif ++/* ++ * pidlist destructions need to be flushed on cgroup destruction. Use a ++ * separate workqueue as flush domain. ++ */ ++static struct workqueue_struct *cgroup_pidlist_destroy_wq; + - if (time_after_eq(jiffies, base->timer_jiffies)) - __run_timers(base); - } -@@ -1566,7 +1630,7 @@ - - BUG_ON(cpu_online(cpu)); - old_base = per_cpu(tvec_bases, cpu); -- new_base = get_cpu_var(tvec_bases); -+ new_base = get_local_var(tvec_bases); - /* - * The caller is globally serialized and nobody else - * takes two locks at once, deadlock is not possible. -@@ -1590,7 +1654,7 @@ - - spin_unlock(&old_base->lock); - spin_unlock_irq(&new_base->lock); -- put_cpu_var(tvec_bases); -+ put_local_var(tvec_bases); - } - - static int timer_cpu_notify(struct notifier_block *self, -@@ -1625,6 +1689,9 @@ - base->cpu = cpu; - per_cpu(tvec_bases, cpu) = base; - spin_lock_init(&base->lock); -+#ifdef CONFIG_PREEMPT_RT_FULL -+ init_waitqueue_head(&base->wait_for_running_timer); -+#endif - - for (j = 0; j < TVN_SIZE; j++) { - INIT_LIST_HEAD(base->tv5.vec + j); -diff -Nur linux-4.1.6.orig/kernel/trace/Kconfig linux-4.1.6/kernel/trace/Kconfig ---- linux-4.1.6.orig/kernel/trace/Kconfig 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/trace/Kconfig 2015-09-08 23:49:08.125865168 +0200 -@@ -187,6 +187,24 @@ - enabled. This option and the preempt-off timing option can be - used together or separately.) - -+config INTERRUPT_OFF_HIST -+ bool "Interrupts-off Latency Histogram" -+ depends on IRQSOFF_TRACER -+ help -+ This option generates continuously updated histograms (one per cpu) -+ of the duration of time periods with interrupts disabled. The -+ histograms are disabled by default. To enable them, write a non-zero -+ number to ++/* generate an array of cgroup subsystem pointers */ ++#define SUBSYS(_x) [_x ## _cgrp_id] = &_x ## _cgrp_subsys, ++static struct cgroup_subsys *cgroup_subsys[] = { ++#include ++}; ++#undef SUBSYS + -+ /sys/kernel/debug/tracing/latency_hist/enable/preemptirqsoff ++/* array of cgroup subsystem names */ ++#define SUBSYS(_x) [_x ## _cgrp_id] = #_x, ++static const char *cgroup_subsys_name[] = { ++#include ++}; ++#undef SUBSYS + -+ If PREEMPT_OFF_HIST is also selected, additional histograms (one -+ per cpu) are generated that accumulate the duration of time periods -+ when both interrupts and preemption are disabled. The histogram data -+ will be located in the debug file system at ++/* ++ * The default hierarchy, reserved for the subsystems that are otherwise ++ * unattached - it never has more than a single cgroup, and all tasks are ++ * part of that cgroup. ++ */ ++struct cgroup_root cgrp_dfl_root; + -+ /sys/kernel/debug/tracing/latency_hist/irqsoff ++/* ++ * The default hierarchy always exists but is hidden until mounted for the ++ * first time. This is for backward compatibility. ++ */ ++static bool cgrp_dfl_root_visible; + - config PREEMPT_TRACER - bool "Preemption-off Latency Tracer" - default n -@@ -211,6 +229,24 @@ - enabled. This option and the irqs-off timing option can be - used together or separately.) - -+config PREEMPT_OFF_HIST -+ bool "Preemption-off Latency Histogram" -+ depends on PREEMPT_TRACER -+ help -+ This option generates continuously updated histograms (one per cpu) -+ of the duration of time periods with preemption disabled. The -+ histograms are disabled by default. To enable them, write a non-zero -+ number to ++/* ++ * Set by the boot param of the same name and makes subsystems with NULL ++ * ->dfl_files to use ->legacy_files on the default hierarchy. ++ */ ++static bool cgroup_legacy_files_on_dfl; + -+ /sys/kernel/debug/tracing/latency_hist/enable/preemptirqsoff ++/* some controllers are not supported in the default hierarchy */ ++static unsigned int cgrp_dfl_root_inhibit_ss_mask; + -+ If INTERRUPT_OFF_HIST is also selected, additional histograms (one -+ per cpu) are generated that accumulate the duration of time periods -+ when both interrupts and preemption are disabled. The histogram data -+ will be located in the debug file system at ++/* The list of hierarchy roots */ + -+ /sys/kernel/debug/tracing/latency_hist/preemptoff ++static LIST_HEAD(cgroup_roots); ++static int cgroup_root_count; + - config SCHED_TRACER - bool "Scheduling Latency Tracer" - select GENERIC_TRACER -@@ -221,6 +257,74 @@ - This tracer tracks the latency of the highest priority task - to be scheduled in, starting from the point it has woken up. - -+config WAKEUP_LATENCY_HIST -+ bool "Scheduling Latency Histogram" -+ depends on SCHED_TRACER -+ help -+ This option generates continuously updated histograms (one per cpu) -+ of the scheduling latency of the highest priority task. -+ The histograms are disabled by default. To enable them, write a -+ non-zero number to ++/* hierarchy ID allocation and mapping, protected by cgroup_mutex */ ++static DEFINE_IDR(cgroup_hierarchy_idr); + -+ /sys/kernel/debug/tracing/latency_hist/enable/wakeup ++/* ++ * Assign a monotonically increasing serial number to csses. It guarantees ++ * cgroups with bigger numbers are newer than those with smaller numbers. ++ * Also, as csses are always appended to the parent's ->children list, it ++ * guarantees that sibling csses are always sorted in the ascending serial ++ * number order on the list. Protected by cgroup_mutex. ++ */ ++static u64 css_serial_nr_next = 1; + -+ Two different algorithms are used, one to determine the latency of -+ processes that exclusively use the highest priority of the system and -+ another one to determine the latency of processes that share the -+ highest system priority with other processes. The former is used to -+ improve hardware and system software, the latter to optimize the -+ priority design of a given system. The histogram data will be -+ located in the debug file system at ++/* This flag indicates whether tasks in the fork and exit paths should ++ * check for fork/exit handlers to call. This avoids us having to do ++ * extra work in the fork/exit path if none of the subsystems need to ++ * be called. ++ */ ++static int need_forkexit_callback __read_mostly; ++ ++static struct cftype cgroup_dfl_base_files[]; ++static struct cftype cgroup_legacy_base_files[]; ++ ++static int rebind_subsystems(struct cgroup_root *dst_root, ++ unsigned int ss_mask); ++static int cgroup_destroy_locked(struct cgroup *cgrp); ++static int create_css(struct cgroup *cgrp, struct cgroup_subsys *ss, ++ bool visible); ++static void css_release(struct percpu_ref *ref); ++static void kill_css(struct cgroup_subsys_state *css); ++static int cgroup_addrm_files(struct cgroup *cgrp, struct cftype cfts[], ++ bool is_add); ++ ++/* IDR wrappers which synchronize using cgroup_idr_lock */ ++static int cgroup_idr_alloc(struct idr *idr, void *ptr, int start, int end, ++ gfp_t gfp_mask) ++{ ++ int ret; + -+ /sys/kernel/debug/tracing/latency_hist/wakeup ++ idr_preload(gfp_mask); ++ spin_lock_bh(&cgroup_idr_lock); ++ ret = idr_alloc(idr, ptr, start, end, gfp_mask); ++ spin_unlock_bh(&cgroup_idr_lock); ++ idr_preload_end(); ++ return ret; ++} + -+ and ++static void *cgroup_idr_replace(struct idr *idr, void *ptr, int id) ++{ ++ void *ret; + -+ /sys/kernel/debug/tracing/latency_hist/wakeup/sharedprio ++ spin_lock_bh(&cgroup_idr_lock); ++ ret = idr_replace(idr, ptr, id); ++ spin_unlock_bh(&cgroup_idr_lock); ++ return ret; ++} + -+ If both Scheduling Latency Histogram and Missed Timer Offsets -+ Histogram are selected, additional histogram data will be collected -+ that contain, in addition to the wakeup latency, the timer latency, in -+ case the wakeup was triggered by an expired timer. These histograms -+ are available in the ++static void cgroup_idr_remove(struct idr *idr, int id) ++{ ++ spin_lock_bh(&cgroup_idr_lock); ++ idr_remove(idr, id); ++ spin_unlock_bh(&cgroup_idr_lock); ++} + -+ /sys/kernel/debug/tracing/latency_hist/timerandwakeup ++static struct cgroup *cgroup_parent(struct cgroup *cgrp) ++{ ++ struct cgroup_subsys_state *parent_css = cgrp->self.parent; + -+ directory. They reflect the apparent interrupt and scheduling latency -+ and are best suitable to determine the worst-case latency of a given -+ system. To enable these histograms, write a non-zero number to ++ if (parent_css) ++ return container_of(parent_css, struct cgroup, self); ++ return NULL; ++} + -+ /sys/kernel/debug/tracing/latency_hist/enable/timerandwakeup ++/** ++ * cgroup_css - obtain a cgroup's css for the specified subsystem ++ * @cgrp: the cgroup of interest ++ * @ss: the subsystem of interest (%NULL returns @cgrp->self) ++ * ++ * Return @cgrp's css (cgroup_subsys_state) associated with @ss. This ++ * function must be called either under cgroup_mutex or rcu_read_lock() and ++ * the caller is responsible for pinning the returned css if it wants to ++ * keep accessing it outside the said locks. This function may return ++ * %NULL if @cgrp doesn't have @subsys_id enabled. ++ */ ++static struct cgroup_subsys_state *cgroup_css(struct cgroup *cgrp, ++ struct cgroup_subsys *ss) ++{ ++ if (ss) ++ return rcu_dereference_check(cgrp->subsys[ss->id], ++ lockdep_is_held(&cgroup_mutex)); ++ else ++ return &cgrp->self; ++} + -+config MISSED_TIMER_OFFSETS_HIST -+ depends on HIGH_RES_TIMERS -+ select GENERIC_TRACER -+ bool "Missed Timer Offsets Histogram" -+ help -+ Generate a histogram of missed timer offsets in microseconds. The -+ histograms are disabled by default. To enable them, write a non-zero -+ number to ++/** ++ * cgroup_e_css - obtain a cgroup's effective css for the specified subsystem ++ * @cgrp: the cgroup of interest ++ * @ss: the subsystem of interest (%NULL returns @cgrp->self) ++ * ++ * Similar to cgroup_css() but returns the effctive css, which is defined ++ * as the matching css of the nearest ancestor including self which has @ss ++ * enabled. If @ss is associated with the hierarchy @cgrp is on, this ++ * function is guaranteed to return non-NULL css. ++ */ ++static struct cgroup_subsys_state *cgroup_e_css(struct cgroup *cgrp, ++ struct cgroup_subsys *ss) ++{ ++ lockdep_assert_held(&cgroup_mutex); + -+ /sys/kernel/debug/tracing/latency_hist/enable/missed_timer_offsets ++ if (!ss) ++ return &cgrp->self; + -+ The histogram data will be located in the debug file system at ++ if (!(cgrp->root->subsys_mask & (1 << ss->id))) ++ return NULL; + -+ /sys/kernel/debug/tracing/latency_hist/missed_timer_offsets ++ /* ++ * This function is used while updating css associations and thus ++ * can't test the csses directly. Use ->child_subsys_mask. ++ */ ++ while (cgroup_parent(cgrp) && ++ !(cgroup_parent(cgrp)->child_subsys_mask & (1 << ss->id))) ++ cgrp = cgroup_parent(cgrp); + -+ If both Scheduling Latency Histogram and Missed Timer Offsets -+ Histogram are selected, additional histogram data will be collected -+ that contain, in addition to the wakeup latency, the timer latency, in -+ case the wakeup was triggered by an expired timer. These histograms -+ are available in the ++ return cgroup_css(cgrp, ss); ++} + -+ /sys/kernel/debug/tracing/latency_hist/timerandwakeup ++/** ++ * cgroup_get_e_css - get a cgroup's effective css for the specified subsystem ++ * @cgrp: the cgroup of interest ++ * @ss: the subsystem of interest ++ * ++ * Find and get the effective css of @cgrp for @ss. The effective css is ++ * defined as the matching css of the nearest ancestor including self which ++ * has @ss enabled. If @ss is not mounted on the hierarchy @cgrp is on, ++ * the root css is returned, so this function always returns a valid css. ++ * The returned css must be put using css_put(). ++ */ ++struct cgroup_subsys_state *cgroup_get_e_css(struct cgroup *cgrp, ++ struct cgroup_subsys *ss) ++{ ++ struct cgroup_subsys_state *css; + -+ directory. They reflect the apparent interrupt and scheduling latency -+ and are best suitable to determine the worst-case latency of a given -+ system. To enable these histograms, write a non-zero number to ++ rcu_read_lock(); + -+ /sys/kernel/debug/tracing/latency_hist/enable/timerandwakeup ++ do { ++ css = cgroup_css(cgrp, ss); + - config ENABLE_DEFAULT_TRACERS - bool "Trace process context switches and events" - depends on !GENERIC_TRACER -diff -Nur linux-4.1.6.orig/kernel/trace/latency_hist.c linux-4.1.6/kernel/trace/latency_hist.c ---- linux-4.1.6.orig/kernel/trace/latency_hist.c 1970-01-01 01:00:00.000000000 +0100 -+++ linux-4.1.6/kernel/trace/latency_hist.c 2015-09-08 23:49:08.125865168 +0200 -@@ -0,0 +1,1178 @@ -+/* -+ * kernel/trace/latency_hist.c -+ * -+ * Add support for histograms of preemption-off latency and -+ * interrupt-off latency and wakeup latency, it depends on -+ * Real-Time Preemption Support. -+ * -+ * Copyright (C) 2005 MontaVista Software, Inc. -+ * Yi Yang ++ if (css && css_tryget_online(css)) ++ goto out_unlock; ++ cgrp = cgroup_parent(cgrp); ++ } while (cgrp); ++ ++ css = init_css_set.subsys[ss->id]; ++ css_get(css); ++out_unlock: ++ rcu_read_unlock(); ++ return css; ++} ++ ++/* convenient tests for these bits */ ++static inline bool cgroup_is_dead(const struct cgroup *cgrp) ++{ ++ return !(cgrp->self.flags & CSS_ONLINE); ++} ++ ++struct cgroup_subsys_state *of_css(struct kernfs_open_file *of) ++{ ++ struct cgroup *cgrp = of->kn->parent->priv; ++ struct cftype *cft = of_cft(of); ++ ++ /* ++ * This is open and unprotected implementation of cgroup_css(). ++ * seq_css() is only called from a kernfs file operation which has ++ * an active reference on the file. Because all the subsystem ++ * files are drained before a css is disassociated with a cgroup, ++ * the matching css from the cgroup's subsys table is guaranteed to ++ * be and stay valid until the enclosing operation is complete. ++ */ ++ if (cft->ss) ++ return rcu_dereference_raw(cgrp->subsys[cft->ss->id]); ++ else ++ return &cgrp->self; ++} ++EXPORT_SYMBOL_GPL(of_css); ++ ++/** ++ * cgroup_is_descendant - test ancestry ++ * @cgrp: the cgroup to be tested ++ * @ancestor: possible ancestor of @cgrp + * -+ * Converted to work with the new latency tracer. -+ * Copyright (C) 2008 Red Hat, Inc. -+ * Steven Rostedt ++ * Test whether @cgrp is a descendant of @ancestor. It also returns %true ++ * if @cgrp == @ancestor. This function is safe to call as long as @cgrp ++ * and @ancestor are accessible. ++ */ ++bool cgroup_is_descendant(struct cgroup *cgrp, struct cgroup *ancestor) ++{ ++ while (cgrp) { ++ if (cgrp == ancestor) ++ return true; ++ cgrp = cgroup_parent(cgrp); ++ } ++ return false; ++} ++ ++static int notify_on_release(const struct cgroup *cgrp) ++{ ++ return test_bit(CGRP_NOTIFY_ON_RELEASE, &cgrp->flags); ++} ++ ++/** ++ * for_each_css - iterate all css's of a cgroup ++ * @css: the iteration cursor ++ * @ssid: the index of the subsystem, CGROUP_SUBSYS_COUNT after reaching the end ++ * @cgrp: the target cgroup to iterate css's of + * ++ * Should be called under cgroup_[tree_]mutex. + */ -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include -+#include ++#define for_each_css(css, ssid, cgrp) \ ++ for ((ssid) = 0; (ssid) < CGROUP_SUBSYS_COUNT; (ssid)++) \ ++ if (!((css) = rcu_dereference_check( \ ++ (cgrp)->subsys[(ssid)], \ ++ lockdep_is_held(&cgroup_mutex)))) { } \ ++ else + -+#include "trace.h" -+#include ++/** ++ * for_each_e_css - iterate all effective css's of a cgroup ++ * @css: the iteration cursor ++ * @ssid: the index of the subsystem, CGROUP_SUBSYS_COUNT after reaching the end ++ * @cgrp: the target cgroup to iterate css's of ++ * ++ * Should be called under cgroup_[tree_]mutex. ++ */ ++#define for_each_e_css(css, ssid, cgrp) \ ++ for ((ssid) = 0; (ssid) < CGROUP_SUBSYS_COUNT; (ssid)++) \ ++ if (!((css) = cgroup_e_css(cgrp, cgroup_subsys[(ssid)]))) \ ++ ; \ ++ else + -+#define NSECS_PER_USECS 1000L ++/** ++ * for_each_subsys - iterate all enabled cgroup subsystems ++ * @ss: the iteration cursor ++ * @ssid: the index of @ss, CGROUP_SUBSYS_COUNT after reaching the end ++ */ ++#define for_each_subsys(ss, ssid) \ ++ for ((ssid) = 0; (ssid) < CGROUP_SUBSYS_COUNT && \ ++ (((ss) = cgroup_subsys[ssid]) || true); (ssid)++) ++ ++/* iterate across the hierarchies */ ++#define for_each_root(root) \ ++ list_for_each_entry((root), &cgroup_roots, root_list) ++ ++/* iterate over child cgrps, lock should be held throughout iteration */ ++#define cgroup_for_each_live_child(child, cgrp) \ ++ list_for_each_entry((child), &(cgrp)->self.children, self.sibling) \ ++ if (({ lockdep_assert_held(&cgroup_mutex); \ ++ cgroup_is_dead(child); })) \ ++ ; \ ++ else + -+#define CREATE_TRACE_POINTS -+#include ++static void cgroup_release_agent(struct work_struct *work); ++static void check_for_release(struct cgroup *cgrp); + -+enum { -+ IRQSOFF_LATENCY = 0, -+ PREEMPTOFF_LATENCY, -+ PREEMPTIRQSOFF_LATENCY, -+ WAKEUP_LATENCY, -+ WAKEUP_LATENCY_SHAREDPRIO, -+ MISSED_TIMER_OFFSETS, -+ TIMERANDWAKEUP_LATENCY, -+ MAX_LATENCY_TYPE, -+}; ++/* ++ * A cgroup can be associated with multiple css_sets as different tasks may ++ * belong to different cgroups on different hierarchies. In the other ++ * direction, a css_set is naturally associated with multiple cgroups. ++ * This M:N relationship is represented by the following link structure ++ * which exists for each association and allows traversing the associations ++ * from both sides. ++ */ ++struct cgrp_cset_link { ++ /* the cgroup and css_set this link associates */ ++ struct cgroup *cgrp; ++ struct css_set *cset; + -+#define MAX_ENTRY_NUM 10240 ++ /* list of cgrp_cset_links anchored at cgrp->cset_links */ ++ struct list_head cset_link; + -+struct hist_data { -+ atomic_t hist_mode; /* 0 log, 1 don't log */ -+ long offset; /* set it to MAX_ENTRY_NUM/2 for a bipolar scale */ -+ long min_lat; -+ long max_lat; -+ unsigned long long below_hist_bound_samples; -+ unsigned long long above_hist_bound_samples; -+ long long accumulate_lat; -+ unsigned long long total_samples; -+ unsigned long long hist_array[MAX_ENTRY_NUM]; ++ /* list of cgrp_cset_links anchored at css_set->cgrp_links */ ++ struct list_head cgrp_link; +}; + -+struct enable_data { -+ int latency_type; -+ int enabled; ++/* ++ * The default css_set - used by init and its children prior to any ++ * hierarchies being mounted. It contains a pointer to the root state ++ * for each subsystem. Also used to anchor the list of css_sets. Not ++ * reference-counted, to improve performance when child cgroups ++ * haven't been created. ++ */ ++struct css_set init_css_set = { ++ .refcount = ATOMIC_INIT(1), ++ .cgrp_links = LIST_HEAD_INIT(init_css_set.cgrp_links), ++ .tasks = LIST_HEAD_INIT(init_css_set.tasks), ++ .mg_tasks = LIST_HEAD_INIT(init_css_set.mg_tasks), ++ .mg_preload_node = LIST_HEAD_INIT(init_css_set.mg_preload_node), ++ .mg_node = LIST_HEAD_INIT(init_css_set.mg_node), +}; + -+static char *latency_hist_dir_root = "latency_hist"; ++static int css_set_count = 1; /* 1 for init_css_set */ + -+#ifdef CONFIG_INTERRUPT_OFF_HIST -+static DEFINE_PER_CPU(struct hist_data, irqsoff_hist); -+static char *irqsoff_hist_dir = "irqsoff"; -+static DEFINE_PER_CPU(cycles_t, hist_irqsoff_start); -+static DEFINE_PER_CPU(int, hist_irqsoff_counting); -+#endif ++/** ++ * cgroup_update_populated - updated populated count of a cgroup ++ * @cgrp: the target cgroup ++ * @populated: inc or dec populated count ++ * ++ * @cgrp is either getting the first task (css_set) or losing the last. ++ * Update @cgrp->populated_cnt accordingly. The count is propagated ++ * towards root so that a given cgroup's populated_cnt is zero iff the ++ * cgroup and all its descendants are empty. ++ * ++ * @cgrp's interface file "cgroup.populated" is zero if ++ * @cgrp->populated_cnt is zero and 1 otherwise. When @cgrp->populated_cnt ++ * changes from or to zero, userland is notified that the content of the ++ * interface file has changed. This can be used to detect when @cgrp and ++ * its descendants become populated or empty. ++ */ ++static void cgroup_update_populated(struct cgroup *cgrp, bool populated) ++{ ++ lockdep_assert_held(&css_set_rwsem); + -+#ifdef CONFIG_PREEMPT_OFF_HIST -+static DEFINE_PER_CPU(struct hist_data, preemptoff_hist); -+static char *preemptoff_hist_dir = "preemptoff"; -+static DEFINE_PER_CPU(cycles_t, hist_preemptoff_start); -+static DEFINE_PER_CPU(int, hist_preemptoff_counting); -+#endif ++ do { ++ bool trigger; + -+#if defined(CONFIG_PREEMPT_OFF_HIST) && defined(CONFIG_INTERRUPT_OFF_HIST) -+static DEFINE_PER_CPU(struct hist_data, preemptirqsoff_hist); -+static char *preemptirqsoff_hist_dir = "preemptirqsoff"; -+static DEFINE_PER_CPU(cycles_t, hist_preemptirqsoff_start); -+static DEFINE_PER_CPU(int, hist_preemptirqsoff_counting); -+#endif ++ if (populated) ++ trigger = !cgrp->populated_cnt++; ++ else ++ trigger = !--cgrp->populated_cnt; + -+#if defined(CONFIG_PREEMPT_OFF_HIST) || defined(CONFIG_INTERRUPT_OFF_HIST) -+static notrace void probe_preemptirqsoff_hist(void *v, int reason, int start); -+static struct enable_data preemptirqsoff_enabled_data = { -+ .latency_type = PREEMPTIRQSOFF_LATENCY, -+ .enabled = 0, -+}; -+#endif ++ if (!trigger) ++ break; + -+#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \ -+ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) -+struct maxlatproc_data { -+ char comm[FIELD_SIZEOF(struct task_struct, comm)]; -+ char current_comm[FIELD_SIZEOF(struct task_struct, comm)]; -+ int pid; -+ int current_pid; -+ int prio; -+ int current_prio; -+ long latency; -+ long timeroffset; -+ cycle_t timestamp; -+}; -+#endif ++ if (cgrp->populated_kn) ++ kernfs_notify(cgrp->populated_kn); ++ cgrp = cgroup_parent(cgrp); ++ } while (cgrp); ++} + -+#ifdef CONFIG_WAKEUP_LATENCY_HIST -+static DEFINE_PER_CPU(struct hist_data, wakeup_latency_hist); -+static DEFINE_PER_CPU(struct hist_data, wakeup_latency_hist_sharedprio); -+static char *wakeup_latency_hist_dir = "wakeup"; -+static char *wakeup_latency_hist_dir_sharedprio = "sharedprio"; -+static notrace void probe_wakeup_latency_hist_start(void *v, -+ struct task_struct *p, int success); -+static notrace void probe_wakeup_latency_hist_stop(void *v, -+ struct task_struct *prev, struct task_struct *next); -+static notrace void probe_sched_migrate_task(void *, -+ struct task_struct *task, int cpu); -+static struct enable_data wakeup_latency_enabled_data = { -+ .latency_type = WAKEUP_LATENCY, -+ .enabled = 0, -+}; -+static DEFINE_PER_CPU(struct maxlatproc_data, wakeup_maxlatproc); -+static DEFINE_PER_CPU(struct maxlatproc_data, wakeup_maxlatproc_sharedprio); -+static DEFINE_PER_CPU(struct task_struct *, wakeup_task); -+static DEFINE_PER_CPU(int, wakeup_sharedprio); -+static unsigned long wakeup_pid; -+#endif ++/* ++ * hash table for cgroup groups. This improves the performance to find ++ * an existing css_set. This hash doesn't (currently) take into ++ * account cgroups in empty hierarchies. ++ */ ++#define CSS_SET_HASH_BITS 7 ++static DEFINE_HASHTABLE(css_set_table, CSS_SET_HASH_BITS); + -+#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST -+static DEFINE_PER_CPU(struct hist_data, missed_timer_offsets); -+static char *missed_timer_offsets_dir = "missed_timer_offsets"; -+static notrace void probe_hrtimer_interrupt(void *v, int cpu, -+ long long offset, struct task_struct *curr, struct task_struct *task); -+static struct enable_data missed_timer_offsets_enabled_data = { -+ .latency_type = MISSED_TIMER_OFFSETS, -+ .enabled = 0, -+}; -+static DEFINE_PER_CPU(struct maxlatproc_data, missed_timer_offsets_maxlatproc); -+static unsigned long missed_timer_offsets_pid; -+#endif ++static unsigned long css_set_hash(struct cgroup_subsys_state *css[]) ++{ ++ unsigned long key = 0UL; ++ struct cgroup_subsys *ss; ++ int i; + -+#if defined(CONFIG_WAKEUP_LATENCY_HIST) && \ -+ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) -+static DEFINE_PER_CPU(struct hist_data, timerandwakeup_latency_hist); -+static char *timerandwakeup_latency_hist_dir = "timerandwakeup"; -+static struct enable_data timerandwakeup_enabled_data = { -+ .latency_type = TIMERANDWAKEUP_LATENCY, -+ .enabled = 0, -+}; -+static DEFINE_PER_CPU(struct maxlatproc_data, timerandwakeup_maxlatproc); -+#endif ++ for_each_subsys(ss, i) ++ key += (unsigned long)css[i]; ++ key = (key >> 16) ^ key; + -+void notrace latency_hist(int latency_type, int cpu, long latency, -+ long timeroffset, cycle_t stop, -+ struct task_struct *p) ++ return key; ++} ++ ++static void put_css_set_locked(struct css_set *cset) +{ -+ struct hist_data *my_hist; -+#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \ -+ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) -+ struct maxlatproc_data *mp = NULL; -+#endif ++ struct cgrp_cset_link *link, *tmp_link; ++ struct cgroup_subsys *ss; ++ int ssid; + -+ if (!cpu_possible(cpu) || latency_type < 0 || -+ latency_type >= MAX_LATENCY_TYPE) ++ lockdep_assert_held(&css_set_rwsem); ++ ++ if (!atomic_dec_and_test(&cset->refcount)) + return; + -+ switch (latency_type) { -+#ifdef CONFIG_INTERRUPT_OFF_HIST -+ case IRQSOFF_LATENCY: -+ my_hist = &per_cpu(irqsoff_hist, cpu); -+ break; -+#endif -+#ifdef CONFIG_PREEMPT_OFF_HIST -+ case PREEMPTOFF_LATENCY: -+ my_hist = &per_cpu(preemptoff_hist, cpu); -+ break; -+#endif -+#if defined(CONFIG_PREEMPT_OFF_HIST) && defined(CONFIG_INTERRUPT_OFF_HIST) -+ case PREEMPTIRQSOFF_LATENCY: -+ my_hist = &per_cpu(preemptirqsoff_hist, cpu); -+ break; -+#endif -+#ifdef CONFIG_WAKEUP_LATENCY_HIST -+ case WAKEUP_LATENCY: -+ my_hist = &per_cpu(wakeup_latency_hist, cpu); -+ mp = &per_cpu(wakeup_maxlatproc, cpu); -+ break; -+ case WAKEUP_LATENCY_SHAREDPRIO: -+ my_hist = &per_cpu(wakeup_latency_hist_sharedprio, cpu); -+ mp = &per_cpu(wakeup_maxlatproc_sharedprio, cpu); -+ break; -+#endif -+#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST -+ case MISSED_TIMER_OFFSETS: -+ my_hist = &per_cpu(missed_timer_offsets, cpu); -+ mp = &per_cpu(missed_timer_offsets_maxlatproc, cpu); -+ break; -+#endif -+#if defined(CONFIG_WAKEUP_LATENCY_HIST) && \ -+ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) -+ case TIMERANDWAKEUP_LATENCY: -+ my_hist = &per_cpu(timerandwakeup_latency_hist, cpu); -+ mp = &per_cpu(timerandwakeup_maxlatproc, cpu); -+ break; -+#endif ++ /* This css_set is dead. unlink it and release cgroup refcounts */ ++ for_each_subsys(ss, ssid) ++ list_del(&cset->e_cset_node[ssid]); ++ hash_del(&cset->hlist); ++ css_set_count--; + -+ default: -+ return; ++ list_for_each_entry_safe(link, tmp_link, &cset->cgrp_links, cgrp_link) { ++ struct cgroup *cgrp = link->cgrp; ++ ++ list_del(&link->cset_link); ++ list_del(&link->cgrp_link); ++ ++ /* @cgrp can't go away while we're holding css_set_rwsem */ ++ if (list_empty(&cgrp->cset_links)) { ++ cgroup_update_populated(cgrp, false); ++ check_for_release(cgrp); ++ } ++ ++ kfree(link); + } + -+ latency += my_hist->offset; ++ kfree_rcu(cset, rcu_head); ++} + -+ if (atomic_read(&my_hist->hist_mode) == 0) ++static void put_css_set(struct css_set *cset) ++{ ++ /* ++ * Ensure that the refcount doesn't hit zero while any readers ++ * can see it. Similar to atomic_dec_and_lock(), but for an ++ * rwlock ++ */ ++ if (atomic_add_unless(&cset->refcount, -1, 1)) + return; + -+ if (latency < 0 || latency >= MAX_ENTRY_NUM) { -+ if (latency < 0) -+ my_hist->below_hist_bound_samples++; -+ else -+ my_hist->above_hist_bound_samples++; -+ } else -+ my_hist->hist_array[latency]++; ++ down_write(&css_set_rwsem); ++ put_css_set_locked(cset); ++ up_write(&css_set_rwsem); ++} + -+ if (unlikely(latency > my_hist->max_lat || -+ my_hist->min_lat == LONG_MAX)) { -+#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \ -+ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) -+ if (latency_type == WAKEUP_LATENCY || -+ latency_type == WAKEUP_LATENCY_SHAREDPRIO || -+ latency_type == MISSED_TIMER_OFFSETS || -+ latency_type == TIMERANDWAKEUP_LATENCY) { -+ strncpy(mp->comm, p->comm, sizeof(mp->comm)); -+ strncpy(mp->current_comm, current->comm, -+ sizeof(mp->current_comm)); -+ mp->pid = task_pid_nr(p); -+ mp->current_pid = task_pid_nr(current); -+ mp->prio = p->prio; -+ mp->current_prio = current->prio; -+ mp->latency = latency; -+ mp->timeroffset = timeroffset; -+ mp->timestamp = stop; -+ } -+#endif -+ my_hist->max_lat = latency; -+ } -+ if (unlikely(latency < my_hist->min_lat)) -+ my_hist->min_lat = latency; -+ my_hist->total_samples++; -+ my_hist->accumulate_lat += latency; ++/* ++ * refcounted get/put for css_set objects ++ */ ++static inline void get_css_set(struct css_set *cset) ++{ ++ atomic_inc(&cset->refcount); +} + -+static void *l_start(struct seq_file *m, loff_t *pos) ++/** ++ * compare_css_sets - helper function for find_existing_css_set(). ++ * @cset: candidate css_set being tested ++ * @old_cset: existing css_set for a task ++ * @new_cgrp: cgroup that's being entered by the task ++ * @template: desired set of css pointers in css_set (pre-calculated) ++ * ++ * Returns true if "cset" matches "old_cset" except for the hierarchy ++ * which "new_cgrp" belongs to, for which it should match "new_cgrp". ++ */ ++static bool compare_css_sets(struct css_set *cset, ++ struct css_set *old_cset, ++ struct cgroup *new_cgrp, ++ struct cgroup_subsys_state *template[]) +{ -+ loff_t *index_ptr = NULL; -+ loff_t index = *pos; -+ struct hist_data *my_hist = m->private; ++ struct list_head *l1, *l2; + -+ if (index == 0) { -+ char minstr[32], avgstr[32], maxstr[32]; ++ /* ++ * On the default hierarchy, there can be csets which are ++ * associated with the same set of cgroups but different csses. ++ * Let's first ensure that csses match. ++ */ ++ if (memcmp(template, cset->subsys, sizeof(cset->subsys))) ++ return false; + -+ atomic_dec(&my_hist->hist_mode); ++ /* ++ * Compare cgroup pointers in order to distinguish between ++ * different cgroups in hierarchies. As different cgroups may ++ * share the same effective css, this comparison is always ++ * necessary. ++ */ ++ l1 = &cset->cgrp_links; ++ l2 = &old_cset->cgrp_links; ++ while (1) { ++ struct cgrp_cset_link *link1, *link2; ++ struct cgroup *cgrp1, *cgrp2; ++ ++ l1 = l1->next; ++ l2 = l2->next; ++ /* See if we reached the end - both lists are equal length. */ ++ if (l1 == &cset->cgrp_links) { ++ BUG_ON(l2 != &old_cset->cgrp_links); ++ break; ++ } else { ++ BUG_ON(l2 == &old_cset->cgrp_links); ++ } ++ /* Locate the cgroups associated with these links. */ ++ link1 = list_entry(l1, struct cgrp_cset_link, cgrp_link); ++ link2 = list_entry(l2, struct cgrp_cset_link, cgrp_link); ++ cgrp1 = link1->cgrp; ++ cgrp2 = link2->cgrp; ++ /* Hierarchies should be linked in the same order. */ ++ BUG_ON(cgrp1->root != cgrp2->root); + -+ if (likely(my_hist->total_samples)) { -+ long avg = (long) div64_s64(my_hist->accumulate_lat, -+ my_hist->total_samples); -+ snprintf(minstr, sizeof(minstr), "%ld", -+ my_hist->min_lat - my_hist->offset); -+ snprintf(avgstr, sizeof(avgstr), "%ld", -+ avg - my_hist->offset); -+ snprintf(maxstr, sizeof(maxstr), "%ld", -+ my_hist->max_lat - my_hist->offset); ++ /* ++ * If this hierarchy is the hierarchy of the cgroup ++ * that's changing, then we need to check that this ++ * css_set points to the new cgroup; if it's any other ++ * hierarchy, then this css_set should point to the ++ * same cgroup as the old css_set. ++ */ ++ if (cgrp1->root == new_cgrp->root) { ++ if (cgrp1 != new_cgrp) ++ return false; + } else { -+ strcpy(minstr, ""); -+ strcpy(avgstr, minstr); -+ strcpy(maxstr, minstr); ++ if (cgrp1 != cgrp2) ++ return false; + } ++ } ++ return true; ++} + -+ seq_printf(m, "#Minimum latency: %s microseconds\n" -+ "#Average latency: %s microseconds\n" -+ "#Maximum latency: %s microseconds\n" -+ "#Total samples: %llu\n" -+ "#There are %llu samples lower than %ld" -+ " microseconds.\n" -+ "#There are %llu samples greater or equal" -+ " than %ld microseconds.\n" -+ "#usecs\t%16s\n", -+ minstr, avgstr, maxstr, -+ my_hist->total_samples, -+ my_hist->below_hist_bound_samples, -+ -my_hist->offset, -+ my_hist->above_hist_bound_samples, -+ MAX_ENTRY_NUM - my_hist->offset, -+ "samples"); ++/** ++ * find_existing_css_set - init css array and find the matching css_set ++ * @old_cset: the css_set that we're using before the cgroup transition ++ * @cgrp: the cgroup that we're moving into ++ * @template: out param for the new set of csses, should be clear on entry ++ */ ++static struct css_set *find_existing_css_set(struct css_set *old_cset, ++ struct cgroup *cgrp, ++ struct cgroup_subsys_state *template[]) ++{ ++ struct cgroup_root *root = cgrp->root; ++ struct cgroup_subsys *ss; ++ struct css_set *cset; ++ unsigned long key; ++ int i; ++ ++ /* ++ * Build the set of subsystem state objects that we want to see in the ++ * new css_set. while subsystems can change globally, the entries here ++ * won't change, so no need for locking. ++ */ ++ for_each_subsys(ss, i) { ++ if (root->subsys_mask & (1UL << i)) { ++ /* ++ * @ss is in this hierarchy, so we want the ++ * effective css from @cgrp. ++ */ ++ template[i] = cgroup_e_css(cgrp, ss); ++ } else { ++ /* ++ * @ss is not in this hierarchy, so we don't want ++ * to change the css. ++ */ ++ template[i] = old_cset->subsys[i]; ++ } + } -+ if (index < MAX_ENTRY_NUM) { -+ index_ptr = kmalloc(sizeof(loff_t), GFP_KERNEL); -+ if (index_ptr) -+ *index_ptr = index; ++ ++ key = css_set_hash(template); ++ hash_for_each_possible(css_set_table, cset, hlist, key) { ++ if (!compare_css_sets(cset, old_cset, cgrp, template)) ++ continue; ++ ++ /* This css_set matches what we need */ ++ return cset; + } + -+ return index_ptr; ++ /* No existing cgroup group matched */ ++ return NULL; +} + -+static void *l_next(struct seq_file *m, void *p, loff_t *pos) ++static void free_cgrp_cset_links(struct list_head *links_to_free) +{ -+ loff_t *index_ptr = p; -+ struct hist_data *my_hist = m->private; ++ struct cgrp_cset_link *link, *tmp_link; + -+ if (++*pos >= MAX_ENTRY_NUM) { -+ atomic_inc(&my_hist->hist_mode); ++ list_for_each_entry_safe(link, tmp_link, links_to_free, cset_link) { ++ list_del(&link->cset_link); ++ kfree(link); ++ } ++} ++ ++/** ++ * allocate_cgrp_cset_links - allocate cgrp_cset_links ++ * @count: the number of links to allocate ++ * @tmp_links: list_head the allocated links are put on ++ * ++ * Allocate @count cgrp_cset_link structures and chain them on @tmp_links ++ * through ->cset_link. Returns 0 on success or -errno. ++ */ ++static int allocate_cgrp_cset_links(int count, struct list_head *tmp_links) ++{ ++ struct cgrp_cset_link *link; ++ int i; ++ ++ INIT_LIST_HEAD(tmp_links); ++ ++ for (i = 0; i < count; i++) { ++ link = kzalloc(sizeof(*link), GFP_KERNEL); ++ if (!link) { ++ free_cgrp_cset_links(tmp_links); ++ return -ENOMEM; ++ } ++ list_add(&link->cset_link, tmp_links); ++ } ++ return 0; ++} ++ ++/** ++ * link_css_set - a helper function to link a css_set to a cgroup ++ * @tmp_links: cgrp_cset_link objects allocated by allocate_cgrp_cset_links() ++ * @cset: the css_set to be linked ++ * @cgrp: the destination cgroup ++ */ ++static void link_css_set(struct list_head *tmp_links, struct css_set *cset, ++ struct cgroup *cgrp) ++{ ++ struct cgrp_cset_link *link; ++ ++ BUG_ON(list_empty(tmp_links)); ++ ++ if (cgroup_on_dfl(cgrp)) ++ cset->dfl_cgrp = cgrp; ++ ++ link = list_first_entry(tmp_links, struct cgrp_cset_link, cset_link); ++ link->cset = cset; ++ link->cgrp = cgrp; ++ ++ if (list_empty(&cgrp->cset_links)) ++ cgroup_update_populated(cgrp, true); ++ list_move(&link->cset_link, &cgrp->cset_links); ++ ++ /* ++ * Always add links to the tail of the list so that the list ++ * is sorted by order of hierarchy creation ++ */ ++ list_add_tail(&link->cgrp_link, &cset->cgrp_links); ++} ++ ++/** ++ * find_css_set - return a new css_set with one cgroup updated ++ * @old_cset: the baseline css_set ++ * @cgrp: the cgroup to be updated ++ * ++ * Return a new css_set that's equivalent to @old_cset, but with @cgrp ++ * substituted into the appropriate hierarchy. ++ */ ++static struct css_set *find_css_set(struct css_set *old_cset, ++ struct cgroup *cgrp) ++{ ++ struct cgroup_subsys_state *template[CGROUP_SUBSYS_COUNT] = { }; ++ struct css_set *cset; ++ struct list_head tmp_links; ++ struct cgrp_cset_link *link; ++ struct cgroup_subsys *ss; ++ unsigned long key; ++ int ssid; ++ ++ lockdep_assert_held(&cgroup_mutex); ++ ++ /* First see if we already have a cgroup group that matches ++ * the desired set */ ++ down_read(&css_set_rwsem); ++ cset = find_existing_css_set(old_cset, cgrp, template); ++ if (cset) ++ get_css_set(cset); ++ up_read(&css_set_rwsem); ++ ++ if (cset) ++ return cset; ++ ++ cset = kzalloc(sizeof(*cset), GFP_KERNEL); ++ if (!cset) ++ return NULL; ++ ++ /* Allocate all the cgrp_cset_link objects that we'll need */ ++ if (allocate_cgrp_cset_links(cgroup_root_count, &tmp_links) < 0) { ++ kfree(cset); + return NULL; + } -+ *index_ptr = *pos; -+ return index_ptr; ++ ++ atomic_set(&cset->refcount, 1); ++ INIT_LIST_HEAD(&cset->cgrp_links); ++ INIT_LIST_HEAD(&cset->tasks); ++ INIT_LIST_HEAD(&cset->mg_tasks); ++ INIT_LIST_HEAD(&cset->mg_preload_node); ++ INIT_LIST_HEAD(&cset->mg_node); ++ INIT_HLIST_NODE(&cset->hlist); ++ ++ /* Copy the set of subsystem state objects generated in ++ * find_existing_css_set() */ ++ memcpy(cset->subsys, template, sizeof(cset->subsys)); ++ ++ down_write(&css_set_rwsem); ++ /* Add reference counts and links from the new css_set. */ ++ list_for_each_entry(link, &old_cset->cgrp_links, cgrp_link) { ++ struct cgroup *c = link->cgrp; ++ ++ if (c->root == cgrp->root) ++ c = cgrp; ++ link_css_set(&tmp_links, cset, c); ++ } ++ ++ BUG_ON(!list_empty(&tmp_links)); ++ ++ css_set_count++; ++ ++ /* Add @cset to the hash table */ ++ key = css_set_hash(cset->subsys); ++ hash_add(css_set_table, &cset->hlist, key); ++ ++ for_each_subsys(ss, ssid) ++ list_add_tail(&cset->e_cset_node[ssid], ++ &cset->subsys[ssid]->cgroup->e_csets[ssid]); ++ ++ up_write(&css_set_rwsem); ++ ++ return cset; +} + -+static void l_stop(struct seq_file *m, void *p) ++static struct cgroup_root *cgroup_root_from_kf(struct kernfs_root *kf_root) +{ -+ kfree(p); ++ struct cgroup *root_cgrp = kf_root->kn->priv; ++ ++ return root_cgrp->root; +} + -+static int l_show(struct seq_file *m, void *p) ++static int cgroup_init_root_id(struct cgroup_root *root) +{ -+ int index = *(loff_t *) p; -+ struct hist_data *my_hist = m->private; ++ int id; + -+ seq_printf(m, "%6ld\t%16llu\n", index - my_hist->offset, -+ my_hist->hist_array[index]); ++ lockdep_assert_held(&cgroup_mutex); ++ ++ id = idr_alloc_cyclic(&cgroup_hierarchy_idr, root, 0, 0, GFP_KERNEL); ++ if (id < 0) ++ return id; ++ ++ root->hierarchy_id = id; + return 0; +} + -+static const struct seq_operations latency_hist_seq_op = { -+ .start = l_start, -+ .next = l_next, -+ .stop = l_stop, -+ .show = l_show -+}; -+ -+static int latency_hist_open(struct inode *inode, struct file *file) ++static void cgroup_exit_root_id(struct cgroup_root *root) +{ -+ int ret; ++ lockdep_assert_held(&cgroup_mutex); + -+ ret = seq_open(file, &latency_hist_seq_op); -+ if (!ret) { -+ struct seq_file *seq = file->private_data; -+ seq->private = inode->i_private; ++ if (root->hierarchy_id) { ++ idr_remove(&cgroup_hierarchy_idr, root->hierarchy_id); ++ root->hierarchy_id = 0; + } -+ return ret; +} + -+static const struct file_operations latency_hist_fops = { -+ .open = latency_hist_open, -+ .read = seq_read, -+ .llseek = seq_lseek, -+ .release = seq_release, -+}; -+ -+#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \ -+ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) -+static void clear_maxlatprocdata(struct maxlatproc_data *mp) ++static void cgroup_free_root(struct cgroup_root *root) +{ -+ mp->comm[0] = mp->current_comm[0] = '\0'; -+ mp->prio = mp->current_prio = mp->pid = mp->current_pid = -+ mp->latency = mp->timeroffset = -1; -+ mp->timestamp = 0; ++ if (root) { ++ /* hierarhcy ID shoulid already have been released */ ++ WARN_ON_ONCE(root->hierarchy_id); ++ ++ idr_destroy(&root->cgroup_idr); ++ kfree(root); ++ } +} -+#endif + -+static void hist_reset(struct hist_data *hist) ++static void cgroup_destroy_root(struct cgroup_root *root) +{ -+ atomic_dec(&hist->hist_mode); ++ struct cgroup *cgrp = &root->cgrp; ++ struct cgrp_cset_link *link, *tmp_link; + -+ memset(hist->hist_array, 0, sizeof(hist->hist_array)); -+ hist->below_hist_bound_samples = 0ULL; -+ hist->above_hist_bound_samples = 0ULL; -+ hist->min_lat = LONG_MAX; -+ hist->max_lat = LONG_MIN; -+ hist->total_samples = 0ULL; -+ hist->accumulate_lat = 0LL; ++ mutex_lock(&cgroup_mutex); + -+ atomic_inc(&hist->hist_mode); ++ BUG_ON(atomic_read(&root->nr_cgrps)); ++ BUG_ON(!list_empty(&cgrp->self.children)); ++ ++ /* Rebind all subsystems back to the default hierarchy */ ++ rebind_subsystems(&cgrp_dfl_root, root->subsys_mask); ++ ++ /* ++ * Release all the links from cset_links to this hierarchy's ++ * root cgroup ++ */ ++ down_write(&css_set_rwsem); ++ ++ list_for_each_entry_safe(link, tmp_link, &cgrp->cset_links, cset_link) { ++ list_del(&link->cset_link); ++ list_del(&link->cgrp_link); ++ kfree(link); ++ } ++ up_write(&css_set_rwsem); ++ ++ if (!list_empty(&root->root_list)) { ++ list_del(&root->root_list); ++ cgroup_root_count--; ++ } ++ ++ cgroup_exit_root_id(root); ++ ++ mutex_unlock(&cgroup_mutex); ++ ++ kernfs_destroy_root(root->kf_root); ++ cgroup_free_root(root); +} + -+static ssize_t -+latency_hist_reset(struct file *file, const char __user *a, -+ size_t size, loff_t *off) ++/* look up cgroup associated with given css_set on the specified hierarchy */ ++static struct cgroup *cset_cgroup_from_root(struct css_set *cset, ++ struct cgroup_root *root) +{ -+ int cpu; -+ struct hist_data *hist = NULL; -+#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \ -+ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) -+ struct maxlatproc_data *mp = NULL; -+#endif -+ off_t latency_type = (off_t) file->private_data; ++ struct cgroup *res = NULL; + -+ for_each_online_cpu(cpu) { ++ lockdep_assert_held(&cgroup_mutex); ++ lockdep_assert_held(&css_set_rwsem); + -+ switch (latency_type) { -+#ifdef CONFIG_PREEMPT_OFF_HIST -+ case PREEMPTOFF_LATENCY: -+ hist = &per_cpu(preemptoff_hist, cpu); -+ break; -+#endif -+#ifdef CONFIG_INTERRUPT_OFF_HIST -+ case IRQSOFF_LATENCY: -+ hist = &per_cpu(irqsoff_hist, cpu); -+ break; -+#endif -+#if defined(CONFIG_INTERRUPT_OFF_HIST) && defined(CONFIG_PREEMPT_OFF_HIST) -+ case PREEMPTIRQSOFF_LATENCY: -+ hist = &per_cpu(preemptirqsoff_hist, cpu); -+ break; -+#endif -+#ifdef CONFIG_WAKEUP_LATENCY_HIST -+ case WAKEUP_LATENCY: -+ hist = &per_cpu(wakeup_latency_hist, cpu); -+ mp = &per_cpu(wakeup_maxlatproc, cpu); -+ break; -+ case WAKEUP_LATENCY_SHAREDPRIO: -+ hist = &per_cpu(wakeup_latency_hist_sharedprio, cpu); -+ mp = &per_cpu(wakeup_maxlatproc_sharedprio, cpu); -+ break; -+#endif -+#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST -+ case MISSED_TIMER_OFFSETS: -+ hist = &per_cpu(missed_timer_offsets, cpu); -+ mp = &per_cpu(missed_timer_offsets_maxlatproc, cpu); -+ break; -+#endif -+#if defined(CONFIG_WAKEUP_LATENCY_HIST) && \ -+ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) -+ case TIMERANDWAKEUP_LATENCY: -+ hist = &per_cpu(timerandwakeup_latency_hist, cpu); -+ mp = &per_cpu(timerandwakeup_maxlatproc, cpu); -+ break; -+#endif -+ } ++ if (cset == &init_css_set) { ++ res = &root->cgrp; ++ } else { ++ struct cgrp_cset_link *link; + -+ hist_reset(hist); -+#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \ -+ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) -+ if (latency_type == WAKEUP_LATENCY || -+ latency_type == WAKEUP_LATENCY_SHAREDPRIO || -+ latency_type == MISSED_TIMER_OFFSETS || -+ latency_type == TIMERANDWAKEUP_LATENCY) -+ clear_maxlatprocdata(mp); -+#endif ++ list_for_each_entry(link, &cset->cgrp_links, cgrp_link) { ++ struct cgroup *c = link->cgrp; ++ ++ if (c->root == root) { ++ res = c; ++ break; ++ } ++ } + } + -+ return size; ++ BUG_ON(!res); ++ return res; +} + -+#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \ -+ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) -+static ssize_t -+show_pid(struct file *file, char __user *ubuf, size_t cnt, loff_t *ppos) ++/* ++ * Return the cgroup for "task" from the given hierarchy. Must be ++ * called with cgroup_mutex and css_set_rwsem held. ++ */ ++static struct cgroup *task_cgroup_from_root(struct task_struct *task, ++ struct cgroup_root *root) +{ -+ char buf[64]; -+ int r; -+ unsigned long *this_pid = file->private_data; ++ /* ++ * No need to lock the task - since we hold cgroup_mutex the ++ * task can't change groups, so the only thing that can happen ++ * is that it exits and its css is set back to init_css_set. ++ */ ++ return cset_cgroup_from_root(task_css_set(task), root); ++} + -+ r = snprintf(buf, sizeof(buf), "%lu\n", *this_pid); -+ return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); ++/* ++ * A task must hold cgroup_mutex to modify cgroups. ++ * ++ * Any task can increment and decrement the count field without lock. ++ * So in general, code holding cgroup_mutex can't rely on the count ++ * field not changing. However, if the count goes to zero, then only ++ * cgroup_attach_task() can increment it again. Because a count of zero ++ * means that no tasks are currently attached, therefore there is no ++ * way a task attached to that cgroup can fork (the other way to ++ * increment the count). So code holding cgroup_mutex can safely ++ * assume that if the count is zero, it will stay zero. Similarly, if ++ * a task holds cgroup_mutex on a cgroup with zero count, it ++ * knows that the cgroup won't be removed, as cgroup_rmdir() ++ * needs that mutex. ++ * ++ * A cgroup can only be deleted if both its 'count' of using tasks ++ * is zero, and its list of 'children' cgroups is empty. Since all ++ * tasks in the system use _some_ cgroup, and since there is always at ++ * least one task in the system (init, pid == 1), therefore, root cgroup ++ * always has either children cgroups and/or using tasks. So we don't ++ * need a special hack to ensure that root cgroup cannot be deleted. ++ * ++ * P.S. One more locking exception. RCU is used to guard the ++ * update of a tasks cgroup pointer by cgroup_attach_task() ++ */ ++ ++static int cgroup_populate_dir(struct cgroup *cgrp, unsigned int subsys_mask); ++static struct kernfs_syscall_ops cgroup_kf_syscall_ops; ++static const struct file_operations proc_cgroupstats_operations; ++ ++static char *cgroup_file_name(struct cgroup *cgrp, const struct cftype *cft, ++ char *buf) ++{ ++ if (cft->ss && !(cft->flags & CFTYPE_NO_PREFIX) && ++ !(cgrp->root->flags & CGRP_ROOT_NOPREFIX)) ++ snprintf(buf, CGROUP_FILE_NAME_MAX, "%s.%s", ++ cft->ss->name, cft->name); ++ else ++ strncpy(buf, cft->name, CGROUP_FILE_NAME_MAX); ++ return buf; +} + -+static ssize_t do_pid(struct file *file, const char __user *ubuf, -+ size_t cnt, loff_t *ppos) ++/** ++ * cgroup_file_mode - deduce file mode of a control file ++ * @cft: the control file in question ++ * ++ * returns cft->mode if ->mode is not 0 ++ * returns S_IRUGO|S_IWUSR if it has both a read and a write handler ++ * returns S_IRUGO if it has only a read handler ++ * returns S_IWUSR if it has only a write hander ++ */ ++static umode_t cgroup_file_mode(const struct cftype *cft) +{ -+ char buf[64]; -+ unsigned long pid; -+ unsigned long *this_pid = file->private_data; ++ umode_t mode = 0; + -+ if (cnt >= sizeof(buf)) -+ return -EINVAL; ++ if (cft->mode) ++ return cft->mode; + -+ if (copy_from_user(&buf, ubuf, cnt)) -+ return -EFAULT; ++ if (cft->read_u64 || cft->read_s64 || cft->seq_show) ++ mode |= S_IRUGO; + -+ buf[cnt] = '\0'; ++ if (cft->write_u64 || cft->write_s64 || cft->write) ++ mode |= S_IWUSR; + -+ if (kstrtoul(buf, 10, &pid)) -+ return -EINVAL; ++ return mode; ++} + -+ *this_pid = pid; ++static void cgroup_get(struct cgroup *cgrp) ++{ ++ WARN_ON_ONCE(cgroup_is_dead(cgrp)); ++ css_get(&cgrp->self); ++} + -+ return cnt; ++static bool cgroup_tryget(struct cgroup *cgrp) ++{ ++ return css_tryget(&cgrp->self); +} -+#endif + -+#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \ -+ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) -+static ssize_t -+show_maxlatproc(struct file *file, char __user *ubuf, size_t cnt, loff_t *ppos) ++static void cgroup_put(struct cgroup *cgrp) +{ -+ int r; -+ struct maxlatproc_data *mp = file->private_data; -+ int strmaxlen = (TASK_COMM_LEN * 2) + (8 * 8); -+ unsigned long long t; -+ unsigned long usecs, secs; -+ char *buf; ++ css_put(&cgrp->self); ++} + -+ if (mp->pid == -1 || mp->current_pid == -1) { -+ buf = "(none)\n"; -+ return simple_read_from_buffer(ubuf, cnt, ppos, buf, -+ strlen(buf)); ++/** ++ * cgroup_calc_child_subsys_mask - calculate child_subsys_mask ++ * @cgrp: the target cgroup ++ * @subtree_control: the new subtree_control mask to consider ++ * ++ * On the default hierarchy, a subsystem may request other subsystems to be ++ * enabled together through its ->depends_on mask. In such cases, more ++ * subsystems than specified in "cgroup.subtree_control" may be enabled. ++ * ++ * This function calculates which subsystems need to be enabled if ++ * @subtree_control is to be applied to @cgrp. The returned mask is always ++ * a superset of @subtree_control and follows the usual hierarchy rules. ++ */ ++static unsigned int cgroup_calc_child_subsys_mask(struct cgroup *cgrp, ++ unsigned int subtree_control) ++{ ++ struct cgroup *parent = cgroup_parent(cgrp); ++ unsigned int cur_ss_mask = subtree_control; ++ struct cgroup_subsys *ss; ++ int ssid; ++ ++ lockdep_assert_held(&cgroup_mutex); ++ ++ if (!cgroup_on_dfl(cgrp)) ++ return cur_ss_mask; ++ ++ while (true) { ++ unsigned int new_ss_mask = cur_ss_mask; ++ ++ for_each_subsys(ss, ssid) ++ if (cur_ss_mask & (1 << ssid)) ++ new_ss_mask |= ss->depends_on; ++ ++ /* ++ * Mask out subsystems which aren't available. This can ++ * happen only if some depended-upon subsystems were bound ++ * to non-default hierarchies. ++ */ ++ if (parent) ++ new_ss_mask &= parent->child_subsys_mask; ++ else ++ new_ss_mask &= cgrp->root->subsys_mask; ++ ++ if (new_ss_mask == cur_ss_mask) ++ break; ++ cur_ss_mask = new_ss_mask; + } + -+ buf = kmalloc(strmaxlen, GFP_KERNEL); -+ if (buf == NULL) -+ return -ENOMEM; ++ return cur_ss_mask; ++} + -+ t = ns2usecs(mp->timestamp); -+ usecs = do_div(t, USEC_PER_SEC); -+ secs = (unsigned long) t; -+ r = snprintf(buf, strmaxlen, -+ "%d %d %ld (%ld) %s <- %d %d %s %lu.%06lu\n", mp->pid, -+ MAX_RT_PRIO-1 - mp->prio, mp->latency, mp->timeroffset, mp->comm, -+ mp->current_pid, MAX_RT_PRIO-1 - mp->current_prio, mp->current_comm, -+ secs, usecs); -+ r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r); -+ kfree(buf); -+ return r; ++/** ++ * cgroup_refresh_child_subsys_mask - update child_subsys_mask ++ * @cgrp: the target cgroup ++ * ++ * Update @cgrp->child_subsys_mask according to the current ++ * @cgrp->subtree_control using cgroup_calc_child_subsys_mask(). ++ */ ++static void cgroup_refresh_child_subsys_mask(struct cgroup *cgrp) ++{ ++ cgrp->child_subsys_mask = ++ cgroup_calc_child_subsys_mask(cgrp, cgrp->subtree_control); +} -+#endif + -+static ssize_t -+show_enable(struct file *file, char __user *ubuf, size_t cnt, loff_t *ppos) ++/** ++ * cgroup_kn_unlock - unlocking helper for cgroup kernfs methods ++ * @kn: the kernfs_node being serviced ++ * ++ * This helper undoes cgroup_kn_lock_live() and should be invoked before ++ * the method finishes if locking succeeded. Note that once this function ++ * returns the cgroup returned by cgroup_kn_lock_live() may become ++ * inaccessible any time. If the caller intends to continue to access the ++ * cgroup, it should pin it before invoking this function. ++ */ ++static void cgroup_kn_unlock(struct kernfs_node *kn) +{ -+ char buf[64]; -+ struct enable_data *ed = file->private_data; -+ int r; ++ struct cgroup *cgrp; + -+ r = snprintf(buf, sizeof(buf), "%d\n", ed->enabled); -+ return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); ++ if (kernfs_type(kn) == KERNFS_DIR) ++ cgrp = kn->priv; ++ else ++ cgrp = kn->parent->priv; ++ ++ mutex_unlock(&cgroup_mutex); ++ ++ kernfs_unbreak_active_protection(kn); ++ cgroup_put(cgrp); +} + -+static ssize_t -+do_enable(struct file *file, const char __user *ubuf, size_t cnt, loff_t *ppos) ++/** ++ * cgroup_kn_lock_live - locking helper for cgroup kernfs methods ++ * @kn: the kernfs_node being serviced ++ * ++ * This helper is to be used by a cgroup kernfs method currently servicing ++ * @kn. It breaks the active protection, performs cgroup locking and ++ * verifies that the associated cgroup is alive. Returns the cgroup if ++ * alive; otherwise, %NULL. A successful return should be undone by a ++ * matching cgroup_kn_unlock() invocation. ++ * ++ * Any cgroup kernfs method implementation which requires locking the ++ * associated cgroup should use this helper. It avoids nesting cgroup ++ * locking under kernfs active protection and allows all kernfs operations ++ * including self-removal. ++ */ ++static struct cgroup *cgroup_kn_lock_live(struct kernfs_node *kn) +{ -+ char buf[64]; -+ long enable; -+ struct enable_data *ed = file->private_data; ++ struct cgroup *cgrp; + -+ if (cnt >= sizeof(buf)) -+ return -EINVAL; ++ if (kernfs_type(kn) == KERNFS_DIR) ++ cgrp = kn->priv; ++ else ++ cgrp = kn->parent->priv; + -+ if (copy_from_user(&buf, ubuf, cnt)) -+ return -EFAULT; ++ /* ++ * We're gonna grab cgroup_mutex which nests outside kernfs ++ * active_ref. cgroup liveliness check alone provides enough ++ * protection against removal. Ensure @cgrp stays accessible and ++ * break the active_ref protection. ++ */ ++ if (!cgroup_tryget(cgrp)) ++ return NULL; ++ kernfs_break_active_protection(kn); + -+ buf[cnt] = 0; ++ mutex_lock(&cgroup_mutex); + -+ if (kstrtoul(buf, 10, &enable)) -+ return -EINVAL; ++ if (!cgroup_is_dead(cgrp)) ++ return cgrp; + -+ if ((enable && ed->enabled) || (!enable && !ed->enabled)) -+ return cnt; ++ cgroup_kn_unlock(kn); ++ return NULL; ++} + -+ if (enable) { -+ int ret; ++static void cgroup_rm_file(struct cgroup *cgrp, const struct cftype *cft) ++{ ++ char name[CGROUP_FILE_NAME_MAX]; + -+ switch (ed->latency_type) { -+#if defined(CONFIG_INTERRUPT_OFF_HIST) || defined(CONFIG_PREEMPT_OFF_HIST) -+ case PREEMPTIRQSOFF_LATENCY: -+ ret = register_trace_preemptirqsoff_hist( -+ probe_preemptirqsoff_hist, NULL); -+ if (ret) { -+ pr_info("wakeup trace: Couldn't assign " -+ "probe_preemptirqsoff_hist " -+ "to trace_preemptirqsoff_hist\n"); -+ return ret; -+ } -+ break; ++ lockdep_assert_held(&cgroup_mutex); ++ kernfs_remove_by_name(cgrp->kn, cgroup_file_name(cgrp, cft, name)); ++} ++ ++/** ++ * cgroup_clear_dir - remove subsys files in a cgroup directory ++ * @cgrp: target cgroup ++ * @subsys_mask: mask of the subsystem ids whose files should be removed ++ */ ++static void cgroup_clear_dir(struct cgroup *cgrp, unsigned int subsys_mask) ++{ ++ struct cgroup_subsys *ss; ++ int i; ++ ++ for_each_subsys(ss, i) { ++ struct cftype *cfts; ++ ++ if (!(subsys_mask & (1 << i))) ++ continue; ++ list_for_each_entry(cfts, &ss->cfts, node) ++ cgroup_addrm_files(cgrp, cfts, false); ++ } ++} ++ ++static int rebind_subsystems(struct cgroup_root *dst_root, unsigned int ss_mask) ++{ ++ struct cgroup_subsys *ss; ++ unsigned int tmp_ss_mask; ++ int ssid, i, ret; ++ ++ lockdep_assert_held(&cgroup_mutex); ++ ++ for_each_subsys(ss, ssid) { ++ if (!(ss_mask & (1 << ssid))) ++ continue; ++ ++ /* if @ss has non-root csses attached to it, can't move */ ++ if (css_next_child(NULL, cgroup_css(&ss->root->cgrp, ss))) ++ return -EBUSY; ++ ++ /* can't move between two non-dummy roots either */ ++ if (ss->root != &cgrp_dfl_root && dst_root != &cgrp_dfl_root) ++ return -EBUSY; ++ } ++ ++ /* skip creating root files on dfl_root for inhibited subsystems */ ++ tmp_ss_mask = ss_mask; ++ if (dst_root == &cgrp_dfl_root) ++ tmp_ss_mask &= ~cgrp_dfl_root_inhibit_ss_mask; ++ ++ ret = cgroup_populate_dir(&dst_root->cgrp, tmp_ss_mask); ++ if (ret) { ++ if (dst_root != &cgrp_dfl_root) ++ return ret; ++ ++ /* ++ * Rebinding back to the default root is not allowed to ++ * fail. Using both default and non-default roots should ++ * be rare. Moving subsystems back and forth even more so. ++ * Just warn about it and continue. ++ */ ++ if (cgrp_dfl_root_visible) { ++ pr_warn("failed to create files (%d) while rebinding 0x%x to default root\n", ++ ret, ss_mask); ++ pr_warn("you may retry by moving them to a different hierarchy and unbinding\n"); ++ } ++ } ++ ++ /* ++ * Nothing can fail from this point on. Remove files for the ++ * removed subsystems and rebind each subsystem. ++ */ ++ for_each_subsys(ss, ssid) ++ if (ss_mask & (1 << ssid)) ++ cgroup_clear_dir(&ss->root->cgrp, 1 << ssid); ++ ++ for_each_subsys(ss, ssid) { ++ struct cgroup_root *src_root; ++ struct cgroup_subsys_state *css; ++ struct css_set *cset; ++ ++ if (!(ss_mask & (1 << ssid))) ++ continue; ++ ++ src_root = ss->root; ++ css = cgroup_css(&src_root->cgrp, ss); ++ ++ WARN_ON(!css || cgroup_css(&dst_root->cgrp, ss)); ++ ++ RCU_INIT_POINTER(src_root->cgrp.subsys[ssid], NULL); ++ rcu_assign_pointer(dst_root->cgrp.subsys[ssid], css); ++ ss->root = dst_root; ++ css->cgroup = &dst_root->cgrp; ++ ++ down_write(&css_set_rwsem); ++ hash_for_each(css_set_table, i, cset, hlist) ++ list_move_tail(&cset->e_cset_node[ss->id], ++ &dst_root->cgrp.e_csets[ss->id]); ++ up_write(&css_set_rwsem); ++ ++ src_root->subsys_mask &= ~(1 << ssid); ++ src_root->cgrp.subtree_control &= ~(1 << ssid); ++ cgroup_refresh_child_subsys_mask(&src_root->cgrp); ++ ++ /* default hierarchy doesn't enable controllers by default */ ++ dst_root->subsys_mask |= 1 << ssid; ++ if (dst_root != &cgrp_dfl_root) { ++ dst_root->cgrp.subtree_control |= 1 << ssid; ++ cgroup_refresh_child_subsys_mask(&dst_root->cgrp); ++ } ++ ++ if (ss->bind) ++ ss->bind(css); ++ } ++ ++ kernfs_activate(dst_root->cgrp.kn); ++ return 0; ++} ++ ++static int cgroup_show_options(struct seq_file *seq, ++ struct kernfs_root *kf_root) ++{ ++ struct cgroup_root *root = cgroup_root_from_kf(kf_root); ++ struct cgroup_subsys *ss; ++ int ssid; ++ ++ for_each_subsys(ss, ssid) ++ if (root->subsys_mask & (1 << ssid)) ++ seq_show_option(seq, ss->name, NULL); ++ if (root->flags & CGRP_ROOT_NOPREFIX) ++ seq_puts(seq, ",noprefix"); ++ if (root->flags & CGRP_ROOT_XATTR) ++ seq_puts(seq, ",xattr"); ++ ++ spin_lock(&release_agent_path_lock); ++ if (strlen(root->release_agent_path)) ++ seq_show_option(seq, "release_agent", ++ root->release_agent_path); ++ spin_unlock(&release_agent_path_lock); ++ ++ if (test_bit(CGRP_CPUSET_CLONE_CHILDREN, &root->cgrp.flags)) ++ seq_puts(seq, ",clone_children"); ++ if (strlen(root->name)) ++ seq_show_option(seq, "name", root->name); ++ return 0; ++} ++ ++struct cgroup_sb_opts { ++ unsigned int subsys_mask; ++ unsigned int flags; ++ char *release_agent; ++ bool cpuset_clone_children; ++ char *name; ++ /* User explicitly requested empty subsystem */ ++ bool none; ++}; ++ ++static int parse_cgroupfs_options(char *data, struct cgroup_sb_opts *opts) ++{ ++ char *token, *o = data; ++ bool all_ss = false, one_ss = false; ++ unsigned int mask = -1U; ++ struct cgroup_subsys *ss; ++ int nr_opts = 0; ++ int i; ++ ++#ifdef CONFIG_CPUSETS ++ mask = ~(1U << cpuset_cgrp_id); +#endif -+#ifdef CONFIG_WAKEUP_LATENCY_HIST -+ case WAKEUP_LATENCY: -+ ret = register_trace_sched_wakeup( -+ probe_wakeup_latency_hist_start, NULL); -+ if (ret) { -+ pr_info("wakeup trace: Couldn't assign " -+ "probe_wakeup_latency_hist_start " -+ "to trace_sched_wakeup\n"); -+ return ret; ++ ++ memset(opts, 0, sizeof(*opts)); ++ ++ while ((token = strsep(&o, ",")) != NULL) { ++ nr_opts++; ++ ++ if (!*token) ++ return -EINVAL; ++ if (!strcmp(token, "none")) { ++ /* Explicitly have no subsystems */ ++ opts->none = true; ++ continue; ++ } ++ if (!strcmp(token, "all")) { ++ /* Mutually exclusive option 'all' + subsystem name */ ++ if (one_ss) ++ return -EINVAL; ++ all_ss = true; ++ continue; ++ } ++ if (!strcmp(token, "__DEVEL__sane_behavior")) { ++ opts->flags |= CGRP_ROOT_SANE_BEHAVIOR; ++ continue; ++ } ++ if (!strcmp(token, "noprefix")) { ++ opts->flags |= CGRP_ROOT_NOPREFIX; ++ continue; ++ } ++ if (!strcmp(token, "clone_children")) { ++ opts->cpuset_clone_children = true; ++ continue; ++ } ++ if (!strcmp(token, "xattr")) { ++ opts->flags |= CGRP_ROOT_XATTR; ++ continue; ++ } ++ if (!strncmp(token, "release_agent=", 14)) { ++ /* Specifying two release agents is forbidden */ ++ if (opts->release_agent) ++ return -EINVAL; ++ opts->release_agent = ++ kstrndup(token + 14, PATH_MAX - 1, GFP_KERNEL); ++ if (!opts->release_agent) ++ return -ENOMEM; ++ continue; ++ } ++ if (!strncmp(token, "name=", 5)) { ++ const char *name = token + 5; ++ /* Can't specify an empty name */ ++ if (!strlen(name)) ++ return -EINVAL; ++ /* Must match [\w.-]+ */ ++ for (i = 0; i < strlen(name); i++) { ++ char c = name[i]; ++ if (isalnum(c)) ++ continue; ++ if ((c == '.') || (c == '-') || (c == '_')) ++ continue; ++ return -EINVAL; + } -+ ret = register_trace_sched_wakeup_new( -+ probe_wakeup_latency_hist_start, NULL); -+ if (ret) { -+ pr_info("wakeup trace: Couldn't assign " -+ "probe_wakeup_latency_hist_start " -+ "to trace_sched_wakeup_new\n"); -+ unregister_trace_sched_wakeup( -+ probe_wakeup_latency_hist_start, NULL); -+ return ret; -+ } -+ ret = register_trace_sched_switch( -+ probe_wakeup_latency_hist_stop, NULL); -+ if (ret) { -+ pr_info("wakeup trace: Couldn't assign " -+ "probe_wakeup_latency_hist_stop " -+ "to trace_sched_switch\n"); -+ unregister_trace_sched_wakeup( -+ probe_wakeup_latency_hist_start, NULL); -+ unregister_trace_sched_wakeup_new( -+ probe_wakeup_latency_hist_start, NULL); -+ return ret; -+ } -+ ret = register_trace_sched_migrate_task( -+ probe_sched_migrate_task, NULL); -+ if (ret) { -+ pr_info("wakeup trace: Couldn't assign " -+ "probe_sched_migrate_task " -+ "to trace_sched_migrate_task\n"); -+ unregister_trace_sched_wakeup( -+ probe_wakeup_latency_hist_start, NULL); -+ unregister_trace_sched_wakeup_new( -+ probe_wakeup_latency_hist_start, NULL); -+ unregister_trace_sched_switch( -+ probe_wakeup_latency_hist_stop, NULL); -+ return ret; -+ } -+ break; -+#endif -+#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST -+ case MISSED_TIMER_OFFSETS: -+ ret = register_trace_hrtimer_interrupt( -+ probe_hrtimer_interrupt, NULL); -+ if (ret) { -+ pr_info("wakeup trace: Couldn't assign " -+ "probe_hrtimer_interrupt " -+ "to trace_hrtimer_interrupt\n"); -+ return ret; -+ } -+ break; -+#endif -+#if defined(CONFIG_WAKEUP_LATENCY_HIST) && \ -+ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) -+ case TIMERANDWAKEUP_LATENCY: -+ if (!wakeup_latency_enabled_data.enabled || -+ !missed_timer_offsets_enabled_data.enabled) ++ /* Specifying two names is forbidden */ ++ if (opts->name) + return -EINVAL; -+ break; -+#endif -+ default: -+ break; ++ opts->name = kstrndup(name, ++ MAX_CGROUP_ROOT_NAMELEN - 1, ++ GFP_KERNEL); ++ if (!opts->name) ++ return -ENOMEM; ++ ++ continue; + } -+ } else { -+ switch (ed->latency_type) { -+#if defined(CONFIG_INTERRUPT_OFF_HIST) || defined(CONFIG_PREEMPT_OFF_HIST) -+ case PREEMPTIRQSOFF_LATENCY: -+ { -+ int cpu; + -+ unregister_trace_preemptirqsoff_hist( -+ probe_preemptirqsoff_hist, NULL); -+ for_each_online_cpu(cpu) { -+#ifdef CONFIG_INTERRUPT_OFF_HIST -+ per_cpu(hist_irqsoff_counting, -+ cpu) = 0; -+#endif -+#ifdef CONFIG_PREEMPT_OFF_HIST -+ per_cpu(hist_preemptoff_counting, -+ cpu) = 0; -+#endif -+#if defined(CONFIG_INTERRUPT_OFF_HIST) && defined(CONFIG_PREEMPT_OFF_HIST) -+ per_cpu(hist_preemptirqsoff_counting, -+ cpu) = 0; -+#endif -+ } -+ } -+ break; -+#endif -+#ifdef CONFIG_WAKEUP_LATENCY_HIST -+ case WAKEUP_LATENCY: -+ { -+ int cpu; ++ for_each_subsys(ss, i) { ++ if (strcmp(token, ss->name)) ++ continue; ++ if (ss->disabled) ++ continue; + -+ unregister_trace_sched_wakeup( -+ probe_wakeup_latency_hist_start, NULL); -+ unregister_trace_sched_wakeup_new( -+ probe_wakeup_latency_hist_start, NULL); -+ unregister_trace_sched_switch( -+ probe_wakeup_latency_hist_stop, NULL); -+ unregister_trace_sched_migrate_task( -+ probe_sched_migrate_task, NULL); ++ /* Mutually exclusive option 'all' + subsystem name */ ++ if (all_ss) ++ return -EINVAL; ++ opts->subsys_mask |= (1 << i); ++ one_ss = true; + -+ for_each_online_cpu(cpu) { -+ per_cpu(wakeup_task, cpu) = NULL; -+ per_cpu(wakeup_sharedprio, cpu) = 0; -+ } -+ } -+#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST -+ timerandwakeup_enabled_data.enabled = 0; -+#endif -+ break; -+#endif -+#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST -+ case MISSED_TIMER_OFFSETS: -+ unregister_trace_hrtimer_interrupt( -+ probe_hrtimer_interrupt, NULL); -+#ifdef CONFIG_WAKEUP_LATENCY_HIST -+ timerandwakeup_enabled_data.enabled = 0; -+#endif -+ break; -+#endif -+ default: + break; + } ++ if (i == CGROUP_SUBSYS_COUNT) ++ return -ENOENT; + } -+ ed->enabled = enable; -+ return cnt; -+} + -+static const struct file_operations latency_hist_reset_fops = { -+ .open = tracing_open_generic, -+ .write = latency_hist_reset, -+}; ++ if (opts->flags & CGRP_ROOT_SANE_BEHAVIOR) { ++ pr_warn("sane_behavior: this is still under development and its behaviors will change, proceed at your own risk\n"); ++ if (nr_opts != 1) { ++ pr_err("sane_behavior: no other mount options allowed\n"); ++ return -EINVAL; ++ } ++ return 0; ++ } + -+static const struct file_operations enable_fops = { -+ .open = tracing_open_generic, -+ .read = show_enable, -+ .write = do_enable, -+}; ++ /* ++ * If the 'all' option was specified select all the subsystems, ++ * otherwise if 'none', 'name=' and a subsystem name options were ++ * not specified, let's default to 'all' ++ */ ++ if (all_ss || (!one_ss && !opts->none && !opts->name)) ++ for_each_subsys(ss, i) ++ if (!ss->disabled) ++ opts->subsys_mask |= (1 << i); + -+#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \ -+ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) -+static const struct file_operations pid_fops = { -+ .open = tracing_open_generic, -+ .read = show_pid, -+ .write = do_pid, -+}; ++ /* ++ * We either have to specify by name or by subsystems. (So all ++ * empty hierarchies must have a name). ++ */ ++ if (!opts->subsys_mask && !opts->name) ++ return -EINVAL; + -+static const struct file_operations maxlatproc_fops = { -+ .open = tracing_open_generic, -+ .read = show_maxlatproc, -+}; -+#endif ++ /* ++ * Option noprefix was introduced just for backward compatibility ++ * with the old cpuset, so we allow noprefix only if mounting just ++ * the cpuset subsystem. ++ */ ++ if ((opts->flags & CGRP_ROOT_NOPREFIX) && (opts->subsys_mask & mask)) ++ return -EINVAL; + -+#if defined(CONFIG_INTERRUPT_OFF_HIST) || defined(CONFIG_PREEMPT_OFF_HIST) -+static notrace void probe_preemptirqsoff_hist(void *v, int reason, -+ int starthist) ++ /* Can't specify "none" and some subsystems */ ++ if (opts->subsys_mask && opts->none) ++ return -EINVAL; ++ ++ return 0; ++} ++ ++static int cgroup_remount(struct kernfs_root *kf_root, int *flags, char *data) +{ -+ int cpu = raw_smp_processor_id(); -+ int time_set = 0; ++ int ret = 0; ++ struct cgroup_root *root = cgroup_root_from_kf(kf_root); ++ struct cgroup_sb_opts opts; ++ unsigned int added_mask, removed_mask; + -+ if (starthist) { -+ cycle_t uninitialized_var(start); ++ if (root == &cgrp_dfl_root) { ++ pr_err("remount is not allowed\n"); ++ return -EINVAL; ++ } + -+ if (!preempt_count() && !irqs_disabled()) -+ return; ++ mutex_lock(&cgroup_mutex); + -+#ifdef CONFIG_INTERRUPT_OFF_HIST -+ if ((reason == IRQS_OFF || reason == TRACE_START) && -+ !per_cpu(hist_irqsoff_counting, cpu)) { -+ per_cpu(hist_irqsoff_counting, cpu) = 1; -+ start = ftrace_now(cpu); -+ time_set++; -+ per_cpu(hist_irqsoff_start, cpu) = start; -+ } -+#endif ++ /* See what subsystems are wanted */ ++ ret = parse_cgroupfs_options(data, &opts); ++ if (ret) ++ goto out_unlock; ++ ++ if (opts.subsys_mask != root->subsys_mask || opts.release_agent) ++ pr_warn("option changes via remount are deprecated (pid=%d comm=%s)\n", ++ task_tgid_nr(current), current->comm); ++ ++ added_mask = opts.subsys_mask & ~root->subsys_mask; ++ removed_mask = root->subsys_mask & ~opts.subsys_mask; ++ ++ /* Don't allow flags or name to change at remount */ ++ if ((opts.flags ^ root->flags) || ++ (opts.name && strcmp(opts.name, root->name))) { ++ pr_err("option or name mismatch, new: 0x%x \"%s\", old: 0x%x \"%s\"\n", ++ opts.flags, opts.name ?: "", root->flags, root->name); ++ ret = -EINVAL; ++ goto out_unlock; ++ } + -+#ifdef CONFIG_PREEMPT_OFF_HIST -+ if ((reason == PREEMPT_OFF || reason == TRACE_START) && -+ !per_cpu(hist_preemptoff_counting, cpu)) { -+ per_cpu(hist_preemptoff_counting, cpu) = 1; -+ if (!(time_set++)) -+ start = ftrace_now(cpu); -+ per_cpu(hist_preemptoff_start, cpu) = start; -+ } -+#endif ++ /* remounting is not allowed for populated hierarchies */ ++ if (!list_empty(&root->cgrp.self.children)) { ++ ret = -EBUSY; ++ goto out_unlock; ++ } + -+#if defined(CONFIG_INTERRUPT_OFF_HIST) && defined(CONFIG_PREEMPT_OFF_HIST) -+ if (per_cpu(hist_irqsoff_counting, cpu) && -+ per_cpu(hist_preemptoff_counting, cpu) && -+ !per_cpu(hist_preemptirqsoff_counting, cpu)) { -+ per_cpu(hist_preemptirqsoff_counting, cpu) = 1; -+ if (!time_set) -+ start = ftrace_now(cpu); -+ per_cpu(hist_preemptirqsoff_start, cpu) = start; -+ } -+#endif -+ } else { -+ cycle_t uninitialized_var(stop); ++ ret = rebind_subsystems(root, added_mask); ++ if (ret) ++ goto out_unlock; + -+#ifdef CONFIG_INTERRUPT_OFF_HIST -+ if ((reason == IRQS_ON || reason == TRACE_STOP) && -+ per_cpu(hist_irqsoff_counting, cpu)) { -+ cycle_t start = per_cpu(hist_irqsoff_start, cpu); ++ rebind_subsystems(&cgrp_dfl_root, removed_mask); + -+ stop = ftrace_now(cpu); -+ time_set++; -+ if (start) { -+ long latency = ((long) (stop - start)) / -+ NSECS_PER_USECS; ++ if (opts.release_agent) { ++ spin_lock(&release_agent_path_lock); ++ strcpy(root->release_agent_path, opts.release_agent); ++ spin_unlock(&release_agent_path_lock); ++ } ++ out_unlock: ++ kfree(opts.release_agent); ++ kfree(opts.name); ++ mutex_unlock(&cgroup_mutex); ++ return ret; ++} + -+ latency_hist(IRQSOFF_LATENCY, cpu, latency, 0, -+ stop, NULL); -+ } -+ per_cpu(hist_irqsoff_counting, cpu) = 0; -+ } -+#endif ++/* ++ * To reduce the fork() overhead for systems that are not actually using ++ * their cgroups capability, we don't maintain the lists running through ++ * each css_set to its tasks until we see the list actually used - in other ++ * words after the first mount. ++ */ ++static bool use_task_css_set_links __read_mostly; + -+#ifdef CONFIG_PREEMPT_OFF_HIST -+ if ((reason == PREEMPT_ON || reason == TRACE_STOP) && -+ per_cpu(hist_preemptoff_counting, cpu)) { -+ cycle_t start = per_cpu(hist_preemptoff_start, cpu); ++static void cgroup_enable_task_cg_lists(void) ++{ ++ struct task_struct *p, *g; + -+ if (!(time_set++)) -+ stop = ftrace_now(cpu); -+ if (start) { -+ long latency = ((long) (stop - start)) / -+ NSECS_PER_USECS; ++ down_write(&css_set_rwsem); + -+ latency_hist(PREEMPTOFF_LATENCY, cpu, latency, -+ 0, stop, NULL); -+ } -+ per_cpu(hist_preemptoff_counting, cpu) = 0; -+ } -+#endif ++ if (use_task_css_set_links) ++ goto out_unlock; + -+#if defined(CONFIG_INTERRUPT_OFF_HIST) && defined(CONFIG_PREEMPT_OFF_HIST) -+ if ((!per_cpu(hist_irqsoff_counting, cpu) || -+ !per_cpu(hist_preemptoff_counting, cpu)) && -+ per_cpu(hist_preemptirqsoff_counting, cpu)) { -+ cycle_t start = per_cpu(hist_preemptirqsoff_start, cpu); ++ use_task_css_set_links = true; + -+ if (!time_set) -+ stop = ftrace_now(cpu); -+ if (start) { -+ long latency = ((long) (stop - start)) / -+ NSECS_PER_USECS; ++ /* ++ * We need tasklist_lock because RCU is not safe against ++ * while_each_thread(). Besides, a forking task that has passed ++ * cgroup_post_fork() without seeing use_task_css_set_links = 1 ++ * is not guaranteed to have its child immediately visible in the ++ * tasklist if we walk through it with RCU. ++ */ ++ read_lock(&tasklist_lock); ++ do_each_thread(g, p) { ++ WARN_ON_ONCE(!list_empty(&p->cg_list) || ++ task_css_set(p) != &init_css_set); + -+ latency_hist(PREEMPTIRQSOFF_LATENCY, cpu, -+ latency, 0, stop, NULL); -+ } -+ per_cpu(hist_preemptirqsoff_counting, cpu) = 0; ++ /* ++ * We should check if the process is exiting, otherwise ++ * it will race with cgroup_exit() in that the list ++ * entry won't be deleted though the process has exited. ++ * Do it while holding siglock so that we don't end up ++ * racing against cgroup_exit(). ++ */ ++ spin_lock_irq(&p->sighand->siglock); ++ if (!(p->flags & PF_EXITING)) { ++ struct css_set *cset = task_css_set(p); ++ ++ list_add(&p->cg_list, &cset->tasks); ++ get_css_set(cset); + } -+#endif -+ } ++ spin_unlock_irq(&p->sighand->siglock); ++ } while_each_thread(g, p); ++ read_unlock(&tasklist_lock); ++out_unlock: ++ up_write(&css_set_rwsem); +} -+#endif + -+#ifdef CONFIG_WAKEUP_LATENCY_HIST -+static DEFINE_RAW_SPINLOCK(wakeup_lock); -+static notrace void probe_sched_migrate_task(void *v, struct task_struct *task, -+ int cpu) ++static void init_cgroup_housekeeping(struct cgroup *cgrp) +{ -+ int old_cpu = task_cpu(task); ++ struct cgroup_subsys *ss; ++ int ssid; ++ ++ INIT_LIST_HEAD(&cgrp->self.sibling); ++ INIT_LIST_HEAD(&cgrp->self.children); ++ INIT_LIST_HEAD(&cgrp->cset_links); ++ INIT_LIST_HEAD(&cgrp->pidlists); ++ mutex_init(&cgrp->pidlist_mutex); ++ cgrp->self.cgroup = cgrp; ++ cgrp->self.flags |= CSS_ONLINE; ++ ++ for_each_subsys(ss, ssid) ++ INIT_LIST_HEAD(&cgrp->e_csets[ssid]); ++ ++ init_waitqueue_head(&cgrp->offline_waitq); ++ INIT_WORK(&cgrp->release_agent_work, cgroup_release_agent); ++} + -+ if (cpu != old_cpu) { -+ unsigned long flags; -+ struct task_struct *cpu_wakeup_task; ++static void init_cgroup_root(struct cgroup_root *root, ++ struct cgroup_sb_opts *opts) ++{ ++ struct cgroup *cgrp = &root->cgrp; ++ ++ INIT_LIST_HEAD(&root->root_list); ++ atomic_set(&root->nr_cgrps, 1); ++ cgrp->root = root; ++ init_cgroup_housekeeping(cgrp); ++ idr_init(&root->cgroup_idr); ++ ++ root->flags = opts->flags; ++ if (opts->release_agent) ++ strcpy(root->release_agent_path, opts->release_agent); ++ if (opts->name) ++ strcpy(root->name, opts->name); ++ if (opts->cpuset_clone_children) ++ set_bit(CGRP_CPUSET_CLONE_CHILDREN, &root->cgrp.flags); ++} + -+ raw_spin_lock_irqsave(&wakeup_lock, flags); ++static int cgroup_setup_root(struct cgroup_root *root, unsigned int ss_mask) ++{ ++ LIST_HEAD(tmp_links); ++ struct cgroup *root_cgrp = &root->cgrp; ++ struct cftype *base_files; ++ struct css_set *cset; ++ int i, ret; + -+ cpu_wakeup_task = per_cpu(wakeup_task, old_cpu); -+ if (task == cpu_wakeup_task) { -+ put_task_struct(cpu_wakeup_task); -+ per_cpu(wakeup_task, old_cpu) = NULL; -+ cpu_wakeup_task = per_cpu(wakeup_task, cpu) = task; -+ get_task_struct(cpu_wakeup_task); -+ } ++ lockdep_assert_held(&cgroup_mutex); + -+ raw_spin_unlock_irqrestore(&wakeup_lock, flags); ++ ret = cgroup_idr_alloc(&root->cgroup_idr, root_cgrp, 1, 2, GFP_NOWAIT); ++ if (ret < 0) ++ goto out; ++ root_cgrp->id = ret; ++ ++ ret = percpu_ref_init(&root_cgrp->self.refcnt, css_release, 0, ++ GFP_KERNEL); ++ if (ret) ++ goto out; ++ ++ /* ++ * We're accessing css_set_count without locking css_set_rwsem here, ++ * but that's OK - it can only be increased by someone holding ++ * cgroup_lock, and that's us. The worst that can happen is that we ++ * have some link structures left over ++ */ ++ ret = allocate_cgrp_cset_links(css_set_count, &tmp_links); ++ if (ret) ++ goto cancel_ref; ++ ++ ret = cgroup_init_root_id(root); ++ if (ret) ++ goto cancel_ref; ++ ++ root->kf_root = kernfs_create_root(&cgroup_kf_syscall_ops, ++ KERNFS_ROOT_CREATE_DEACTIVATED, ++ root_cgrp); ++ if (IS_ERR(root->kf_root)) { ++ ret = PTR_ERR(root->kf_root); ++ goto exit_root_id; + } -+} ++ root_cgrp->kn = root->kf_root->kn; + -+static notrace void probe_wakeup_latency_hist_start(void *v, -+ struct task_struct *p, int success) -+{ -+ unsigned long flags; -+ struct task_struct *curr = current; -+ int cpu = task_cpu(p); -+ struct task_struct *cpu_wakeup_task; ++ if (root == &cgrp_dfl_root) ++ base_files = cgroup_dfl_base_files; ++ else ++ base_files = cgroup_legacy_base_files; + -+ raw_spin_lock_irqsave(&wakeup_lock, flags); ++ ret = cgroup_addrm_files(root_cgrp, base_files, true); ++ if (ret) ++ goto destroy_root; + -+ cpu_wakeup_task = per_cpu(wakeup_task, cpu); ++ ret = rebind_subsystems(root, ss_mask); ++ if (ret) ++ goto destroy_root; + -+ if (wakeup_pid) { -+ if ((cpu_wakeup_task && p->prio == cpu_wakeup_task->prio) || -+ p->prio == curr->prio) -+ per_cpu(wakeup_sharedprio, cpu) = 1; -+ if (likely(wakeup_pid != task_pid_nr(p))) -+ goto out; -+ } else { -+ if (likely(!rt_task(p)) || -+ (cpu_wakeup_task && p->prio > cpu_wakeup_task->prio) || -+ p->prio > curr->prio) -+ goto out; -+ if ((cpu_wakeup_task && p->prio == cpu_wakeup_task->prio) || -+ p->prio == curr->prio) -+ per_cpu(wakeup_sharedprio, cpu) = 1; -+ } ++ /* ++ * There must be no failure case after here, since rebinding takes ++ * care of subsystems' refcounts, which are explicitly dropped in ++ * the failure exit path. ++ */ ++ list_add(&root->root_list, &cgroup_roots); ++ cgroup_root_count++; + -+ if (cpu_wakeup_task) -+ put_task_struct(cpu_wakeup_task); -+ cpu_wakeup_task = per_cpu(wakeup_task, cpu) = p; -+ get_task_struct(cpu_wakeup_task); -+ cpu_wakeup_task->preempt_timestamp_hist = -+ ftrace_now(raw_smp_processor_id()); ++ /* ++ * Link the root cgroup in this hierarchy into all the css_set ++ * objects. ++ */ ++ down_write(&css_set_rwsem); ++ hash_for_each(css_set_table, i, cset, hlist) ++ link_css_set(&tmp_links, cset, root_cgrp); ++ up_write(&css_set_rwsem); ++ ++ BUG_ON(!list_empty(&root_cgrp->self.children)); ++ BUG_ON(atomic_read(&root->nr_cgrps) != 1); ++ ++ kernfs_activate(root_cgrp->kn); ++ ret = 0; ++ goto out; ++ ++destroy_root: ++ kernfs_destroy_root(root->kf_root); ++ root->kf_root = NULL; ++exit_root_id: ++ cgroup_exit_root_id(root); ++cancel_ref: ++ percpu_ref_exit(&root_cgrp->self.refcnt); +out: -+ raw_spin_unlock_irqrestore(&wakeup_lock, flags); ++ free_cgrp_cset_links(&tmp_links); ++ return ret; +} + -+static notrace void probe_wakeup_latency_hist_stop(void *v, -+ struct task_struct *prev, struct task_struct *next) ++static struct dentry *cgroup_mount(struct file_system_type *fs_type, ++ int flags, const char *unused_dev_name, ++ void *data) +{ -+ unsigned long flags; -+ int cpu = task_cpu(next); -+ long latency; -+ cycle_t stop; -+ struct task_struct *cpu_wakeup_task; -+ -+ raw_spin_lock_irqsave(&wakeup_lock, flags); ++ struct super_block *pinned_sb = NULL; ++ struct cgroup_subsys *ss; ++ struct cgroup_root *root; ++ struct cgroup_sb_opts opts; ++ struct dentry *dentry; ++ int ret; ++ int i; ++ bool new_sb; + -+ cpu_wakeup_task = per_cpu(wakeup_task, cpu); ++ /* ++ * The first time anyone tries to mount a cgroup, enable the list ++ * linking each css_set to its tasks and fix up all existing tasks. ++ */ ++ if (!use_task_css_set_links) ++ cgroup_enable_task_cg_lists(); + -+ if (cpu_wakeup_task == NULL) -+ goto out; ++ mutex_lock(&cgroup_mutex); + -+ /* Already running? */ -+ if (unlikely(current == cpu_wakeup_task)) -+ goto out_reset; ++ /* First find the desired set of subsystems */ ++ ret = parse_cgroupfs_options(data, &opts); ++ if (ret) ++ goto out_unlock; + -+ if (next != cpu_wakeup_task) { -+ if (next->prio < cpu_wakeup_task->prio) -+ goto out_reset; ++ /* look for a matching existing root */ ++ if (opts.flags & CGRP_ROOT_SANE_BEHAVIOR) { ++ cgrp_dfl_root_visible = true; ++ root = &cgrp_dfl_root; ++ cgroup_get(&root->cgrp); ++ ret = 0; ++ goto out_unlock; ++ } + -+ if (next->prio == cpu_wakeup_task->prio) -+ per_cpu(wakeup_sharedprio, cpu) = 1; ++ /* ++ * Destruction of cgroup root is asynchronous, so subsystems may ++ * still be dying after the previous unmount. Let's drain the ++ * dying subsystems. We just need to ensure that the ones ++ * unmounted previously finish dying and don't care about new ones ++ * starting. Testing ref liveliness is good enough. ++ */ ++ for_each_subsys(ss, i) { ++ if (!(opts.subsys_mask & (1 << i)) || ++ ss->root == &cgrp_dfl_root) ++ continue; + -+ goto out; ++ if (!percpu_ref_tryget_live(&ss->root->cgrp.self.refcnt)) { ++ mutex_unlock(&cgroup_mutex); ++ msleep(10); ++ ret = restart_syscall(); ++ goto out_free; ++ } ++ cgroup_put(&ss->root->cgrp); + } + -+ if (current->prio == cpu_wakeup_task->prio) -+ per_cpu(wakeup_sharedprio, cpu) = 1; ++ for_each_root(root) { ++ bool name_match = false; ++ ++ if (root == &cgrp_dfl_root) ++ continue; ++ ++ /* ++ * If we asked for a name then it must match. Also, if ++ * name matches but sybsys_mask doesn't, we should fail. ++ * Remember whether name matched. ++ */ ++ if (opts.name) { ++ if (strcmp(opts.name, root->name)) ++ continue; ++ name_match = true; ++ } ++ ++ /* ++ * If we asked for subsystems (or explicitly for no ++ * subsystems) then they must match. ++ */ ++ if ((opts.subsys_mask || opts.none) && ++ (opts.subsys_mask != root->subsys_mask)) { ++ if (!name_match) ++ continue; ++ ret = -EBUSY; ++ goto out_unlock; ++ } ++ ++ if (root->flags ^ opts.flags) ++ pr_warn("new mount options do not match the existing superblock, will be ignored\n"); ++ ++ /* ++ * We want to reuse @root whose lifetime is governed by its ++ * ->cgrp. Let's check whether @root is alive and keep it ++ * that way. As cgroup_kill_sb() can happen anytime, we ++ * want to block it by pinning the sb so that @root doesn't ++ * get killed before mount is complete. ++ * ++ * With the sb pinned, tryget_live can reliably indicate ++ * whether @root can be reused. If it's being killed, ++ * drain it. We can use wait_queue for the wait but this ++ * path is super cold. Let's just sleep a bit and retry. ++ */ ++ pinned_sb = kernfs_pin_sb(root->kf_root, NULL); ++ if (IS_ERR(pinned_sb) || ++ !percpu_ref_tryget_live(&root->cgrp.self.refcnt)) { ++ mutex_unlock(&cgroup_mutex); ++ if (!IS_ERR_OR_NULL(pinned_sb)) ++ deactivate_super(pinned_sb); ++ msleep(10); ++ ret = restart_syscall(); ++ goto out_free; ++ } ++ ++ ret = 0; ++ goto out_unlock; ++ } + + /* -+ * The task we are waiting for is about to be switched to. -+ * Calculate latency and store it in histogram. ++ * No such thing, create a new one. name= matching without subsys ++ * specification is allowed for already existing hierarchies but we ++ * can't create new one without subsys specification. + */ -+ stop = ftrace_now(raw_smp_processor_id()); ++ if (!opts.subsys_mask && !opts.none) { ++ ret = -EINVAL; ++ goto out_unlock; ++ } + -+ latency = ((long) (stop - next->preempt_timestamp_hist)) / -+ NSECS_PER_USECS; ++ root = kzalloc(sizeof(*root), GFP_KERNEL); ++ if (!root) { ++ ret = -ENOMEM; ++ goto out_unlock; ++ } + -+ if (per_cpu(wakeup_sharedprio, cpu)) { -+ latency_hist(WAKEUP_LATENCY_SHAREDPRIO, cpu, latency, 0, stop, -+ next); -+ per_cpu(wakeup_sharedprio, cpu) = 0; -+ } else { -+ latency_hist(WAKEUP_LATENCY, cpu, latency, 0, stop, next); -+#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST -+ if (timerandwakeup_enabled_data.enabled) { -+ latency_hist(TIMERANDWAKEUP_LATENCY, cpu, -+ next->timer_offset + latency, next->timer_offset, -+ stop, next); -+ } -+#endif ++ init_cgroup_root(root, &opts); ++ ++ ret = cgroup_setup_root(root, opts.subsys_mask); ++ if (ret) ++ cgroup_free_root(root); ++ ++out_unlock: ++ mutex_unlock(&cgroup_mutex); ++out_free: ++ kfree(opts.release_agent); ++ kfree(opts.name); ++ ++ if (ret) ++ return ERR_PTR(ret); ++ ++ dentry = kernfs_mount(fs_type, flags, root->kf_root, ++ CGROUP_SUPER_MAGIC, &new_sb); ++ if (IS_ERR(dentry) || !new_sb) ++ cgroup_put(&root->cgrp); ++ ++ /* ++ * If @pinned_sb, we're reusing an existing root and holding an ++ * extra ref on its sb. Mount is complete. Put the extra ref. ++ */ ++ if (pinned_sb) { ++ WARN_ON(new_sb); ++ deactivate_super(pinned_sb); + } + -+out_reset: -+#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST -+ next->timer_offset = 0; -+#endif -+ put_task_struct(cpu_wakeup_task); -+ per_cpu(wakeup_task, cpu) = NULL; -+out: -+ raw_spin_unlock_irqrestore(&wakeup_lock, flags); ++ return dentry; +} -+#endif + -+#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST -+static notrace void probe_hrtimer_interrupt(void *v, int cpu, -+ long long latency_ns, struct task_struct *curr, -+ struct task_struct *task) ++static void cgroup_kill_sb(struct super_block *sb) +{ -+ if (latency_ns <= 0 && task != NULL && rt_task(task) && -+ (task->prio < curr->prio || -+ (task->prio == curr->prio && -+ !cpumask_test_cpu(cpu, &task->cpus_allowed)))) { -+ long latency; -+ cycle_t now; ++ struct kernfs_root *kf_root = kernfs_root_from_sb(sb); ++ struct cgroup_root *root = cgroup_root_from_kf(kf_root); + -+ if (missed_timer_offsets_pid) { -+ if (likely(missed_timer_offsets_pid != -+ task_pid_nr(task))) -+ return; -+ } ++ /* ++ * If @root doesn't have any mounts or children, start killing it. ++ * This prevents new mounts by disabling percpu_ref_tryget_live(). ++ * cgroup_mount() may wait for @root's release. ++ * ++ * And don't kill the default root. ++ */ ++ if (!list_empty(&root->cgrp.self.children) || ++ root == &cgrp_dfl_root) ++ cgroup_put(&root->cgrp); ++ else ++ percpu_ref_kill(&root->cgrp.self.refcnt); + -+ now = ftrace_now(cpu); -+ latency = (long) div_s64(-latency_ns, NSECS_PER_USECS); -+ latency_hist(MISSED_TIMER_OFFSETS, cpu, latency, latency, now, -+ task); -+#ifdef CONFIG_WAKEUP_LATENCY_HIST -+ task->timer_offset = latency; -+#endif -+ } ++ kernfs_kill_sb(sb); +} -+#endif + -+static __init int latency_hist_init(void) ++static struct file_system_type cgroup_fs_type = { ++ .name = "cgroup", ++ .mount = cgroup_mount, ++ .kill_sb = cgroup_kill_sb, ++}; ++ ++/** ++ * task_cgroup_path - cgroup path of a task in the first cgroup hierarchy ++ * @task: target task ++ * @buf: the buffer to write the path into ++ * @buflen: the length of the buffer ++ * ++ * Determine @task's cgroup on the first (the one with the lowest non-zero ++ * hierarchy_id) cgroup hierarchy and copy its path into @buf. This ++ * function grabs cgroup_mutex and shouldn't be used inside locks used by ++ * cgroup controller callbacks. ++ * ++ * Return value is the same as kernfs_path(). ++ */ ++char *task_cgroup_path(struct task_struct *task, char *buf, size_t buflen) +{ -+ struct dentry *latency_hist_root = NULL; -+ struct dentry *dentry; -+#ifdef CONFIG_WAKEUP_LATENCY_HIST -+ struct dentry *dentry_sharedprio; -+#endif -+ struct dentry *entry; -+ struct dentry *enable_root; -+ int i = 0; -+ struct hist_data *my_hist; -+ char name[64]; -+ char *cpufmt = "CPU%d"; -+#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \ -+ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) -+ char *cpufmt_maxlatproc = "max_latency-CPU%d"; -+ struct maxlatproc_data *mp = NULL; -+#endif ++ struct cgroup_root *root; ++ struct cgroup *cgrp; ++ int hierarchy_id = 1; ++ char *path = NULL; + -+ dentry = tracing_init_dentry(); -+ latency_hist_root = debugfs_create_dir(latency_hist_dir_root, dentry); -+ enable_root = debugfs_create_dir("enable", latency_hist_root); ++ mutex_lock(&cgroup_mutex); ++ down_read(&css_set_rwsem); + -+#ifdef CONFIG_INTERRUPT_OFF_HIST -+ dentry = debugfs_create_dir(irqsoff_hist_dir, latency_hist_root); -+ for_each_possible_cpu(i) { -+ sprintf(name, cpufmt, i); -+ entry = debugfs_create_file(name, 0444, dentry, -+ &per_cpu(irqsoff_hist, i), &latency_hist_fops); -+ my_hist = &per_cpu(irqsoff_hist, i); -+ atomic_set(&my_hist->hist_mode, 1); -+ my_hist->min_lat = LONG_MAX; -+ } -+ entry = debugfs_create_file("reset", 0644, dentry, -+ (void *)IRQSOFF_LATENCY, &latency_hist_reset_fops); -+#endif ++ root = idr_get_next(&cgroup_hierarchy_idr, &hierarchy_id); + -+#ifdef CONFIG_PREEMPT_OFF_HIST -+ dentry = debugfs_create_dir(preemptoff_hist_dir, -+ latency_hist_root); -+ for_each_possible_cpu(i) { -+ sprintf(name, cpufmt, i); -+ entry = debugfs_create_file(name, 0444, dentry, -+ &per_cpu(preemptoff_hist, i), &latency_hist_fops); -+ my_hist = &per_cpu(preemptoff_hist, i); -+ atomic_set(&my_hist->hist_mode, 1); -+ my_hist->min_lat = LONG_MAX; ++ if (root) { ++ cgrp = task_cgroup_from_root(task, root); ++ path = cgroup_path(cgrp, buf, buflen); ++ } else { ++ /* if no hierarchy exists, everyone is in "/" */ ++ if (strlcpy(buf, "/", buflen) < buflen) ++ path = buf; + } -+ entry = debugfs_create_file("reset", 0644, dentry, -+ (void *)PREEMPTOFF_LATENCY, &latency_hist_reset_fops); -+#endif + -+#if defined(CONFIG_INTERRUPT_OFF_HIST) && defined(CONFIG_PREEMPT_OFF_HIST) -+ dentry = debugfs_create_dir(preemptirqsoff_hist_dir, -+ latency_hist_root); -+ for_each_possible_cpu(i) { -+ sprintf(name, cpufmt, i); -+ entry = debugfs_create_file(name, 0444, dentry, -+ &per_cpu(preemptirqsoff_hist, i), &latency_hist_fops); -+ my_hist = &per_cpu(preemptirqsoff_hist, i); -+ atomic_set(&my_hist->hist_mode, 1); -+ my_hist->min_lat = LONG_MAX; -+ } -+ entry = debugfs_create_file("reset", 0644, dentry, -+ (void *)PREEMPTIRQSOFF_LATENCY, &latency_hist_reset_fops); -+#endif ++ up_read(&css_set_rwsem); ++ mutex_unlock(&cgroup_mutex); ++ return path; ++} ++EXPORT_SYMBOL_GPL(task_cgroup_path); + -+#if defined(CONFIG_INTERRUPT_OFF_HIST) || defined(CONFIG_PREEMPT_OFF_HIST) -+ entry = debugfs_create_file("preemptirqsoff", 0644, -+ enable_root, (void *)&preemptirqsoff_enabled_data, -+ &enable_fops); -+#endif ++/* used to track tasks and other necessary states during migration */ ++struct cgroup_taskset { ++ /* the src and dst cset list running through cset->mg_node */ ++ struct list_head src_csets; ++ struct list_head dst_csets; + -+#ifdef CONFIG_WAKEUP_LATENCY_HIST -+ dentry = debugfs_create_dir(wakeup_latency_hist_dir, -+ latency_hist_root); -+ dentry_sharedprio = debugfs_create_dir( -+ wakeup_latency_hist_dir_sharedprio, dentry); -+ for_each_possible_cpu(i) { -+ sprintf(name, cpufmt, i); ++ /* ++ * Fields for cgroup_taskset_*() iteration. ++ * ++ * Before migration is committed, the target migration tasks are on ++ * ->mg_tasks of the csets on ->src_csets. After, on ->mg_tasks of ++ * the csets on ->dst_csets. ->csets point to either ->src_csets ++ * or ->dst_csets depending on whether migration is committed. ++ * ++ * ->cur_csets and ->cur_task point to the current task position ++ * during iteration. ++ */ ++ struct list_head *csets; ++ struct css_set *cur_cset; ++ struct task_struct *cur_task; ++}; + -+ entry = debugfs_create_file(name, 0444, dentry, -+ &per_cpu(wakeup_latency_hist, i), -+ &latency_hist_fops); -+ my_hist = &per_cpu(wakeup_latency_hist, i); -+ atomic_set(&my_hist->hist_mode, 1); -+ my_hist->min_lat = LONG_MAX; ++/** ++ * cgroup_taskset_first - reset taskset and return the first task ++ * @tset: taskset of interest ++ * ++ * @tset iteration is initialized and the first task is returned. ++ */ ++struct task_struct *cgroup_taskset_first(struct cgroup_taskset *tset) ++{ ++ tset->cur_cset = list_first_entry(tset->csets, struct css_set, mg_node); ++ tset->cur_task = NULL; + -+ entry = debugfs_create_file(name, 0444, dentry_sharedprio, -+ &per_cpu(wakeup_latency_hist_sharedprio, i), -+ &latency_hist_fops); -+ my_hist = &per_cpu(wakeup_latency_hist_sharedprio, i); -+ atomic_set(&my_hist->hist_mode, 1); -+ my_hist->min_lat = LONG_MAX; ++ return cgroup_taskset_next(tset); ++} + -+ sprintf(name, cpufmt_maxlatproc, i); ++/** ++ * cgroup_taskset_next - iterate to the next task in taskset ++ * @tset: taskset of interest ++ * ++ * Return the next task in @tset. Iteration must have been initialized ++ * with cgroup_taskset_first(). ++ */ ++struct task_struct *cgroup_taskset_next(struct cgroup_taskset *tset) ++{ ++ struct css_set *cset = tset->cur_cset; ++ struct task_struct *task = tset->cur_task; + -+ mp = &per_cpu(wakeup_maxlatproc, i); -+ entry = debugfs_create_file(name, 0444, dentry, mp, -+ &maxlatproc_fops); -+ clear_maxlatprocdata(mp); ++ while (&cset->mg_node != tset->csets) { ++ if (!task) ++ task = list_first_entry(&cset->mg_tasks, ++ struct task_struct, cg_list); ++ else ++ task = list_next_entry(task, cg_list); + -+ mp = &per_cpu(wakeup_maxlatproc_sharedprio, i); -+ entry = debugfs_create_file(name, 0444, dentry_sharedprio, mp, -+ &maxlatproc_fops); -+ clear_maxlatprocdata(mp); ++ if (&task->cg_list != &cset->mg_tasks) { ++ tset->cur_cset = cset; ++ tset->cur_task = task; ++ return task; ++ } ++ ++ cset = list_next_entry(cset, mg_node); ++ task = NULL; + } -+ entry = debugfs_create_file("pid", 0644, dentry, -+ (void *)&wakeup_pid, &pid_fops); -+ entry = debugfs_create_file("reset", 0644, dentry, -+ (void *)WAKEUP_LATENCY, &latency_hist_reset_fops); -+ entry = debugfs_create_file("reset", 0644, dentry_sharedprio, -+ (void *)WAKEUP_LATENCY_SHAREDPRIO, &latency_hist_reset_fops); -+ entry = debugfs_create_file("wakeup", 0644, -+ enable_root, (void *)&wakeup_latency_enabled_data, -+ &enable_fops); -+#endif + -+#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST -+ dentry = debugfs_create_dir(missed_timer_offsets_dir, -+ latency_hist_root); -+ for_each_possible_cpu(i) { -+ sprintf(name, cpufmt, i); -+ entry = debugfs_create_file(name, 0444, dentry, -+ &per_cpu(missed_timer_offsets, i), &latency_hist_fops); -+ my_hist = &per_cpu(missed_timer_offsets, i); -+ atomic_set(&my_hist->hist_mode, 1); -+ my_hist->min_lat = LONG_MAX; ++ return NULL; ++} + -+ sprintf(name, cpufmt_maxlatproc, i); -+ mp = &per_cpu(missed_timer_offsets_maxlatproc, i); -+ entry = debugfs_create_file(name, 0444, dentry, mp, -+ &maxlatproc_fops); -+ clear_maxlatprocdata(mp); ++/** ++ * cgroup_task_migrate - move a task from one cgroup to another. ++ * @old_cgrp: the cgroup @tsk is being migrated from ++ * @tsk: the task being migrated ++ * @new_cset: the new css_set @tsk is being attached to ++ * ++ * Must be called with cgroup_mutex, threadgroup and css_set_rwsem locked. ++ */ ++static void cgroup_task_migrate(struct cgroup *old_cgrp, ++ struct task_struct *tsk, ++ struct css_set *new_cset) ++{ ++ struct css_set *old_cset; ++ ++ lockdep_assert_held(&cgroup_mutex); ++ lockdep_assert_held(&css_set_rwsem); ++ ++ /* ++ * We are synchronized through threadgroup_lock() against PF_EXITING ++ * setting such that we can't race against cgroup_exit() changing the ++ * css_set to init_css_set and dropping the old one. ++ */ ++ WARN_ON_ONCE(tsk->flags & PF_EXITING); ++ old_cset = task_css_set(tsk); ++ ++ get_css_set(new_cset); ++ rcu_assign_pointer(tsk->cgroups, new_cset); ++ ++ /* ++ * Use move_tail so that cgroup_taskset_first() still returns the ++ * leader after migration. This works because cgroup_migrate() ++ * ensures that the dst_cset of the leader is the first on the ++ * tset's dst_csets list. ++ */ ++ list_move_tail(&tsk->cg_list, &new_cset->mg_tasks); ++ ++ /* ++ * We just gained a reference on old_cset by taking it from the ++ * task. As trading it for new_cset is protected by cgroup_mutex, ++ * we're safe to drop it here; it will be freed under RCU. ++ */ ++ put_css_set_locked(old_cset); ++} ++ ++/** ++ * cgroup_migrate_finish - cleanup after attach ++ * @preloaded_csets: list of preloaded css_sets ++ * ++ * Undo cgroup_migrate_add_src() and cgroup_migrate_prepare_dst(). See ++ * those functions for details. ++ */ ++static void cgroup_migrate_finish(struct list_head *preloaded_csets) ++{ ++ struct css_set *cset, *tmp_cset; ++ ++ lockdep_assert_held(&cgroup_mutex); ++ ++ down_write(&css_set_rwsem); ++ list_for_each_entry_safe(cset, tmp_cset, preloaded_csets, mg_preload_node) { ++ cset->mg_src_cgrp = NULL; ++ cset->mg_dst_cset = NULL; ++ list_del_init(&cset->mg_preload_node); ++ put_css_set_locked(cset); + } -+ entry = debugfs_create_file("pid", 0644, dentry, -+ (void *)&missed_timer_offsets_pid, &pid_fops); -+ entry = debugfs_create_file("reset", 0644, dentry, -+ (void *)MISSED_TIMER_OFFSETS, &latency_hist_reset_fops); -+ entry = debugfs_create_file("missed_timer_offsets", 0644, -+ enable_root, (void *)&missed_timer_offsets_enabled_data, -+ &enable_fops); -+#endif ++ up_write(&css_set_rwsem); ++} + -+#if defined(CONFIG_WAKEUP_LATENCY_HIST) && \ -+ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) -+ dentry = debugfs_create_dir(timerandwakeup_latency_hist_dir, -+ latency_hist_root); -+ for_each_possible_cpu(i) { -+ sprintf(name, cpufmt, i); -+ entry = debugfs_create_file(name, 0444, dentry, -+ &per_cpu(timerandwakeup_latency_hist, i), -+ &latency_hist_fops); -+ my_hist = &per_cpu(timerandwakeup_latency_hist, i); -+ atomic_set(&my_hist->hist_mode, 1); -+ my_hist->min_lat = LONG_MAX; ++/** ++ * cgroup_migrate_add_src - add a migration source css_set ++ * @src_cset: the source css_set to add ++ * @dst_cgrp: the destination cgroup ++ * @preloaded_csets: list of preloaded css_sets ++ * ++ * Tasks belonging to @src_cset are about to be migrated to @dst_cgrp. Pin ++ * @src_cset and add it to @preloaded_csets, which should later be cleaned ++ * up by cgroup_migrate_finish(). ++ * ++ * This function may be called without holding threadgroup_lock even if the ++ * target is a process. Threads may be created and destroyed but as long ++ * as cgroup_mutex is not dropped, no new css_set can be put into play and ++ * the preloaded css_sets are guaranteed to cover all migrations. ++ */ ++static void cgroup_migrate_add_src(struct css_set *src_cset, ++ struct cgroup *dst_cgrp, ++ struct list_head *preloaded_csets) ++{ ++ struct cgroup *src_cgrp; + -+ sprintf(name, cpufmt_maxlatproc, i); -+ mp = &per_cpu(timerandwakeup_maxlatproc, i); -+ entry = debugfs_create_file(name, 0444, dentry, mp, -+ &maxlatproc_fops); -+ clear_maxlatprocdata(mp); ++ lockdep_assert_held(&cgroup_mutex); ++ lockdep_assert_held(&css_set_rwsem); ++ ++ src_cgrp = cset_cgroup_from_root(src_cset, dst_cgrp->root); ++ ++ if (!list_empty(&src_cset->mg_preload_node)) ++ return; ++ ++ WARN_ON(src_cset->mg_src_cgrp); ++ WARN_ON(!list_empty(&src_cset->mg_tasks)); ++ WARN_ON(!list_empty(&src_cset->mg_node)); ++ ++ src_cset->mg_src_cgrp = src_cgrp; ++ get_css_set(src_cset); ++ list_add(&src_cset->mg_preload_node, preloaded_csets); ++} ++ ++/** ++ * cgroup_migrate_prepare_dst - prepare destination css_sets for migration ++ * @dst_cgrp: the destination cgroup (may be %NULL) ++ * @preloaded_csets: list of preloaded source css_sets ++ * ++ * Tasks are about to be moved to @dst_cgrp and all the source css_sets ++ * have been preloaded to @preloaded_csets. This function looks up and ++ * pins all destination css_sets, links each to its source, and append them ++ * to @preloaded_csets. If @dst_cgrp is %NULL, the destination of each ++ * source css_set is assumed to be its cgroup on the default hierarchy. ++ * ++ * This function must be called after cgroup_migrate_add_src() has been ++ * called on each migration source css_set. After migration is performed ++ * using cgroup_migrate(), cgroup_migrate_finish() must be called on ++ * @preloaded_csets. ++ */ ++static int cgroup_migrate_prepare_dst(struct cgroup *dst_cgrp, ++ struct list_head *preloaded_csets) ++{ ++ LIST_HEAD(csets); ++ struct css_set *src_cset, *tmp_cset; ++ ++ lockdep_assert_held(&cgroup_mutex); ++ ++ /* ++ * Except for the root, child_subsys_mask must be zero for a cgroup ++ * with tasks so that child cgroups don't compete against tasks. ++ */ ++ if (dst_cgrp && cgroup_on_dfl(dst_cgrp) && cgroup_parent(dst_cgrp) && ++ dst_cgrp->child_subsys_mask) ++ return -EBUSY; ++ ++ /* look up the dst cset for each src cset and link it to src */ ++ list_for_each_entry_safe(src_cset, tmp_cset, preloaded_csets, mg_preload_node) { ++ struct css_set *dst_cset; ++ ++ dst_cset = find_css_set(src_cset, ++ dst_cgrp ?: src_cset->dfl_cgrp); ++ if (!dst_cset) ++ goto err; ++ ++ WARN_ON_ONCE(src_cset->mg_dst_cset || dst_cset->mg_dst_cset); ++ ++ /* ++ * If src cset equals dst, it's noop. Drop the src. ++ * cgroup_migrate() will skip the cset too. Note that we ++ * can't handle src == dst as some nodes are used by both. ++ */ ++ if (src_cset == dst_cset) { ++ src_cset->mg_src_cgrp = NULL; ++ list_del_init(&src_cset->mg_preload_node); ++ put_css_set(src_cset); ++ put_css_set(dst_cset); ++ continue; ++ } ++ ++ src_cset->mg_dst_cset = dst_cset; ++ ++ if (list_empty(&dst_cset->mg_preload_node)) ++ list_add(&dst_cset->mg_preload_node, &csets); ++ else ++ put_css_set(dst_cset); + } -+ entry = debugfs_create_file("reset", 0644, dentry, -+ (void *)TIMERANDWAKEUP_LATENCY, &latency_hist_reset_fops); -+ entry = debugfs_create_file("timerandwakeup", 0644, -+ enable_root, (void *)&timerandwakeup_enabled_data, -+ &enable_fops); -+#endif ++ ++ list_splice_tail(&csets, preloaded_csets); + return 0; ++err: ++ cgroup_migrate_finish(&csets); ++ return -ENOMEM; +} + -+device_initcall(latency_hist_init); -diff -Nur linux-4.1.6.orig/kernel/trace/Makefile linux-4.1.6/kernel/trace/Makefile ---- linux-4.1.6.orig/kernel/trace/Makefile 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/trace/Makefile 2015-09-08 23:49:08.125865168 +0200 -@@ -36,6 +36,10 @@ - obj-$(CONFIG_IRQSOFF_TRACER) += trace_irqsoff.o - obj-$(CONFIG_PREEMPT_TRACER) += trace_irqsoff.o - obj-$(CONFIG_SCHED_TRACER) += trace_sched_wakeup.o -+obj-$(CONFIG_INTERRUPT_OFF_HIST) += latency_hist.o -+obj-$(CONFIG_PREEMPT_OFF_HIST) += latency_hist.o -+obj-$(CONFIG_WAKEUP_LATENCY_HIST) += latency_hist.o -+obj-$(CONFIG_MISSED_TIMER_OFFSETS_HIST) += latency_hist.o - obj-$(CONFIG_NOP_TRACER) += trace_nop.o - obj-$(CONFIG_STACK_TRACER) += trace_stack.o - obj-$(CONFIG_MMIOTRACE) += trace_mmiotrace.o -diff -Nur linux-4.1.6.orig/kernel/trace/trace.c linux-4.1.6/kernel/trace/trace.c ---- linux-4.1.6.orig/kernel/trace/trace.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/trace/trace.c 2015-09-08 23:49:08.125865168 +0200 -@@ -1630,6 +1630,7 @@ - struct task_struct *tsk = current; - - entry->preempt_count = pc & 0xff; -+ entry->preempt_lazy_count = preempt_lazy_count(); - entry->pid = (tsk) ? tsk->pid : 0; - entry->flags = - #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT -@@ -1639,8 +1640,11 @@ - #endif - ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) | - ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) | -- (tif_need_resched() ? TRACE_FLAG_NEED_RESCHED : 0) | -+ (tif_need_resched_now() ? TRACE_FLAG_NEED_RESCHED : 0) | -+ (need_resched_lazy() ? TRACE_FLAG_NEED_RESCHED_LAZY : 0) | - (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED : 0); ++/** ++ * cgroup_migrate - migrate a process or task to a cgroup ++ * @cgrp: the destination cgroup ++ * @leader: the leader of the process or the task to migrate ++ * @threadgroup: whether @leader points to the whole process or a single task ++ * ++ * Migrate a process or task denoted by @leader to @cgrp. If migrating a ++ * process, the caller must be holding threadgroup_lock of @leader. The ++ * caller is also responsible for invoking cgroup_migrate_add_src() and ++ * cgroup_migrate_prepare_dst() on the targets before invoking this ++ * function and following up with cgroup_migrate_finish(). ++ * ++ * As long as a controller's ->can_attach() doesn't fail, this function is ++ * guaranteed to succeed. This means that, excluding ->can_attach() ++ * failure, when migrating multiple targets, the success or failure can be ++ * decided for all targets by invoking group_migrate_prepare_dst() before ++ * actually starting migrating. ++ */ ++static int cgroup_migrate(struct cgroup *cgrp, struct task_struct *leader, ++ bool threadgroup) ++{ ++ struct cgroup_taskset tset = { ++ .src_csets = LIST_HEAD_INIT(tset.src_csets), ++ .dst_csets = LIST_HEAD_INIT(tset.dst_csets), ++ .csets = &tset.src_csets, ++ }; ++ struct cgroup_subsys_state *css, *failed_css = NULL; ++ struct css_set *cset, *tmp_cset; ++ struct task_struct *task, *tmp_task; ++ int i, ret; + -+ entry->migrate_disable = (tsk) ? __migrate_disabled(tsk) & 0xFF : 0; - } - EXPORT_SYMBOL_GPL(tracing_generic_entry_update); - -@@ -2558,14 +2562,17 @@ - - static void print_lat_help_header(struct seq_file *m) - { -- seq_puts(m, "# _------=> CPU# \n" -- "# / _-----=> irqs-off \n" -- "# | / _----=> need-resched \n" -- "# || / _---=> hardirq/softirq \n" -- "# ||| / _--=> preempt-depth \n" -- "# |||| / delay \n" -- "# cmd pid ||||| time | caller \n" -- "# \\ / ||||| \\ | / \n"); -+ seq_puts(m, "# _--------=> CPU# \n" -+ "# / _-------=> irqs-off \n" -+ "# | / _------=> need-resched \n" -+ "# || / _-----=> need-resched_lazy \n" -+ "# ||| / _----=> hardirq/softirq \n" -+ "# |||| / _---=> preempt-depth \n" -+ "# ||||| / _--=> preempt-lazy-depth\n" -+ "# |||||| / _-=> migrate-disable \n" -+ "# ||||||| / delay \n" -+ "# cmd pid |||||||| time | caller \n" -+ "# \\ / |||||||| \\ | / \n"); - } - - static void print_event_info(struct trace_buffer *buf, struct seq_file *m) -@@ -2591,11 +2598,14 @@ - print_event_info(buf, m); - seq_puts(m, "# _-----=> irqs-off\n" - "# / _----=> need-resched\n" -- "# | / _---=> hardirq/softirq\n" -- "# || / _--=> preempt-depth\n" -- "# ||| / delay\n" -- "# TASK-PID CPU# |||| TIMESTAMP FUNCTION\n" -- "# | | | |||| | |\n"); -+ "# |/ _-----=> need-resched_lazy\n" -+ "# || / _---=> hardirq/softirq\n" -+ "# ||| / _--=> preempt-depth\n" -+ "# |||| /_--=> preempt-lazy-depth\n" -+ "# ||||| _-=> migrate-disable \n" -+ "# ||||| / delay\n" -+ "# TASK-PID CPU# |||||| TIMESTAMP FUNCTION\n" -+ "# | | | |||||| | |\n"); - } - - void -diff -Nur linux-4.1.6.orig/kernel/trace/trace_events.c linux-4.1.6/kernel/trace/trace_events.c ---- linux-4.1.6.orig/kernel/trace/trace_events.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/trace/trace_events.c 2015-09-08 23:49:08.125865168 +0200 -@@ -162,6 +162,8 @@ - __common_field(unsigned char, flags); - __common_field(unsigned char, preempt_count); - __common_field(int, pid); -+ __common_field(unsigned short, migrate_disable); -+ __common_field(unsigned short, padding); - - return ret; - } -diff -Nur linux-4.1.6.orig/kernel/trace/trace.h linux-4.1.6/kernel/trace/trace.h ---- linux-4.1.6.orig/kernel/trace/trace.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/trace/trace.h 2015-09-08 23:49:08.125865168 +0200 -@@ -120,6 +120,7 @@ - * NEED_RESCHED - reschedule is requested - * HARDIRQ - inside an interrupt handler - * SOFTIRQ - inside a softirq handler -+ * NEED_RESCHED_LAZY - lazy reschedule is requested - */ - enum trace_flag_type { - TRACE_FLAG_IRQS_OFF = 0x01, -@@ -128,6 +129,7 @@ - TRACE_FLAG_HARDIRQ = 0x08, - TRACE_FLAG_SOFTIRQ = 0x10, - TRACE_FLAG_PREEMPT_RESCHED = 0x20, -+ TRACE_FLAG_NEED_RESCHED_LAZY = 0x40, - }; - - #define TRACE_BUF_SIZE 1024 -diff -Nur linux-4.1.6.orig/kernel/trace/trace_irqsoff.c linux-4.1.6/kernel/trace/trace_irqsoff.c ---- linux-4.1.6.orig/kernel/trace/trace_irqsoff.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/trace/trace_irqsoff.c 2015-09-08 23:49:08.129864725 +0200 -@@ -13,6 +13,7 @@ - #include - #include - #include -+#include - - #include "trace.h" - -@@ -433,11 +434,13 @@ - { - if (preempt_trace() || irq_trace()) - start_critical_timing(CALLER_ADDR0, CALLER_ADDR1); -+ trace_preemptirqsoff_hist(TRACE_START, 1); - } - EXPORT_SYMBOL_GPL(start_critical_timings); - - void stop_critical_timings(void) - { -+ trace_preemptirqsoff_hist(TRACE_STOP, 0); - if (preempt_trace() || irq_trace()) - stop_critical_timing(CALLER_ADDR0, CALLER_ADDR1); - } -@@ -447,6 +450,7 @@ - #ifdef CONFIG_PROVE_LOCKING - void time_hardirqs_on(unsigned long a0, unsigned long a1) - { -+ trace_preemptirqsoff_hist(IRQS_ON, 0); - if (!preempt_trace() && irq_trace()) - stop_critical_timing(a0, a1); - } -@@ -455,6 +459,7 @@ - { - if (!preempt_trace() && irq_trace()) - start_critical_timing(a0, a1); -+ trace_preemptirqsoff_hist(IRQS_OFF, 1); - } - - #else /* !CONFIG_PROVE_LOCKING */ -@@ -480,6 +485,7 @@ - */ - void trace_hardirqs_on(void) - { -+ trace_preemptirqsoff_hist(IRQS_ON, 0); - if (!preempt_trace() && irq_trace()) - stop_critical_timing(CALLER_ADDR0, CALLER_ADDR1); - } -@@ -489,11 +495,13 @@ - { - if (!preempt_trace() && irq_trace()) - start_critical_timing(CALLER_ADDR0, CALLER_ADDR1); -+ trace_preemptirqsoff_hist(IRQS_OFF, 1); - } - EXPORT_SYMBOL(trace_hardirqs_off); - - __visible void trace_hardirqs_on_caller(unsigned long caller_addr) - { -+ trace_preemptirqsoff_hist(IRQS_ON, 0); - if (!preempt_trace() && irq_trace()) - stop_critical_timing(CALLER_ADDR0, caller_addr); - } -@@ -503,6 +511,7 @@ - { - if (!preempt_trace() && irq_trace()) - start_critical_timing(CALLER_ADDR0, caller_addr); -+ trace_preemptirqsoff_hist(IRQS_OFF, 1); - } - EXPORT_SYMBOL(trace_hardirqs_off_caller); - -@@ -512,12 +521,14 @@ - #ifdef CONFIG_PREEMPT_TRACER - void trace_preempt_on(unsigned long a0, unsigned long a1) - { -+ trace_preemptirqsoff_hist(PREEMPT_ON, 0); - if (preempt_trace() && !irq_trace()) - stop_critical_timing(a0, a1); - } - - void trace_preempt_off(unsigned long a0, unsigned long a1) - { -+ trace_preemptirqsoff_hist(PREEMPT_ON, 1); - if (preempt_trace() && !irq_trace()) - start_critical_timing(a0, a1); - } -diff -Nur linux-4.1.6.orig/kernel/trace/trace_output.c linux-4.1.6/kernel/trace/trace_output.c ---- linux-4.1.6.orig/kernel/trace/trace_output.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/trace/trace_output.c 2015-09-08 23:49:08.137863836 +0200 -@@ -430,6 +430,7 @@ - { - char hardsoft_irq; - char need_resched; -+ char need_resched_lazy; - char irqs_off; - int hardirq; - int softirq; -@@ -457,6 +458,8 @@ - need_resched = '.'; - break; - } -+ need_resched_lazy = -+ (entry->flags & TRACE_FLAG_NEED_RESCHED_LAZY) ? 'L' : '.'; - - hardsoft_irq = - (hardirq && softirq) ? 'H' : -@@ -464,14 +467,25 @@ - softirq ? 's' : - '.'; - -- trace_seq_printf(s, "%c%c%c", -- irqs_off, need_resched, hardsoft_irq); -+ trace_seq_printf(s, "%c%c%c%c", -+ irqs_off, need_resched, need_resched_lazy, -+ hardsoft_irq); - - if (entry->preempt_count) - trace_seq_printf(s, "%x", entry->preempt_count); - else - trace_seq_putc(s, '.'); - -+ if (entry->preempt_lazy_count) -+ trace_seq_printf(s, "%x", entry->preempt_lazy_count); -+ else -+ trace_seq_putc(s, '.'); ++ /* ++ * Prevent freeing of tasks while we take a snapshot. Tasks that are ++ * already PF_EXITING could be freed from underneath us unless we ++ * take an rcu_read_lock. ++ */ ++ down_write(&css_set_rwsem); ++ rcu_read_lock(); ++ task = leader; ++ do { ++ /* @task either already exited or can't exit until the end */ ++ if (task->flags & PF_EXITING) ++ goto next; + -+ if (entry->migrate_disable) -+ trace_seq_printf(s, "%x", entry->migrate_disable); -+ else -+ trace_seq_putc(s, '.'); ++ /* leave @task alone if post_fork() hasn't linked it yet */ ++ if (list_empty(&task->cg_list)) ++ goto next; + - return !trace_seq_has_overflowed(s); - } - -diff -Nur linux-4.1.6.orig/kernel/user.c linux-4.1.6/kernel/user.c ---- linux-4.1.6.orig/kernel/user.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/user.c 2015-09-08 23:49:08.137863836 +0200 -@@ -161,11 +161,11 @@ - if (!up) - return; - -- local_irq_save(flags); -+ local_irq_save_nort(flags); - if (atomic_dec_and_lock(&up->__count, &uidhash_lock)) - free_user(up, flags); - else -- local_irq_restore(flags); -+ local_irq_restore_nort(flags); - } - - struct user_struct *alloc_uid(kuid_t uid) -diff -Nur linux-4.1.6.orig/kernel/watchdog.c linux-4.1.6/kernel/watchdog.c ---- linux-4.1.6.orig/kernel/watchdog.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/watchdog.c 2015-09-08 23:49:08.421832347 +0200 -@@ -262,6 +262,8 @@ - - #ifdef CONFIG_HARDLOCKUP_DETECTOR - -+static DEFINE_RAW_SPINLOCK(watchdog_output_lock); ++ cset = task_css_set(task); ++ if (!cset->mg_src_cgrp) ++ goto next; + - static struct perf_event_attr wd_hw_attr = { - .type = PERF_TYPE_HARDWARE, - .config = PERF_COUNT_HW_CPU_CYCLES, -@@ -295,13 +297,21 @@ - /* only print hardlockups once */ - if (__this_cpu_read(hard_watchdog_warn) == true) - return; + /* -+ * If early-printk is enabled then make sure we do not -+ * lock up in printk() and kill console logging: ++ * cgroup_taskset_first() must always return the leader. ++ * Take care to avoid disturbing the ordering. + */ -+ printk_kill(); - -- if (hardlockup_panic) -+ if (hardlockup_panic) { - panic("Watchdog detected hard LOCKUP on cpu %d", - this_cpu); -- else -+ } else { -+ raw_spin_lock(&watchdog_output_lock); - WARN(1, "Watchdog detected hard LOCKUP on cpu %d", - this_cpu); -+ raw_spin_unlock(&watchdog_output_lock); ++ list_move_tail(&task->cg_list, &cset->mg_tasks); ++ if (list_empty(&cset->mg_node)) ++ list_add_tail(&cset->mg_node, &tset.src_csets); ++ if (list_empty(&cset->mg_dst_cset->mg_node)) ++ list_move_tail(&cset->mg_dst_cset->mg_node, ++ &tset.dst_csets); ++ next: ++ if (!threadgroup) ++ break; ++ } while_each_thread(leader, task); ++ rcu_read_unlock(); ++ up_write(&css_set_rwsem); ++ ++ /* methods shouldn't be called if no task is actually migrating */ ++ if (list_empty(&tset.src_csets)) ++ return 0; ++ ++ /* check that we can legitimately attach to the cgroup */ ++ for_each_e_css(css, i, cgrp) { ++ if (css->ss->can_attach) { ++ ret = css->ss->can_attach(css, &tset); ++ if (ret) { ++ failed_css = css; ++ goto out_cancel_attach; ++ } + } - - __this_cpu_write(hard_watchdog_warn, true); - return; -@@ -444,6 +454,7 @@ - /* kick off the timer for the hardlockup detector */ - hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); - hrtimer->function = watchdog_timer_fn; -+ hrtimer->irqsafe = 1; - - /* Enable the perf event */ - watchdog_nmi_enable(cpu); -diff -Nur linux-4.1.6.orig/kernel/workqueue.c linux-4.1.6/kernel/workqueue.c ---- linux-4.1.6.orig/kernel/workqueue.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/workqueue.c 2015-09-08 23:49:08.425831903 +0200 -@@ -48,6 +48,8 @@ - #include - #include - #include -+#include -+#include - - #include "workqueue_internal.h" - -@@ -121,15 +123,20 @@ - * cpu or grabbing pool->lock is enough for read access. If - * POOL_DISASSOCIATED is set, it's identical to L. - * -+ * On RT we need the extra protection via rt_lock_idle_list() for -+ * the list manipulations against read access from -+ * wq_worker_sleeping(). All other places are nicely serialized via -+ * pool->lock. -+ * - * A: pool->attach_mutex protected. - * - * PL: wq_pool_mutex protected. - * -- * PR: wq_pool_mutex protected for writes. Sched-RCU protected for reads. -+ * PR: wq_pool_mutex protected for writes. RCU protected for reads. - * - * WQ: wq->mutex protected. - * -- * WR: wq->mutex protected for writes. Sched-RCU protected for reads. -+ * WR: wq->mutex protected for writes. RCU protected for reads. - * - * MD: wq_mayday_lock protected. - */ -@@ -178,7 +185,7 @@ - atomic_t nr_running ____cacheline_aligned_in_smp; - - /* -- * Destruction of pool is sched-RCU protected to allow dereferences -+ * Destruction of pool is RCU protected to allow dereferences - * from get_work_pool(). - */ - struct rcu_head rcu; -@@ -207,7 +214,7 @@ - /* - * Release of unbound pwq is punted to system_wq. See put_pwq() - * and pwq_unbound_release_workfn() for details. pool_workqueue -- * itself is also sched-RCU protected so that the first pwq can be -+ * itself is also RCU protected so that the first pwq can be - * determined without grabbing wq->mutex. - */ - struct work_struct unbound_release_work; -@@ -329,6 +336,8 @@ - struct workqueue_struct *system_freezable_power_efficient_wq __read_mostly; - EXPORT_SYMBOL_GPL(system_freezable_power_efficient_wq); - -+static DEFINE_LOCAL_IRQ_LOCK(pendingb_lock); ++ } + - static int worker_thread(void *__worker); - static void copy_workqueue_attrs(struct workqueue_attrs *to, - const struct workqueue_attrs *from); -@@ -338,14 +347,14 @@ - #include - - #define assert_rcu_or_pool_mutex() \ -- rcu_lockdep_assert(rcu_read_lock_sched_held() || \ -+ rcu_lockdep_assert(rcu_read_lock_held() || \ - lockdep_is_held(&wq_pool_mutex), \ -- "sched RCU or wq_pool_mutex should be held") -+ "RCU or wq_pool_mutex should be held") - - #define assert_rcu_or_wq_mutex(wq) \ -- rcu_lockdep_assert(rcu_read_lock_sched_held() || \ -+ rcu_lockdep_assert(rcu_read_lock_held() || \ - lockdep_is_held(&wq->mutex), \ -- "sched RCU or wq->mutex should be held") -+ "RCU or wq->mutex should be held") - - #define for_each_cpu_worker_pool(pool, cpu) \ - for ((pool) = &per_cpu(cpu_worker_pools, cpu)[0]; \ -@@ -357,7 +366,7 @@ - * @pool: iteration cursor - * @pi: integer used for iteration - * -- * This must be called either with wq_pool_mutex held or sched RCU read -+ * This must be called either with wq_pool_mutex held or RCU read - * locked. If the pool needs to be used beyond the locking in effect, the - * caller is responsible for guaranteeing that the pool stays online. - * -@@ -389,7 +398,7 @@ - * @pwq: iteration cursor - * @wq: the target workqueue - * -- * This must be called either with wq->mutex held or sched RCU read locked. -+ * This must be called either with wq->mutex held or RCU read locked. - * If the pwq needs to be used beyond the locking in effect, the caller is - * responsible for guaranteeing that the pwq stays online. - * -@@ -401,6 +410,31 @@ - if (({ assert_rcu_or_wq_mutex(wq); false; })) { } \ - else - -+#ifdef CONFIG_PREEMPT_RT_BASE -+static inline void rt_lock_idle_list(struct worker_pool *pool) ++ /* ++ * Now that we're guaranteed success, proceed to move all tasks to ++ * the new cgroup. There are no failure cases after here, so this ++ * is the commit point. ++ */ ++ down_write(&css_set_rwsem); ++ list_for_each_entry(cset, &tset.src_csets, mg_node) { ++ list_for_each_entry_safe(task, tmp_task, &cset->mg_tasks, cg_list) ++ cgroup_task_migrate(cset->mg_src_cgrp, task, ++ cset->mg_dst_cset); ++ } ++ up_write(&css_set_rwsem); ++ ++ /* ++ * Migration is committed, all target tasks are now on dst_csets. ++ * Nothing is sensitive to fork() after this point. Notify ++ * controllers that migration is complete. ++ */ ++ tset.csets = &tset.dst_csets; ++ ++ for_each_e_css(css, i, cgrp) ++ if (css->ss->attach) ++ css->ss->attach(css, &tset); ++ ++ ret = 0; ++ goto out_release_tset; ++ ++out_cancel_attach: ++ for_each_e_css(css, i, cgrp) { ++ if (css == failed_css) ++ break; ++ if (css->ss->cancel_attach) ++ css->ss->cancel_attach(css, &tset); ++ } ++out_release_tset: ++ down_write(&css_set_rwsem); ++ list_splice_init(&tset.dst_csets, &tset.src_csets); ++ list_for_each_entry_safe(cset, tmp_cset, &tset.src_csets, mg_node) { ++ list_splice_tail_init(&cset->mg_tasks, &cset->tasks); ++ list_del_init(&cset->mg_node); ++ } ++ up_write(&css_set_rwsem); ++ return ret; ++} ++ ++/** ++ * cgroup_attach_task - attach a task or a whole threadgroup to a cgroup ++ * @dst_cgrp: the cgroup to attach to ++ * @leader: the task or the leader of the threadgroup to be attached ++ * @threadgroup: attach the whole threadgroup? ++ * ++ * Call holding cgroup_mutex and threadgroup_lock of @leader. ++ */ ++static int cgroup_attach_task(struct cgroup *dst_cgrp, ++ struct task_struct *leader, bool threadgroup) +{ -+ preempt_disable(); ++ LIST_HEAD(preloaded_csets); ++ struct task_struct *task; ++ int ret; ++ ++ /* look up all src csets */ ++ down_read(&css_set_rwsem); ++ rcu_read_lock(); ++ task = leader; ++ do { ++ cgroup_migrate_add_src(task_css_set(task), dst_cgrp, ++ &preloaded_csets); ++ if (!threadgroup) ++ break; ++ } while_each_thread(leader, task); ++ rcu_read_unlock(); ++ up_read(&css_set_rwsem); ++ ++ /* prepare dst csets and commit */ ++ ret = cgroup_migrate_prepare_dst(dst_cgrp, &preloaded_csets); ++ if (!ret) ++ ret = cgroup_migrate(dst_cgrp, leader, threadgroup); ++ ++ cgroup_migrate_finish(&preloaded_csets); ++ return ret; +} -+static inline void rt_unlock_idle_list(struct worker_pool *pool) ++ ++/* ++ * Find the task_struct of the task to attach by vpid and pass it along to the ++ * function to attach either it or all tasks in its threadgroup. Will lock ++ * cgroup_mutex and threadgroup. ++ */ ++static ssize_t __cgroup_procs_write(struct kernfs_open_file *of, char *buf, ++ size_t nbytes, loff_t off, bool threadgroup) +{ -+ preempt_enable(); ++ struct task_struct *tsk; ++ const struct cred *cred = current_cred(), *tcred; ++ struct cgroup *cgrp; ++ pid_t pid; ++ int ret; ++ ++ if (kstrtoint(strstrip(buf), 0, &pid) || pid < 0) ++ return -EINVAL; ++ ++ cgrp = cgroup_kn_lock_live(of->kn); ++ if (!cgrp) ++ return -ENODEV; ++ ++retry_find_task: ++ rcu_read_lock(); ++ if (pid) { ++ tsk = find_task_by_vpid(pid); ++ if (!tsk) { ++ rcu_read_unlock(); ++ ret = -ESRCH; ++ goto out_unlock_cgroup; ++ } ++ /* ++ * even if we're attaching all tasks in the thread group, we ++ * only need to check permissions on one of them. ++ */ ++ tcred = __task_cred(tsk); ++ if (!uid_eq(cred->euid, GLOBAL_ROOT_UID) && ++ !uid_eq(cred->euid, tcred->uid) && ++ !uid_eq(cred->euid, tcred->suid)) { ++ rcu_read_unlock(); ++ ret = -EACCES; ++ goto out_unlock_cgroup; ++ } ++ } else ++ tsk = current; ++ ++ if (threadgroup) ++ tsk = tsk->group_leader; ++ ++ /* ++ * Workqueue threads may acquire PF_NO_SETAFFINITY and become ++ * trapped in a cpuset, or RT worker may be born in a cgroup ++ * with no rt_runtime allocated. Just say no. ++ */ ++ if (tsk == kthreadd_task || (tsk->flags & PF_NO_SETAFFINITY)) { ++ ret = -EINVAL; ++ rcu_read_unlock(); ++ goto out_unlock_cgroup; ++ } ++ ++ get_task_struct(tsk); ++ rcu_read_unlock(); ++ ++ threadgroup_lock(tsk); ++ if (threadgroup) { ++ if (!thread_group_leader(tsk)) { ++ /* ++ * a race with de_thread from another thread's exec() ++ * may strip us of our leadership, if this happens, ++ * there is no choice but to throw this task away and ++ * try again; this is ++ * "double-double-toil-and-trouble-check locking". ++ */ ++ threadgroup_unlock(tsk); ++ put_task_struct(tsk); ++ goto retry_find_task; ++ } ++ } ++ ++ ret = cgroup_attach_task(cgrp, tsk, threadgroup); ++ ++ threadgroup_unlock(tsk); ++ ++ put_task_struct(tsk); ++out_unlock_cgroup: ++ cgroup_kn_unlock(of->kn); ++ return ret ?: nbytes; +} -+static inline void sched_lock_idle_list(struct worker_pool *pool) { } -+static inline void sched_unlock_idle_list(struct worker_pool *pool) { } -+#else -+static inline void rt_lock_idle_list(struct worker_pool *pool) { } -+static inline void rt_unlock_idle_list(struct worker_pool *pool) { } -+static inline void sched_lock_idle_list(struct worker_pool *pool) ++ ++/** ++ * cgroup_attach_task_all - attach task 'tsk' to all cgroups of task 'from' ++ * @from: attach to all cgroups of a given task ++ * @tsk: the task to be attached ++ */ ++int cgroup_attach_task_all(struct task_struct *from, struct task_struct *tsk) +{ -+ spin_lock_irq(&pool->lock); ++ struct cgroup_root *root; ++ int retval = 0; ++ ++ mutex_lock(&cgroup_mutex); ++ for_each_root(root) { ++ struct cgroup *from_cgrp; ++ ++ if (root == &cgrp_dfl_root) ++ continue; ++ ++ down_read(&css_set_rwsem); ++ from_cgrp = task_cgroup_from_root(from, root); ++ up_read(&css_set_rwsem); ++ ++ retval = cgroup_attach_task(from_cgrp, tsk, false); ++ if (retval) ++ break; ++ } ++ mutex_unlock(&cgroup_mutex); ++ ++ return retval; +} -+static inline void sched_unlock_idle_list(struct worker_pool *pool) ++EXPORT_SYMBOL_GPL(cgroup_attach_task_all); ++ ++static ssize_t cgroup_tasks_write(struct kernfs_open_file *of, ++ char *buf, size_t nbytes, loff_t off) +{ -+ spin_unlock_irq(&pool->lock); ++ return __cgroup_procs_write(of, buf, nbytes, off, false); +} -+#endif + ++static ssize_t cgroup_procs_write(struct kernfs_open_file *of, ++ char *buf, size_t nbytes, loff_t off) ++{ ++ return __cgroup_procs_write(of, buf, nbytes, off, true); ++} + - #ifdef CONFIG_DEBUG_OBJECTS_WORK - - static struct debug_obj_descr work_debug_descr; -@@ -551,7 +585,7 @@ - * @wq: the target workqueue - * @node: the node ID - * -- * This must be called either with pwq_lock held or sched RCU read locked. -+ * This must be called either with pwq_lock held or RCU read locked. - * If the pwq needs to be used beyond the locking in effect, the caller is - * responsible for guaranteeing that the pwq stays online. - * -@@ -655,8 +689,8 @@ - * @work: the work item of interest - * - * Pools are created and destroyed under wq_pool_mutex, and allows read -- * access under sched-RCU read lock. As such, this function should be -- * called under wq_pool_mutex or with preemption disabled. -+ * access under RCU read lock. As such, this function should be -+ * called under wq_pool_mutex or inside of a rcu_read_lock() region. - * - * All fields of the returned pool are accessible as long as the above - * mentioned locking is in effect. If the returned pool needs to be used -@@ -793,51 +827,44 @@ - */ - static void wake_up_worker(struct worker_pool *pool) - { -- struct worker *worker = first_idle_worker(pool); -+ struct worker *worker; ++static ssize_t cgroup_release_agent_write(struct kernfs_open_file *of, ++ char *buf, size_t nbytes, loff_t off) ++{ ++ struct cgroup *cgrp; + -+ rt_lock_idle_list(pool); ++ BUILD_BUG_ON(sizeof(cgrp->root->release_agent_path) < PATH_MAX); + -+ worker = first_idle_worker(pool); - - if (likely(worker)) - wake_up_process(worker->task); ++ cgrp = cgroup_kn_lock_live(of->kn); ++ if (!cgrp) ++ return -ENODEV; ++ spin_lock(&release_agent_path_lock); ++ strlcpy(cgrp->root->release_agent_path, strstrip(buf), ++ sizeof(cgrp->root->release_agent_path)); ++ spin_unlock(&release_agent_path_lock); ++ cgroup_kn_unlock(of->kn); ++ return nbytes; ++} + -+ rt_unlock_idle_list(pool); - } - - /** -- * wq_worker_waking_up - a worker is waking up -- * @task: task waking up -- * @cpu: CPU @task is waking up to -+ * wq_worker_running - a worker is running again -+ * @task: task returning from sleep - * -- * This function is called during try_to_wake_up() when a worker is -- * being awoken. -- * -- * CONTEXT: -- * spin_lock_irq(rq->lock) -+ * This function is called when a worker returns from schedule() - */ --void wq_worker_waking_up(struct task_struct *task, int cpu) -+void wq_worker_running(struct task_struct *task) - { - struct worker *worker = kthread_data(task); - -- if (!(worker->flags & WORKER_NOT_RUNNING)) { -- WARN_ON_ONCE(worker->pool->cpu != cpu); -+ if (!worker->sleeping) -+ return; -+ if (!(worker->flags & WORKER_NOT_RUNNING)) - atomic_inc(&worker->pool->nr_running); -- } -+ worker->sleeping = 0; - } - - /** - * wq_worker_sleeping - a worker is going to sleep - * @task: task going to sleep -- * @cpu: CPU in question, must be the current CPU number -- * -- * This function is called during schedule() when a busy worker is -- * going to sleep. Worker on the same cpu can be woken up by -- * returning pointer to its task. -- * -- * CONTEXT: -- * spin_lock_irq(rq->lock) -- * -- * Return: -- * Worker task on @cpu to wake up, %NULL if none. -+ * This function is called from schedule() when a busy worker is -+ * going to sleep. - */ --struct task_struct *wq_worker_sleeping(struct task_struct *task, int cpu) -+void wq_worker_sleeping(struct task_struct *task) - { -- struct worker *worker = kthread_data(task), *to_wakeup = NULL; -+ struct worker *worker = kthread_data(task); - struct worker_pool *pool; - - /* -@@ -846,29 +873,26 @@ - * checking NOT_RUNNING. - */ - if (worker->flags & WORKER_NOT_RUNNING) -- return NULL; -+ return; - - pool = worker->pool; - -- /* this can only happen on the local cpu */ -- if (WARN_ON_ONCE(cpu != raw_smp_processor_id() || pool->cpu != cpu)) -- return NULL; -+ if (WARN_ON_ONCE(worker->sleeping)) -+ return; ++static int cgroup_release_agent_show(struct seq_file *seq, void *v) ++{ ++ struct cgroup *cgrp = seq_css(seq)->cgroup; + -+ worker->sleeping = 1; - - /* - * The counterpart of the following dec_and_test, implied mb, - * worklist not empty test sequence is in insert_work(). - * Please read comment there. -- * -- * NOT_RUNNING is clear. This means that we're bound to and -- * running on the local cpu w/ rq lock held and preemption -- * disabled, which in turn means that none else could be -- * manipulating idle_list, so dereferencing idle_list without pool -- * lock is safe. - */ - if (atomic_dec_and_test(&pool->nr_running) && -- !list_empty(&pool->worklist)) -- to_wakeup = first_idle_worker(pool); -- return to_wakeup ? to_wakeup->task : NULL; -+ !list_empty(&pool->worklist)) { -+ sched_lock_idle_list(pool); -+ wake_up_worker(pool); -+ sched_unlock_idle_list(pool); ++ spin_lock(&release_agent_path_lock); ++ seq_puts(seq, cgrp->root->release_agent_path); ++ spin_unlock(&release_agent_path_lock); ++ seq_putc(seq, '\n'); ++ return 0; ++} ++ ++static int cgroup_sane_behavior_show(struct seq_file *seq, void *v) ++{ ++ seq_puts(seq, "0\n"); ++ return 0; ++} ++ ++static void cgroup_print_ss_mask(struct seq_file *seq, unsigned int ss_mask) ++{ ++ struct cgroup_subsys *ss; ++ bool printed = false; ++ int ssid; ++ ++ for_each_subsys(ss, ssid) { ++ if (ss_mask & (1 << ssid)) { ++ if (printed) ++ seq_putc(seq, ' '); ++ seq_printf(seq, "%s", ss->name); ++ printed = true; ++ } + } - } - - /** -@@ -1062,12 +1086,12 @@ - { - if (pwq) { - /* -- * As both pwqs and pools are sched-RCU protected, the -+ * As both pwqs and pools are RCU protected, the - * following lock operations are safe. - */ -- spin_lock_irq(&pwq->pool->lock); -+ local_spin_lock_irq(pendingb_lock, &pwq->pool->lock); - put_pwq(pwq); -- spin_unlock_irq(&pwq->pool->lock); -+ local_spin_unlock_irq(pendingb_lock, &pwq->pool->lock); - } - } - -@@ -1169,7 +1193,7 @@ - struct worker_pool *pool; - struct pool_workqueue *pwq; - -- local_irq_save(*flags); -+ local_lock_irqsave(pendingb_lock, *flags); - - /* try to steal the timer if it exists */ - if (is_dwork) { -@@ -1188,6 +1212,7 @@ - if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) - return 0; - ++ if (printed) ++ seq_putc(seq, '\n'); ++} ++ ++/* show controllers which are currently attached to the default hierarchy */ ++static int cgroup_root_controllers_show(struct seq_file *seq, void *v) ++{ ++ struct cgroup *cgrp = seq_css(seq)->cgroup; ++ ++ cgroup_print_ss_mask(seq, cgrp->root->subsys_mask & ++ ~cgrp_dfl_root_inhibit_ss_mask); ++ return 0; ++} ++ ++/* show controllers which are enabled from the parent */ ++static int cgroup_controllers_show(struct seq_file *seq, void *v) ++{ ++ struct cgroup *cgrp = seq_css(seq)->cgroup; ++ ++ cgroup_print_ss_mask(seq, cgroup_parent(cgrp)->subtree_control); ++ return 0; ++} ++ ++/* show controllers which are enabled for a given cgroup's children */ ++static int cgroup_subtree_control_show(struct seq_file *seq, void *v) ++{ ++ struct cgroup *cgrp = seq_css(seq)->cgroup; ++ ++ cgroup_print_ss_mask(seq, cgrp->subtree_control); ++ return 0; ++} ++ ++/** ++ * cgroup_update_dfl_csses - update css assoc of a subtree in default hierarchy ++ * @cgrp: root of the subtree to update csses for ++ * ++ * @cgrp's child_subsys_mask has changed and its subtree's (self excluded) ++ * css associations need to be updated accordingly. This function looks up ++ * all css_sets which are attached to the subtree, creates the matching ++ * updated css_sets and migrates the tasks to the new ones. ++ */ ++static int cgroup_update_dfl_csses(struct cgroup *cgrp) ++{ ++ LIST_HEAD(preloaded_csets); ++ struct cgroup_subsys_state *css; ++ struct css_set *src_cset; ++ int ret; ++ ++ lockdep_assert_held(&cgroup_mutex); ++ ++ /* look up all csses currently attached to @cgrp's subtree */ ++ down_read(&css_set_rwsem); ++ css_for_each_descendant_pre(css, cgroup_css(cgrp, NULL)) { ++ struct cgrp_cset_link *link; ++ ++ /* self is not affected by child_subsys_mask change */ ++ if (css->cgroup == cgrp) ++ continue; ++ ++ list_for_each_entry(link, &css->cgroup->cset_links, cset_link) ++ cgroup_migrate_add_src(link->cset, cgrp, ++ &preloaded_csets); ++ } ++ up_read(&css_set_rwsem); ++ ++ /* NULL dst indicates self on default hierarchy */ ++ ret = cgroup_migrate_prepare_dst(NULL, &preloaded_csets); ++ if (ret) ++ goto out_finish; ++ ++ list_for_each_entry(src_cset, &preloaded_csets, mg_preload_node) { ++ struct task_struct *last_task = NULL, *task; ++ ++ /* src_csets precede dst_csets, break on the first dst_cset */ ++ if (!src_cset->mg_src_cgrp) ++ break; ++ ++ /* ++ * All tasks in src_cset need to be migrated to the ++ * matching dst_cset. Empty it process by process. We ++ * walk tasks but migrate processes. The leader might even ++ * belong to a different cset but such src_cset would also ++ * be among the target src_csets because the default ++ * hierarchy enforces per-process membership. ++ */ ++ while (true) { ++ down_read(&css_set_rwsem); ++ task = list_first_entry_or_null(&src_cset->tasks, ++ struct task_struct, cg_list); ++ if (task) { ++ task = task->group_leader; ++ WARN_ON_ONCE(!task_css_set(task)->mg_src_cgrp); ++ get_task_struct(task); ++ } ++ up_read(&css_set_rwsem); ++ ++ if (!task) ++ break; ++ ++ /* guard against possible infinite loop */ ++ if (WARN(last_task == task, ++ "cgroup: update_dfl_csses failed to make progress, aborting in inconsistent state\n")) ++ goto out_finish; ++ last_task = task; ++ ++ threadgroup_lock(task); ++ /* raced against de_thread() from another thread? */ ++ if (!thread_group_leader(task)) { ++ threadgroup_unlock(task); ++ put_task_struct(task); ++ continue; ++ } ++ ++ ret = cgroup_migrate(src_cset->dfl_cgrp, task, true); ++ ++ threadgroup_unlock(task); ++ put_task_struct(task); ++ ++ if (WARN(ret, "cgroup: failed to update controllers for the default hierarchy (%d), further operations may crash or hang\n", ret)) ++ goto out_finish; ++ } ++ } ++ ++out_finish: ++ cgroup_migrate_finish(&preloaded_csets); ++ return ret; ++} ++ ++/* change the enabled child controllers for a cgroup in the default hierarchy */ ++static ssize_t cgroup_subtree_control_write(struct kernfs_open_file *of, ++ char *buf, size_t nbytes, ++ loff_t off) ++{ ++ unsigned int enable = 0, disable = 0; ++ unsigned int css_enable, css_disable, old_sc, new_sc, old_ss, new_ss; ++ struct cgroup *cgrp, *child; ++ struct cgroup_subsys *ss; ++ char *tok; ++ int ssid, ret; ++ ++ /* ++ * Parse input - space separated list of subsystem names prefixed ++ * with either + or -. ++ */ ++ buf = strstrip(buf); ++ while ((tok = strsep(&buf, " "))) { ++ if (tok[0] == '\0') ++ continue; ++ for_each_subsys(ss, ssid) { ++ if (ss->disabled || strcmp(tok + 1, ss->name) || ++ ((1 << ss->id) & cgrp_dfl_root_inhibit_ss_mask)) ++ continue; ++ ++ if (*tok == '+') { ++ enable |= 1 << ssid; ++ disable &= ~(1 << ssid); ++ } else if (*tok == '-') { ++ disable |= 1 << ssid; ++ enable &= ~(1 << ssid); ++ } else { ++ return -EINVAL; ++ } ++ break; ++ } ++ if (ssid == CGROUP_SUBSYS_COUNT) ++ return -EINVAL; ++ } ++ ++ cgrp = cgroup_kn_lock_live(of->kn); ++ if (!cgrp) ++ return -ENODEV; ++ ++ for_each_subsys(ss, ssid) { ++ if (enable & (1 << ssid)) { ++ if (cgrp->subtree_control & (1 << ssid)) { ++ enable &= ~(1 << ssid); ++ continue; ++ } ++ ++ /* unavailable or not enabled on the parent? */ ++ if (!(cgrp_dfl_root.subsys_mask & (1 << ssid)) || ++ (cgroup_parent(cgrp) && ++ !(cgroup_parent(cgrp)->subtree_control & (1 << ssid)))) { ++ ret = -ENOENT; ++ goto out_unlock; ++ } ++ } else if (disable & (1 << ssid)) { ++ if (!(cgrp->subtree_control & (1 << ssid))) { ++ disable &= ~(1 << ssid); ++ continue; ++ } ++ ++ /* a child has it enabled? */ ++ cgroup_for_each_live_child(child, cgrp) { ++ if (child->subtree_control & (1 << ssid)) { ++ ret = -EBUSY; ++ goto out_unlock; ++ } ++ } ++ } ++ } ++ ++ if (!enable && !disable) { ++ ret = 0; ++ goto out_unlock; ++ } ++ ++ /* ++ * Except for the root, subtree_control must be zero for a cgroup ++ * with tasks so that child cgroups don't compete against tasks. ++ */ ++ if (enable && cgroup_parent(cgrp) && !list_empty(&cgrp->cset_links)) { ++ ret = -EBUSY; ++ goto out_unlock; ++ } ++ ++ /* ++ * Update subsys masks and calculate what needs to be done. More ++ * subsystems than specified may need to be enabled or disabled ++ * depending on subsystem dependencies. ++ */ ++ old_sc = cgrp->subtree_control; ++ old_ss = cgrp->child_subsys_mask; ++ new_sc = (old_sc | enable) & ~disable; ++ new_ss = cgroup_calc_child_subsys_mask(cgrp, new_sc); ++ ++ css_enable = ~old_ss & new_ss; ++ css_disable = old_ss & ~new_ss; ++ enable |= css_enable; ++ disable |= css_disable; ++ ++ /* ++ * Because css offlining is asynchronous, userland might try to ++ * re-enable the same controller while the previous instance is ++ * still around. In such cases, wait till it's gone using ++ * offline_waitq. ++ */ ++ for_each_subsys(ss, ssid) { ++ if (!(css_enable & (1 << ssid))) ++ continue; ++ ++ cgroup_for_each_live_child(child, cgrp) { ++ DEFINE_WAIT(wait); ++ ++ if (!cgroup_css(child, ss)) ++ continue; ++ ++ cgroup_get(child); ++ prepare_to_wait(&child->offline_waitq, &wait, ++ TASK_UNINTERRUPTIBLE); ++ cgroup_kn_unlock(of->kn); ++ schedule(); ++ finish_wait(&child->offline_waitq, &wait); ++ cgroup_put(child); ++ ++ return restart_syscall(); ++ } ++ } ++ ++ cgrp->subtree_control = new_sc; ++ cgrp->child_subsys_mask = new_ss; ++ ++ /* ++ * Create new csses or make the existing ones visible. A css is ++ * created invisible if it's being implicitly enabled through ++ * dependency. An invisible css is made visible when the userland ++ * explicitly enables it. ++ */ ++ for_each_subsys(ss, ssid) { ++ if (!(enable & (1 << ssid))) ++ continue; ++ ++ cgroup_for_each_live_child(child, cgrp) { ++ if (css_enable & (1 << ssid)) ++ ret = create_css(child, ss, ++ cgrp->subtree_control & (1 << ssid)); ++ else ++ ret = cgroup_populate_dir(child, 1 << ssid); ++ if (ret) ++ goto err_undo_css; ++ } ++ } ++ ++ /* ++ * At this point, cgroup_e_css() results reflect the new csses ++ * making the following cgroup_update_dfl_csses() properly update ++ * css associations of all tasks in the subtree. ++ */ ++ ret = cgroup_update_dfl_csses(cgrp); ++ if (ret) ++ goto err_undo_css; ++ ++ /* ++ * All tasks are migrated out of disabled csses. Kill or hide ++ * them. A css is hidden when the userland requests it to be ++ * disabled while other subsystems are still depending on it. The ++ * css must not actively control resources and be in the vanilla ++ * state if it's made visible again later. Controllers which may ++ * be depended upon should provide ->css_reset() for this purpose. ++ */ ++ for_each_subsys(ss, ssid) { ++ if (!(disable & (1 << ssid))) ++ continue; ++ ++ cgroup_for_each_live_child(child, cgrp) { ++ struct cgroup_subsys_state *css = cgroup_css(child, ss); ++ ++ if (css_disable & (1 << ssid)) { ++ kill_css(css); ++ } else { ++ cgroup_clear_dir(child, 1 << ssid); ++ if (ss->css_reset) ++ ss->css_reset(css); ++ } ++ } ++ } ++ ++ /* ++ * The effective csses of all the descendants (excluding @cgrp) may ++ * have changed. Subsystems can optionally subscribe to this event ++ * by implementing ->css_e_css_changed() which is invoked if any of ++ * the effective csses seen from the css's cgroup may have changed. ++ */ ++ for_each_subsys(ss, ssid) { ++ struct cgroup_subsys_state *this_css = cgroup_css(cgrp, ss); ++ struct cgroup_subsys_state *css; ++ ++ if (!ss->css_e_css_changed || !this_css) ++ continue; ++ ++ css_for_each_descendant_pre(css, this_css) ++ if (css != this_css) ++ ss->css_e_css_changed(css); ++ } ++ ++ kernfs_activate(cgrp->kn); ++ ret = 0; ++out_unlock: ++ cgroup_kn_unlock(of->kn); ++ return ret ?: nbytes; ++ ++err_undo_css: ++ cgrp->subtree_control = old_sc; ++ cgrp->child_subsys_mask = old_ss; ++ ++ for_each_subsys(ss, ssid) { ++ if (!(enable & (1 << ssid))) ++ continue; ++ ++ cgroup_for_each_live_child(child, cgrp) { ++ struct cgroup_subsys_state *css = cgroup_css(child, ss); ++ ++ if (!css) ++ continue; ++ ++ if (css_enable & (1 << ssid)) ++ kill_css(css); ++ else ++ cgroup_clear_dir(child, 1 << ssid); ++ } ++ } ++ goto out_unlock; ++} ++ ++static int cgroup_populated_show(struct seq_file *seq, void *v) ++{ ++ seq_printf(seq, "%d\n", (bool)seq_css(seq)->cgroup->populated_cnt); ++ return 0; ++} ++ ++static ssize_t cgroup_file_write(struct kernfs_open_file *of, char *buf, ++ size_t nbytes, loff_t off) ++{ ++ struct cgroup *cgrp = of->kn->parent->priv; ++ struct cftype *cft = of->kn->priv; ++ struct cgroup_subsys_state *css; ++ int ret; ++ ++ if (cft->write) ++ return cft->write(of, buf, nbytes, off); ++ ++ /* ++ * kernfs guarantees that a file isn't deleted with operations in ++ * flight, which means that the matching css is and stays alive and ++ * doesn't need to be pinned. The RCU locking is not necessary ++ * either. It's just for the convenience of using cgroup_css(). ++ */ + rcu_read_lock(); - /* - * The queueing is in progress, or it is already queued. Try to - * steal it from ->worklist without clearing WORK_STRUCT_PENDING. -@@ -1226,14 +1251,16 @@ - set_work_pool_and_keep_pending(work, pool->id); - - spin_unlock(&pool->lock); -+ rcu_read_unlock(); - return 1; - } - spin_unlock(&pool->lock); - fail: -- local_irq_restore(*flags); ++ css = cgroup_css(cgrp, cft->ss); + rcu_read_unlock(); -+ local_unlock_irqrestore(pendingb_lock, *flags); - if (work_is_canceling(work)) - return -ENOENT; -- cpu_relax(); -+ cpu_chill(); - return -EAGAIN; - } - -@@ -1302,7 +1329,7 @@ - * queued or lose PENDING. Grabbing PENDING and queueing should - * happen with IRQ disabled. - */ -- WARN_ON_ONCE(!irqs_disabled()); -+ WARN_ON_ONCE_NONRT(!irqs_disabled()); - - debug_work_activate(work); - -@@ -1310,6 +1337,8 @@ - if (unlikely(wq->flags & __WQ_DRAINING) && - WARN_ON_ONCE(!is_chained_work(wq))) - return; + -+ rcu_read_lock(); - retry: - if (req_cpu == WORK_CPU_UNBOUND) - cpu = raw_smp_processor_id(); -@@ -1366,10 +1395,8 @@ - /* pwq determined, queue */ - trace_workqueue_queue_work(req_cpu, pwq, work); - -- if (WARN_ON(!list_empty(&work->entry))) { -- spin_unlock(&pwq->pool->lock); -- return; -- } -+ if (WARN_ON(!list_empty(&work->entry))) -+ goto out; - - pwq->nr_in_flight[pwq->work_color]++; - work_flags = work_color_to_flags(pwq->work_color); -@@ -1385,7 +1412,9 @@ - - insert_work(pwq, work, worklist, work_flags); - -+out: - spin_unlock(&pwq->pool->lock); -+ rcu_read_unlock(); - } - - /** -@@ -1405,14 +1434,14 @@ - bool ret = false; - unsigned long flags; - -- local_irq_save(flags); -+ local_lock_irqsave(pendingb_lock,flags); - - if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) { - __queue_work(cpu, wq, work); - ret = true; - } - -- local_irq_restore(flags); -+ local_unlock_irqrestore(pendingb_lock, flags); - return ret; - } - EXPORT_SYMBOL(queue_work_on); -@@ -1479,14 +1508,14 @@ - unsigned long flags; - - /* read the comment in __queue_work() */ -- local_irq_save(flags); -+ local_lock_irqsave(pendingb_lock, flags); - - if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) { - __queue_delayed_work(cpu, wq, dwork, delay); - ret = true; - } - -- local_irq_restore(flags); -+ local_unlock_irqrestore(pendingb_lock, flags); - return ret; - } - EXPORT_SYMBOL(queue_delayed_work_on); -@@ -1521,7 +1550,7 @@ - - if (likely(ret >= 0)) { - __queue_delayed_work(cpu, wq, dwork, delay); -- local_irq_restore(flags); -+ local_unlock_irqrestore(pendingb_lock, flags); - } - - /* -ENOENT from try_to_grab_pending() becomes %true */ -@@ -1554,7 +1583,9 @@ - worker->last_active = jiffies; - - /* idle_list is LIFO */ -+ rt_lock_idle_list(pool); - list_add(&worker->entry, &pool->idle_list); -+ rt_unlock_idle_list(pool); - - if (too_many_workers(pool) && !timer_pending(&pool->idle_timer)) - mod_timer(&pool->idle_timer, jiffies + IDLE_WORKER_TIMEOUT); -@@ -1587,7 +1618,9 @@ - return; - worker_clr_flags(worker, WORKER_IDLE); - pool->nr_idle--; -+ rt_lock_idle_list(pool); - list_del_init(&worker->entry); -+ rt_unlock_idle_list(pool); - } - - static struct worker *alloc_worker(int node) -@@ -1755,7 +1788,9 @@ - pool->nr_workers--; - pool->nr_idle--; - -+ rt_lock_idle_list(pool); - list_del_init(&worker->entry); -+ rt_unlock_idle_list(pool); - worker->flags |= WORKER_DIE; - wake_up_process(worker->task); - } -@@ -2672,14 +2707,14 @@ - - might_sleep(); - -- local_irq_disable(); -+ rcu_read_lock(); - pool = get_work_pool(work); - if (!pool) { -- local_irq_enable(); -+ rcu_read_unlock(); - return false; - } - -- spin_lock(&pool->lock); -+ spin_lock_irq(&pool->lock); - /* see the comment in try_to_grab_pending() with the same code */ - pwq = get_work_pwq(work); - if (pwq) { -@@ -2706,10 +2741,11 @@ - else - lock_map_acquire_read(&pwq->wq->lockdep_map); - lock_map_release(&pwq->wq->lockdep_map); -- -+ rcu_read_unlock(); - return true; - already_gone: - spin_unlock_irq(&pool->lock); -+ rcu_read_unlock(); - return false; - } - -@@ -2796,7 +2832,7 @@ - - /* tell other tasks trying to grab @work to back off */ - mark_work_canceling(work); -- local_irq_restore(flags); -+ local_unlock_irqrestore(pendingb_lock, flags); - - flush_work(work); - clear_work_data(work); -@@ -2851,10 +2887,10 @@ - */ - bool flush_delayed_work(struct delayed_work *dwork) - { -- local_irq_disable(); -+ local_lock_irq(pendingb_lock); - if (del_timer_sync(&dwork->timer)) - __queue_work(dwork->cpu, dwork->wq, &dwork->work); -- local_irq_enable(); -+ local_unlock_irq(pendingb_lock); - return flush_work(&dwork->work); - } - EXPORT_SYMBOL(flush_delayed_work); -@@ -2889,7 +2925,7 @@ - - set_work_pool_and_clear_pending(&dwork->work, - get_work_pool_id(&dwork->work)); -- local_irq_restore(flags); -+ local_unlock_irqrestore(pendingb_lock, flags); - return ret; - } - EXPORT_SYMBOL(cancel_delayed_work); -@@ -3147,7 +3183,7 @@ - * put_unbound_pool - put a worker_pool - * @pool: worker_pool to put - * -- * Put @pool. If its refcnt reaches zero, it gets destroyed in sched-RCU -+ * Put @pool. If its refcnt reaches zero, it gets destroyed in RCU - * safe manner. get_unbound_pool() calls this function on its failure path - * and this function should be able to release pools which went through, - * successfully or not, init_worker_pool(). -@@ -3201,8 +3237,8 @@ - del_timer_sync(&pool->idle_timer); - del_timer_sync(&pool->mayday_timer); - -- /* sched-RCU protected to allow dereferences from get_work_pool() */ -- call_rcu_sched(&pool->rcu, rcu_free_pool); -+ /* RCU protected to allow dereferences from get_work_pool() */ -+ call_rcu(&pool->rcu, rcu_free_pool); - } - - /** -@@ -3307,14 +3343,14 @@ - put_unbound_pool(pool); - mutex_unlock(&wq_pool_mutex); - -- call_rcu_sched(&pwq->rcu, rcu_free_pwq); -+ call_rcu(&pwq->rcu, rcu_free_pwq); - - /* - * If we're the last pwq going away, @wq is already dead and no one - * is gonna access it anymore. Schedule RCU free. - */ - if (is_last) -- call_rcu_sched(&wq->rcu, rcu_free_wq); -+ call_rcu(&wq->rcu, rcu_free_wq); - } - - /** -@@ -3920,7 +3956,7 @@ - * The base ref is never dropped on per-cpu pwqs. Directly - * schedule RCU free. - */ -- call_rcu_sched(&wq->rcu, rcu_free_wq); -+ call_rcu(&wq->rcu, rcu_free_wq); - } else { - /* - * We're the sole accessor of @wq at this point. Directly -@@ -4013,7 +4049,8 @@ - struct pool_workqueue *pwq; - bool ret; - -- rcu_read_lock_sched(); -+ rcu_read_lock(); -+ preempt_disable(); - - if (cpu == WORK_CPU_UNBOUND) - cpu = smp_processor_id(); -@@ -4024,7 +4061,8 @@ - pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu)); - - ret = !list_empty(&pwq->delayed_works); -- rcu_read_unlock_sched(); -+ preempt_enable(); -+ rcu_read_unlock(); - - return ret; - } -@@ -4050,15 +4088,15 @@ - if (work_pending(work)) - ret |= WORK_BUSY_PENDING; - -- local_irq_save(flags); -+ rcu_read_lock(); - pool = get_work_pool(work); - if (pool) { -- spin_lock(&pool->lock); -+ spin_lock_irqsave(&pool->lock, flags); - if (find_worker_executing_work(pool, work)) - ret |= WORK_BUSY_RUNNING; -- spin_unlock(&pool->lock); -+ spin_unlock_irqrestore(&pool->lock, flags); - } -- local_irq_restore(flags); -+ rcu_read_unlock(); - - return ret; - } -@@ -4247,7 +4285,7 @@ - unsigned long flags; - int pi; - -- rcu_read_lock_sched(); -+ rcu_read_lock(); - - pr_info("Showing busy workqueues and worker pools:\n"); - -@@ -4298,7 +4336,7 @@ - spin_unlock_irqrestore(&pool->lock, flags); - } - -- rcu_read_unlock_sched(); -+ rcu_read_unlock(); - } - - /* -@@ -4648,16 +4686,16 @@ - * nr_active is monotonically decreasing. It's safe - * to peek without lock. - */ -- rcu_read_lock_sched(); -+ rcu_read_lock(); - for_each_pwq(pwq, wq) { - WARN_ON_ONCE(pwq->nr_active < 0); - if (pwq->nr_active) { - busy = true; -- rcu_read_unlock_sched(); -+ rcu_read_unlock(); - goto out_unlock; - } - } -- rcu_read_unlock_sched(); -+ rcu_read_unlock(); - } - out_unlock: - mutex_unlock(&wq_pool_mutex); -@@ -4771,7 +4809,8 @@ - const char *delim = ""; - int node, written = 0; - -- rcu_read_lock_sched(); -+ get_online_cpus(); -+ rcu_read_lock(); - for_each_node(node) { - written += scnprintf(buf + written, PAGE_SIZE - written, - "%s%d:%d", delim, node, -@@ -4779,7 +4818,8 @@ - delim = " "; - } - written += scnprintf(buf + written, PAGE_SIZE - written, "\n"); -- rcu_read_unlock_sched(); -+ rcu_read_unlock(); -+ put_online_cpus(); - - return written; - } -diff -Nur linux-4.1.6.orig/kernel/workqueue_internal.h linux-4.1.6/kernel/workqueue_internal.h ---- linux-4.1.6.orig/kernel/workqueue_internal.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/kernel/workqueue_internal.h 2015-09-08 23:49:08.425831903 +0200 -@@ -43,6 +43,7 @@ - unsigned long last_active; /* L: last active timestamp */ - unsigned int flags; /* X: flags */ - int id; /* I: worker id */ -+ int sleeping; /* None */ - - /* - * Opaque string set with work_set_desc(). Printed out with task -@@ -68,7 +69,7 @@ - * Scheduler hooks for concurrency managed workqueue. Only to be used from - * sched/core.c and workqueue.c. - */ --void wq_worker_waking_up(struct task_struct *task, int cpu); --struct task_struct *wq_worker_sleeping(struct task_struct *task, int cpu); -+void wq_worker_running(struct task_struct *task); -+void wq_worker_sleeping(struct task_struct *task); - - #endif /* _KERNEL_WORKQUEUE_INTERNAL_H */ -diff -Nur linux-4.1.6.orig/lib/debugobjects.c linux-4.1.6/lib/debugobjects.c ---- linux-4.1.6.orig/lib/debugobjects.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/lib/debugobjects.c 2015-09-08 23:49:08.425831903 +0200 -@@ -309,7 +309,10 @@ - struct debug_obj *obj; - unsigned long flags; - -- fill_pool(); -+#ifdef CONFIG_PREEMPT_RT_FULL -+ if (preempt_count() == 0 && !irqs_disabled()) -+#endif -+ fill_pool(); - - db = get_bucket((unsigned long) addr); - -diff -Nur linux-4.1.6.orig/lib/dump_stack.c linux-4.1.6/lib/dump_stack.c ---- linux-4.1.6.orig/lib/dump_stack.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/lib/dump_stack.c 2015-09-08 23:49:08.429831459 +0200 -@@ -33,7 +33,7 @@ - * Permit this cpu to perform nested stack dumps while serialising - * against other CPUs - */ -- preempt_disable(); -+ migrate_disable(); - - retry: - cpu = smp_processor_id(); -@@ -52,7 +52,7 @@ - if (!was_locked) - atomic_set(&dump_lock, -1); - -- preempt_enable(); -+ migrate_enable(); - } - #else - asmlinkage __visible void dump_stack(void) -diff -Nur linux-4.1.6.orig/lib/idr.c linux-4.1.6/lib/idr.c ---- linux-4.1.6.orig/lib/idr.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/lib/idr.c 2015-09-08 23:49:08.429831459 +0200 -@@ -30,6 +30,7 @@ - #include - #include - #include -+#include - - #define MAX_IDR_SHIFT (sizeof(int) * 8 - 1) - #define MAX_IDR_BIT (1U << MAX_IDR_SHIFT) -@@ -366,6 +367,35 @@ - idr_mark_full(pa, id); - } - -+#ifdef CONFIG_PREEMPT_RT_FULL -+static DEFINE_LOCAL_IRQ_LOCK(idr_lock); ++ if (cft->write_u64) { ++ unsigned long long v; ++ ret = kstrtoull(buf, 0, &v); ++ if (!ret) ++ ret = cft->write_u64(css, cft, v); ++ } else if (cft->write_s64) { ++ long long v; ++ ret = kstrtoll(buf, 0, &v); ++ if (!ret) ++ ret = cft->write_s64(css, cft, v); ++ } else { ++ ret = -EINVAL; ++ } + -+static inline void idr_preload_lock(void) -+{ -+ local_lock(idr_lock); ++ return ret ?: nbytes; +} + -+static inline void idr_preload_unlock(void) ++static void *cgroup_seqfile_start(struct seq_file *seq, loff_t *ppos) +{ -+ local_unlock(idr_lock); ++ return seq_cft(seq)->seq_start(seq, ppos); +} + -+void idr_preload_end(void) ++static void *cgroup_seqfile_next(struct seq_file *seq, void *v, loff_t *ppos) +{ -+ idr_preload_unlock(); ++ return seq_cft(seq)->seq_next(seq, v, ppos); +} -+EXPORT_SYMBOL(idr_preload_end); -+#else -+static inline void idr_preload_lock(void) ++ ++static void cgroup_seqfile_stop(struct seq_file *seq, void *v) +{ -+ preempt_disable(); ++ seq_cft(seq)->seq_stop(seq, v); +} + -+static inline void idr_preload_unlock(void) ++static int cgroup_seqfile_show(struct seq_file *m, void *arg) +{ -+ preempt_enable(); ++ struct cftype *cft = seq_cft(m); ++ struct cgroup_subsys_state *css = seq_css(m); ++ ++ if (cft->seq_show) ++ return cft->seq_show(m, arg); ++ ++ if (cft->read_u64) ++ seq_printf(m, "%llu\n", cft->read_u64(css, cft)); ++ else if (cft->read_s64) ++ seq_printf(m, "%lld\n", cft->read_s64(css, cft)); ++ else ++ return -EINVAL; ++ return 0; +} -+#endif - - /** - * idr_preload - preload for idr_alloc() -@@ -401,7 +431,7 @@ - WARN_ON_ONCE(in_interrupt()); - might_sleep_if(gfp_mask & __GFP_WAIT); - -- preempt_disable(); -+ idr_preload_lock(); - - /* - * idr_alloc() is likely to succeed w/o full idr_layer buffer and -@@ -413,9 +443,9 @@ - while (__this_cpu_read(idr_preload_cnt) < MAX_IDR_FREE) { - struct idr_layer *new; - -- preempt_enable(); -+ idr_preload_unlock(); - new = kmem_cache_zalloc(idr_layer_cache, gfp_mask); -- preempt_disable(); -+ idr_preload_lock(); - if (!new) - break; - -diff -Nur linux-4.1.6.orig/lib/Kconfig linux-4.1.6/lib/Kconfig ---- linux-4.1.6.orig/lib/Kconfig 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/lib/Kconfig 2015-09-08 23:49:08.425831903 +0200 -@@ -391,6 +391,7 @@ - - config CPUMASK_OFFSTACK - bool "Force CPU masks off stack" if DEBUG_PER_CPU_MAPS -+ depends on !PREEMPT_RT_FULL - help - Use dynamic allocation for cpumask_var_t, instead of putting - them on the stack. This is a bit more expensive, but avoids -diff -Nur linux-4.1.6.orig/lib/locking-selftest.c linux-4.1.6/lib/locking-selftest.c ---- linux-4.1.6.orig/lib/locking-selftest.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/lib/locking-selftest.c 2015-09-08 23:49:08.429831459 +0200 -@@ -590,6 +590,8 @@ - #include "locking-selftest-spin-hardirq.h" - GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_hard_spin) - -+#ifndef CONFIG_PREEMPT_RT_FULL + - #include "locking-selftest-rlock-hardirq.h" - GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_hard_rlock) - -@@ -605,9 +607,12 @@ - #include "locking-selftest-wlock-softirq.h" - GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_soft_wlock) - -+#endif ++static struct kernfs_ops cgroup_kf_single_ops = { ++ .atomic_write_len = PAGE_SIZE, ++ .write = cgroup_file_write, ++ .seq_show = cgroup_seqfile_show, ++}; + - #undef E1 - #undef E2 - -+#ifndef CONFIG_PREEMPT_RT_FULL - /* - * Enabling hardirqs with a softirq-safe lock held: - */ -@@ -640,6 +645,8 @@ - #undef E1 - #undef E2 - -+#endif ++static struct kernfs_ops cgroup_kf_ops = { ++ .atomic_write_len = PAGE_SIZE, ++ .write = cgroup_file_write, ++ .seq_start = cgroup_seqfile_start, ++ .seq_next = cgroup_seqfile_next, ++ .seq_stop = cgroup_seqfile_stop, ++ .seq_show = cgroup_seqfile_show, ++}; + - /* - * Enabling irqs with an irq-safe lock held: - */ -@@ -663,6 +670,8 @@ - #include "locking-selftest-spin-hardirq.h" - GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_hard_spin) - -+#ifndef CONFIG_PREEMPT_RT_FULL ++/* ++ * cgroup_rename - Only allow simple rename of directories in place. ++ */ ++static int cgroup_rename(struct kernfs_node *kn, struct kernfs_node *new_parent, ++ const char *new_name_str) ++{ ++ struct cgroup *cgrp = kn->priv; ++ int ret; + - #include "locking-selftest-rlock-hardirq.h" - GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_hard_rlock) - -@@ -678,6 +687,8 @@ - #include "locking-selftest-wlock-softirq.h" - GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_soft_wlock) - -+#endif ++ if (kernfs_type(kn) != KERNFS_DIR) ++ return -ENOTDIR; ++ if (kn->parent != new_parent) ++ return -EIO; + - #undef E1 - #undef E2 - -@@ -709,6 +720,8 @@ - #include "locking-selftest-spin-hardirq.h" - GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_hard_spin) - -+#ifndef CONFIG_PREEMPT_RT_FULL ++ /* ++ * This isn't a proper migration and its usefulness is very ++ * limited. Disallow on the default hierarchy. ++ */ ++ if (cgroup_on_dfl(cgrp)) ++ return -EPERM; + - #include "locking-selftest-rlock-hardirq.h" - GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_hard_rlock) - -@@ -724,6 +737,8 @@ - #include "locking-selftest-wlock-softirq.h" - GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_soft_wlock) - -+#endif ++ /* ++ * We're gonna grab cgroup_mutex which nests outside kernfs ++ * active_ref. kernfs_rename() doesn't require active_ref ++ * protection. Break them before grabbing cgroup_mutex. ++ */ ++ kernfs_break_active_protection(new_parent); ++ kernfs_break_active_protection(kn); + - #undef E1 - #undef E2 - #undef E3 -@@ -757,6 +772,8 @@ - #include "locking-selftest-spin-hardirq.h" - GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_hard_spin) - -+#ifndef CONFIG_PREEMPT_RT_FULL ++ mutex_lock(&cgroup_mutex); + - #include "locking-selftest-rlock-hardirq.h" - GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_hard_rlock) - -@@ -772,10 +789,14 @@ - #include "locking-selftest-wlock-softirq.h" - GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_soft_wlock) - -+#endif ++ ret = kernfs_rename(kn, new_parent, new_name_str); + - #undef E1 - #undef E2 - #undef E3 - -+#ifndef CONFIG_PREEMPT_RT_FULL ++ mutex_unlock(&cgroup_mutex); + - /* - * read-lock / write-lock irq inversion. - * -@@ -838,6 +859,10 @@ - #undef E2 - #undef E3 - -+#endif ++ kernfs_unbreak_active_protection(kn); ++ kernfs_unbreak_active_protection(new_parent); ++ return ret; ++} + -+#ifndef CONFIG_PREEMPT_RT_FULL ++/* set uid and gid of cgroup dirs and files to that of the creator */ ++static int cgroup_kn_set_ugid(struct kernfs_node *kn) ++{ ++ struct iattr iattr = { .ia_valid = ATTR_UID | ATTR_GID, ++ .ia_uid = current_fsuid(), ++ .ia_gid = current_fsgid(), }; + - /* - * read-lock / write-lock recursion that is actually safe. - */ -@@ -876,6 +901,8 @@ - #undef E2 - #undef E3 - ++ if (uid_eq(iattr.ia_uid, GLOBAL_ROOT_UID) && ++ gid_eq(iattr.ia_gid, GLOBAL_ROOT_GID)) ++ return 0; ++ ++ return kernfs_setattr(kn, &iattr); ++} ++ ++static int cgroup_add_file(struct cgroup *cgrp, struct cftype *cft) ++{ ++ char name[CGROUP_FILE_NAME_MAX]; ++ struct kernfs_node *kn; ++ struct lock_class_key *key = NULL; ++ int ret; ++ ++#ifdef CONFIG_DEBUG_LOCK_ALLOC ++ key = &cft->lockdep_key; +#endif ++ kn = __kernfs_create_file(cgrp->kn, cgroup_file_name(cgrp, cft, name), ++ cgroup_file_mode(cft), 0, cft->kf_ops, cft, ++ NULL, key); ++ if (IS_ERR(kn)) ++ return PTR_ERR(kn); + - /* - * read-lock / write-lock recursion that is unsafe. - */ -@@ -1858,6 +1885,7 @@ - - printk(" --------------------------------------------------------------------------\n"); - -+#ifndef CONFIG_PREEMPT_RT_FULL - /* - * irq-context testcases: - */ -@@ -1870,6 +1898,28 @@ - - DO_TESTCASE_6x2("irq read-recursion", irq_read_recursion); - // DO_TESTCASE_6x2B("irq read-recursion #2", irq_read_recursion2); -+#else -+ /* On -rt, we only do hardirq context test for raw spinlock */ -+ DO_TESTCASE_1B("hard-irqs-on + irq-safe-A", irqsafe1_hard_spin, 12); -+ DO_TESTCASE_1B("hard-irqs-on + irq-safe-A", irqsafe1_hard_spin, 21); ++ ret = cgroup_kn_set_ugid(kn); ++ if (ret) { ++ kernfs_remove(kn); ++ return ret; ++ } + -+ DO_TESTCASE_1B("hard-safe-A + irqs-on", irqsafe2B_hard_spin, 12); -+ DO_TESTCASE_1B("hard-safe-A + irqs-on", irqsafe2B_hard_spin, 21); ++ if (cft->seq_show == cgroup_populated_show) ++ cgrp->populated_kn = kn; ++ return 0; ++} + -+ DO_TESTCASE_1B("hard-safe-A + unsafe-B #1", irqsafe3_hard_spin, 123); -+ DO_TESTCASE_1B("hard-safe-A + unsafe-B #1", irqsafe3_hard_spin, 132); -+ DO_TESTCASE_1B("hard-safe-A + unsafe-B #1", irqsafe3_hard_spin, 213); -+ DO_TESTCASE_1B("hard-safe-A + unsafe-B #1", irqsafe3_hard_spin, 231); -+ DO_TESTCASE_1B("hard-safe-A + unsafe-B #1", irqsafe3_hard_spin, 312); -+ DO_TESTCASE_1B("hard-safe-A + unsafe-B #1", irqsafe3_hard_spin, 321); ++/** ++ * cgroup_addrm_files - add or remove files to a cgroup directory ++ * @cgrp: the target cgroup ++ * @cfts: array of cftypes to be added ++ * @is_add: whether to add or remove ++ * ++ * Depending on @is_add, add or remove files defined by @cfts on @cgrp. ++ * For removals, this function never fails. If addition fails, this ++ * function doesn't remove files already added. The caller is responsible ++ * for cleaning up. ++ */ ++static int cgroup_addrm_files(struct cgroup *cgrp, struct cftype cfts[], ++ bool is_add) ++{ ++ struct cftype *cft; ++ int ret; + -+ DO_TESTCASE_1B("hard-safe-A + unsafe-B #2", irqsafe4_hard_spin, 123); -+ DO_TESTCASE_1B("hard-safe-A + unsafe-B #2", irqsafe4_hard_spin, 132); -+ DO_TESTCASE_1B("hard-safe-A + unsafe-B #2", irqsafe4_hard_spin, 213); -+ DO_TESTCASE_1B("hard-safe-A + unsafe-B #2", irqsafe4_hard_spin, 231); -+ DO_TESTCASE_1B("hard-safe-A + unsafe-B #2", irqsafe4_hard_spin, 312); -+ DO_TESTCASE_1B("hard-safe-A + unsafe-B #2", irqsafe4_hard_spin, 321); -+#endif - - ww_tests(); - -diff -Nur linux-4.1.6.orig/lib/percpu_ida.c linux-4.1.6/lib/percpu_ida.c ---- linux-4.1.6.orig/lib/percpu_ida.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/lib/percpu_ida.c 2015-09-08 23:49:08.433831016 +0200 -@@ -26,6 +26,9 @@ - #include - #include - #include -+#include ++ lockdep_assert_held(&cgroup_mutex); + -+static DEFINE_LOCAL_IRQ_LOCK(irq_off_lock); - - struct percpu_ida_cpu { - /* -@@ -148,13 +151,13 @@ - unsigned long flags; - int tag; - -- local_irq_save(flags); -+ local_lock_irqsave(irq_off_lock, flags); - tags = this_cpu_ptr(pool->tag_cpu); - - /* Fastpath */ - tag = alloc_local_tag(tags); - if (likely(tag >= 0)) { -- local_irq_restore(flags); -+ local_unlock_irqrestore(irq_off_lock, flags); - return tag; - } - -@@ -173,6 +176,7 @@ - - if (!tags->nr_free) - alloc_global_tags(pool, tags); ++ for (cft = cfts; cft->name[0] != '\0'; cft++) { ++ /* does cft->flags tell us to skip this file on @cgrp? */ ++ if ((cft->flags & __CFTYPE_ONLY_ON_DFL) && !cgroup_on_dfl(cgrp)) ++ continue; ++ if ((cft->flags & __CFTYPE_NOT_ON_DFL) && cgroup_on_dfl(cgrp)) ++ continue; ++ if ((cft->flags & CFTYPE_NOT_ON_ROOT) && !cgroup_parent(cgrp)) ++ continue; ++ if ((cft->flags & CFTYPE_ONLY_ON_ROOT) && cgroup_parent(cgrp)) ++ continue; + - if (!tags->nr_free) - steal_tags(pool, tags); - -@@ -184,7 +188,7 @@ - } - - spin_unlock(&pool->lock); -- local_irq_restore(flags); -+ local_unlock_irqrestore(irq_off_lock, flags); - - if (tag >= 0 || state == TASK_RUNNING) - break; -@@ -196,7 +200,7 @@ - - schedule(); - -- local_irq_save(flags); -+ local_lock_irqsave(irq_off_lock, flags); - tags = this_cpu_ptr(pool->tag_cpu); - } - if (state != TASK_RUNNING) -@@ -221,7 +225,7 @@ - - BUG_ON(tag >= pool->nr_tags); - -- local_irq_save(flags); -+ local_lock_irqsave(irq_off_lock, flags); - tags = this_cpu_ptr(pool->tag_cpu); - - spin_lock(&tags->lock); -@@ -253,7 +257,7 @@ - spin_unlock(&pool->lock); - } - -- local_irq_restore(flags); -+ local_unlock_irqrestore(irq_off_lock, flags); - } - EXPORT_SYMBOL_GPL(percpu_ida_free); - -@@ -345,7 +349,7 @@ - struct percpu_ida_cpu *remote; - unsigned cpu, i, err = 0; - -- local_irq_save(flags); -+ local_lock_irqsave(irq_off_lock, flags); - for_each_possible_cpu(cpu) { - remote = per_cpu_ptr(pool->tag_cpu, cpu); - spin_lock(&remote->lock); -@@ -367,7 +371,7 @@ - } - spin_unlock(&pool->lock); - out: -- local_irq_restore(flags); -+ local_unlock_irqrestore(irq_off_lock, flags); - return err; - } - EXPORT_SYMBOL_GPL(percpu_ida_for_each_free); -diff -Nur linux-4.1.6.orig/lib/radix-tree.c linux-4.1.6/lib/radix-tree.c ---- linux-4.1.6.orig/lib/radix-tree.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/lib/radix-tree.c 2015-09-08 23:49:08.433831016 +0200 -@@ -195,12 +195,13 @@ - * succeed in getting a node here (and never reach - * kmem_cache_alloc) - */ -- rtp = this_cpu_ptr(&radix_tree_preloads); -+ rtp = &get_cpu_var(radix_tree_preloads); - if (rtp->nr) { - ret = rtp->nodes[rtp->nr - 1]; - rtp->nodes[rtp->nr - 1] = NULL; - rtp->nr--; - } -+ put_cpu_var(radix_tree_preloads); - /* - * Update the allocation stack trace as this is more useful - * for debugging. -@@ -240,6 +241,7 @@ - call_rcu(&node->rcu_head, radix_tree_node_rcu_free); - } - -+#ifndef CONFIG_PREEMPT_RT_FULL - /* - * Load up this CPU's radix_tree_node buffer with sufficient objects to - * ensure that the addition of a single element in the tree cannot fail. On -@@ -305,6 +307,7 @@ - return 0; - } - EXPORT_SYMBOL(radix_tree_maybe_preload); -+#endif - - /* - * Return the maximum key which can be store into a -diff -Nur linux-4.1.6.orig/lib/scatterlist.c linux-4.1.6/lib/scatterlist.c ---- linux-4.1.6.orig/lib/scatterlist.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/lib/scatterlist.c 2015-09-08 23:49:08.433831016 +0200 -@@ -592,7 +592,7 @@ - flush_kernel_dcache_page(miter->page); - - if (miter->__flags & SG_MITER_ATOMIC) { -- WARN_ON_ONCE(preemptible()); -+ WARN_ON_ONCE(!pagefault_disabled()); - kunmap_atomic(miter->addr); - } else - kunmap(miter->page); -@@ -637,7 +637,7 @@ - if (!sg_miter_skip(&miter, skip)) - return false; - -- local_irq_save(flags); -+ local_irq_save_nort(flags); - - while (sg_miter_next(&miter) && offset < buflen) { - unsigned int len; -@@ -654,7 +654,7 @@ - - sg_miter_stop(&miter); - -- local_irq_restore(flags); -+ local_irq_restore_nort(flags); - return offset; - } - -diff -Nur linux-4.1.6.orig/lib/smp_processor_id.c linux-4.1.6/lib/smp_processor_id.c ---- linux-4.1.6.orig/lib/smp_processor_id.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/lib/smp_processor_id.c 2015-09-08 23:49:08.433831016 +0200 -@@ -39,8 +39,9 @@ - if (!printk_ratelimit()) - goto out_enable; - -- printk(KERN_ERR "BUG: using %s%s() in preemptible [%08x] code: %s/%d\n", -- what1, what2, preempt_count() - 1, current->comm, current->pid); -+ printk(KERN_ERR "BUG: using %s%s() in preemptible [%08x %08x] code: %s/%d\n", -+ what1, what2, preempt_count() - 1, __migrate_disabled(current), -+ current->comm, current->pid); - - print_symbol("caller is %s\n", (long)__builtin_return_address(0)); - dump_stack(); -diff -Nur linux-4.1.6.orig/lib/strnlen_user.c linux-4.1.6/lib/strnlen_user.c ---- linux-4.1.6.orig/lib/strnlen_user.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/lib/strnlen_user.c 2015-09-08 23:49:08.433831016 +0200 -@@ -85,7 +85,8 @@ - * @str: The string to measure. - * @count: Maximum count (including NUL character) - * -- * Context: User context only. This function may sleep. -+ * Context: User context only. This function may sleep if pagefaults are -+ * enabled. - * - * Get the size of a NUL-terminated string in user space. - * -@@ -121,7 +122,8 @@ - * strlen_user: - Get the size of a user string INCLUDING final NUL. - * @str: The string to measure. - * -- * Context: User context only. This function may sleep. -+ * Context: User context only. This function may sleep if pagefaults are -+ * enabled. - * - * Get the size of a NUL-terminated string in user space. - * -diff -Nur linux-4.1.6.orig/mm/compaction.c linux-4.1.6/mm/compaction.c ---- linux-4.1.6.orig/mm/compaction.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/mm/compaction.c 2015-09-08 23:49:08.437830574 +0200 -@@ -1406,10 +1406,12 @@ - cc->migrate_pfn & ~((1UL << cc->order) - 1); - - if (last_migrated_pfn < current_block_start) { -- cpu = get_cpu(); -+ cpu = get_cpu_light(); -+ local_lock_irq(swapvec_lock); - lru_add_drain_cpu(cpu); -+ local_unlock_irq(swapvec_lock); - drain_local_pages(zone); -- put_cpu(); -+ put_cpu_light(); - /* No more flushing until we migrate again */ - last_migrated_pfn = 0; - } -diff -Nur linux-4.1.6.orig/mm/filemap.c linux-4.1.6/mm/filemap.c ---- linux-4.1.6.orig/mm/filemap.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/mm/filemap.c 2015-09-08 23:49:08.437830574 +0200 -@@ -167,7 +167,9 @@ - if (!workingset_node_pages(node) && - list_empty(&node->private_list)) { - node->private_data = mapping; -- list_lru_add(&workingset_shadow_nodes, &node->private_list); -+ local_lock(workingset_shadow_lock); -+ list_lru_add(&__workingset_shadow_nodes, &node->private_list); -+ local_unlock(workingset_shadow_lock); - } - } - -@@ -533,9 +535,12 @@ - * node->private_list is protected by - * mapping->tree_lock. - */ -- if (!list_empty(&node->private_list)) -- list_lru_del(&workingset_shadow_nodes, -+ if (!list_empty(&node->private_list)) { -+ local_lock(workingset_shadow_lock); -+ list_lru_del(&__workingset_shadow_nodes, - &node->private_list); -+ local_unlock(workingset_shadow_lock); ++ if (is_add) { ++ ret = cgroup_add_file(cgrp, cft); ++ if (ret) { ++ pr_warn("%s: failed to add %s, err=%d\n", ++ __func__, cft->name, ret); ++ return ret; ++ } ++ } else { ++ cgroup_rm_file(cgrp, cft); + } - } - return 0; - } -diff -Nur linux-4.1.6.orig/mm/highmem.c linux-4.1.6/mm/highmem.c ---- linux-4.1.6.orig/mm/highmem.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/mm/highmem.c 2015-09-08 23:49:08.437830574 +0200 -@@ -29,10 +29,11 @@ - #include - #include - -- -+#ifndef CONFIG_PREEMPT_RT_FULL - #if defined(CONFIG_HIGHMEM) || defined(CONFIG_X86_32) - DEFINE_PER_CPU(int, __kmap_atomic_idx); - #endif -+#endif - - /* - * Virtual_count is not a pure "count". -@@ -107,8 +108,9 @@ - unsigned long totalhigh_pages __read_mostly; - EXPORT_SYMBOL(totalhigh_pages); - -- -+#ifndef CONFIG_PREEMPT_RT_FULL - EXPORT_PER_CPU_SYMBOL(__kmap_atomic_idx); -+#endif - - unsigned int nr_free_highpages (void) - { -diff -Nur linux-4.1.6.orig/mm/Kconfig linux-4.1.6/mm/Kconfig ---- linux-4.1.6.orig/mm/Kconfig 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/mm/Kconfig 2015-09-08 23:49:08.437830574 +0200 -@@ -409,7 +409,7 @@ - - config TRANSPARENT_HUGEPAGE - bool "Transparent Hugepage Support" -- depends on HAVE_ARCH_TRANSPARENT_HUGEPAGE -+ depends on HAVE_ARCH_TRANSPARENT_HUGEPAGE && !PREEMPT_RT_FULL - select COMPACTION - help - Transparent Hugepages allows the kernel to use huge pages and -diff -Nur linux-4.1.6.orig/mm/memcontrol.c linux-4.1.6/mm/memcontrol.c ---- linux-4.1.6.orig/mm/memcontrol.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/mm/memcontrol.c 2015-09-08 23:49:08.437830574 +0200 -@@ -66,6 +66,8 @@ - #include - #include - #include -+#include ++ } ++ return 0; ++} + - #include "slab.h" - - #include -@@ -85,6 +87,7 @@ - #define do_swap_account 0 - #endif - -+static DEFINE_LOCAL_IRQ_LOCK(event_lock); - static const char * const mem_cgroup_stat_names[] = { - "cache", - "rss", -@@ -2124,14 +2127,17 @@ - */ - static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages) - { -- struct memcg_stock_pcp *stock = &get_cpu_var(memcg_stock); -+ struct memcg_stock_pcp *stock; -+ int cpu = get_cpu_light(); ++static int cgroup_apply_cftypes(struct cftype *cfts, bool is_add) ++{ ++ LIST_HEAD(pending); ++ struct cgroup_subsys *ss = cfts[0].ss; ++ struct cgroup *root = &ss->root->cgrp; ++ struct cgroup_subsys_state *css; ++ int ret = 0; + -+ stock = &per_cpu(memcg_stock, cpu); - - if (stock->cached != memcg) { /* reset if necessary */ - drain_stock(stock); - stock->cached = memcg; - } - stock->nr_pages += nr_pages; -- put_cpu_var(memcg_stock); -+ put_cpu_light(); - } - - /* -@@ -2147,7 +2153,7 @@ - return; - /* Notify other cpus that system-wide "drain" is running */ - get_online_cpus(); -- curcpu = get_cpu(); -+ curcpu = get_cpu_light(); - for_each_online_cpu(cpu) { - struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu); - struct mem_cgroup *memcg; -@@ -2164,7 +2170,7 @@ - schedule_work_on(cpu, &stock->work); - } - } -- put_cpu(); -+ put_cpu_light(); - put_online_cpus(); - mutex_unlock(&percpu_charge_mutex); - } -@@ -4801,12 +4807,12 @@ - - ret = 0; - -- local_irq_disable(); -+ local_lock_irq(event_lock); - mem_cgroup_charge_statistics(to, page, nr_pages); - memcg_check_events(to, page); - mem_cgroup_charge_statistics(from, page, -nr_pages); - memcg_check_events(from, page); -- local_irq_enable(); -+ local_unlock_irq(event_lock); - out_unlock: - unlock_page(page); - out: -@@ -5543,10 +5549,10 @@ - VM_BUG_ON_PAGE(!PageTransHuge(page), page); - } - -- local_irq_disable(); -+ local_lock_irq(event_lock); - mem_cgroup_charge_statistics(memcg, page, nr_pages); - memcg_check_events(memcg, page); -- local_irq_enable(); -+ local_unlock_irq(event_lock); - - if (do_swap_account && PageSwapCache(page)) { - swp_entry_t entry = { .val = page_private(page) }; -@@ -5602,14 +5608,14 @@ - memcg_oom_recover(memcg); - } - -- local_irq_save(flags); -+ local_lock_irqsave(event_lock, flags); - __this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_RSS], nr_anon); - __this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_CACHE], nr_file); - __this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_RSS_HUGE], nr_huge); - __this_cpu_add(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGOUT], pgpgout); - __this_cpu_add(memcg->stat->nr_page_events, nr_pages); - memcg_check_events(memcg, dummy_page); -- local_irq_restore(flags); -+ local_unlock_irqrestore(event_lock, flags); - - if (!mem_cgroup_is_root(memcg)) - css_put_many(&memcg->css, nr_pages); -@@ -5813,6 +5819,7 @@ - { - struct mem_cgroup *memcg; - unsigned short oldid; -+ unsigned long flags; - - VM_BUG_ON_PAGE(PageLRU(page), page); - VM_BUG_ON_PAGE(page_count(page), page); -@@ -5835,9 +5842,11 @@ - if (!mem_cgroup_is_root(memcg)) - page_counter_uncharge(&memcg->memory, 1); - -+ local_lock_irqsave(event_lock, flags); - /* Caller disabled preemption with mapping->tree_lock */ - mem_cgroup_charge_statistics(memcg, page, -1); - memcg_check_events(memcg, page); -+ local_unlock_irqrestore(event_lock, flags); - } - - /** -diff -Nur linux-4.1.6.orig/mm/memory.c linux-4.1.6/mm/memory.c ---- linux-4.1.6.orig/mm/memory.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/mm/memory.c 2015-09-08 23:49:08.437830574 +0200 -@@ -3743,7 +3743,7 @@ - } - - #if defined(CONFIG_PROVE_LOCKING) || defined(CONFIG_DEBUG_ATOMIC_SLEEP) --void might_fault(void) -+void __might_fault(const char *file, int line) - { - /* - * Some code (nfs/sunrpc) uses socket ops on kernel memory while -@@ -3753,21 +3753,15 @@ - */ - if (segment_eq(get_fs(), KERNEL_DS)) - return; -- -- /* -- * it would be nicer only to annotate paths which are not under -- * pagefault_disable, however that requires a larger audit and -- * providing helpers like get_user_atomic. -- */ -- if (in_atomic()) -+ if (pagefault_disabled()) - return; -- -- __might_sleep(__FILE__, __LINE__, 0); -- -+ __might_sleep(file, line, 0); -+#if defined(CONFIG_DEBUG_ATOMIC_SLEEP) - if (current->mm) - might_lock_read(¤t->mm->mmap_sem); -+#endif - } --EXPORT_SYMBOL(might_fault); -+EXPORT_SYMBOL(__might_fault); - #endif - - #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS) -diff -Nur linux-4.1.6.orig/mm/mmu_context.c linux-4.1.6/mm/mmu_context.c ---- linux-4.1.6.orig/mm/mmu_context.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/mm/mmu_context.c 2015-09-08 23:49:08.437830574 +0200 -@@ -23,6 +23,7 @@ - struct task_struct *tsk = current; - - task_lock(tsk); -+ preempt_disable_rt(); - active_mm = tsk->active_mm; - if (active_mm != mm) { - atomic_inc(&mm->mm_count); -@@ -30,6 +31,7 @@ - } - tsk->mm = mm; - switch_mm(active_mm, mm, tsk); -+ preempt_enable_rt(); - task_unlock(tsk); - #ifdef finish_arch_post_lock_switch - finish_arch_post_lock_switch(); -diff -Nur linux-4.1.6.orig/mm/page_alloc.c linux-4.1.6/mm/page_alloc.c ---- linux-4.1.6.orig/mm/page_alloc.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/mm/page_alloc.c 2015-09-08 23:49:08.437830574 +0200 -@@ -60,6 +60,7 @@ - #include - #include - #include -+#include - #include - - #include -@@ -233,6 +234,18 @@ - EXPORT_SYMBOL(nr_online_nodes); - #endif - -+static DEFINE_LOCAL_IRQ_LOCK(pa_lock); ++ lockdep_assert_held(&cgroup_mutex); + -+#ifdef CONFIG_PREEMPT_RT_BASE -+# define cpu_lock_irqsave(cpu, flags) \ -+ local_lock_irqsave_on(pa_lock, flags, cpu) -+# define cpu_unlock_irqrestore(cpu, flags) \ -+ local_unlock_irqrestore_on(pa_lock, flags, cpu) -+#else -+# define cpu_lock_irqsave(cpu, flags) local_irq_save(flags) -+# define cpu_unlock_irqrestore(cpu, flags) local_irq_restore(flags) -+#endif ++ /* add/rm files for all cgroups created before */ ++ css_for_each_descendant_pre(css, cgroup_css(root, ss)) { ++ struct cgroup *cgrp = css->cgroup; + - int page_group_by_mobility_disabled __read_mostly; - - void set_pageblock_migratetype(struct page *page, int migratetype) -@@ -681,7 +694,7 @@ - } - - /* -- * Frees a number of pages from the PCP lists -+ * Frees a number of pages which have been collected from the pcp lists. - * Assumes all pages on list are in same zone, and of same order. - * count is the number of pages to free. - * -@@ -692,18 +705,51 @@ - * pinned" detection logic. - */ - static void free_pcppages_bulk(struct zone *zone, int count, -- struct per_cpu_pages *pcp) -+ struct list_head *list) - { -- int migratetype = 0; -- int batch_free = 0; - int to_free = count; - unsigned long nr_scanned; -+ unsigned long flags; ++ if (cgroup_is_dead(cgrp)) ++ continue; + -+ spin_lock_irqsave(&zone->lock, flags); - -- spin_lock(&zone->lock); - nr_scanned = zone_page_state(zone, NR_PAGES_SCANNED); - if (nr_scanned) - __mod_zone_page_state(zone, NR_PAGES_SCANNED, -nr_scanned); - -+ while (!list_empty(list)) { -+ struct page *page = list_first_entry(list, struct page, lru); -+ int mt; /* migratetype of the to-be-freed page */ ++ ret = cgroup_addrm_files(cgrp, cfts, is_add); ++ if (ret) ++ break; ++ } + -+ /* must delete as __free_one_page list manipulates */ -+ list_del(&page->lru); ++ if (is_add && !ret) ++ kernfs_activate(root->kn); ++ return ret; ++} + -+ mt = get_freepage_migratetype(page); -+ if (unlikely(has_isolate_pageblock(zone))) -+ mt = get_pageblock_migratetype(page); ++static void cgroup_exit_cftypes(struct cftype *cfts) ++{ ++ struct cftype *cft; + -+ /* MIGRATE_MOVABLE list may include MIGRATE_RESERVEs */ -+ __free_one_page(page, page_to_pfn(page), zone, 0, mt); -+ trace_mm_page_pcpu_drain(page, 0, mt); -+ to_free--; ++ for (cft = cfts; cft->name[0] != '\0'; cft++) { ++ /* free copy for custom atomic_write_len, see init_cftypes() */ ++ if (cft->max_write_len && cft->max_write_len != PAGE_SIZE) ++ kfree(cft->kf_ops); ++ cft->kf_ops = NULL; ++ cft->ss = NULL; ++ ++ /* revert flags set by cgroup core while adding @cfts */ ++ cft->flags &= ~(__CFTYPE_ONLY_ON_DFL | __CFTYPE_NOT_ON_DFL); + } -+ WARN_ON(to_free != 0); -+ spin_unlock_irqrestore(&zone->lock, flags); +} + -+/* -+ * Moves a number of pages from the PCP lists to free list which -+ * is freed outside of the locked region. -+ * -+ * Assumes all pages on list are in same zone, and of same order. -+ * count is the number of pages to free. -+ */ -+static void isolate_pcp_pages(int to_free, struct per_cpu_pages *src, -+ struct list_head *dst) ++static int cgroup_init_cftypes(struct cgroup_subsys *ss, struct cftype *cfts) +{ -+ int migratetype = 0; -+ int batch_free = 0; ++ struct cftype *cft; + - while (to_free) { - struct page *page; - struct list_head *list; -@@ -719,7 +765,7 @@ - batch_free++; - if (++migratetype == MIGRATE_PCPTYPES) - migratetype = 0; -- list = &pcp->lists[migratetype]; -+ list = &src->lists[migratetype]; - } while (list_empty(list)); - - /* This is the only non-empty list. Free them all. */ -@@ -727,21 +773,11 @@ - batch_free = to_free; - - do { -- int mt; /* migratetype of the to-be-freed page */ -- -- page = list_entry(list->prev, struct page, lru); -- /* must delete as __free_one_page list manipulates */ -+ page = list_last_entry(list, struct page, lru); - list_del(&page->lru); -- mt = get_freepage_migratetype(page); -- if (unlikely(has_isolate_pageblock(zone))) -- mt = get_pageblock_migratetype(page); -- -- /* MIGRATE_MOVABLE list may include MIGRATE_RESERVEs */ -- __free_one_page(page, page_to_pfn(page), zone, 0, mt); -- trace_mm_page_pcpu_drain(page, 0, mt); -+ list_add(&page->lru, dst); - } while (--to_free && --batch_free && !list_empty(list)); - } -- spin_unlock(&zone->lock); - } - - static void free_one_page(struct zone *zone, -@@ -750,7 +786,9 @@ - int migratetype) - { - unsigned long nr_scanned; -- spin_lock(&zone->lock); -+ unsigned long flags; ++ for (cft = cfts; cft->name[0] != '\0'; cft++) { ++ struct kernfs_ops *kf_ops; + -+ spin_lock_irqsave(&zone->lock, flags); - nr_scanned = zone_page_state(zone, NR_PAGES_SCANNED); - if (nr_scanned) - __mod_zone_page_state(zone, NR_PAGES_SCANNED, -nr_scanned); -@@ -760,7 +798,7 @@ - migratetype = get_pfnblock_migratetype(page, pfn); - } - __free_one_page(page, pfn, zone, order, migratetype); -- spin_unlock(&zone->lock); -+ spin_unlock_irqrestore(&zone->lock, flags); - } - - static int free_tail_pages_check(struct page *head_page, struct page *page) -@@ -825,11 +863,11 @@ - return; - - migratetype = get_pfnblock_migratetype(page, pfn); -- local_irq_save(flags); -+ local_lock_irqsave(pa_lock, flags); - __count_vm_events(PGFREE, 1 << order); - set_freepage_migratetype(page, migratetype); - free_one_page(page_zone(page), page, pfn, order, migratetype); -- local_irq_restore(flags); -+ local_unlock_irqrestore(pa_lock, flags); - } - - void __init __free_pages_bootmem(struct page *page, unsigned int order) -@@ -1368,16 +1406,18 @@ - void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp) - { - unsigned long flags; -+ LIST_HEAD(dst); - int to_drain, batch; - -- local_irq_save(flags); -+ local_lock_irqsave(pa_lock, flags); - batch = READ_ONCE(pcp->batch); - to_drain = min(pcp->count, batch); - if (to_drain > 0) { -- free_pcppages_bulk(zone, to_drain, pcp); -+ isolate_pcp_pages(to_drain, pcp, &dst); - pcp->count -= to_drain; - } -- local_irq_restore(flags); -+ local_unlock_irqrestore(pa_lock, flags); -+ free_pcppages_bulk(zone, to_drain, &dst); - } - #endif - -@@ -1393,16 +1433,21 @@ - unsigned long flags; - struct per_cpu_pageset *pset; - struct per_cpu_pages *pcp; -+ LIST_HEAD(dst); -+ int count; - -- local_irq_save(flags); -+ cpu_lock_irqsave(cpu, flags); - pset = per_cpu_ptr(zone->pageset, cpu); - - pcp = &pset->pcp; -- if (pcp->count) { -- free_pcppages_bulk(zone, pcp->count, pcp); -+ count = pcp->count; -+ if (count) { -+ isolate_pcp_pages(count, pcp, &dst); - pcp->count = 0; - } -- local_irq_restore(flags); -+ cpu_unlock_irqrestore(cpu, flags); -+ if (count) -+ free_pcppages_bulk(zone, count, &dst); - } - - /* -@@ -1488,8 +1533,17 @@ - else - cpumask_clear_cpu(cpu, &cpus_with_pcps); - } -+#ifndef CONFIG_PREEMPT_RT_BASE - on_each_cpu_mask(&cpus_with_pcps, (smp_call_func_t) drain_local_pages, - zone, 1); -+#else -+ for_each_cpu(cpu, &cpus_with_pcps) { -+ if (zone) -+ drain_pages_zone(cpu, zone); ++ WARN_ON(cft->ss || cft->kf_ops); ++ ++ if (cft->seq_start) ++ kf_ops = &cgroup_kf_ops; + else -+ drain_pages(cpu); -+ } -+#endif - } - - #ifdef CONFIG_HIBERNATION -@@ -1545,7 +1599,7 @@ - - migratetype = get_pfnblock_migratetype(page, pfn); - set_freepage_migratetype(page, migratetype); -- local_irq_save(flags); -+ local_lock_irqsave(pa_lock, flags); - __count_vm_event(PGFREE); - - /* -@@ -1571,12 +1625,17 @@ - pcp->count++; - if (pcp->count >= pcp->high) { - unsigned long batch = READ_ONCE(pcp->batch); -- free_pcppages_bulk(zone, batch, pcp); -+ LIST_HEAD(dst); ++ kf_ops = &cgroup_kf_single_ops; + -+ isolate_pcp_pages(batch, pcp, &dst); - pcp->count -= batch; -+ local_unlock_irqrestore(pa_lock, flags); -+ free_pcppages_bulk(zone, batch, &dst); -+ return; - } - - out: -- local_irq_restore(flags); -+ local_unlock_irqrestore(pa_lock, flags); - } - - /* -@@ -1707,7 +1766,7 @@ - struct per_cpu_pages *pcp; - struct list_head *list; - -- local_irq_save(flags); -+ local_lock_irqsave(pa_lock, flags); - pcp = &this_cpu_ptr(zone->pageset)->pcp; - list = &pcp->lists[migratetype]; - if (list_empty(list)) { -@@ -1739,13 +1798,15 @@ - */ - WARN_ON_ONCE(order > 1); - } -- spin_lock_irqsave(&zone->lock, flags); -+ local_spin_lock_irqsave(pa_lock, &zone->lock, flags); - page = __rmqueue(zone, order, migratetype); -- spin_unlock(&zone->lock); -- if (!page) -+ if (!page) { -+ spin_unlock(&zone->lock); - goto failed; ++ /* ++ * Ugh... if @cft wants a custom max_write_len, we need to ++ * make a copy of kf_ops to set its atomic_write_len. ++ */ ++ if (cft->max_write_len && cft->max_write_len != PAGE_SIZE) { ++ kf_ops = kmemdup(kf_ops, sizeof(*kf_ops), GFP_KERNEL); ++ if (!kf_ops) { ++ cgroup_exit_cftypes(cfts); ++ return -ENOMEM; ++ } ++ kf_ops->atomic_write_len = cft->max_write_len; + } - __mod_zone_freepage_state(zone, -(1 << order), - get_freepage_migratetype(page)); -+ spin_unlock(&zone->lock); - } - - __mod_zone_page_state(zone, NR_ALLOC_BATCH, -(1 << order)); -@@ -1755,13 +1816,13 @@ - - __count_zone_vm_events(PGALLOC, zone, 1 << order); - zone_statistics(preferred_zone, zone, gfp_flags); -- local_irq_restore(flags); -+ local_unlock_irqrestore(pa_lock, flags); - - VM_BUG_ON_PAGE(bad_range(zone, page), page); - return page; - - failed: -- local_irq_restore(flags); -+ local_unlock_irqrestore(pa_lock, flags); - return NULL; - } - -@@ -5650,6 +5711,7 @@ - void __init page_alloc_init(void) - { - hotcpu_notifier(page_alloc_cpu_notify, 0); -+ local_irq_lock_init(pa_lock); - } - - /* -@@ -6544,7 +6606,7 @@ - struct per_cpu_pageset *pset; - - /* avoid races with drain_pages() */ -- local_irq_save(flags); -+ local_lock_irqsave(pa_lock, flags); - if (zone->pageset != &boot_pageset) { - for_each_online_cpu(cpu) { - pset = per_cpu_ptr(zone->pageset, cpu); -@@ -6553,7 +6615,7 @@ - free_percpu(zone->pageset); - zone->pageset = &boot_pageset; - } -- local_irq_restore(flags); -+ local_unlock_irqrestore(pa_lock, flags); - } - - #ifdef CONFIG_MEMORY_HOTREMOVE -diff -Nur linux-4.1.6.orig/mm/slab.h linux-4.1.6/mm/slab.h ---- linux-4.1.6.orig/mm/slab.h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/mm/slab.h 2015-09-08 23:49:08.441830131 +0200 -@@ -330,7 +330,11 @@ - * The slab lists for all objects. - */ - struct kmem_cache_node { -+#ifdef CONFIG_SLUB -+ raw_spinlock_t list_lock; -+#else - spinlock_t list_lock; -+#endif - - #ifdef CONFIG_SLAB - struct list_head slabs_partial; /* partial list first, better asm code */ -diff -Nur linux-4.1.6.orig/mm/slub.c linux-4.1.6/mm/slub.c ---- linux-4.1.6.orig/mm/slub.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/mm/slub.c 2015-09-08 23:49:08.441830131 +0200 -@@ -1069,7 +1069,7 @@ - { - struct kmem_cache_node *n = get_node(s, page_to_nid(page)); - -- spin_lock_irqsave(&n->list_lock, *flags); -+ raw_spin_lock_irqsave(&n->list_lock, *flags); - slab_lock(page); - - if (!check_slab(s, page)) -@@ -1116,7 +1116,7 @@ - - fail: - slab_unlock(page); -- spin_unlock_irqrestore(&n->list_lock, *flags); -+ raw_spin_unlock_irqrestore(&n->list_lock, *flags); - slab_fix(s, "Object at 0x%p not freed", object); - return NULL; - } -@@ -1242,6 +1242,12 @@ - - #endif /* CONFIG_SLUB_DEBUG */ - -+struct slub_free_list { -+ raw_spinlock_t lock; -+ struct list_head list; -+}; -+static DEFINE_PER_CPU(struct slub_free_list, slub_free_list); + - /* - * Hooks for other subsystems that check memory allocations. In a typical - * production configuration these hooks all should produce no code at all. -@@ -1306,6 +1312,17 @@ - kasan_slab_free(s, x); - } - -+static void setup_object(struct kmem_cache *s, struct page *page, -+ void *object) -+{ -+ setup_object_debug(s, page, object); -+ if (unlikely(s->ctor)) { -+ kasan_unpoison_object_data(s, object); -+ s->ctor(object); -+ kasan_poison_object_data(s, object); ++ cft->kf_ops = kf_ops; ++ cft->ss = ss; + } -+} + - /* - * Slab allocation and freeing - */ -@@ -1336,10 +1353,17 @@ - struct page *page; - struct kmem_cache_order_objects oo = s->oo; - gfp_t alloc_gfp; -+ void *start, *p; -+ int idx, order; -+ bool enableirqs; - - flags &= gfp_allowed_mask; - -- if (flags & __GFP_WAIT) -+ enableirqs = (flags & __GFP_WAIT) != 0; -+#ifdef CONFIG_PREEMPT_RT_FULL -+ enableirqs |= system_state == SYSTEM_RUNNING; -+#endif -+ if (enableirqs) - local_irq_enable(); - - flags |= s->allocflags; -@@ -1359,13 +1383,13 @@ - * Try a lower order alloc if possible - */ - page = alloc_slab_page(s, alloc_gfp, node, oo); -- -- if (page) -- stat(s, ORDER_FALLBACK); -+ if (unlikely(!page)) -+ goto out; -+ stat(s, ORDER_FALLBACK); - } - -- if (kmemcheck_enabled && page -- && !(s->flags & (SLAB_NOTRACK | DEBUG_DEFAULT_FLAGS))) { -+ if (kmemcheck_enabled && -+ !(s->flags & (SLAB_NOTRACK | DEBUG_DEFAULT_FLAGS))) { - int pages = 1 << oo_order(oo); - - kmemcheck_alloc_shadow(page, oo_order(oo), alloc_gfp, node); -@@ -1380,51 +1404,9 @@ - kmemcheck_mark_unallocated_pages(page, pages); - } - -- if (flags & __GFP_WAIT) -- local_irq_disable(); -- if (!page) -- return NULL; -- - page->objects = oo_objects(oo); -- mod_zone_page_state(page_zone(page), -- (s->flags & SLAB_RECLAIM_ACCOUNT) ? -- NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE, -- 1 << oo_order(oo)); -- -- return page; --} -- --static void setup_object(struct kmem_cache *s, struct page *page, -- void *object) --{ -- setup_object_debug(s, page, object); -- if (unlikely(s->ctor)) { -- kasan_unpoison_object_data(s, object); -- s->ctor(object); -- kasan_poison_object_data(s, object); -- } --} -- --static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node) --{ -- struct page *page; -- void *start; -- void *p; -- int order; -- int idx; -- -- if (unlikely(flags & GFP_SLAB_BUG_MASK)) { -- pr_emerg("gfp: %u\n", flags & GFP_SLAB_BUG_MASK); -- BUG(); -- } -- -- page = allocate_slab(s, -- flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node); -- if (!page) -- goto out; - - order = compound_order(page); -- inc_slabs_node(s, page_to_nid(page), page->objects); - page->slab_cache = s; - __SetPageSlab(page); - if (page->pfmemalloc) -@@ -1448,10 +1430,34 @@ - page->freelist = start; - page->inuse = page->objects; - page->frozen = 1; ++ return 0; ++} + - out: -+ if (enableirqs) -+ local_irq_disable(); -+ if (!page) -+ return NULL; ++static int cgroup_rm_cftypes_locked(struct cftype *cfts) ++{ ++ lockdep_assert_held(&cgroup_mutex); + -+ mod_zone_page_state(page_zone(page), -+ (s->flags & SLAB_RECLAIM_ACCOUNT) ? -+ NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE, -+ 1 << oo_order(oo)); ++ if (!cfts || !cfts[0].ss) ++ return -ENOENT; + -+ inc_slabs_node(s, page_to_nid(page), page->objects); ++ list_del(&cfts->node); ++ cgroup_apply_cftypes(cfts, false); ++ cgroup_exit_cftypes(cfts); ++ return 0; ++} + - return page; - } - -+static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node) ++/** ++ * cgroup_rm_cftypes - remove an array of cftypes from a subsystem ++ * @cfts: zero-length name terminated array of cftypes ++ * ++ * Unregister @cfts. Files described by @cfts are removed from all ++ * existing cgroups and all future cgroups won't have them either. This ++ * function can be called anytime whether @cfts' subsys is attached or not. ++ * ++ * Returns 0 on successful unregistration, -ENOENT if @cfts is not ++ * registered. ++ */ ++int cgroup_rm_cftypes(struct cftype *cfts) +{ -+ if (unlikely(flags & GFP_SLAB_BUG_MASK)) { -+ pr_emerg("gfp: %u\n", flags & GFP_SLAB_BUG_MASK); -+ BUG(); -+ } ++ int ret; + -+ return allocate_slab(s, -+ flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node); ++ mutex_lock(&cgroup_mutex); ++ ret = cgroup_rm_cftypes_locked(cfts); ++ mutex_unlock(&cgroup_mutex); ++ return ret; +} + - static void __free_slab(struct kmem_cache *s, struct page *page) - { - int order = compound_order(page); -@@ -1483,6 +1489,16 @@ - memcg_uncharge_slab(s, order); - } - -+static void free_delayed(struct list_head *h) ++/** ++ * cgroup_add_cftypes - add an array of cftypes to a subsystem ++ * @ss: target cgroup subsystem ++ * @cfts: zero-length name terminated array of cftypes ++ * ++ * Register @cfts to @ss. Files described by @cfts are created for all ++ * existing cgroups to which @ss is attached and all future cgroups will ++ * have them too. This function can be called anytime whether @ss is ++ * attached or not. ++ * ++ * Returns 0 on successful registration, -errno on failure. Note that this ++ * function currently returns 0 as long as @cfts registration is successful ++ * even if some file creation attempts on existing cgroups fail. ++ */ ++static int cgroup_add_cftypes(struct cgroup_subsys *ss, struct cftype *cfts) +{ -+ while(!list_empty(h)) { -+ struct page *page = list_first_entry(h, struct page, lru); ++ int ret; + -+ list_del(&page->lru); -+ __free_slab(page->slab_cache, page); -+ } -+} ++ if (ss->disabled) ++ return 0; + - #define need_reserve_slab_rcu \ - (sizeof(((struct page *)NULL)->lru) < sizeof(struct rcu_head)) - -@@ -1517,6 +1533,12 @@ - } - - call_rcu(head, rcu_free_slab); -+ } else if (irqs_disabled()) { -+ struct slub_free_list *f = this_cpu_ptr(&slub_free_list); ++ if (!cfts || cfts[0].name[0] == '\0') ++ return 0; + -+ raw_spin_lock(&f->lock); -+ list_add(&page->lru, &f->list); -+ raw_spin_unlock(&f->lock); - } else - __free_slab(s, page); - } -@@ -1630,7 +1652,7 @@ - if (!n || !n->nr_partial) - return NULL; - -- spin_lock(&n->list_lock); -+ raw_spin_lock(&n->list_lock); - list_for_each_entry_safe(page, page2, &n->partial, lru) { - void *t; - -@@ -1655,7 +1677,7 @@ - break; - - } -- spin_unlock(&n->list_lock); -+ raw_spin_unlock(&n->list_lock); - return object; - } - -@@ -1901,7 +1923,7 @@ - * that acquire_slab() will see a slab page that - * is frozen - */ -- spin_lock(&n->list_lock); -+ raw_spin_lock(&n->list_lock); - } - } else { - m = M_FULL; -@@ -1912,7 +1934,7 @@ - * slabs from diagnostic functions will not see - * any frozen slabs. - */ -- spin_lock(&n->list_lock); -+ raw_spin_lock(&n->list_lock); - } - } - -@@ -1947,7 +1969,7 @@ - goto redo; - - if (lock) -- spin_unlock(&n->list_lock); -+ raw_spin_unlock(&n->list_lock); - - if (m == M_FREE) { - stat(s, DEACTIVATE_EMPTY); -@@ -1979,10 +2001,10 @@ - n2 = get_node(s, page_to_nid(page)); - if (n != n2) { - if (n) -- spin_unlock(&n->list_lock); -+ raw_spin_unlock(&n->list_lock); - - n = n2; -- spin_lock(&n->list_lock); -+ raw_spin_lock(&n->list_lock); - } - - do { -@@ -2011,7 +2033,7 @@ - } - - if (n) -- spin_unlock(&n->list_lock); -+ raw_spin_unlock(&n->list_lock); - - while (discard_page) { - page = discard_page; -@@ -2050,14 +2072,21 @@ - pobjects = oldpage->pobjects; - pages = oldpage->pages; - if (drain && pobjects > s->cpu_partial) { -+ struct slub_free_list *f; - unsigned long flags; -+ LIST_HEAD(tofree); - /* - * partial array is full. Move the existing - * set to the per node partial list. - */ - local_irq_save(flags); - unfreeze_partials(s, this_cpu_ptr(s->cpu_slab)); -+ f = this_cpu_ptr(&slub_free_list); -+ raw_spin_lock(&f->lock); -+ list_splice_init(&f->list, &tofree); -+ raw_spin_unlock(&f->lock); - local_irq_restore(flags); -+ free_delayed(&tofree); - oldpage = NULL; - pobjects = 0; - pages = 0; -@@ -2129,7 +2158,22 @@ - - static void flush_all(struct kmem_cache *s) - { -+ LIST_HEAD(tofree); -+ int cpu; ++ ret = cgroup_init_cftypes(ss, cfts); ++ if (ret) ++ return ret; + - on_each_cpu_cond(has_cpu_slab, flush_cpu_slab, s, 1, GFP_ATOMIC); -+ for_each_online_cpu(cpu) { -+ struct slub_free_list *f; ++ mutex_lock(&cgroup_mutex); + -+ if (!has_cpu_slab(cpu, s)) -+ continue; ++ list_add_tail(&cfts->node, &ss->cfts); ++ ret = cgroup_apply_cftypes(cfts, true); ++ if (ret) ++ cgroup_rm_cftypes_locked(cfts); + -+ f = &per_cpu(slub_free_list, cpu); -+ raw_spin_lock_irq(&f->lock); -+ list_splice_init(&f->list, &tofree); -+ raw_spin_unlock_irq(&f->lock); -+ free_delayed(&tofree); ++ mutex_unlock(&cgroup_mutex); ++ return ret; ++} ++ ++/** ++ * cgroup_add_dfl_cftypes - add an array of cftypes for default hierarchy ++ * @ss: target cgroup subsystem ++ * @cfts: zero-length name terminated array of cftypes ++ * ++ * Similar to cgroup_add_cftypes() but the added files are only used for ++ * the default hierarchy. ++ */ ++int cgroup_add_dfl_cftypes(struct cgroup_subsys *ss, struct cftype *cfts) ++{ ++ struct cftype *cft; ++ ++ for (cft = cfts; cft && cft->name[0] != '\0'; cft++) ++ cft->flags |= __CFTYPE_ONLY_ON_DFL; ++ return cgroup_add_cftypes(ss, cfts); ++} ++ ++/** ++ * cgroup_add_legacy_cftypes - add an array of cftypes for legacy hierarchies ++ * @ss: target cgroup subsystem ++ * @cfts: zero-length name terminated array of cftypes ++ * ++ * Similar to cgroup_add_cftypes() but the added files are only used for ++ * the legacy hierarchies. ++ */ ++int cgroup_add_legacy_cftypes(struct cgroup_subsys *ss, struct cftype *cfts) ++{ ++ struct cftype *cft; ++ ++ /* ++ * If legacy_flies_on_dfl, we want to show the legacy files on the ++ * dfl hierarchy but iff the target subsystem hasn't been updated ++ * for the dfl hierarchy yet. ++ */ ++ if (!cgroup_legacy_files_on_dfl || ++ ss->dfl_cftypes != ss->legacy_cftypes) { ++ for (cft = cfts; cft && cft->name[0] != '\0'; cft++) ++ cft->flags |= __CFTYPE_NOT_ON_DFL; + } - } - - /* -@@ -2165,10 +2209,10 @@ - unsigned long x = 0; - struct page *page; - -- spin_lock_irqsave(&n->list_lock, flags); -+ raw_spin_lock_irqsave(&n->list_lock, flags); - list_for_each_entry(page, &n->partial, lru) - x += get_count(page); -- spin_unlock_irqrestore(&n->list_lock, flags); -+ raw_spin_unlock_irqrestore(&n->list_lock, flags); - return x; - } - #endif /* CONFIG_SLUB_DEBUG || CONFIG_SYSFS */ -@@ -2305,9 +2349,11 @@ - static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, - unsigned long addr, struct kmem_cache_cpu *c) - { -+ struct slub_free_list *f; - void *freelist; - struct page *page; - unsigned long flags; -+ LIST_HEAD(tofree); - - local_irq_save(flags); - #ifdef CONFIG_PREEMPT -@@ -2375,7 +2421,13 @@ - VM_BUG_ON(!c->page->frozen); - c->freelist = get_freepointer(s, freelist); - c->tid = next_tid(c->tid); -+out: -+ f = this_cpu_ptr(&slub_free_list); -+ raw_spin_lock(&f->lock); -+ list_splice_init(&f->list, &tofree); -+ raw_spin_unlock(&f->lock); - local_irq_restore(flags); -+ free_delayed(&tofree); - return freelist; - - new_slab: -@@ -2392,8 +2444,7 @@ - - if (unlikely(!freelist)) { - slab_out_of_memory(s, gfpflags, node); -- local_irq_restore(flags); -- return NULL; -+ goto out; - } - - page = c->page; -@@ -2408,8 +2459,7 @@ - deactivate_slab(s, page, get_freepointer(s, freelist)); - c->page = NULL; - c->freelist = NULL; -- local_irq_restore(flags); -- return freelist; -+ goto out; - } - - /* -@@ -2593,7 +2643,7 @@ - - do { - if (unlikely(n)) { -- spin_unlock_irqrestore(&n->list_lock, flags); -+ raw_spin_unlock_irqrestore(&n->list_lock, flags); - n = NULL; - } - prior = page->freelist; -@@ -2625,7 +2675,7 @@ - * Otherwise the list_lock will synchronize with - * other processors updating the list of slabs. - */ -- spin_lock_irqsave(&n->list_lock, flags); -+ raw_spin_lock_irqsave(&n->list_lock, flags); - - } - } -@@ -2667,7 +2717,7 @@ - add_partial(n, page, DEACTIVATE_TO_TAIL); - stat(s, FREE_ADD_PARTIAL); - } -- spin_unlock_irqrestore(&n->list_lock, flags); -+ raw_spin_unlock_irqrestore(&n->list_lock, flags); - return; - - slab_empty: -@@ -2682,7 +2732,7 @@ - remove_full(s, n, page); - } - -- spin_unlock_irqrestore(&n->list_lock, flags); -+ raw_spin_unlock_irqrestore(&n->list_lock, flags); - stat(s, FREE_SLAB); - discard_slab(s, page); - } -@@ -2881,7 +2931,7 @@ - init_kmem_cache_node(struct kmem_cache_node *n) - { - n->nr_partial = 0; -- spin_lock_init(&n->list_lock); -+ raw_spin_lock_init(&n->list_lock); - INIT_LIST_HEAD(&n->partial); - #ifdef CONFIG_SLUB_DEBUG - atomic_long_set(&n->nr_slabs, 0); -@@ -3463,7 +3513,7 @@ - for (i = 0; i < SHRINK_PROMOTE_MAX; i++) - INIT_LIST_HEAD(promote + i); - -- spin_lock_irqsave(&n->list_lock, flags); -+ raw_spin_lock_irqsave(&n->list_lock, flags); - - /* - * Build lists of slabs to discard or promote. -@@ -3494,7 +3544,7 @@ - for (i = SHRINK_PROMOTE_MAX - 1; i >= 0; i--) - list_splice(promote + i, &n->partial); - -- spin_unlock_irqrestore(&n->list_lock, flags); -+ raw_spin_unlock_irqrestore(&n->list_lock, flags); - - /* Release empty slabs */ - list_for_each_entry_safe(page, t, &discard, lru) -@@ -3670,6 +3720,12 @@ - { - static __initdata struct kmem_cache boot_kmem_cache, - boot_kmem_cache_node; -+ int cpu; + -+ for_each_possible_cpu(cpu) { -+ raw_spin_lock_init(&per_cpu(slub_free_list, cpu).lock); -+ INIT_LIST_HEAD(&per_cpu(slub_free_list, cpu).list); ++ return cgroup_add_cftypes(ss, cfts); ++} ++ ++/** ++ * cgroup_task_count - count the number of tasks in a cgroup. ++ * @cgrp: the cgroup in question ++ * ++ * Return the number of tasks in the cgroup. ++ */ ++static int cgroup_task_count(const struct cgroup *cgrp) ++{ ++ int count = 0; ++ struct cgrp_cset_link *link; ++ ++ down_read(&css_set_rwsem); ++ list_for_each_entry(link, &cgrp->cset_links, cset_link) ++ count += atomic_read(&link->cset->refcount); ++ up_read(&css_set_rwsem); ++ return count; ++} ++ ++/** ++ * css_next_child - find the next child of a given css ++ * @pos: the current position (%NULL to initiate traversal) ++ * @parent: css whose children to walk ++ * ++ * This function returns the next child of @parent and should be called ++ * under either cgroup_mutex or RCU read lock. The only requirement is ++ * that @parent and @pos are accessible. The next sibling is guaranteed to ++ * be returned regardless of their states. ++ * ++ * If a subsystem synchronizes ->css_online() and the start of iteration, a ++ * css which finished ->css_online() is guaranteed to be visible in the ++ * future iterations and will stay visible until the last reference is put. ++ * A css which hasn't finished ->css_online() or already finished ++ * ->css_offline() may show up during traversal. It's each subsystem's ++ * responsibility to synchronize against on/offlining. ++ */ ++struct cgroup_subsys_state *css_next_child(struct cgroup_subsys_state *pos, ++ struct cgroup_subsys_state *parent) ++{ ++ struct cgroup_subsys_state *next; ++ ++ cgroup_assert_mutex_or_rcu_locked(); ++ ++ /* ++ * @pos could already have been unlinked from the sibling list. ++ * Once a cgroup is removed, its ->sibling.next is no longer ++ * updated when its next sibling changes. CSS_RELEASED is set when ++ * @pos is taken off list, at which time its next pointer is valid, ++ * and, as releases are serialized, the one pointed to by the next ++ * pointer is guaranteed to not have started release yet. This ++ * implies that if we observe !CSS_RELEASED on @pos in this RCU ++ * critical section, the one pointed to by its next pointer is ++ * guaranteed to not have finished its RCU grace period even if we ++ * have dropped rcu_read_lock() inbetween iterations. ++ * ++ * If @pos has CSS_RELEASED set, its next pointer can't be ++ * dereferenced; however, as each css is given a monotonically ++ * increasing unique serial number and always appended to the ++ * sibling list, the next one can be found by walking the parent's ++ * children until the first css with higher serial number than ++ * @pos's. While this path can be slower, it happens iff iteration ++ * races against release and the race window is very small. ++ */ ++ if (!pos) { ++ next = list_entry_rcu(parent->children.next, struct cgroup_subsys_state, sibling); ++ } else if (likely(!(pos->flags & CSS_RELEASED))) { ++ next = list_entry_rcu(pos->sibling.next, struct cgroup_subsys_state, sibling); ++ } else { ++ list_for_each_entry_rcu(next, &parent->children, sibling) ++ if (next->serial_nr > pos->serial_nr) ++ break; + } - - if (debug_guardpage_minorder()) - slub_max_order = 0; -@@ -3912,7 +3968,7 @@ - struct page *page; - unsigned long flags; - -- spin_lock_irqsave(&n->list_lock, flags); -+ raw_spin_lock_irqsave(&n->list_lock, flags); - - list_for_each_entry(page, &n->partial, lru) { - validate_slab_slab(s, page, map); -@@ -3934,7 +3990,7 @@ - s->name, count, atomic_long_read(&n->nr_slabs)); - - out: -- spin_unlock_irqrestore(&n->list_lock, flags); -+ raw_spin_unlock_irqrestore(&n->list_lock, flags); - return count; - } - -@@ -4122,12 +4178,12 @@ - if (!atomic_long_read(&n->nr_slabs)) - continue; - -- spin_lock_irqsave(&n->list_lock, flags); -+ raw_spin_lock_irqsave(&n->list_lock, flags); - list_for_each_entry(page, &n->partial, lru) - process_slab(&t, s, page, alloc, map); - list_for_each_entry(page, &n->full, lru) - process_slab(&t, s, page, alloc, map); -- spin_unlock_irqrestore(&n->list_lock, flags); -+ raw_spin_unlock_irqrestore(&n->list_lock, flags); - } - - for (i = 0; i < t.count; i++) { -diff -Nur linux-4.1.6.orig/mm/swap.c linux-4.1.6/mm/swap.c ---- linux-4.1.6.orig/mm/swap.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/mm/swap.c 2015-09-08 23:49:08.441830131 +0200 -@@ -32,6 +32,7 @@ - #include - #include - #include -+#include - - #include "internal.h" - -@@ -45,6 +46,9 @@ - static DEFINE_PER_CPU(struct pagevec, lru_rotate_pvecs); - static DEFINE_PER_CPU(struct pagevec, lru_deactivate_file_pvecs); - -+static DEFINE_LOCAL_IRQ_LOCK(rotate_lock); -+DEFINE_LOCAL_IRQ_LOCK(swapvec_lock); + - /* - * This path almost never happens for VM activity - pages are normally - * freed via pagevecs. But it gets used by networking. -@@ -481,11 +485,11 @@ - unsigned long flags; - - page_cache_get(page); -- local_irq_save(flags); -+ local_lock_irqsave(rotate_lock, flags); - pvec = this_cpu_ptr(&lru_rotate_pvecs); - if (!pagevec_add(pvec, page)) - pagevec_move_tail(pvec); -- local_irq_restore(flags); -+ local_unlock_irqrestore(rotate_lock, flags); - } - } - -@@ -536,12 +540,13 @@ - void activate_page(struct page *page) - { - if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) { -- struct pagevec *pvec = &get_cpu_var(activate_page_pvecs); -+ struct pagevec *pvec = &get_locked_var(swapvec_lock, -+ activate_page_pvecs); - - page_cache_get(page); - if (!pagevec_add(pvec, page)) - pagevec_lru_move_fn(pvec, __activate_page, NULL); -- put_cpu_var(activate_page_pvecs); -+ put_locked_var(swapvec_lock, activate_page_pvecs); - } - } - -@@ -567,7 +572,7 @@ - - static void __lru_cache_activate_page(struct page *page) - { -- struct pagevec *pvec = &get_cpu_var(lru_add_pvec); -+ struct pagevec *pvec = &get_locked_var(swapvec_lock, lru_add_pvec); - int i; - - /* -@@ -589,7 +594,7 @@ - } - } - -- put_cpu_var(lru_add_pvec); -+ put_locked_var(swapvec_lock, lru_add_pvec); - } - - /* -@@ -628,13 +633,13 @@ - - static void __lru_cache_add(struct page *page) - { -- struct pagevec *pvec = &get_cpu_var(lru_add_pvec); -+ struct pagevec *pvec = &get_locked_var(swapvec_lock, lru_add_pvec); - - page_cache_get(page); - if (!pagevec_space(pvec)) - __pagevec_lru_add(pvec); - pagevec_add(pvec, page); -- put_cpu_var(lru_add_pvec); -+ put_locked_var(swapvec_lock, lru_add_pvec); - } - - /** -@@ -814,9 +819,9 @@ - unsigned long flags; - - /* No harm done if a racing interrupt already did this */ -- local_irq_save(flags); -+ local_lock_irqsave(rotate_lock, flags); - pagevec_move_tail(pvec); -- local_irq_restore(flags); -+ local_unlock_irqrestore(rotate_lock, flags); - } - - pvec = &per_cpu(lru_deactivate_file_pvecs, cpu); -@@ -844,18 +849,19 @@ - return; - - if (likely(get_page_unless_zero(page))) { -- struct pagevec *pvec = &get_cpu_var(lru_deactivate_file_pvecs); -+ struct pagevec *pvec = &get_locked_var(swapvec_lock, -+ lru_deactivate_file_pvecs); - - if (!pagevec_add(pvec, page)) - pagevec_lru_move_fn(pvec, lru_deactivate_file_fn, NULL); -- put_cpu_var(lru_deactivate_file_pvecs); -+ put_locked_var(swapvec_lock, lru_deactivate_file_pvecs); - } - } - - void lru_add_drain(void) - { -- lru_add_drain_cpu(get_cpu()); -- put_cpu(); -+ lru_add_drain_cpu(local_lock_cpu(swapvec_lock)); -+ local_unlock_cpu(swapvec_lock); - } - - static void lru_add_drain_per_cpu(struct work_struct *dummy) -diff -Nur linux-4.1.6.orig/mm/truncate.c linux-4.1.6/mm/truncate.c ---- linux-4.1.6.orig/mm/truncate.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/mm/truncate.c 2015-09-08 23:49:08.441830131 +0200 -@@ -56,8 +56,11 @@ - * protected by mapping->tree_lock. - */ - if (!workingset_node_shadows(node) && -- !list_empty(&node->private_list)) -- list_lru_del(&workingset_shadow_nodes, &node->private_list); -+ !list_empty(&node->private_list)) { -+ local_lock(workingset_shadow_lock); -+ list_lru_del(&__workingset_shadow_nodes, &node->private_list); -+ local_unlock(workingset_shadow_lock); ++ /* ++ * @next, if not pointing to the head, can be dereferenced and is ++ * the next sibling. ++ */ ++ if (&next->sibling != &parent->children) ++ return next; ++ return NULL; ++} ++ ++/** ++ * css_next_descendant_pre - find the next descendant for pre-order walk ++ * @pos: the current position (%NULL to initiate traversal) ++ * @root: css whose descendants to walk ++ * ++ * To be used by css_for_each_descendant_pre(). Find the next descendant ++ * to visit for pre-order traversal of @root's descendants. @root is ++ * included in the iteration and the first node to be visited. ++ * ++ * While this function requires cgroup_mutex or RCU read locking, it ++ * doesn't require the whole traversal to be contained in a single critical ++ * section. This function will return the correct next descendant as long ++ * as both @pos and @root are accessible and @pos is a descendant of @root. ++ * ++ * If a subsystem synchronizes ->css_online() and the start of iteration, a ++ * css which finished ->css_online() is guaranteed to be visible in the ++ * future iterations and will stay visible until the last reference is put. ++ * A css which hasn't finished ->css_online() or already finished ++ * ->css_offline() may show up during traversal. It's each subsystem's ++ * responsibility to synchronize against on/offlining. ++ */ ++struct cgroup_subsys_state * ++css_next_descendant_pre(struct cgroup_subsys_state *pos, ++ struct cgroup_subsys_state *root) ++{ ++ struct cgroup_subsys_state *next; ++ ++ cgroup_assert_mutex_or_rcu_locked(); ++ ++ /* if first iteration, visit @root */ ++ if (!pos) ++ return root; ++ ++ /* visit the first child if exists */ ++ next = css_next_child(NULL, pos); ++ if (next) ++ return next; ++ ++ /* no child, visit my or the closest ancestor's next sibling */ ++ while (pos != root) { ++ next = css_next_child(pos, pos->parent); ++ if (next) ++ return next; ++ pos = pos->parent; + } - __radix_tree_delete_node(&mapping->page_tree, node); - unlock: - spin_unlock_irq(&mapping->tree_lock); -diff -Nur linux-4.1.6.orig/mm/vmalloc.c linux-4.1.6/mm/vmalloc.c ---- linux-4.1.6.orig/mm/vmalloc.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/mm/vmalloc.c 2015-09-08 23:49:08.441830131 +0200 -@@ -819,7 +819,7 @@ - struct vmap_block *vb; - struct vmap_area *va; - unsigned long vb_idx; -- int node, err; -+ int node, err, cpu; - void *vaddr; - - node = numa_node_id(); -@@ -862,11 +862,12 @@ - BUG_ON(err); - radix_tree_preload_end(); - -- vbq = &get_cpu_var(vmap_block_queue); -+ cpu = get_cpu_light(); -+ vbq = this_cpu_ptr(&vmap_block_queue); - spin_lock(&vbq->lock); - list_add_tail_rcu(&vb->free_list, &vbq->free); - spin_unlock(&vbq->lock); -- put_cpu_var(vmap_block_queue); -+ put_cpu_light(); - - return vaddr; - } -@@ -935,6 +936,7 @@ - struct vmap_block *vb; - void *vaddr = NULL; - unsigned int order; -+ int cpu; - - BUG_ON(size & ~PAGE_MASK); - BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC); -@@ -949,7 +951,8 @@ - order = get_order(size); - - rcu_read_lock(); -- vbq = &get_cpu_var(vmap_block_queue); -+ cpu = get_cpu_light(); -+ vbq = this_cpu_ptr(&vmap_block_queue); - list_for_each_entry_rcu(vb, &vbq->free, free_list) { - unsigned long pages_off; - ++ ++ return NULL; ++} ++ ++/** ++ * css_rightmost_descendant - return the rightmost descendant of a css ++ * @pos: css of interest ++ * ++ * Return the rightmost descendant of @pos. If there's no descendant, @pos ++ * is returned. This can be used during pre-order traversal to skip ++ * subtree of @pos. ++ * ++ * While this function requires cgroup_mutex or RCU read locking, it ++ * doesn't require the whole traversal to be contained in a single critical ++ * section. This function will return the correct rightmost descendant as ++ * long as @pos is accessible. ++ */ ++struct cgroup_subsys_state * ++css_rightmost_descendant(struct cgroup_subsys_state *pos) ++{ ++ struct cgroup_subsys_state *last, *tmp; ++ ++ cgroup_assert_mutex_or_rcu_locked(); ++ ++ do { ++ last = pos; ++ /* ->prev isn't RCU safe, walk ->next till the end */ ++ pos = NULL; ++ css_for_each_child(tmp, last) ++ pos = tmp; ++ } while (pos); ++ ++ return last; ++} ++ ++static struct cgroup_subsys_state * ++css_leftmost_descendant(struct cgroup_subsys_state *pos) ++{ ++ struct cgroup_subsys_state *last; ++ ++ do { ++ last = pos; ++ pos = css_next_child(NULL, pos); ++ } while (pos); ++ ++ return last; ++} ++ ++/** ++ * css_next_descendant_post - find the next descendant for post-order walk ++ * @pos: the current position (%NULL to initiate traversal) ++ * @root: css whose descendants to walk ++ * ++ * To be used by css_for_each_descendant_post(). Find the next descendant ++ * to visit for post-order traversal of @root's descendants. @root is ++ * included in the iteration and the last node to be visited. ++ * ++ * While this function requires cgroup_mutex or RCU read locking, it ++ * doesn't require the whole traversal to be contained in a single critical ++ * section. This function will return the correct next descendant as long ++ * as both @pos and @cgroup are accessible and @pos is a descendant of ++ * @cgroup. ++ * ++ * If a subsystem synchronizes ->css_online() and the start of iteration, a ++ * css which finished ->css_online() is guaranteed to be visible in the ++ * future iterations and will stay visible until the last reference is put. ++ * A css which hasn't finished ->css_online() or already finished ++ * ->css_offline() may show up during traversal. It's each subsystem's ++ * responsibility to synchronize against on/offlining. ++ */ ++struct cgroup_subsys_state * ++css_next_descendant_post(struct cgroup_subsys_state *pos, ++ struct cgroup_subsys_state *root) ++{ ++ struct cgroup_subsys_state *next; ++ ++ cgroup_assert_mutex_or_rcu_locked(); ++ ++ /* if first iteration, visit leftmost descendant which may be @root */ ++ if (!pos) ++ return css_leftmost_descendant(root); ++ ++ /* if we visited @root, we're done */ ++ if (pos == root) ++ return NULL; ++ ++ /* if there's an unvisited sibling, visit its leftmost descendant */ ++ next = css_next_child(pos, pos->parent); ++ if (next) ++ return css_leftmost_descendant(next); ++ ++ /* no sibling left, visit parent */ ++ return pos->parent; ++} ++ ++/** ++ * css_has_online_children - does a css have online children ++ * @css: the target css ++ * ++ * Returns %true if @css has any online children; otherwise, %false. This ++ * function can be called from any context but the caller is responsible ++ * for synchronizing against on/offlining as necessary. ++ */ ++bool css_has_online_children(struct cgroup_subsys_state *css) ++{ ++ struct cgroup_subsys_state *child; ++ bool ret = false; ++ ++ rcu_read_lock(); ++ css_for_each_child(child, css) { ++ if (child->flags & CSS_ONLINE) { ++ ret = true; ++ break; ++ } ++ } ++ rcu_read_unlock(); ++ return ret; ++} ++ ++/** ++ * css_advance_task_iter - advance a task itererator to the next css_set ++ * @it: the iterator to advance ++ * ++ * Advance @it to the next css_set to walk. ++ */ ++static void css_advance_task_iter(struct css_task_iter *it) ++{ ++ struct list_head *l = it->cset_pos; ++ struct cgrp_cset_link *link; ++ struct css_set *cset; ++ ++ /* Advance to the next non-empty css_set */ ++ do { ++ l = l->next; ++ if (l == it->cset_head) { ++ it->cset_pos = NULL; ++ return; ++ } ++ ++ if (it->ss) { ++ cset = container_of(l, struct css_set, ++ e_cset_node[it->ss->id]); ++ } else { ++ link = list_entry(l, struct cgrp_cset_link, cset_link); ++ cset = link->cset; ++ } ++ } while (list_empty(&cset->tasks) && list_empty(&cset->mg_tasks)); ++ ++ it->cset_pos = l; ++ ++ if (!list_empty(&cset->tasks)) ++ it->task_pos = cset->tasks.next; ++ else ++ it->task_pos = cset->mg_tasks.next; ++ ++ it->tasks_head = &cset->tasks; ++ it->mg_tasks_head = &cset->mg_tasks; ++} ++ ++/** ++ * css_task_iter_start - initiate task iteration ++ * @css: the css to walk tasks of ++ * @it: the task iterator to use ++ * ++ * Initiate iteration through the tasks of @css. The caller can call ++ * css_task_iter_next() to walk through the tasks until the function ++ * returns NULL. On completion of iteration, css_task_iter_end() must be ++ * called. ++ * ++ * Note that this function acquires a lock which is released when the ++ * iteration finishes. The caller can't sleep while iteration is in ++ * progress. ++ */ ++void css_task_iter_start(struct cgroup_subsys_state *css, ++ struct css_task_iter *it) ++ __acquires(css_set_rwsem) ++{ ++ /* no one should try to iterate before mounting cgroups */ ++ WARN_ON_ONCE(!use_task_css_set_links); ++ ++ down_read(&css_set_rwsem); ++ ++ it->ss = css->ss; ++ ++ if (it->ss) ++ it->cset_pos = &css->cgroup->e_csets[css->ss->id]; ++ else ++ it->cset_pos = &css->cgroup->cset_links; ++ ++ it->cset_head = it->cset_pos; ++ ++ css_advance_task_iter(it); ++} ++ ++/** ++ * css_task_iter_next - return the next task for the iterator ++ * @it: the task iterator being iterated ++ * ++ * The "next" function for task iteration. @it should have been ++ * initialized via css_task_iter_start(). Returns NULL when the iteration ++ * reaches the end. ++ */ ++struct task_struct *css_task_iter_next(struct css_task_iter *it) ++{ ++ struct task_struct *res; ++ struct list_head *l = it->task_pos; ++ ++ /* If the iterator cg is NULL, we have no tasks */ ++ if (!it->cset_pos) ++ return NULL; ++ res = list_entry(l, struct task_struct, cg_list); ++ ++ /* ++ * Advance iterator to find next entry. cset->tasks is consumed ++ * first and then ->mg_tasks. After ->mg_tasks, we move onto the ++ * next cset. ++ */ ++ l = l->next; ++ ++ if (l == it->tasks_head) ++ l = it->mg_tasks_head->next; ++ ++ if (l == it->mg_tasks_head) ++ css_advance_task_iter(it); ++ else ++ it->task_pos = l; ++ ++ return res; ++} ++ ++/** ++ * css_task_iter_end - finish task iteration ++ * @it: the task iterator to finish ++ * ++ * Finish task iteration started by css_task_iter_start(). ++ */ ++void css_task_iter_end(struct css_task_iter *it) ++ __releases(css_set_rwsem) ++{ ++ up_read(&css_set_rwsem); ++} ++ ++/** ++ * cgroup_trasnsfer_tasks - move tasks from one cgroup to another ++ * @to: cgroup to which the tasks will be moved ++ * @from: cgroup in which the tasks currently reside ++ * ++ * Locking rules between cgroup_post_fork() and the migration path ++ * guarantee that, if a task is forking while being migrated, the new child ++ * is guaranteed to be either visible in the source cgroup after the ++ * parent's migration is complete or put into the target cgroup. No task ++ * can slip out of migration through forking. ++ */ ++int cgroup_transfer_tasks(struct cgroup *to, struct cgroup *from) ++{ ++ LIST_HEAD(preloaded_csets); ++ struct cgrp_cset_link *link; ++ struct css_task_iter it; ++ struct task_struct *task; ++ int ret; ++ ++ mutex_lock(&cgroup_mutex); ++ ++ /* all tasks in @from are being moved, all csets are source */ ++ down_read(&css_set_rwsem); ++ list_for_each_entry(link, &from->cset_links, cset_link) ++ cgroup_migrate_add_src(link->cset, to, &preloaded_csets); ++ up_read(&css_set_rwsem); ++ ++ ret = cgroup_migrate_prepare_dst(to, &preloaded_csets); ++ if (ret) ++ goto out_err; ++ ++ /* ++ * Migrate tasks one-by-one until @form is empty. This fails iff ++ * ->can_attach() fails. ++ */ ++ do { ++ css_task_iter_start(&from->self, &it); ++ task = css_task_iter_next(&it); ++ if (task) ++ get_task_struct(task); ++ css_task_iter_end(&it); ++ ++ if (task) { ++ ret = cgroup_migrate(to, task, false); ++ put_task_struct(task); ++ } ++ } while (task && !ret); ++out_err: ++ cgroup_migrate_finish(&preloaded_csets); ++ mutex_unlock(&cgroup_mutex); ++ return ret; ++} ++ ++/* ++ * Stuff for reading the 'tasks'/'procs' files. ++ * ++ * Reading this file can return large amounts of data if a cgroup has ++ * *lots* of attached tasks. So it may need several calls to read(), ++ * but we cannot guarantee that the information we produce is correct ++ * unless we produce it entirely atomically. ++ * ++ */ ++ ++/* which pidlist file are we talking about? */ ++enum cgroup_filetype { ++ CGROUP_FILE_PROCS, ++ CGROUP_FILE_TASKS, ++}; ++ ++/* ++ * A pidlist is a list of pids that virtually represents the contents of one ++ * of the cgroup files ("procs" or "tasks"). We keep a list of such pidlists, ++ * a pair (one each for procs, tasks) for each pid namespace that's relevant ++ * to the cgroup. ++ */ ++struct cgroup_pidlist { ++ /* ++ * used to find which pidlist is wanted. doesn't change as long as ++ * this particular list stays in the list. ++ */ ++ struct { enum cgroup_filetype type; struct pid_namespace *ns; } key; ++ /* array of xids */ ++ pid_t *list; ++ /* how many elements the above list has */ ++ int length; ++ /* each of these stored in a list by its cgroup */ ++ struct list_head links; ++ /* pointer to the cgroup we belong to, for list removal purposes */ ++ struct cgroup *owner; ++ /* for delayed destruction */ ++ struct delayed_work destroy_dwork; ++}; ++ ++/* ++ * The following two functions "fix" the issue where there are more pids ++ * than kmalloc will give memory for; in such cases, we use vmalloc/vfree. ++ * TODO: replace with a kernel-wide solution to this problem ++ */ ++#define PIDLIST_TOO_LARGE(c) ((c) * sizeof(pid_t) > (PAGE_SIZE * 2)) ++static void *pidlist_allocate(int count) ++{ ++ if (PIDLIST_TOO_LARGE(count)) ++ return vmalloc(count * sizeof(pid_t)); ++ else ++ return kmalloc(count * sizeof(pid_t), GFP_KERNEL); ++} ++ ++static void pidlist_free(void *p) ++{ ++ kvfree(p); ++} ++ ++/* ++ * Used to destroy all pidlists lingering waiting for destroy timer. None ++ * should be left afterwards. ++ */ ++static void cgroup_pidlist_destroy_all(struct cgroup *cgrp) ++{ ++ struct cgroup_pidlist *l, *tmp_l; ++ ++ mutex_lock(&cgrp->pidlist_mutex); ++ list_for_each_entry_safe(l, tmp_l, &cgrp->pidlists, links) ++ mod_delayed_work(cgroup_pidlist_destroy_wq, &l->destroy_dwork, 0); ++ mutex_unlock(&cgrp->pidlist_mutex); ++ ++ flush_workqueue(cgroup_pidlist_destroy_wq); ++ BUG_ON(!list_empty(&cgrp->pidlists)); ++} ++ ++static void cgroup_pidlist_destroy_work_fn(struct work_struct *work) ++{ ++ struct delayed_work *dwork = to_delayed_work(work); ++ struct cgroup_pidlist *l = container_of(dwork, struct cgroup_pidlist, ++ destroy_dwork); ++ struct cgroup_pidlist *tofree = NULL; ++ ++ mutex_lock(&l->owner->pidlist_mutex); ++ ++ /* ++ * Destroy iff we didn't get queued again. The state won't change ++ * as destroy_dwork can only be queued while locked. ++ */ ++ if (!delayed_work_pending(dwork)) { ++ list_del(&l->links); ++ pidlist_free(l->list); ++ put_pid_ns(l->key.ns); ++ tofree = l; ++ } ++ ++ mutex_unlock(&l->owner->pidlist_mutex); ++ kfree(tofree); ++} ++ ++/* ++ * pidlist_uniq - given a kmalloc()ed list, strip out all duplicate entries ++ * Returns the number of unique elements. ++ */ ++static int pidlist_uniq(pid_t *list, int length) ++{ ++ int src, dest = 1; ++ ++ /* ++ * we presume the 0th element is unique, so i starts at 1. trivial ++ * edge cases first; no work needs to be done for either ++ */ ++ if (length == 0 || length == 1) ++ return length; ++ /* src and dest walk down the list; dest counts unique elements */ ++ for (src = 1; src < length; src++) { ++ /* find next unique element */ ++ while (list[src] == list[src-1]) { ++ src++; ++ if (src == length) ++ goto after; ++ } ++ /* dest always points to where the next unique element goes */ ++ list[dest] = list[src]; ++ dest++; ++ } ++after: ++ return dest; ++} ++ ++/* ++ * The two pid files - task and cgroup.procs - guaranteed that the result ++ * is sorted, which forced this whole pidlist fiasco. As pid order is ++ * different per namespace, each namespace needs differently sorted list, ++ * making it impossible to use, for example, single rbtree of member tasks ++ * sorted by task pointer. As pidlists can be fairly large, allocating one ++ * per open file is dangerous, so cgroup had to implement shared pool of ++ * pidlists keyed by cgroup and namespace. ++ * ++ * All this extra complexity was caused by the original implementation ++ * committing to an entirely unnecessary property. In the long term, we ++ * want to do away with it. Explicitly scramble sort order if on the ++ * default hierarchy so that no such expectation exists in the new ++ * interface. ++ * ++ * Scrambling is done by swapping every two consecutive bits, which is ++ * non-identity one-to-one mapping which disturbs sort order sufficiently. ++ */ ++static pid_t pid_fry(pid_t pid) ++{ ++ unsigned a = pid & 0x55555555; ++ unsigned b = pid & 0xAAAAAAAA; ++ ++ return (a << 1) | (b >> 1); ++} ++ ++static pid_t cgroup_pid_fry(struct cgroup *cgrp, pid_t pid) ++{ ++ if (cgroup_on_dfl(cgrp)) ++ return pid_fry(pid); ++ else ++ return pid; ++} ++ ++static int cmppid(const void *a, const void *b) ++{ ++ return *(pid_t *)a - *(pid_t *)b; ++} ++ ++static int fried_cmppid(const void *a, const void *b) ++{ ++ return pid_fry(*(pid_t *)a) - pid_fry(*(pid_t *)b); ++} ++ ++static struct cgroup_pidlist *cgroup_pidlist_find(struct cgroup *cgrp, ++ enum cgroup_filetype type) ++{ ++ struct cgroup_pidlist *l; ++ /* don't need task_nsproxy() if we're looking at ourself */ ++ struct pid_namespace *ns = task_active_pid_ns(current); ++ ++ lockdep_assert_held(&cgrp->pidlist_mutex); ++ ++ list_for_each_entry(l, &cgrp->pidlists, links) ++ if (l->key.type == type && l->key.ns == ns) ++ return l; ++ return NULL; ++} ++ ++/* ++ * find the appropriate pidlist for our purpose (given procs vs tasks) ++ * returns with the lock on that pidlist already held, and takes care ++ * of the use count, or returns NULL with no locks held if we're out of ++ * memory. ++ */ ++static struct cgroup_pidlist *cgroup_pidlist_find_create(struct cgroup *cgrp, ++ enum cgroup_filetype type) ++{ ++ struct cgroup_pidlist *l; ++ ++ lockdep_assert_held(&cgrp->pidlist_mutex); ++ ++ l = cgroup_pidlist_find(cgrp, type); ++ if (l) ++ return l; ++ ++ /* entry not found; create a new one */ ++ l = kzalloc(sizeof(struct cgroup_pidlist), GFP_KERNEL); ++ if (!l) ++ return l; ++ ++ INIT_DELAYED_WORK(&l->destroy_dwork, cgroup_pidlist_destroy_work_fn); ++ l->key.type = type; ++ /* don't need task_nsproxy() if we're looking at ourself */ ++ l->key.ns = get_pid_ns(task_active_pid_ns(current)); ++ l->owner = cgrp; ++ list_add(&l->links, &cgrp->pidlists); ++ return l; ++} ++ ++/* ++ * Load a cgroup's pidarray with either procs' tgids or tasks' pids ++ */ ++static int pidlist_array_load(struct cgroup *cgrp, enum cgroup_filetype type, ++ struct cgroup_pidlist **lp) ++{ ++ pid_t *array; ++ int length; ++ int pid, n = 0; /* used for populating the array */ ++ struct css_task_iter it; ++ struct task_struct *tsk; ++ struct cgroup_pidlist *l; ++ ++ lockdep_assert_held(&cgrp->pidlist_mutex); ++ ++ /* ++ * If cgroup gets more users after we read count, we won't have ++ * enough space - tough. This race is indistinguishable to the ++ * caller from the case that the additional cgroup users didn't ++ * show up until sometime later on. ++ */ ++ length = cgroup_task_count(cgrp); ++ array = pidlist_allocate(length); ++ if (!array) ++ return -ENOMEM; ++ /* now, populate the array */ ++ css_task_iter_start(&cgrp->self, &it); ++ while ((tsk = css_task_iter_next(&it))) { ++ if (unlikely(n == length)) ++ break; ++ /* get tgid or pid for procs or tasks file respectively */ ++ if (type == CGROUP_FILE_PROCS) ++ pid = task_tgid_vnr(tsk); ++ else ++ pid = task_pid_vnr(tsk); ++ if (pid > 0) /* make sure to only use valid results */ ++ array[n++] = pid; ++ } ++ css_task_iter_end(&it); ++ length = n; ++ /* now sort & (if procs) strip out duplicates */ ++ if (cgroup_on_dfl(cgrp)) ++ sort(array, length, sizeof(pid_t), fried_cmppid, NULL); ++ else ++ sort(array, length, sizeof(pid_t), cmppid, NULL); ++ if (type == CGROUP_FILE_PROCS) ++ length = pidlist_uniq(array, length); ++ ++ l = cgroup_pidlist_find_create(cgrp, type); ++ if (!l) { ++ pidlist_free(array); ++ return -ENOMEM; ++ } ++ ++ /* store array, freeing old if necessary */ ++ pidlist_free(l->list); ++ l->list = array; ++ l->length = length; ++ *lp = l; ++ return 0; ++} ++ ++/** ++ * cgroupstats_build - build and fill cgroupstats ++ * @stats: cgroupstats to fill information into ++ * @dentry: A dentry entry belonging to the cgroup for which stats have ++ * been requested. ++ * ++ * Build and fill cgroupstats so that taskstats can export it to user ++ * space. ++ */ ++int cgroupstats_build(struct cgroupstats *stats, struct dentry *dentry) ++{ ++ struct kernfs_node *kn = kernfs_node_from_dentry(dentry); ++ struct cgroup *cgrp; ++ struct css_task_iter it; ++ struct task_struct *tsk; ++ ++ /* it should be kernfs_node belonging to cgroupfs and is a directory */ ++ if (dentry->d_sb->s_type != &cgroup_fs_type || !kn || ++ kernfs_type(kn) != KERNFS_DIR) ++ return -EINVAL; ++ ++ mutex_lock(&cgroup_mutex); ++ ++ /* ++ * We aren't being called from kernfs and there's no guarantee on ++ * @kn->priv's validity. For this and css_tryget_online_from_dir(), ++ * @kn->priv is RCU safe. Let's do the RCU dancing. ++ */ ++ rcu_read_lock(); ++ cgrp = rcu_dereference(kn->priv); ++ if (!cgrp || cgroup_is_dead(cgrp)) { ++ rcu_read_unlock(); ++ mutex_unlock(&cgroup_mutex); ++ return -ENOENT; ++ } ++ rcu_read_unlock(); ++ ++ css_task_iter_start(&cgrp->self, &it); ++ while ((tsk = css_task_iter_next(&it))) { ++ switch (tsk->state) { ++ case TASK_RUNNING: ++ stats->nr_running++; ++ break; ++ case TASK_INTERRUPTIBLE: ++ stats->nr_sleeping++; ++ break; ++ case TASK_UNINTERRUPTIBLE: ++ stats->nr_uninterruptible++; ++ break; ++ case TASK_STOPPED: ++ stats->nr_stopped++; ++ break; ++ default: ++ if (delayacct_is_task_waiting_on_io(tsk)) ++ stats->nr_io_wait++; ++ break; ++ } ++ } ++ css_task_iter_end(&it); ++ ++ mutex_unlock(&cgroup_mutex); ++ return 0; ++} ++ ++ ++/* ++ * seq_file methods for the tasks/procs files. The seq_file position is the ++ * next pid to display; the seq_file iterator is a pointer to the pid ++ * in the cgroup->l->list array. ++ */ ++ ++static void *cgroup_pidlist_start(struct seq_file *s, loff_t *pos) ++{ ++ /* ++ * Initially we receive a position value that corresponds to ++ * one more than the last pid shown (or 0 on the first call or ++ * after a seek to the start). Use a binary-search to find the ++ * next pid to display, if any ++ */ ++ struct kernfs_open_file *of = s->private; ++ struct cgroup *cgrp = seq_css(s)->cgroup; ++ struct cgroup_pidlist *l; ++ enum cgroup_filetype type = seq_cft(s)->private; ++ int index = 0, pid = *pos; ++ int *iter, ret; ++ ++ mutex_lock(&cgrp->pidlist_mutex); ++ ++ /* ++ * !NULL @of->priv indicates that this isn't the first start() ++ * after open. If the matching pidlist is around, we can use that. ++ * Look for it. Note that @of->priv can't be used directly. It ++ * could already have been destroyed. ++ */ ++ if (of->priv) ++ of->priv = cgroup_pidlist_find(cgrp, type); ++ ++ /* ++ * Either this is the first start() after open or the matching ++ * pidlist has been destroyed inbetween. Create a new one. ++ */ ++ if (!of->priv) { ++ ret = pidlist_array_load(cgrp, type, ++ (struct cgroup_pidlist **)&of->priv); ++ if (ret) ++ return ERR_PTR(ret); ++ } ++ l = of->priv; ++ ++ if (pid) { ++ int end = l->length; ++ ++ while (index < end) { ++ int mid = (index + end) / 2; ++ if (cgroup_pid_fry(cgrp, l->list[mid]) == pid) { ++ index = mid; ++ break; ++ } else if (cgroup_pid_fry(cgrp, l->list[mid]) <= pid) ++ index = mid + 1; ++ else ++ end = mid; ++ } ++ } ++ /* If we're off the end of the array, we're done */ ++ if (index >= l->length) ++ return NULL; ++ /* Update the abstract position to be the actual pid that we found */ ++ iter = l->list + index; ++ *pos = cgroup_pid_fry(cgrp, *iter); ++ return iter; ++} ++ ++static void cgroup_pidlist_stop(struct seq_file *s, void *v) ++{ ++ struct kernfs_open_file *of = s->private; ++ struct cgroup_pidlist *l = of->priv; ++ ++ if (l) ++ mod_delayed_work(cgroup_pidlist_destroy_wq, &l->destroy_dwork, ++ CGROUP_PIDLIST_DESTROY_DELAY); ++ mutex_unlock(&seq_css(s)->cgroup->pidlist_mutex); ++} ++ ++static void *cgroup_pidlist_next(struct seq_file *s, void *v, loff_t *pos) ++{ ++ struct kernfs_open_file *of = s->private; ++ struct cgroup_pidlist *l = of->priv; ++ pid_t *p = v; ++ pid_t *end = l->list + l->length; ++ /* ++ * Advance to the next pid in the array. If this goes off the ++ * end, we're done ++ */ ++ p++; ++ if (p >= end) { ++ return NULL; ++ } else { ++ *pos = cgroup_pid_fry(seq_css(s)->cgroup, *p); ++ return p; ++ } ++} ++ ++static int cgroup_pidlist_show(struct seq_file *s, void *v) ++{ ++ seq_printf(s, "%d\n", *(int *)v); ++ ++ return 0; ++} ++ ++static u64 cgroup_read_notify_on_release(struct cgroup_subsys_state *css, ++ struct cftype *cft) ++{ ++ return notify_on_release(css->cgroup); ++} ++ ++static int cgroup_write_notify_on_release(struct cgroup_subsys_state *css, ++ struct cftype *cft, u64 val) ++{ ++ if (val) ++ set_bit(CGRP_NOTIFY_ON_RELEASE, &css->cgroup->flags); ++ else ++ clear_bit(CGRP_NOTIFY_ON_RELEASE, &css->cgroup->flags); ++ return 0; ++} ++ ++static u64 cgroup_clone_children_read(struct cgroup_subsys_state *css, ++ struct cftype *cft) ++{ ++ return test_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags); ++} ++ ++static int cgroup_clone_children_write(struct cgroup_subsys_state *css, ++ struct cftype *cft, u64 val) ++{ ++ if (val) ++ set_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags); ++ else ++ clear_bit(CGRP_CPUSET_CLONE_CHILDREN, &css->cgroup->flags); ++ return 0; ++} ++ ++/* cgroup core interface files for the default hierarchy */ ++static struct cftype cgroup_dfl_base_files[] = { ++ { ++ .name = "cgroup.procs", ++ .seq_start = cgroup_pidlist_start, ++ .seq_next = cgroup_pidlist_next, ++ .seq_stop = cgroup_pidlist_stop, ++ .seq_show = cgroup_pidlist_show, ++ .private = CGROUP_FILE_PROCS, ++ .write = cgroup_procs_write, ++ .mode = S_IRUGO | S_IWUSR, ++ }, ++ { ++ .name = "cgroup.controllers", ++ .flags = CFTYPE_ONLY_ON_ROOT, ++ .seq_show = cgroup_root_controllers_show, ++ }, ++ { ++ .name = "cgroup.controllers", ++ .flags = CFTYPE_NOT_ON_ROOT, ++ .seq_show = cgroup_controllers_show, ++ }, ++ { ++ .name = "cgroup.subtree_control", ++ .seq_show = cgroup_subtree_control_show, ++ .write = cgroup_subtree_control_write, ++ }, ++ { ++ .name = "cgroup.populated", ++ .flags = CFTYPE_NOT_ON_ROOT, ++ .seq_show = cgroup_populated_show, ++ }, ++ { } /* terminate */ ++}; ++ ++/* cgroup core interface files for the legacy hierarchies */ ++static struct cftype cgroup_legacy_base_files[] = { ++ { ++ .name = "cgroup.procs", ++ .seq_start = cgroup_pidlist_start, ++ .seq_next = cgroup_pidlist_next, ++ .seq_stop = cgroup_pidlist_stop, ++ .seq_show = cgroup_pidlist_show, ++ .private = CGROUP_FILE_PROCS, ++ .write = cgroup_procs_write, ++ .mode = S_IRUGO | S_IWUSR, ++ }, ++ { ++ .name = "cgroup.clone_children", ++ .read_u64 = cgroup_clone_children_read, ++ .write_u64 = cgroup_clone_children_write, ++ }, ++ { ++ .name = "cgroup.sane_behavior", ++ .flags = CFTYPE_ONLY_ON_ROOT, ++ .seq_show = cgroup_sane_behavior_show, ++ }, ++ { ++ .name = "tasks", ++ .seq_start = cgroup_pidlist_start, ++ .seq_next = cgroup_pidlist_next, ++ .seq_stop = cgroup_pidlist_stop, ++ .seq_show = cgroup_pidlist_show, ++ .private = CGROUP_FILE_TASKS, ++ .write = cgroup_tasks_write, ++ .mode = S_IRUGO | S_IWUSR, ++ }, ++ { ++ .name = "notify_on_release", ++ .read_u64 = cgroup_read_notify_on_release, ++ .write_u64 = cgroup_write_notify_on_release, ++ }, ++ { ++ .name = "release_agent", ++ .flags = CFTYPE_ONLY_ON_ROOT, ++ .seq_show = cgroup_release_agent_show, ++ .write = cgroup_release_agent_write, ++ .max_write_len = PATH_MAX - 1, ++ }, ++ { } /* terminate */ ++}; ++ ++/** ++ * cgroup_populate_dir - create subsys files in a cgroup directory ++ * @cgrp: target cgroup ++ * @subsys_mask: mask of the subsystem ids whose files should be added ++ * ++ * On failure, no file is added. ++ */ ++static int cgroup_populate_dir(struct cgroup *cgrp, unsigned int subsys_mask) ++{ ++ struct cgroup_subsys *ss; ++ int i, ret = 0; ++ ++ /* process cftsets of each subsystem */ ++ for_each_subsys(ss, i) { ++ struct cftype *cfts; ++ ++ if (!(subsys_mask & (1 << i))) ++ continue; ++ ++ list_for_each_entry(cfts, &ss->cfts, node) { ++ ret = cgroup_addrm_files(cgrp, cfts, true); ++ if (ret < 0) ++ goto err; ++ } ++ } ++ return 0; ++err: ++ cgroup_clear_dir(cgrp, subsys_mask); ++ return ret; ++} ++ ++/* ++ * css destruction is four-stage process. ++ * ++ * 1. Destruction starts. Killing of the percpu_ref is initiated. ++ * Implemented in kill_css(). ++ * ++ * 2. When the percpu_ref is confirmed to be visible as killed on all CPUs ++ * and thus css_tryget_online() is guaranteed to fail, the css can be ++ * offlined by invoking offline_css(). After offlining, the base ref is ++ * put. Implemented in css_killed_work_fn(). ++ * ++ * 3. When the percpu_ref reaches zero, the only possible remaining ++ * accessors are inside RCU read sections. css_release() schedules the ++ * RCU callback. ++ * ++ * 4. After the grace period, the css can be freed. Implemented in ++ * css_free_work_fn(). ++ * ++ * It is actually hairier because both step 2 and 4 require process context ++ * and thus involve punting to css->destroy_work adding two additional ++ * steps to the already complex sequence. ++ */ ++static void css_free_work_fn(struct work_struct *work) ++{ ++ struct cgroup_subsys_state *css = ++ container_of(work, struct cgroup_subsys_state, destroy_work); ++ struct cgroup_subsys *ss = css->ss; ++ struct cgroup *cgrp = css->cgroup; ++ ++ percpu_ref_exit(&css->refcnt); ++ ++ if (ss) { ++ /* css free path */ ++ int id = css->id; ++ ++ if (css->parent) ++ css_put(css->parent); ++ ++ ss->css_free(css); ++ cgroup_idr_remove(&ss->css_idr, id); ++ cgroup_put(cgrp); ++ } else { ++ /* cgroup free path */ ++ atomic_dec(&cgrp->root->nr_cgrps); ++ cgroup_pidlist_destroy_all(cgrp); ++ cancel_work_sync(&cgrp->release_agent_work); ++ ++ if (cgroup_parent(cgrp)) { ++ /* ++ * We get a ref to the parent, and put the ref when ++ * this cgroup is being freed, so it's guaranteed ++ * that the parent won't be destroyed before its ++ * children. ++ */ ++ cgroup_put(cgroup_parent(cgrp)); ++ kernfs_put(cgrp->kn); ++ kfree(cgrp); ++ } else { ++ /* ++ * This is root cgroup's refcnt reaching zero, ++ * which indicates that the root should be ++ * released. ++ */ ++ cgroup_destroy_root(cgrp->root); ++ } ++ } ++} ++ ++static void css_free_rcu_fn(struct rcu_head *rcu_head) ++{ ++ struct cgroup_subsys_state *css = ++ container_of(rcu_head, struct cgroup_subsys_state, rcu_head); ++ ++ INIT_WORK(&css->destroy_work, css_free_work_fn); ++ queue_work(cgroup_destroy_wq, &css->destroy_work); ++} ++ ++static void css_release_work_fn(struct work_struct *work) ++{ ++ struct cgroup_subsys_state *css = ++ container_of(work, struct cgroup_subsys_state, destroy_work); ++ struct cgroup_subsys *ss = css->ss; ++ struct cgroup *cgrp = css->cgroup; ++ ++ mutex_lock(&cgroup_mutex); ++ ++ css->flags |= CSS_RELEASED; ++ list_del_rcu(&css->sibling); ++ ++ if (ss) { ++ /* css release path */ ++ cgroup_idr_replace(&ss->css_idr, NULL, css->id); ++ if (ss->css_released) ++ ss->css_released(css); ++ } else { ++ /* cgroup release path */ ++ cgroup_idr_remove(&cgrp->root->cgroup_idr, cgrp->id); ++ cgrp->id = -1; ++ ++ /* ++ * There are two control paths which try to determine ++ * cgroup from dentry without going through kernfs - ++ * cgroupstats_build() and css_tryget_online_from_dir(). ++ * Those are supported by RCU protecting clearing of ++ * cgrp->kn->priv backpointer. ++ */ ++ RCU_INIT_POINTER(*(void __rcu __force **)&cgrp->kn->priv, NULL); ++ } ++ ++ mutex_unlock(&cgroup_mutex); ++ ++ call_rcu(&css->rcu_head, css_free_rcu_fn); ++} ++ ++static void css_release(struct percpu_ref *ref) ++{ ++ struct cgroup_subsys_state *css = ++ container_of(ref, struct cgroup_subsys_state, refcnt); ++ ++ INIT_WORK(&css->destroy_work, css_release_work_fn); ++ queue_work(cgroup_destroy_wq, &css->destroy_work); ++} ++ ++static void init_and_link_css(struct cgroup_subsys_state *css, ++ struct cgroup_subsys *ss, struct cgroup *cgrp) ++{ ++ lockdep_assert_held(&cgroup_mutex); ++ ++ cgroup_get(cgrp); ++ ++ memset(css, 0, sizeof(*css)); ++ css->cgroup = cgrp; ++ css->ss = ss; ++ INIT_LIST_HEAD(&css->sibling); ++ INIT_LIST_HEAD(&css->children); ++ css->serial_nr = css_serial_nr_next++; ++ ++ if (cgroup_parent(cgrp)) { ++ css->parent = cgroup_css(cgroup_parent(cgrp), ss); ++ css_get(css->parent); ++ } ++ ++ BUG_ON(cgroup_css(cgrp, ss)); ++} ++ ++/* invoke ->css_online() on a new CSS and mark it online if successful */ ++static int online_css(struct cgroup_subsys_state *css) ++{ ++ struct cgroup_subsys *ss = css->ss; ++ int ret = 0; ++ ++ lockdep_assert_held(&cgroup_mutex); ++ ++ if (ss->css_online) ++ ret = ss->css_online(css); ++ if (!ret) { ++ css->flags |= CSS_ONLINE; ++ rcu_assign_pointer(css->cgroup->subsys[ss->id], css); ++ } ++ return ret; ++} ++ ++/* if the CSS is online, invoke ->css_offline() on it and mark it offline */ ++static void offline_css(struct cgroup_subsys_state *css) ++{ ++ struct cgroup_subsys *ss = css->ss; ++ ++ lockdep_assert_held(&cgroup_mutex); ++ ++ if (!(css->flags & CSS_ONLINE)) ++ return; ++ ++ if (ss->css_offline) ++ ss->css_offline(css); ++ ++ css->flags &= ~CSS_ONLINE; ++ RCU_INIT_POINTER(css->cgroup->subsys[ss->id], NULL); ++ ++ wake_up_all(&css->cgroup->offline_waitq); ++} ++ ++/** ++ * create_css - create a cgroup_subsys_state ++ * @cgrp: the cgroup new css will be associated with ++ * @ss: the subsys of new css ++ * @visible: whether to create control knobs for the new css or not ++ * ++ * Create a new css associated with @cgrp - @ss pair. On success, the new ++ * css is online and installed in @cgrp with all interface files created if ++ * @visible. Returns 0 on success, -errno on failure. ++ */ ++static int create_css(struct cgroup *cgrp, struct cgroup_subsys *ss, ++ bool visible) ++{ ++ struct cgroup *parent = cgroup_parent(cgrp); ++ struct cgroup_subsys_state *parent_css = cgroup_css(parent, ss); ++ struct cgroup_subsys_state *css; ++ int err; ++ ++ lockdep_assert_held(&cgroup_mutex); ++ ++ css = ss->css_alloc(parent_css); ++ if (IS_ERR(css)) ++ return PTR_ERR(css); ++ ++ init_and_link_css(css, ss, cgrp); ++ ++ err = percpu_ref_init(&css->refcnt, css_release, 0, GFP_KERNEL); ++ if (err) ++ goto err_free_css; ++ ++ err = cgroup_idr_alloc(&ss->css_idr, NULL, 2, 0, GFP_NOWAIT); ++ if (err < 0) ++ goto err_free_percpu_ref; ++ css->id = err; ++ ++ if (visible) { ++ err = cgroup_populate_dir(cgrp, 1 << ss->id); ++ if (err) ++ goto err_free_id; ++ } ++ ++ /* @css is ready to be brought online now, make it visible */ ++ list_add_tail_rcu(&css->sibling, &parent_css->children); ++ cgroup_idr_replace(&ss->css_idr, css, css->id); ++ ++ err = online_css(css); ++ if (err) ++ goto err_list_del; ++ ++ if (ss->broken_hierarchy && !ss->warned_broken_hierarchy && ++ cgroup_parent(parent)) { ++ pr_warn("%s (%d) created nested cgroup for controller \"%s\" which has incomplete hierarchy support. Nested cgroups may change behavior in the future.\n", ++ current->comm, current->pid, ss->name); ++ if (!strcmp(ss->name, "memory")) ++ pr_warn("\"memory\" requires setting use_hierarchy to 1 on the root\n"); ++ ss->warned_broken_hierarchy = true; ++ } ++ ++ return 0; ++ ++err_list_del: ++ list_del_rcu(&css->sibling); ++ cgroup_clear_dir(css->cgroup, 1 << css->ss->id); ++err_free_id: ++ cgroup_idr_remove(&ss->css_idr, css->id); ++err_free_percpu_ref: ++ percpu_ref_exit(&css->refcnt); ++err_free_css: ++ call_rcu(&css->rcu_head, css_free_rcu_fn); ++ return err; ++} ++ ++static int cgroup_mkdir(struct kernfs_node *parent_kn, const char *name, ++ umode_t mode) ++{ ++ struct cgroup *parent, *cgrp; ++ struct cgroup_root *root; ++ struct cgroup_subsys *ss; ++ struct kernfs_node *kn; ++ struct cftype *base_files; ++ int ssid, ret; ++ ++ /* Do not accept '\n' to prevent making /proc//cgroup unparsable. ++ */ ++ if (strchr(name, '\n')) ++ return -EINVAL; ++ ++ parent = cgroup_kn_lock_live(parent_kn); ++ if (!parent) ++ return -ENODEV; ++ root = parent->root; ++ ++ /* allocate the cgroup and its ID, 0 is reserved for the root */ ++ cgrp = kzalloc(sizeof(*cgrp), GFP_KERNEL); ++ if (!cgrp) { ++ ret = -ENOMEM; ++ goto out_unlock; ++ } ++ ++ ret = percpu_ref_init(&cgrp->self.refcnt, css_release, 0, GFP_KERNEL); ++ if (ret) ++ goto out_free_cgrp; ++ ++ /* ++ * Temporarily set the pointer to NULL, so idr_find() won't return ++ * a half-baked cgroup. ++ */ ++ cgrp->id = cgroup_idr_alloc(&root->cgroup_idr, NULL, 2, 0, GFP_NOWAIT); ++ if (cgrp->id < 0) { ++ ret = -ENOMEM; ++ goto out_cancel_ref; ++ } ++ ++ init_cgroup_housekeeping(cgrp); ++ ++ cgrp->self.parent = &parent->self; ++ cgrp->root = root; ++ ++ if (notify_on_release(parent)) ++ set_bit(CGRP_NOTIFY_ON_RELEASE, &cgrp->flags); ++ ++ if (test_bit(CGRP_CPUSET_CLONE_CHILDREN, &parent->flags)) ++ set_bit(CGRP_CPUSET_CLONE_CHILDREN, &cgrp->flags); ++ ++ /* create the directory */ ++ kn = kernfs_create_dir(parent->kn, name, mode, cgrp); ++ if (IS_ERR(kn)) { ++ ret = PTR_ERR(kn); ++ goto out_free_id; ++ } ++ cgrp->kn = kn; ++ ++ /* ++ * This extra ref will be put in cgroup_free_fn() and guarantees ++ * that @cgrp->kn is always accessible. ++ */ ++ kernfs_get(kn); ++ ++ cgrp->self.serial_nr = css_serial_nr_next++; ++ ++ /* allocation complete, commit to creation */ ++ list_add_tail_rcu(&cgrp->self.sibling, &cgroup_parent(cgrp)->self.children); ++ atomic_inc(&root->nr_cgrps); ++ cgroup_get(parent); ++ ++ /* ++ * @cgrp is now fully operational. If something fails after this ++ * point, it'll be released via the normal destruction path. ++ */ ++ cgroup_idr_replace(&root->cgroup_idr, cgrp, cgrp->id); ++ ++ ret = cgroup_kn_set_ugid(kn); ++ if (ret) ++ goto out_destroy; ++ ++ if (cgroup_on_dfl(cgrp)) ++ base_files = cgroup_dfl_base_files; ++ else ++ base_files = cgroup_legacy_base_files; ++ ++ ret = cgroup_addrm_files(cgrp, base_files, true); ++ if (ret) ++ goto out_destroy; ++ ++ /* let's create and online css's */ ++ for_each_subsys(ss, ssid) { ++ if (parent->child_subsys_mask & (1 << ssid)) { ++ ret = create_css(cgrp, ss, ++ parent->subtree_control & (1 << ssid)); ++ if (ret) ++ goto out_destroy; ++ } ++ } ++ ++ /* ++ * On the default hierarchy, a child doesn't automatically inherit ++ * subtree_control from the parent. Each is configured manually. ++ */ ++ if (!cgroup_on_dfl(cgrp)) { ++ cgrp->subtree_control = parent->subtree_control; ++ cgroup_refresh_child_subsys_mask(cgrp); ++ } ++ ++ kernfs_activate(kn); ++ ++ ret = 0; ++ goto out_unlock; ++ ++out_free_id: ++ cgroup_idr_remove(&root->cgroup_idr, cgrp->id); ++out_cancel_ref: ++ percpu_ref_exit(&cgrp->self.refcnt); ++out_free_cgrp: ++ kfree(cgrp); ++out_unlock: ++ cgroup_kn_unlock(parent_kn); ++ return ret; ++ ++out_destroy: ++ cgroup_destroy_locked(cgrp); ++ goto out_unlock; ++} ++ ++/* ++ * This is called when the refcnt of a css is confirmed to be killed. ++ * css_tryget_online() is now guaranteed to fail. Tell the subsystem to ++ * initate destruction and put the css ref from kill_css(). ++ */ ++static void css_killed_work_fn(struct work_struct *work) ++{ ++ struct cgroup_subsys_state *css = ++ container_of(work, struct cgroup_subsys_state, destroy_work); ++ ++ mutex_lock(&cgroup_mutex); ++ offline_css(css); ++ mutex_unlock(&cgroup_mutex); ++ ++ css_put(css); ++} ++ ++/* css kill confirmation processing requires process context, bounce */ ++static void css_killed_ref_fn(struct percpu_ref *ref) ++{ ++ struct cgroup_subsys_state *css = ++ container_of(ref, struct cgroup_subsys_state, refcnt); ++ ++ INIT_WORK(&css->destroy_work, css_killed_work_fn); ++ queue_work(cgroup_destroy_wq, &css->destroy_work); ++} ++ ++/** ++ * kill_css - destroy a css ++ * @css: css to destroy ++ * ++ * This function initiates destruction of @css by removing cgroup interface ++ * files and putting its base reference. ->css_offline() will be invoked ++ * asynchronously once css_tryget_online() is guaranteed to fail and when ++ * the reference count reaches zero, @css will be released. ++ */ ++static void kill_css(struct cgroup_subsys_state *css) ++{ ++ lockdep_assert_held(&cgroup_mutex); ++ ++ /* ++ * This must happen before css is disassociated with its cgroup. ++ * See seq_css() for details. ++ */ ++ cgroup_clear_dir(css->cgroup, 1 << css->ss->id); ++ ++ /* ++ * Killing would put the base ref, but we need to keep it alive ++ * until after ->css_offline(). ++ */ ++ css_get(css); ++ ++ /* ++ * cgroup core guarantees that, by the time ->css_offline() is ++ * invoked, no new css reference will be given out via ++ * css_tryget_online(). We can't simply call percpu_ref_kill() and ++ * proceed to offlining css's because percpu_ref_kill() doesn't ++ * guarantee that the ref is seen as killed on all CPUs on return. ++ * ++ * Use percpu_ref_kill_and_confirm() to get notifications as each ++ * css is confirmed to be seen as killed on all CPUs. ++ */ ++ percpu_ref_kill_and_confirm(&css->refcnt, css_killed_ref_fn); ++} ++ ++/** ++ * cgroup_destroy_locked - the first stage of cgroup destruction ++ * @cgrp: cgroup to be destroyed ++ * ++ * css's make use of percpu refcnts whose killing latency shouldn't be ++ * exposed to userland and are RCU protected. Also, cgroup core needs to ++ * guarantee that css_tryget_online() won't succeed by the time ++ * ->css_offline() is invoked. To satisfy all the requirements, ++ * destruction is implemented in the following two steps. ++ * ++ * s1. Verify @cgrp can be destroyed and mark it dying. Remove all ++ * userland visible parts and start killing the percpu refcnts of ++ * css's. Set up so that the next stage will be kicked off once all ++ * the percpu refcnts are confirmed to be killed. ++ * ++ * s2. Invoke ->css_offline(), mark the cgroup dead and proceed with the ++ * rest of destruction. Once all cgroup references are gone, the ++ * cgroup is RCU-freed. ++ * ++ * This function implements s1. After this step, @cgrp is gone as far as ++ * the userland is concerned and a new cgroup with the same name may be ++ * created. As cgroup doesn't care about the names internally, this ++ * doesn't cause any problem. ++ */ ++static int cgroup_destroy_locked(struct cgroup *cgrp) ++ __releases(&cgroup_mutex) __acquires(&cgroup_mutex) ++{ ++ struct cgroup_subsys_state *css; ++ bool empty; ++ int ssid; ++ ++ lockdep_assert_held(&cgroup_mutex); ++ ++ /* ++ * css_set_rwsem synchronizes access to ->cset_links and prevents ++ * @cgrp from being removed while put_css_set() is in progress. ++ */ ++ down_read(&css_set_rwsem); ++ empty = list_empty(&cgrp->cset_links); ++ up_read(&css_set_rwsem); ++ if (!empty) ++ return -EBUSY; ++ ++ /* ++ * Make sure there's no live children. We can't test emptiness of ++ * ->self.children as dead children linger on it while being ++ * drained; otherwise, "rmdir parent/child parent" may fail. ++ */ ++ if (css_has_online_children(&cgrp->self)) ++ return -EBUSY; ++ ++ /* ++ * Mark @cgrp dead. This prevents further task migration and child ++ * creation by disabling cgroup_lock_live_group(). ++ */ ++ cgrp->self.flags &= ~CSS_ONLINE; ++ ++ /* initiate massacre of all css's */ ++ for_each_css(css, ssid, cgrp) ++ kill_css(css); ++ ++ /* ++ * Remove @cgrp directory along with the base files. @cgrp has an ++ * extra ref on its kn. ++ */ ++ kernfs_remove(cgrp->kn); ++ ++ check_for_release(cgroup_parent(cgrp)); ++ ++ /* put the base reference */ ++ percpu_ref_kill(&cgrp->self.refcnt); ++ ++ return 0; ++}; ++ ++static int cgroup_rmdir(struct kernfs_node *kn) ++{ ++ struct cgroup *cgrp; ++ int ret = 0; ++ ++ cgrp = cgroup_kn_lock_live(kn); ++ if (!cgrp) ++ return 0; ++ ++ ret = cgroup_destroy_locked(cgrp); ++ ++ cgroup_kn_unlock(kn); ++ return ret; ++} ++ ++static struct kernfs_syscall_ops cgroup_kf_syscall_ops = { ++ .remount_fs = cgroup_remount, ++ .show_options = cgroup_show_options, ++ .mkdir = cgroup_mkdir, ++ .rmdir = cgroup_rmdir, ++ .rename = cgroup_rename, ++}; ++ ++static void __init cgroup_init_subsys(struct cgroup_subsys *ss, bool early) ++{ ++ struct cgroup_subsys_state *css; ++ ++ printk(KERN_INFO "Initializing cgroup subsys %s\n", ss->name); ++ ++ mutex_lock(&cgroup_mutex); ++ ++ idr_init(&ss->css_idr); ++ INIT_LIST_HEAD(&ss->cfts); ++ ++ /* Create the root cgroup state for this subsystem */ ++ ss->root = &cgrp_dfl_root; ++ css = ss->css_alloc(cgroup_css(&cgrp_dfl_root.cgrp, ss)); ++ /* We don't handle early failures gracefully */ ++ BUG_ON(IS_ERR(css)); ++ init_and_link_css(css, ss, &cgrp_dfl_root.cgrp); ++ ++ /* ++ * Root csses are never destroyed and we can't initialize ++ * percpu_ref during early init. Disable refcnting. ++ */ ++ css->flags |= CSS_NO_REF; ++ ++ if (early) { ++ /* allocation can't be done safely during early init */ ++ css->id = 1; ++ } else { ++ css->id = cgroup_idr_alloc(&ss->css_idr, css, 1, 2, GFP_KERNEL); ++ BUG_ON(css->id < 0); ++ } ++ ++ /* Update the init_css_set to contain a subsys ++ * pointer to this state - since the subsystem is ++ * newly registered, all tasks and hence the ++ * init_css_set is in the subsystem's root cgroup. */ ++ init_css_set.subsys[ss->id] = css; ++ ++ need_forkexit_callback |= ss->fork || ss->exit; ++ ++ /* At system boot, before all subsystems have been ++ * registered, no tasks have been forked, so we don't ++ * need to invoke fork callbacks here. */ ++ BUG_ON(!list_empty(&init_task.tasks)); ++ ++ BUG_ON(online_css(css)); ++ ++ mutex_unlock(&cgroup_mutex); ++} ++ ++/** ++ * cgroup_init_early - cgroup initialization at system boot ++ * ++ * Initialize cgroups at system boot, and initialize any ++ * subsystems that request early init. ++ */ ++int __init cgroup_init_early(void) ++{ ++ static struct cgroup_sb_opts __initdata opts; ++ struct cgroup_subsys *ss; ++ int i; ++ ++ init_cgroup_root(&cgrp_dfl_root, &opts); ++ cgrp_dfl_root.cgrp.self.flags |= CSS_NO_REF; ++ ++ RCU_INIT_POINTER(init_task.cgroups, &init_css_set); ++ ++ for_each_subsys(ss, i) { ++ WARN(!ss->css_alloc || !ss->css_free || ss->name || ss->id, ++ "invalid cgroup_subsys %d:%s css_alloc=%p css_free=%p name:id=%d:%s\n", ++ i, cgroup_subsys_name[i], ss->css_alloc, ss->css_free, ++ ss->id, ss->name); ++ WARN(strlen(cgroup_subsys_name[i]) > MAX_CGROUP_TYPE_NAMELEN, ++ "cgroup_subsys_name %s too long\n", cgroup_subsys_name[i]); ++ ++ ss->id = i; ++ ss->name = cgroup_subsys_name[i]; ++ ++ if (ss->early_init) ++ cgroup_init_subsys(ss, true); ++ } ++ return 0; ++} ++ ++/** ++ * cgroup_init - cgroup initialization ++ * ++ * Register cgroup filesystem and /proc file, and initialize ++ * any subsystems that didn't request early init. ++ */ ++int __init cgroup_init(void) ++{ ++ struct cgroup_subsys *ss; ++ unsigned long key; ++ int ssid, err; ++ ++ BUG_ON(cgroup_init_cftypes(NULL, cgroup_dfl_base_files)); ++ BUG_ON(cgroup_init_cftypes(NULL, cgroup_legacy_base_files)); ++ ++ mutex_lock(&cgroup_mutex); ++ ++ /* Add init_css_set to the hash table */ ++ key = css_set_hash(init_css_set.subsys); ++ hash_add(css_set_table, &init_css_set.hlist, key); ++ ++ BUG_ON(cgroup_setup_root(&cgrp_dfl_root, 0)); ++ ++ mutex_unlock(&cgroup_mutex); ++ ++ for_each_subsys(ss, ssid) { ++ if (ss->early_init) { ++ struct cgroup_subsys_state *css = ++ init_css_set.subsys[ss->id]; ++ ++ css->id = cgroup_idr_alloc(&ss->css_idr, css, 1, 2, ++ GFP_KERNEL); ++ BUG_ON(css->id < 0); ++ } else { ++ cgroup_init_subsys(ss, false); ++ } ++ ++ list_add_tail(&init_css_set.e_cset_node[ssid], ++ &cgrp_dfl_root.cgrp.e_csets[ssid]); ++ ++ /* ++ * Setting dfl_root subsys_mask needs to consider the ++ * disabled flag and cftype registration needs kmalloc, ++ * both of which aren't available during early_init. ++ */ ++ if (ss->disabled) ++ continue; ++ ++ cgrp_dfl_root.subsys_mask |= 1 << ss->id; ++ ++ if (cgroup_legacy_files_on_dfl && !ss->dfl_cftypes) ++ ss->dfl_cftypes = ss->legacy_cftypes; ++ ++ if (!ss->dfl_cftypes) ++ cgrp_dfl_root_inhibit_ss_mask |= 1 << ss->id; ++ ++ if (ss->dfl_cftypes == ss->legacy_cftypes) { ++ WARN_ON(cgroup_add_cftypes(ss, ss->dfl_cftypes)); ++ } else { ++ WARN_ON(cgroup_add_dfl_cftypes(ss, ss->dfl_cftypes)); ++ WARN_ON(cgroup_add_legacy_cftypes(ss, ss->legacy_cftypes)); ++ } ++ ++ if (ss->bind) ++ ss->bind(init_css_set.subsys[ssid]); ++ } ++ ++ err = sysfs_create_mount_point(fs_kobj, "cgroup"); ++ if (err) ++ return err; ++ ++ err = register_filesystem(&cgroup_fs_type); ++ if (err < 0) { ++ sysfs_remove_mount_point(fs_kobj, "cgroup"); ++ return err; ++ } ++ ++ proc_create("cgroups", 0, NULL, &proc_cgroupstats_operations); ++ return 0; ++} ++ ++static int __init cgroup_wq_init(void) ++{ ++ /* ++ * There isn't much point in executing destruction path in ++ * parallel. Good chunk is serialized with cgroup_mutex anyway. ++ * Use 1 for @max_active. ++ * ++ * We would prefer to do this in cgroup_init() above, but that ++ * is called before init_workqueues(): so leave this until after. ++ */ ++ cgroup_destroy_wq = alloc_workqueue("cgroup_destroy", 0, 1); ++ BUG_ON(!cgroup_destroy_wq); ++ ++ /* ++ * Used to destroy pidlists and separate to serve as flush domain. ++ * Cap @max_active to 1 too. ++ */ ++ cgroup_pidlist_destroy_wq = alloc_workqueue("cgroup_pidlist_destroy", ++ 0, 1); ++ BUG_ON(!cgroup_pidlist_destroy_wq); ++ ++ return 0; ++} ++core_initcall(cgroup_wq_init); ++ ++/* ++ * proc_cgroup_show() ++ * - Print task's cgroup paths into seq_file, one line for each hierarchy ++ * - Used for /proc//cgroup. ++ */ ++int proc_cgroup_show(struct seq_file *m, struct pid_namespace *ns, ++ struct pid *pid, struct task_struct *tsk) ++{ ++ char *buf, *path; ++ int retval; ++ struct cgroup_root *root; ++ ++ retval = -ENOMEM; ++ buf = kmalloc(PATH_MAX, GFP_KERNEL); ++ if (!buf) ++ goto out; ++ ++ mutex_lock(&cgroup_mutex); ++ down_read(&css_set_rwsem); ++ ++ for_each_root(root) { ++ struct cgroup_subsys *ss; ++ struct cgroup *cgrp; ++ int ssid, count = 0; ++ ++ if (root == &cgrp_dfl_root && !cgrp_dfl_root_visible) ++ continue; ++ ++ seq_printf(m, "%d:", root->hierarchy_id); ++ for_each_subsys(ss, ssid) ++ if (root->subsys_mask & (1 << ssid)) ++ seq_printf(m, "%s%s", count++ ? "," : "", ss->name); ++ if (strlen(root->name)) ++ seq_printf(m, "%sname=%s", count ? "," : "", ++ root->name); ++ seq_putc(m, ':'); ++ cgrp = task_cgroup_from_root(tsk, root); ++ path = cgroup_path(cgrp, buf, PATH_MAX); ++ if (!path) { ++ retval = -ENAMETOOLONG; ++ goto out_unlock; ++ } ++ seq_puts(m, path); ++ seq_putc(m, '\n'); ++ } ++ ++ retval = 0; ++out_unlock: ++ up_read(&css_set_rwsem); ++ mutex_unlock(&cgroup_mutex); ++ kfree(buf); ++out: ++ return retval; ++} ++ ++/* Display information about each subsystem and each hierarchy */ ++static int proc_cgroupstats_show(struct seq_file *m, void *v) ++{ ++ struct cgroup_subsys *ss; ++ int i; ++ ++ seq_puts(m, "#subsys_name\thierarchy\tnum_cgroups\tenabled\n"); ++ /* ++ * ideally we don't want subsystems moving around while we do this. ++ * cgroup_mutex is also necessary to guarantee an atomic snapshot of ++ * subsys/hierarchy state. ++ */ ++ mutex_lock(&cgroup_mutex); ++ ++ for_each_subsys(ss, i) ++ seq_printf(m, "%s\t%d\t%d\t%d\n", ++ ss->name, ss->root->hierarchy_id, ++ atomic_read(&ss->root->nr_cgrps), !ss->disabled); ++ ++ mutex_unlock(&cgroup_mutex); ++ return 0; ++} ++ ++static int cgroupstats_open(struct inode *inode, struct file *file) ++{ ++ return single_open(file, proc_cgroupstats_show, NULL); ++} ++ ++static const struct file_operations proc_cgroupstats_operations = { ++ .open = cgroupstats_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = single_release, ++}; ++ ++/** ++ * cgroup_fork - initialize cgroup related fields during copy_process() ++ * @child: pointer to task_struct of forking parent process. ++ * ++ * A task is associated with the init_css_set until cgroup_post_fork() ++ * attaches it to the parent's css_set. Empty cg_list indicates that ++ * @child isn't holding reference to its css_set. ++ */ ++void cgroup_fork(struct task_struct *child) ++{ ++ RCU_INIT_POINTER(child->cgroups, &init_css_set); ++ INIT_LIST_HEAD(&child->cg_list); ++} ++ ++/** ++ * cgroup_post_fork - called on a new task after adding it to the task list ++ * @child: the task in question ++ * ++ * Adds the task to the list running through its css_set if necessary and ++ * call the subsystem fork() callbacks. Has to be after the task is ++ * visible on the task list in case we race with the first call to ++ * cgroup_task_iter_start() - to guarantee that the new task ends up on its ++ * list. ++ */ ++void cgroup_post_fork(struct task_struct *child) ++{ ++ struct cgroup_subsys *ss; ++ int i; ++ ++ /* ++ * This may race against cgroup_enable_task_cg_lists(). As that ++ * function sets use_task_css_set_links before grabbing ++ * tasklist_lock and we just went through tasklist_lock to add ++ * @child, it's guaranteed that either we see the set ++ * use_task_css_set_links or cgroup_enable_task_cg_lists() sees ++ * @child during its iteration. ++ * ++ * If we won the race, @child is associated with %current's ++ * css_set. Grabbing css_set_rwsem guarantees both that the ++ * association is stable, and, on completion of the parent's ++ * migration, @child is visible in the source of migration or ++ * already in the destination cgroup. This guarantee is necessary ++ * when implementing operations which need to migrate all tasks of ++ * a cgroup to another. ++ * ++ * Note that if we lose to cgroup_enable_task_cg_lists(), @child ++ * will remain in init_css_set. This is safe because all tasks are ++ * in the init_css_set before cg_links is enabled and there's no ++ * operation which transfers all tasks out of init_css_set. ++ */ ++ if (use_task_css_set_links) { ++ struct css_set *cset; ++ ++ down_write(&css_set_rwsem); ++ cset = task_css_set(current); ++ if (list_empty(&child->cg_list)) { ++ rcu_assign_pointer(child->cgroups, cset); ++ list_add(&child->cg_list, &cset->tasks); ++ get_css_set(cset); ++ } ++ up_write(&css_set_rwsem); ++ } ++ ++ /* ++ * Call ss->fork(). This must happen after @child is linked on ++ * css_set; otherwise, @child might change state between ->fork() ++ * and addition to css_set. ++ */ ++ if (need_forkexit_callback) { ++ for_each_subsys(ss, i) ++ if (ss->fork) ++ ss->fork(child); ++ } ++} ++ ++/** ++ * cgroup_exit - detach cgroup from exiting task ++ * @tsk: pointer to task_struct of exiting process ++ * ++ * Description: Detach cgroup from @tsk and release it. ++ * ++ * Note that cgroups marked notify_on_release force every task in ++ * them to take the global cgroup_mutex mutex when exiting. ++ * This could impact scaling on very large systems. Be reluctant to ++ * use notify_on_release cgroups where very high task exit scaling ++ * is required on large systems. ++ * ++ * We set the exiting tasks cgroup to the root cgroup (top_cgroup). We ++ * call cgroup_exit() while the task is still competent to handle ++ * notify_on_release(), then leave the task attached to the root cgroup in ++ * each hierarchy for the remainder of its exit. No need to bother with ++ * init_css_set refcnting. init_css_set never goes away and we can't race ++ * with migration path - PF_EXITING is visible to migration path. ++ */ ++void cgroup_exit(struct task_struct *tsk) ++{ ++ struct cgroup_subsys *ss; ++ struct css_set *cset; ++ bool put_cset = false; ++ int i; ++ ++ /* ++ * Unlink from @tsk from its css_set. As migration path can't race ++ * with us, we can check cg_list without grabbing css_set_rwsem. ++ */ ++ if (!list_empty(&tsk->cg_list)) { ++ down_write(&css_set_rwsem); ++ list_del_init(&tsk->cg_list); ++ up_write(&css_set_rwsem); ++ put_cset = true; ++ } ++ ++ /* Reassign the task to the init_css_set. */ ++ cset = task_css_set(tsk); ++ RCU_INIT_POINTER(tsk->cgroups, &init_css_set); ++ ++ if (need_forkexit_callback) { ++ /* see cgroup_post_fork() for details */ ++ for_each_subsys(ss, i) { ++ if (ss->exit) { ++ struct cgroup_subsys_state *old_css = cset->subsys[i]; ++ struct cgroup_subsys_state *css = task_css(tsk, i); ++ ++ ss->exit(css, old_css, tsk); ++ } ++ } ++ } ++ ++ if (put_cset) ++ put_css_set(cset); ++} ++ ++static void check_for_release(struct cgroup *cgrp) ++{ ++ if (notify_on_release(cgrp) && !cgroup_has_tasks(cgrp) && ++ !css_has_online_children(&cgrp->self) && !cgroup_is_dead(cgrp)) ++ schedule_work(&cgrp->release_agent_work); ++} ++ ++/* ++ * Notify userspace when a cgroup is released, by running the ++ * configured release agent with the name of the cgroup (path ++ * relative to the root of cgroup file system) as the argument. ++ * ++ * Most likely, this user command will try to rmdir this cgroup. ++ * ++ * This races with the possibility that some other task will be ++ * attached to this cgroup before it is removed, or that some other ++ * user task will 'mkdir' a child cgroup of this cgroup. That's ok. ++ * The presumed 'rmdir' will fail quietly if this cgroup is no longer ++ * unused, and this cgroup will be reprieved from its death sentence, ++ * to continue to serve a useful existence. Next time it's released, ++ * we will get notified again, if it still has 'notify_on_release' set. ++ * ++ * The final arg to call_usermodehelper() is UMH_WAIT_EXEC, which ++ * means only wait until the task is successfully execve()'d. The ++ * separate release agent task is forked by call_usermodehelper(), ++ * then control in this thread returns here, without waiting for the ++ * release agent task. We don't bother to wait because the caller of ++ * this routine has no use for the exit status of the release agent ++ * task, so no sense holding our caller up for that. ++ */ ++static void cgroup_release_agent(struct work_struct *work) ++{ ++ struct cgroup *cgrp = ++ container_of(work, struct cgroup, release_agent_work); ++ char *pathbuf = NULL, *agentbuf = NULL, *path; ++ char *argv[3], *envp[3]; ++ ++ mutex_lock(&cgroup_mutex); ++ ++ pathbuf = kmalloc(PATH_MAX, GFP_KERNEL); ++ agentbuf = kstrdup(cgrp->root->release_agent_path, GFP_KERNEL); ++ if (!pathbuf || !agentbuf) ++ goto out; ++ ++ path = cgroup_path(cgrp, pathbuf, PATH_MAX); ++ if (!path) ++ goto out; ++ ++ argv[0] = agentbuf; ++ argv[1] = path; ++ argv[2] = NULL; ++ ++ /* minimal command environment */ ++ envp[0] = "HOME=/"; ++ envp[1] = "PATH=/sbin:/bin:/usr/sbin:/usr/bin"; ++ envp[2] = NULL; ++ ++ mutex_unlock(&cgroup_mutex); ++ call_usermodehelper(argv[0], argv, envp, UMH_WAIT_EXEC); ++ goto out_free; ++out: ++ mutex_unlock(&cgroup_mutex); ++out_free: ++ kfree(agentbuf); ++ kfree(pathbuf); ++} ++ ++static int __init cgroup_disable(char *str) ++{ ++ struct cgroup_subsys *ss; ++ char *token; ++ int i; ++ ++ while ((token = strsep(&str, ",")) != NULL) { ++ if (!*token) ++ continue; ++ ++ for_each_subsys(ss, i) { ++ if (!strcmp(token, ss->name)) { ++ ss->disabled = 1; ++ printk(KERN_INFO "Disabling %s control group" ++ " subsystem\n", ss->name); ++ break; ++ } ++ } ++ } ++ return 1; ++} ++__setup("cgroup_disable=", cgroup_disable); ++ ++static int __init cgroup_set_legacy_files_on_dfl(char *str) ++{ ++ printk("cgroup: using legacy files on the default hierarchy\n"); ++ cgroup_legacy_files_on_dfl = true; ++ return 0; ++} ++__setup("cgroup__DEVEL__legacy_files_on_dfl", cgroup_set_legacy_files_on_dfl); ++ ++/** ++ * css_tryget_online_from_dir - get corresponding css from a cgroup dentry ++ * @dentry: directory dentry of interest ++ * @ss: subsystem of interest ++ * ++ * If @dentry is a directory for a cgroup which has @ss enabled on it, try ++ * to get the corresponding css and return it. If such css doesn't exist ++ * or can't be pinned, an ERR_PTR value is returned. ++ */ ++struct cgroup_subsys_state *css_tryget_online_from_dir(struct dentry *dentry, ++ struct cgroup_subsys *ss) ++{ ++ struct kernfs_node *kn = kernfs_node_from_dentry(dentry); ++ struct cgroup_subsys_state *css = NULL; ++ struct cgroup *cgrp; ++ ++ /* is @dentry a cgroup dir? */ ++ if (dentry->d_sb->s_type != &cgroup_fs_type || !kn || ++ kernfs_type(kn) != KERNFS_DIR) ++ return ERR_PTR(-EBADF); ++ ++ rcu_read_lock(); ++ ++ /* ++ * This path doesn't originate from kernfs and @kn could already ++ * have been or be removed at any point. @kn->priv is RCU ++ * protected for this access. See css_release_work_fn() for details. ++ */ ++ cgrp = rcu_dereference(kn->priv); ++ if (cgrp) ++ css = cgroup_css(cgrp, ss); ++ ++ if (!css || !css_tryget_online(css)) ++ css = ERR_PTR(-ENOENT); ++ ++ rcu_read_unlock(); ++ return css; ++} ++ ++/** ++ * css_from_id - lookup css by id ++ * @id: the cgroup id ++ * @ss: cgroup subsys to be looked into ++ * ++ * Returns the css if there's valid one with @id, otherwise returns NULL. ++ * Should be called under rcu_read_lock(). ++ */ ++struct cgroup_subsys_state *css_from_id(int id, struct cgroup_subsys *ss) ++{ ++ WARN_ON_ONCE(!rcu_read_lock_held()); ++ return id > 0 ? idr_find(&ss->css_idr, id) : NULL; ++} ++ ++#ifdef CONFIG_CGROUP_DEBUG ++static struct cgroup_subsys_state * ++debug_css_alloc(struct cgroup_subsys_state *parent_css) ++{ ++ struct cgroup_subsys_state *css = kzalloc(sizeof(*css), GFP_KERNEL); ++ ++ if (!css) ++ return ERR_PTR(-ENOMEM); ++ ++ return css; ++} ++ ++static void debug_css_free(struct cgroup_subsys_state *css) ++{ ++ kfree(css); ++} ++ ++static u64 debug_taskcount_read(struct cgroup_subsys_state *css, ++ struct cftype *cft) ++{ ++ return cgroup_task_count(css->cgroup); ++} ++ ++static u64 current_css_set_read(struct cgroup_subsys_state *css, ++ struct cftype *cft) ++{ ++ return (u64)(unsigned long)current->cgroups; ++} ++ ++static u64 current_css_set_refcount_read(struct cgroup_subsys_state *css, ++ struct cftype *cft) ++{ ++ u64 count; ++ ++ rcu_read_lock(); ++ count = atomic_read(&task_css_set(current)->refcount); ++ rcu_read_unlock(); ++ return count; ++} ++ ++static int current_css_set_cg_links_read(struct seq_file *seq, void *v) ++{ ++ struct cgrp_cset_link *link; ++ struct css_set *cset; ++ char *name_buf; ++ ++ name_buf = kmalloc(NAME_MAX + 1, GFP_KERNEL); ++ if (!name_buf) ++ return -ENOMEM; ++ ++ down_read(&css_set_rwsem); ++ rcu_read_lock(); ++ cset = rcu_dereference(current->cgroups); ++ list_for_each_entry(link, &cset->cgrp_links, cgrp_link) { ++ struct cgroup *c = link->cgrp; ++ ++ cgroup_name(c, name_buf, NAME_MAX + 1); ++ seq_printf(seq, "Root %d group %s\n", ++ c->root->hierarchy_id, name_buf); ++ } ++ rcu_read_unlock(); ++ up_read(&css_set_rwsem); ++ kfree(name_buf); ++ return 0; ++} ++ ++#define MAX_TASKS_SHOWN_PER_CSS 25 ++static int cgroup_css_links_read(struct seq_file *seq, void *v) ++{ ++ struct cgroup_subsys_state *css = seq_css(seq); ++ struct cgrp_cset_link *link; ++ ++ down_read(&css_set_rwsem); ++ list_for_each_entry(link, &css->cgroup->cset_links, cset_link) { ++ struct css_set *cset = link->cset; ++ struct task_struct *task; ++ int count = 0; ++ ++ seq_printf(seq, "css_set %p\n", cset); ++ ++ list_for_each_entry(task, &cset->tasks, cg_list) { ++ if (count++ > MAX_TASKS_SHOWN_PER_CSS) ++ goto overflow; ++ seq_printf(seq, " task %d\n", task_pid_vnr(task)); ++ } ++ ++ list_for_each_entry(task, &cset->mg_tasks, cg_list) { ++ if (count++ > MAX_TASKS_SHOWN_PER_CSS) ++ goto overflow; ++ seq_printf(seq, " task %d\n", task_pid_vnr(task)); ++ } ++ continue; ++ overflow: ++ seq_puts(seq, " ...\n"); ++ } ++ up_read(&css_set_rwsem); ++ return 0; ++} ++ ++static u64 releasable_read(struct cgroup_subsys_state *css, struct cftype *cft) ++{ ++ return (!cgroup_has_tasks(css->cgroup) && ++ !css_has_online_children(&css->cgroup->self)); ++} ++ ++static struct cftype debug_files[] = { ++ { ++ .name = "taskcount", ++ .read_u64 = debug_taskcount_read, ++ }, ++ ++ { ++ .name = "current_css_set", ++ .read_u64 = current_css_set_read, ++ }, ++ ++ { ++ .name = "current_css_set_refcount", ++ .read_u64 = current_css_set_refcount_read, ++ }, ++ ++ { ++ .name = "current_css_set_cg_links", ++ .seq_show = current_css_set_cg_links_read, ++ }, ++ ++ { ++ .name = "cgroup_css_links", ++ .seq_show = cgroup_css_links_read, ++ }, ++ ++ { ++ .name = "releasable", ++ .read_u64 = releasable_read, ++ }, ++ ++ { } /* terminate */ ++}; ++ ++struct cgroup_subsys debug_cgrp_subsys = { ++ .css_alloc = debug_css_alloc, ++ .css_free = debug_css_free, ++ .legacy_cftypes = debug_files, ++}; ++#endif /* CONFIG_CGROUP_DEBUG */ +diff -Nur linux-4.1.10.orig/kernel/cpu.c linux-4.1.10/kernel/cpu.c +--- linux-4.1.10.orig/kernel/cpu.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/cpu.c 2015-10-07 18:00:08.000000000 +0200 +@@ -74,8 +74,8 @@ + #endif + } cpu_hotplug = { + .active_writer = NULL, +- .wq = __WAIT_QUEUE_HEAD_INITIALIZER(cpu_hotplug.wq), + .lock = __MUTEX_INITIALIZER(cpu_hotplug.lock), ++ .wq = __WAIT_QUEUE_HEAD_INITIALIZER(cpu_hotplug.wq), + #ifdef CONFIG_DEBUG_LOCK_ALLOC + .dep_map = {.name = "cpu_hotplug.lock" }, + #endif +@@ -88,6 +88,289 @@ + #define cpuhp_lock_acquire() lock_map_acquire(&cpu_hotplug.dep_map) + #define cpuhp_lock_release() lock_map_release(&cpu_hotplug.dep_map) + ++/** ++ * hotplug_pcp - per cpu hotplug descriptor ++ * @unplug: set when pin_current_cpu() needs to sync tasks ++ * @sync_tsk: the task that waits for tasks to finish pinned sections ++ * @refcount: counter of tasks in pinned sections ++ * @grab_lock: set when the tasks entering pinned sections should wait ++ * @synced: notifier for @sync_tsk to tell cpu_down it's finished ++ * @mutex: the mutex to make tasks wait (used when @grab_lock is true) ++ * @mutex_init: zero if the mutex hasn't been initialized yet. ++ * ++ * Although @unplug and @sync_tsk may point to the same task, the @unplug ++ * is used as a flag and still exists after @sync_tsk has exited and ++ * @sync_tsk set to NULL. ++ */ ++struct hotplug_pcp { ++ struct task_struct *unplug; ++ struct task_struct *sync_tsk; ++ int refcount; ++ int grab_lock; ++ struct completion synced; ++ struct completion unplug_wait; ++#ifdef CONFIG_PREEMPT_RT_FULL ++ /* ++ * Note, on PREEMPT_RT, the hotplug lock must save the state of ++ * the task, otherwise the mutex will cause the task to fail ++ * to sleep when required. (Because it's called from migrate_disable()) ++ * ++ * The spinlock_t on PREEMPT_RT is a mutex that saves the task's ++ * state. ++ */ ++ spinlock_t lock; ++#else ++ struct mutex mutex; ++#endif ++ int mutex_init; ++}; ++ ++#ifdef CONFIG_PREEMPT_RT_FULL ++# define hotplug_lock(hp) rt_spin_lock(&(hp)->lock) ++# define hotplug_unlock(hp) rt_spin_unlock(&(hp)->lock) ++#else ++# define hotplug_lock(hp) mutex_lock(&(hp)->mutex) ++# define hotplug_unlock(hp) mutex_unlock(&(hp)->mutex) ++#endif ++ ++static DEFINE_PER_CPU(struct hotplug_pcp, hotplug_pcp); ++ ++/** ++ * pin_current_cpu - Prevent the current cpu from being unplugged ++ * ++ * Lightweight version of get_online_cpus() to prevent cpu from being ++ * unplugged when code runs in a migration disabled region. ++ * ++ * Must be called with preemption disabled (preempt_count = 1)! ++ */ ++void pin_current_cpu(void) ++{ ++ struct hotplug_pcp *hp; ++ int force = 0; ++ ++retry: ++ hp = this_cpu_ptr(&hotplug_pcp); ++ ++ if (!hp->unplug || hp->refcount || force || preempt_count() > 1 || ++ hp->unplug == current) { ++ hp->refcount++; ++ return; ++ } ++ if (hp->grab_lock) { ++ preempt_enable(); ++ hotplug_lock(hp); ++ hotplug_unlock(hp); ++ } else { ++ preempt_enable(); ++ /* ++ * Try to push this task off of this CPU. ++ */ ++ if (!migrate_me()) { ++ preempt_disable(); ++ hp = this_cpu_ptr(&hotplug_pcp); ++ if (!hp->grab_lock) { ++ /* ++ * Just let it continue it's already pinned ++ * or about to sleep. ++ */ ++ force = 1; ++ goto retry; ++ } ++ preempt_enable(); ++ } ++ } ++ preempt_disable(); ++ goto retry; ++} ++ ++/** ++ * unpin_current_cpu - Allow unplug of current cpu ++ * ++ * Must be called with preemption or interrupts disabled! ++ */ ++void unpin_current_cpu(void) ++{ ++ struct hotplug_pcp *hp = this_cpu_ptr(&hotplug_pcp); ++ ++ WARN_ON(hp->refcount <= 0); ++ ++ /* This is safe. sync_unplug_thread is pinned to this cpu */ ++ if (!--hp->refcount && hp->unplug && hp->unplug != current) ++ wake_up_process(hp->unplug); ++} ++ ++static void wait_for_pinned_cpus(struct hotplug_pcp *hp) ++{ ++ set_current_state(TASK_UNINTERRUPTIBLE); ++ while (hp->refcount) { ++ schedule_preempt_disabled(); ++ set_current_state(TASK_UNINTERRUPTIBLE); ++ } ++} ++ ++static int sync_unplug_thread(void *data) ++{ ++ struct hotplug_pcp *hp = data; ++ ++ wait_for_completion(&hp->unplug_wait); ++ preempt_disable(); ++ hp->unplug = current; ++ wait_for_pinned_cpus(hp); ++ ++ /* ++ * This thread will synchronize the cpu_down() with threads ++ * that have pinned the CPU. When the pinned CPU count reaches ++ * zero, we inform the cpu_down code to continue to the next step. ++ */ ++ set_current_state(TASK_UNINTERRUPTIBLE); ++ preempt_enable(); ++ complete(&hp->synced); ++ ++ /* ++ * If all succeeds, the next step will need tasks to wait till ++ * the CPU is offline before continuing. To do this, the grab_lock ++ * is set and tasks going into pin_current_cpu() will block on the ++ * mutex. But we still need to wait for those that are already in ++ * pinned CPU sections. If the cpu_down() failed, the kthread_should_stop() ++ * will kick this thread out. ++ */ ++ while (!hp->grab_lock && !kthread_should_stop()) { ++ schedule(); ++ set_current_state(TASK_UNINTERRUPTIBLE); ++ } ++ ++ /* Make sure grab_lock is seen before we see a stale completion */ ++ smp_mb(); ++ ++ /* ++ * Now just before cpu_down() enters stop machine, we need to make ++ * sure all tasks that are in pinned CPU sections are out, and new ++ * tasks will now grab the lock, keeping them from entering pinned ++ * CPU sections. ++ */ ++ if (!kthread_should_stop()) { ++ preempt_disable(); ++ wait_for_pinned_cpus(hp); ++ preempt_enable(); ++ complete(&hp->synced); ++ } ++ ++ set_current_state(TASK_UNINTERRUPTIBLE); ++ while (!kthread_should_stop()) { ++ schedule(); ++ set_current_state(TASK_UNINTERRUPTIBLE); ++ } ++ set_current_state(TASK_RUNNING); ++ ++ /* ++ * Force this thread off this CPU as it's going down and ++ * we don't want any more work on this CPU. ++ */ ++ current->flags &= ~PF_NO_SETAFFINITY; ++ set_cpus_allowed_ptr(current, cpu_present_mask); ++ migrate_me(); ++ return 0; ++} ++ ++static void __cpu_unplug_sync(struct hotplug_pcp *hp) ++{ ++ wake_up_process(hp->sync_tsk); ++ wait_for_completion(&hp->synced); ++} ++ ++static void __cpu_unplug_wait(unsigned int cpu) ++{ ++ struct hotplug_pcp *hp = &per_cpu(hotplug_pcp, cpu); ++ ++ complete(&hp->unplug_wait); ++ wait_for_completion(&hp->synced); ++} ++ ++/* ++ * Start the sync_unplug_thread on the target cpu and wait for it to ++ * complete. ++ */ ++static int cpu_unplug_begin(unsigned int cpu) ++{ ++ struct hotplug_pcp *hp = &per_cpu(hotplug_pcp, cpu); ++ int err; ++ ++ /* Protected by cpu_hotplug.lock */ ++ if (!hp->mutex_init) { ++#ifdef CONFIG_PREEMPT_RT_FULL ++ spin_lock_init(&hp->lock); ++#else ++ mutex_init(&hp->mutex); ++#endif ++ hp->mutex_init = 1; ++ } ++ ++ /* Inform the scheduler to migrate tasks off this CPU */ ++ tell_sched_cpu_down_begin(cpu); ++ ++ init_completion(&hp->synced); ++ init_completion(&hp->unplug_wait); ++ ++ hp->sync_tsk = kthread_create(sync_unplug_thread, hp, "sync_unplug/%d", cpu); ++ if (IS_ERR(hp->sync_tsk)) { ++ err = PTR_ERR(hp->sync_tsk); ++ hp->sync_tsk = NULL; ++ return err; ++ } ++ kthread_bind(hp->sync_tsk, cpu); ++ ++ /* ++ * Wait for tasks to get out of the pinned sections, ++ * it's still OK if new tasks enter. Some CPU notifiers will ++ * wait for tasks that are going to enter these sections and ++ * we must not have them block. ++ */ ++ wake_up_process(hp->sync_tsk); ++ return 0; ++} ++ ++static void cpu_unplug_sync(unsigned int cpu) ++{ ++ struct hotplug_pcp *hp = &per_cpu(hotplug_pcp, cpu); ++ ++ init_completion(&hp->synced); ++ /* The completion needs to be initialzied before setting grab_lock */ ++ smp_wmb(); ++ ++ /* Grab the mutex before setting grab_lock */ ++ hotplug_lock(hp); ++ hp->grab_lock = 1; ++ ++ /* ++ * The CPU notifiers have been completed. ++ * Wait for tasks to get out of pinned CPU sections and have new ++ * tasks block until the CPU is completely down. ++ */ ++ __cpu_unplug_sync(hp); ++ ++ /* All done with the sync thread */ ++ kthread_stop(hp->sync_tsk); ++ hp->sync_tsk = NULL; ++} ++ ++static void cpu_unplug_done(unsigned int cpu) ++{ ++ struct hotplug_pcp *hp = &per_cpu(hotplug_pcp, cpu); ++ ++ hp->unplug = NULL; ++ /* Let all tasks know cpu unplug is finished before cleaning up */ ++ smp_wmb(); ++ ++ if (hp->sync_tsk) ++ kthread_stop(hp->sync_tsk); ++ ++ if (hp->grab_lock) { ++ hotplug_unlock(hp); ++ /* protected by cpu_hotplug.lock */ ++ hp->grab_lock = 0; ++ } ++ tell_sched_cpu_down_done(cpu); ++} + + void get_online_cpus(void) + { +@@ -349,13 +632,15 @@ + /* Requires cpu_add_remove_lock to be held */ + static int __ref _cpu_down(unsigned int cpu, int tasks_frozen) + { +- int err, nr_calls = 0; ++ int mycpu, err, nr_calls = 0; + void *hcpu = (void *)(long)cpu; + unsigned long mod = tasks_frozen ? CPU_TASKS_FROZEN : 0; + struct take_cpu_down_param tcd_param = { + .mod = mod, + .hcpu = hcpu, + }; ++ cpumask_var_t cpumask; ++ cpumask_var_t cpumask_org; + + if (num_online_cpus() == 1) + return -EBUSY; +@@ -363,7 +648,34 @@ + if (!cpu_online(cpu)) + return -EINVAL; + ++ /* Move the downtaker off the unplug cpu */ ++ if (!alloc_cpumask_var(&cpumask, GFP_KERNEL)) ++ return -ENOMEM; ++ if (!alloc_cpumask_var(&cpumask_org, GFP_KERNEL)) { ++ free_cpumask_var(cpumask); ++ return -ENOMEM; ++ } ++ ++ cpumask_copy(cpumask_org, tsk_cpus_allowed(current)); ++ cpumask_andnot(cpumask, cpu_online_mask, cpumask_of(cpu)); ++ set_cpus_allowed_ptr(current, cpumask); ++ free_cpumask_var(cpumask); ++ migrate_disable(); ++ mycpu = smp_processor_id(); ++ if (mycpu == cpu) { ++ printk(KERN_ERR "Yuck! Still on unplug CPU\n!"); ++ migrate_enable(); ++ err = -EBUSY; ++ goto restore_cpus; ++ } ++ migrate_enable(); ++ + cpu_hotplug_begin(); ++ err = cpu_unplug_begin(cpu); ++ if (err) { ++ printk("cpu_unplug_begin(%d) failed\n", cpu); ++ goto out_cancel; ++ } + + err = __cpu_notify(CPU_DOWN_PREPARE | mod, hcpu, -1, &nr_calls); + if (err) { +@@ -389,8 +701,12 @@ + #endif + synchronize_rcu(); + ++ __cpu_unplug_wait(cpu); + smpboot_park_threads(cpu); + ++ /* Notifiers are done. Don't let any more tasks pin this CPU. */ ++ cpu_unplug_sync(cpu); ++ + /* + * So now all preempt/rcu users must observe !cpu_active(). + */ +@@ -427,9 +743,14 @@ + check_for_tasks(cpu); + + out_release: ++ cpu_unplug_done(cpu); ++out_cancel: + cpu_hotplug_done(); + if (!err) + cpu_notify_nofail(CPU_POST_DEAD | mod, hcpu); ++restore_cpus: ++ set_cpus_allowed_ptr(current, cpumask_org); ++ free_cpumask_var(cpumask_org); + return err; + } + +diff -Nur linux-4.1.10.orig/kernel/debug/kdb/kdb_io.c linux-4.1.10/kernel/debug/kdb/kdb_io.c +--- linux-4.1.10.orig/kernel/debug/kdb/kdb_io.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/debug/kdb/kdb_io.c 2015-10-07 18:00:08.000000000 +0200 +@@ -554,7 +554,6 @@ + int linecount; + int colcount; + int logging, saved_loglevel = 0; +- int saved_trap_printk; + int got_printf_lock = 0; + int retlen = 0; + int fnd, len; +@@ -565,8 +564,6 @@ + unsigned long uninitialized_var(flags); + + preempt_disable(); +- saved_trap_printk = kdb_trap_printk; +- kdb_trap_printk = 0; + + /* Serialize kdb_printf if multiple cpus try to write at once. + * But if any cpu goes recursive in kdb, just print the output, +@@ -855,7 +852,6 @@ + } else { + __release(kdb_printf_lock); + } +- kdb_trap_printk = saved_trap_printk; + preempt_enable(); + return retlen; + } +@@ -865,9 +861,11 @@ + va_list ap; + int r; + ++ kdb_trap_printk++; + va_start(ap, fmt); + r = vkdb_printf(KDB_MSGSRC_INTERNAL, fmt, ap); + va_end(ap); ++ kdb_trap_printk--; + + return r; + } +diff -Nur linux-4.1.10.orig/kernel/events/core.c linux-4.1.10/kernel/events/core.c +--- linux-4.1.10.orig/kernel/events/core.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/events/core.c 2015-10-07 18:00:08.000000000 +0200 +@@ -6933,6 +6933,7 @@ + + hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + hwc->hrtimer.function = perf_swevent_hrtimer; ++ hwc->hrtimer.irqsafe = 1; + + /* + * Since hrtimers have a fixed rate, we can do a static freq->period +diff -Nur linux-4.1.10.orig/kernel/exit.c linux-4.1.10/kernel/exit.c +--- linux-4.1.10.orig/kernel/exit.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/exit.c 2015-10-07 18:00:08.000000000 +0200 +@@ -144,7 +144,7 @@ + * Do this under ->siglock, we can race with another thread + * doing sigqueue_free() if we have SIGQUEUE_PREALLOC signals. + */ +- flush_sigqueue(&tsk->pending); ++ flush_task_sigqueue(tsk); + tsk->sighand = NULL; + spin_unlock(&sighand->siglock); + +diff -Nur linux-4.1.10.orig/kernel/fork.c linux-4.1.10/kernel/fork.c +--- linux-4.1.10.orig/kernel/fork.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/fork.c 2015-10-07 18:00:08.000000000 +0200 +@@ -108,7 +108,7 @@ + + DEFINE_PER_CPU(unsigned long, process_counts) = 0; + +-__cacheline_aligned DEFINE_RWLOCK(tasklist_lock); /* outer */ ++DEFINE_RWLOCK(tasklist_lock); /* outer */ + + #ifdef CONFIG_PROVE_RCU + int lockdep_tasklist_lock_is_held(void) +@@ -244,7 +244,9 @@ + if (atomic_dec_and_test(&sig->sigcnt)) + free_signal_struct(sig); + } +- ++#ifdef CONFIG_PREEMPT_RT_BASE ++static ++#endif + void __put_task_struct(struct task_struct *tsk) + { + WARN_ON(!tsk->exit_state); +@@ -260,7 +262,18 @@ + if (!profile_handoff_task(tsk)) + free_task(tsk); + } ++#ifndef CONFIG_PREEMPT_RT_BASE + EXPORT_SYMBOL_GPL(__put_task_struct); ++#else ++void __put_task_struct_cb(struct rcu_head *rhp) ++{ ++ struct task_struct *tsk = container_of(rhp, struct task_struct, put_rcu); ++ ++ __put_task_struct(tsk); ++ ++} ++EXPORT_SYMBOL_GPL(__put_task_struct_cb); ++#endif + + void __init __weak arch_task_cache_init(void) { } + +@@ -680,6 +693,19 @@ + } + EXPORT_SYMBOL_GPL(__mmdrop); + ++#ifdef CONFIG_PREEMPT_RT_BASE ++/* ++ * RCU callback for delayed mm drop. Not strictly rcu, but we don't ++ * want another facility to make this work. ++ */ ++void __mmdrop_delayed(struct rcu_head *rhp) ++{ ++ struct mm_struct *mm = container_of(rhp, struct mm_struct, delayed_drop); ++ ++ __mmdrop(mm); ++} ++#endif ++ + /* + * Decrement the use count and release all resources for an mm. + */ +@@ -1214,6 +1240,9 @@ + */ + static void posix_cpu_timers_init(struct task_struct *tsk) + { ++#ifdef CONFIG_PREEMPT_RT_BASE ++ tsk->posix_timer_list = NULL; ++#endif + tsk->cputime_expires.prof_exp = 0; + tsk->cputime_expires.virt_exp = 0; + tsk->cputime_expires.sched_exp = 0; +@@ -1338,6 +1367,7 @@ + spin_lock_init(&p->alloc_lock); + + init_sigpending(&p->pending); ++ p->sigqueue_cache = NULL; + + p->utime = p->stime = p->gtime = 0; + p->utimescaled = p->stimescaled = 0; +@@ -1345,7 +1375,8 @@ + p->prev_cputime.utime = p->prev_cputime.stime = 0; + #endif + #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN +- seqlock_init(&p->vtime_seqlock); ++ raw_spin_lock_init(&p->vtime_lock); ++ seqcount_init(&p->vtime_seq); + p->vtime_snap = 0; + p->vtime_snap_whence = VTIME_SLEEPING; + #endif +@@ -1396,6 +1427,9 @@ + p->hardirq_context = 0; + p->softirq_context = 0; + #endif ++ ++ p->pagefault_disabled = 0; ++ + #ifdef CONFIG_LOCKDEP + p->lockdep_depth = 0; /* no locks held yet */ + p->curr_chain_key = 0; +diff -Nur linux-4.1.10.orig/kernel/futex.c linux-4.1.10/kernel/futex.c +--- linux-4.1.10.orig/kernel/futex.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/futex.c 2015-10-07 18:00:08.000000000 +0200 +@@ -738,7 +738,9 @@ + * task still owns the PI-state: + */ + if (head->next != next) { ++ raw_spin_unlock_irq(&curr->pi_lock); + spin_unlock(&hb->lock); ++ raw_spin_lock_irq(&curr->pi_lock); + continue; + } + +@@ -1090,9 +1092,11 @@ + + /* + * The hash bucket lock must be held when this is called. +- * Afterwards, the futex_q must not be accessed. ++ * Afterwards, the futex_q must not be accessed. Callers ++ * must ensure to later call wake_up_q() for the actual ++ * wakeups to occur. + */ +-static void wake_futex(struct futex_q *q) ++static void mark_wake_futex(struct wake_q_head *wake_q, struct futex_q *q) + { + struct task_struct *p = q->task; + +@@ -1100,14 +1104,10 @@ + return; + + /* +- * We set q->lock_ptr = NULL _before_ we wake up the task. If +- * a non-futex wake up happens on another CPU then the task +- * might exit and p would dereference a non-existing task +- * struct. Prevent this by holding a reference on p across the +- * wake up. ++ * Queue the task for later wakeup for after we've released ++ * the hb->lock. wake_q_add() grabs reference to p. + */ +- get_task_struct(p); +- ++ wake_q_add(wake_q, p); + __unqueue_futex(q); + /* + * The waiting task can free the futex_q as soon as +@@ -1117,16 +1117,15 @@ + */ + smp_wmb(); + q->lock_ptr = NULL; +- +- wake_up_state(p, TASK_NORMAL); +- put_task_struct(p); + } + +-static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_q *this) ++static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_q *this, ++ struct futex_hash_bucket *hb) + { + struct task_struct *new_owner; + struct futex_pi_state *pi_state = this->pi_state; + u32 uninitialized_var(curval), newval; ++ bool deboost; + int ret = 0; + + if (!pi_state) +@@ -1178,7 +1177,17 @@ + raw_spin_unlock_irq(&new_owner->pi_lock); + + raw_spin_unlock(&pi_state->pi_mutex.wait_lock); +- rt_mutex_unlock(&pi_state->pi_mutex); ++ ++ deboost = rt_mutex_futex_unlock(&pi_state->pi_mutex); ++ ++ /* ++ * We deboost after dropping hb->lock. That prevents a double ++ * wakeup on RT. ++ */ ++ spin_unlock(&hb->lock); ++ ++ if (deboost) ++ rt_mutex_adjust_prio(current); + + return 0; + } +@@ -1217,6 +1226,7 @@ + struct futex_q *this, *next; + union futex_key key = FUTEX_KEY_INIT; + int ret; ++ WAKE_Q(wake_q); + + if (!bitset) + return -EINVAL; +@@ -1244,13 +1254,14 @@ + if (!(this->bitset & bitset)) + continue; + +- wake_futex(this); ++ mark_wake_futex(&wake_q, this); + if (++ret >= nr_wake) + break; + } + } + + spin_unlock(&hb->lock); ++ wake_up_q(&wake_q); + out_put_key: + put_futex_key(&key); + out: +@@ -1269,6 +1280,7 @@ + struct futex_hash_bucket *hb1, *hb2; + struct futex_q *this, *next; + int ret, op_ret; ++ WAKE_Q(wake_q); + + retry: + ret = get_futex_key(uaddr1, flags & FLAGS_SHARED, &key1, VERIFY_READ); +@@ -1320,7 +1332,7 @@ + ret = -EINVAL; + goto out_unlock; + } +- wake_futex(this); ++ mark_wake_futex(&wake_q, this); + if (++ret >= nr_wake) + break; + } +@@ -1334,7 +1346,7 @@ + ret = -EINVAL; + goto out_unlock; + } +- wake_futex(this); ++ mark_wake_futex(&wake_q, this); + if (++op_ret >= nr_wake2) + break; + } +@@ -1344,6 +1356,7 @@ + + out_unlock: + double_unlock_hb(hb1, hb2); ++ wake_up_q(&wake_q); + out_put_keys: + put_futex_key(&key2); + out_put_key1: +@@ -1503,6 +1516,7 @@ + struct futex_pi_state *pi_state = NULL; + struct futex_hash_bucket *hb1, *hb2; + struct futex_q *this, *next; ++ WAKE_Q(wake_q); + + if (requeue_pi) { + /* +@@ -1679,7 +1693,7 @@ + * woken by futex_unlock_pi(). + */ + if (++task_count <= nr_wake && !requeue_pi) { +- wake_futex(this); ++ mark_wake_futex(&wake_q, this); + continue; + } + +@@ -1705,6 +1719,16 @@ + requeue_pi_wake_futex(this, &key2, hb2); + drop_count++; + continue; ++ } else if (ret == -EAGAIN) { ++ /* ++ * Waiter was woken by timeout or ++ * signal and has set pi_blocked_on to ++ * PI_WAKEUP_INPROGRESS before we ++ * tried to enqueue it on the rtmutex. ++ */ ++ this->pi_state = NULL; ++ free_pi_state(pi_state); ++ continue; + } else if (ret) { + /* -EDEADLK */ + this->pi_state = NULL; +@@ -1719,6 +1743,7 @@ + out_unlock: + free_pi_state(pi_state); + double_unlock_hb(hb1, hb2); ++ wake_up_q(&wake_q); + hb_waiters_dec(hb2); + + /* +@@ -2412,13 +2437,26 @@ + */ + match = futex_top_waiter(hb, &key); + if (match) { +- ret = wake_futex_pi(uaddr, uval, match); ++ ret = wake_futex_pi(uaddr, uval, match, hb); ++ ++ /* ++ * In case of success wake_futex_pi dropped the hash ++ * bucket lock. ++ */ ++ if (!ret) ++ goto out_putkey; ++ + /* + * The atomic access to the futex value generated a + * pagefault, so retry the user-access and the wakeup: + */ + if (ret == -EFAULT) + goto pi_faulted; ++ ++ /* ++ * wake_futex_pi has detected invalid state. Tell user ++ * space. ++ */ + goto out_unlock; + } + +@@ -2439,6 +2477,7 @@ + + out_unlock: + spin_unlock(&hb->lock); ++out_putkey: + put_futex_key(&key); + return ret; + +@@ -2549,7 +2588,7 @@ + struct hrtimer_sleeper timeout, *to = NULL; + struct rt_mutex_waiter rt_waiter; + struct rt_mutex *pi_mutex = NULL; +- struct futex_hash_bucket *hb; ++ struct futex_hash_bucket *hb, *hb2; + union futex_key key2 = FUTEX_KEY_INIT; + struct futex_q q = futex_q_init; + int res, ret; +@@ -2574,10 +2613,7 @@ + * The waiter is allocated on our stack, manipulated by the requeue + * code while we sleep on uaddr. + */ +- debug_rt_mutex_init_waiter(&rt_waiter); +- RB_CLEAR_NODE(&rt_waiter.pi_tree_entry); +- RB_CLEAR_NODE(&rt_waiter.tree_entry); +- rt_waiter.task = NULL; ++ rt_mutex_init_waiter(&rt_waiter, false); + + ret = get_futex_key(uaddr2, flags & FLAGS_SHARED, &key2, VERIFY_WRITE); + if (unlikely(ret != 0)) +@@ -2608,20 +2644,55 @@ + /* Queue the futex_q, drop the hb lock, wait for wakeup. */ + futex_wait_queue_me(hb, &q, to); + +- spin_lock(&hb->lock); +- ret = handle_early_requeue_pi_wakeup(hb, &q, &key2, to); +- spin_unlock(&hb->lock); +- if (ret) +- goto out_put_keys; ++ /* ++ * On RT we must avoid races with requeue and trying to block ++ * on two mutexes (hb->lock and uaddr2's rtmutex) by ++ * serializing access to pi_blocked_on with pi_lock. ++ */ ++ raw_spin_lock_irq(¤t->pi_lock); ++ if (current->pi_blocked_on) { ++ /* ++ * We have been requeued or are in the process of ++ * being requeued. ++ */ ++ raw_spin_unlock_irq(¤t->pi_lock); ++ } else { ++ /* ++ * Setting pi_blocked_on to PI_WAKEUP_INPROGRESS ++ * prevents a concurrent requeue from moving us to the ++ * uaddr2 rtmutex. After that we can safely acquire ++ * (and possibly block on) hb->lock. ++ */ ++ current->pi_blocked_on = PI_WAKEUP_INPROGRESS; ++ raw_spin_unlock_irq(¤t->pi_lock); ++ ++ spin_lock(&hb->lock); ++ ++ /* ++ * Clean up pi_blocked_on. We might leak it otherwise ++ * when we succeeded with the hb->lock in the fast ++ * path. ++ */ ++ raw_spin_lock_irq(¤t->pi_lock); ++ current->pi_blocked_on = NULL; ++ raw_spin_unlock_irq(¤t->pi_lock); ++ ++ ret = handle_early_requeue_pi_wakeup(hb, &q, &key2, to); ++ spin_unlock(&hb->lock); ++ if (ret) ++ goto out_put_keys; ++ } + + /* +- * In order for us to be here, we know our q.key == key2, and since +- * we took the hb->lock above, we also know that futex_requeue() has +- * completed and we no longer have to concern ourselves with a wakeup +- * race with the atomic proxy lock acquisition by the requeue code. The +- * futex_requeue dropped our key1 reference and incremented our key2 +- * reference count. ++ * In order to be here, we have either been requeued, are in ++ * the process of being requeued, or requeue successfully ++ * acquired uaddr2 on our behalf. If pi_blocked_on was ++ * non-null above, we may be racing with a requeue. Do not ++ * rely on q->lock_ptr to be hb2->lock until after blocking on ++ * hb->lock or hb2->lock. The futex_requeue dropped our key1 ++ * reference and incremented our key2 reference count. + */ ++ hb2 = hash_futex(&key2); + + /* Check if the requeue code acquired the second futex for us. */ + if (!q.rt_waiter) { +@@ -2630,9 +2701,10 @@ + * did a lock-steal - fix up the PI-state in that case. + */ + if (q.pi_state && (q.pi_state->owner != current)) { +- spin_lock(q.lock_ptr); ++ spin_lock(&hb2->lock); ++ BUG_ON(&hb2->lock != q.lock_ptr); + ret = fixup_pi_state_owner(uaddr2, &q, current); +- spin_unlock(q.lock_ptr); ++ spin_unlock(&hb2->lock); + } + } else { + /* +@@ -2645,7 +2717,8 @@ + ret = rt_mutex_finish_proxy_lock(pi_mutex, to, &rt_waiter); + debug_rt_mutex_free_waiter(&rt_waiter); + +- spin_lock(q.lock_ptr); ++ spin_lock(&hb2->lock); ++ BUG_ON(&hb2->lock != q.lock_ptr); + /* + * Fixup the pi_state owner and possibly acquire the lock if we + * haven't already. +diff -Nur linux-4.1.10.orig/kernel/irq/handle.c linux-4.1.10/kernel/irq/handle.c +--- linux-4.1.10.orig/kernel/irq/handle.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/irq/handle.c 2015-10-07 18:00:08.000000000 +0200 +@@ -133,6 +133,8 @@ + irqreturn_t + handle_irq_event_percpu(struct irq_desc *desc, struct irqaction *action) + { ++ struct pt_regs *regs = get_irq_regs(); ++ u64 ip = regs ? instruction_pointer(regs) : 0; + irqreturn_t retval = IRQ_NONE; + unsigned int flags = 0, irq = desc->irq_data.irq; + +@@ -173,7 +175,11 @@ + action = action->next; + } while (action); + +- add_interrupt_randomness(irq, flags); ++#ifndef CONFIG_PREEMPT_RT_FULL ++ add_interrupt_randomness(irq, flags, ip); ++#else ++ desc->random_ip = ip; ++#endif + + if (!noirqdebug) + note_interrupt(irq, desc, retval); +diff -Nur linux-4.1.10.orig/kernel/irq/manage.c linux-4.1.10/kernel/irq/manage.c +--- linux-4.1.10.orig/kernel/irq/manage.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/irq/manage.c 2015-10-07 18:00:08.000000000 +0200 +@@ -22,6 +22,7 @@ + #include "internals.h" + + #ifdef CONFIG_IRQ_FORCED_THREADING ++# ifndef CONFIG_PREEMPT_RT_BASE + __read_mostly bool force_irqthreads; + + static int __init setup_forced_irqthreads(char *arg) +@@ -30,6 +31,7 @@ + return 0; + } + early_param("threadirqs", setup_forced_irqthreads); ++# endif + #endif + + static void __synchronize_hardirq(struct irq_desc *desc) +@@ -179,6 +181,62 @@ + irq_get_pending(struct cpumask *mask, struct irq_desc *desc) { } + #endif + ++#ifdef CONFIG_PREEMPT_RT_FULL ++static void _irq_affinity_notify(struct irq_affinity_notify *notify); ++static struct task_struct *set_affinity_helper; ++static LIST_HEAD(affinity_list); ++static DEFINE_RAW_SPINLOCK(affinity_list_lock); ++ ++static int set_affinity_thread(void *unused) ++{ ++ while (1) { ++ struct irq_affinity_notify *notify; ++ int empty; ++ ++ set_current_state(TASK_INTERRUPTIBLE); ++ ++ raw_spin_lock_irq(&affinity_list_lock); ++ empty = list_empty(&affinity_list); ++ raw_spin_unlock_irq(&affinity_list_lock); ++ ++ if (empty) ++ schedule(); ++ if (kthread_should_stop()) ++ break; ++ set_current_state(TASK_RUNNING); ++try_next: ++ notify = NULL; ++ ++ raw_spin_lock_irq(&affinity_list_lock); ++ if (!list_empty(&affinity_list)) { ++ notify = list_first_entry(&affinity_list, ++ struct irq_affinity_notify, list); ++ list_del_init(¬ify->list); ++ } ++ raw_spin_unlock_irq(&affinity_list_lock); ++ ++ if (!notify) ++ continue; ++ _irq_affinity_notify(notify); ++ goto try_next; ++ } ++ return 0; ++} ++ ++static void init_helper_thread(void) ++{ ++ if (set_affinity_helper) ++ return; ++ set_affinity_helper = kthread_run(set_affinity_thread, NULL, ++ "affinity-cb"); ++ WARN_ON(IS_ERR(set_affinity_helper)); ++} ++#else ++ ++static inline void init_helper_thread(void) { } ++ ++#endif ++ + int irq_do_set_affinity(struct irq_data *data, const struct cpumask *mask, + bool force) + { +@@ -218,7 +276,17 @@ + + if (desc->affinity_notify) { + kref_get(&desc->affinity_notify->kref); ++ ++#ifdef CONFIG_PREEMPT_RT_FULL ++ raw_spin_lock(&affinity_list_lock); ++ if (list_empty(&desc->affinity_notify->list)) ++ list_add_tail(&affinity_list, ++ &desc->affinity_notify->list); ++ raw_spin_unlock(&affinity_list_lock); ++ wake_up_process(set_affinity_helper); ++#else + schedule_work(&desc->affinity_notify->work); ++#endif + } + irqd_set(data, IRQD_AFFINITY_SET); + +@@ -256,10 +324,8 @@ + } + EXPORT_SYMBOL_GPL(irq_set_affinity_hint); + +-static void irq_affinity_notify(struct work_struct *work) ++static void _irq_affinity_notify(struct irq_affinity_notify *notify) + { +- struct irq_affinity_notify *notify = +- container_of(work, struct irq_affinity_notify, work); + struct irq_desc *desc = irq_to_desc(notify->irq); + cpumask_var_t cpumask; + unsigned long flags; +@@ -281,6 +347,13 @@ + kref_put(¬ify->kref, notify->release); + } + ++static void irq_affinity_notify(struct work_struct *work) ++{ ++ struct irq_affinity_notify *notify = ++ container_of(work, struct irq_affinity_notify, work); ++ _irq_affinity_notify(notify); ++} ++ + /** + * irq_set_affinity_notifier - control notification of IRQ affinity changes + * @irq: Interrupt for which to enable/disable notification +@@ -310,6 +383,8 @@ + notify->irq = irq; + kref_init(¬ify->kref); + INIT_WORK(¬ify->work, irq_affinity_notify); ++ INIT_LIST_HEAD(¬ify->list); ++ init_helper_thread(); + } + + raw_spin_lock_irqsave(&desc->lock, flags); +@@ -697,6 +772,12 @@ + return IRQ_NONE; + } + ++static irqreturn_t irq_forced_secondary_handler(int irq, void *dev_id) ++{ ++ WARN(1, "Secondary action handler called for irq %d\n", irq); ++ return IRQ_NONE; ++} ++ + static int irq_wait_for_interrupt(struct irqaction *action) + { + set_current_state(TASK_INTERRUPTIBLE); +@@ -723,7 +804,8 @@ + static void irq_finalize_oneshot(struct irq_desc *desc, + struct irqaction *action) + { +- if (!(desc->istate & IRQS_ONESHOT)) ++ if (!(desc->istate & IRQS_ONESHOT) || ++ action->handler == irq_forced_secondary_handler) + return; + again: + chip_bus_lock(desc); +@@ -825,7 +907,15 @@ + local_bh_disable(); + ret = action->thread_fn(action->irq, action->dev_id); + irq_finalize_oneshot(desc, action); +- local_bh_enable(); ++ /* ++ * Interrupts which have real time requirements can be set up ++ * to avoid softirq processing in the thread handler. This is ++ * safe as these interrupts do not raise soft interrupts. ++ */ ++ if (irq_settings_no_softirq_call(desc)) ++ _local_bh_enable(); ++ else ++ local_bh_enable(); + return ret; + } + +@@ -877,6 +967,18 @@ + irq_finalize_oneshot(desc, action); + } + ++static void irq_wake_secondary(struct irq_desc *desc, struct irqaction *action) ++{ ++ struct irqaction *secondary = action->secondary; ++ ++ if (WARN_ON_ONCE(!secondary)) ++ return; ++ ++ raw_spin_lock_irq(&desc->lock); ++ __irq_wake_thread(desc, secondary); ++ raw_spin_unlock_irq(&desc->lock); ++} ++ + /* + * Interrupt handler thread + */ +@@ -907,7 +1009,15 @@ + action_ret = handler_fn(desc, action); + if (action_ret == IRQ_HANDLED) + atomic_inc(&desc->threads_handled); ++ if (action_ret == IRQ_WAKE_THREAD) ++ irq_wake_secondary(desc, action); + ++#ifdef CONFIG_PREEMPT_RT_FULL ++ migrate_disable(); ++ add_interrupt_randomness(action->irq, 0, ++ desc->random_ip ^ (unsigned long) action); ++ migrate_enable(); ++#endif + wake_threads_waitq(desc); + } + +@@ -951,20 +1061,36 @@ + } + EXPORT_SYMBOL_GPL(irq_wake_thread); + +-static void irq_setup_forced_threading(struct irqaction *new) ++static int irq_setup_forced_threading(struct irqaction *new) + { + if (!force_irqthreads) +- return; ++ return 0; + if (new->flags & (IRQF_NO_THREAD | IRQF_PERCPU | IRQF_ONESHOT)) +- return; ++ return 0; + + new->flags |= IRQF_ONESHOT; + +- if (!new->thread_fn) { +- set_bit(IRQTF_FORCED_THREAD, &new->thread_flags); +- new->thread_fn = new->handler; +- new->handler = irq_default_primary_handler; +- } ++ /* ++ * Handle the case where we have a real primary handler and a ++ * thread handler. We force thread them as well by creating a ++ * secondary action. ++ */ ++ if (new->handler != irq_default_primary_handler && new->thread_fn) { ++ /* Allocate the secondary action */ ++ new->secondary = kzalloc(sizeof(struct irqaction), GFP_KERNEL); ++ if (!new->secondary) ++ return -ENOMEM; ++ new->secondary->handler = irq_forced_secondary_handler; ++ new->secondary->thread_fn = new->thread_fn; ++ new->secondary->dev_id = new->dev_id; ++ new->secondary->irq = new->irq; ++ new->secondary->name = new->name; ++ } ++ /* Deal with the primary handler */ ++ set_bit(IRQTF_FORCED_THREAD, &new->thread_flags); ++ new->thread_fn = new->handler; ++ new->handler = irq_default_primary_handler; ++ return 0; + } + + static int irq_request_resources(struct irq_desc *desc) +@@ -984,6 +1110,48 @@ + c->irq_release_resources(d); + } + ++static int ++setup_irq_thread(struct irqaction *new, unsigned int irq, bool secondary) ++{ ++ struct task_struct *t; ++ struct sched_param param = { ++ .sched_priority = MAX_USER_RT_PRIO/2, ++ }; ++ ++ if (!secondary) { ++ t = kthread_create(irq_thread, new, "irq/%d-%s", irq, ++ new->name); ++ } else { ++ t = kthread_create(irq_thread, new, "irq/%d-s-%s", irq, ++ new->name); ++ param.sched_priority += 1; ++ } ++ ++ if (IS_ERR(t)) ++ return PTR_ERR(t); ++ ++ sched_setscheduler_nocheck(t, SCHED_FIFO, ¶m); ++ ++ /* ++ * We keep the reference to the task struct even if ++ * the thread dies to avoid that the interrupt code ++ * references an already freed task_struct. ++ */ ++ get_task_struct(t); ++ new->thread = t; ++ /* ++ * Tell the thread to set its affinity. This is ++ * important for shared interrupt handlers as we do ++ * not invoke setup_affinity() for the secondary ++ * handlers as everything is already set up. Even for ++ * interrupts marked with IRQF_NO_BALANCE this is ++ * correct as we want the thread to move to the cpu(s) ++ * on which the requesting code placed the interrupt. ++ */ ++ set_bit(IRQTF_AFFINITY, &new->thread_flags); ++ return 0; ++} ++ + /* + * Internal function to register an irqaction - typically used to + * allocate special interrupts that are part of the architecture. +@@ -1004,6 +1172,8 @@ + if (!try_module_get(desc->owner)) + return -ENODEV; + ++ new->irq = irq; ++ + /* + * Check whether the interrupt nests into another interrupt + * thread. +@@ -1021,8 +1191,11 @@ + */ + new->handler = irq_nested_primary_handler; + } else { +- if (irq_settings_can_thread(desc)) +- irq_setup_forced_threading(new); ++ if (irq_settings_can_thread(desc)) { ++ ret = irq_setup_forced_threading(new); ++ if (ret) ++ goto out_mput; ++ } + } + + /* +@@ -1031,37 +1204,14 @@ + * thread. + */ + if (new->thread_fn && !nested) { +- struct task_struct *t; +- static const struct sched_param param = { +- .sched_priority = MAX_USER_RT_PRIO/2, +- }; +- +- t = kthread_create(irq_thread, new, "irq/%d-%s", irq, +- new->name); +- if (IS_ERR(t)) { +- ret = PTR_ERR(t); ++ ret = setup_irq_thread(new, irq, false); ++ if (ret) + goto out_mput; ++ if (new->secondary) { ++ ret = setup_irq_thread(new->secondary, irq, true); ++ if (ret) ++ goto out_thread; + } +- +- sched_setscheduler_nocheck(t, SCHED_FIFO, ¶m); +- +- /* +- * We keep the reference to the task struct even if +- * the thread dies to avoid that the interrupt code +- * references an already freed task_struct. +- */ +- get_task_struct(t); +- new->thread = t; +- /* +- * Tell the thread to set its affinity. This is +- * important for shared interrupt handlers as we do +- * not invoke setup_affinity() for the secondary +- * handlers as everything is already set up. Even for +- * interrupts marked with IRQF_NO_BALANCE this is +- * correct as we want the thread to move to the cpu(s) +- * on which the requesting code placed the interrupt. +- */ +- set_bit(IRQTF_AFFINITY, &new->thread_flags); + } + + if (!alloc_cpumask_var(&mask, GFP_KERNEL)) { +@@ -1221,6 +1371,9 @@ + irqd_set(&desc->irq_data, IRQD_NO_BALANCING); + } + ++ if (new->flags & IRQF_NO_SOFTIRQ_CALL) ++ irq_settings_set_no_softirq_call(desc); ++ + /* Set default affinity mask once everything is setup */ + setup_affinity(irq, desc, mask); + +@@ -1234,7 +1387,6 @@ + irq, nmsk, omsk); + } + +- new->irq = irq; + *old_ptr = new; + + irq_pm_install_action(desc, new); +@@ -1260,6 +1412,8 @@ + */ + if (new->thread) + wake_up_process(new->thread); ++ if (new->secondary) ++ wake_up_process(new->secondary->thread); + + register_irq_proc(irq, desc); + new->dir = NULL; +@@ -1290,6 +1444,13 @@ + kthread_stop(t); + put_task_struct(t); + } ++ if (new->secondary && new->secondary->thread) { ++ struct task_struct *t = new->secondary->thread; ++ ++ new->secondary->thread = NULL; ++ kthread_stop(t); ++ put_task_struct(t); ++ } + out_mput: + module_put(desc->owner); + return ret; +@@ -1397,9 +1558,14 @@ + if (action->thread) { + kthread_stop(action->thread); + put_task_struct(action->thread); ++ if (action->secondary && action->secondary->thread) { ++ kthread_stop(action->secondary->thread); ++ put_task_struct(action->secondary->thread); ++ } + } + + module_put(desc->owner); ++ kfree(action->secondary); + return action; + } + +@@ -1543,8 +1709,10 @@ + retval = __setup_irq(irq, desc, action); + chip_bus_sync_unlock(desc); + +- if (retval) ++ if (retval) { ++ kfree(action->secondary); + kfree(action); ++ } + + #ifdef CONFIG_DEBUG_SHIRQ_FIXME + if (!retval && (irqflags & IRQF_SHARED)) { +diff -Nur linux-4.1.10.orig/kernel/irq/settings.h linux-4.1.10/kernel/irq/settings.h +--- linux-4.1.10.orig/kernel/irq/settings.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/irq/settings.h 2015-10-07 18:00:08.000000000 +0200 +@@ -15,6 +15,7 @@ + _IRQ_NESTED_THREAD = IRQ_NESTED_THREAD, + _IRQ_PER_CPU_DEVID = IRQ_PER_CPU_DEVID, + _IRQ_IS_POLLED = IRQ_IS_POLLED, ++ _IRQ_NO_SOFTIRQ_CALL = IRQ_NO_SOFTIRQ_CALL, + _IRQF_MODIFY_MASK = IRQF_MODIFY_MASK, + }; + +@@ -28,6 +29,7 @@ + #define IRQ_NESTED_THREAD GOT_YOU_MORON + #define IRQ_PER_CPU_DEVID GOT_YOU_MORON + #define IRQ_IS_POLLED GOT_YOU_MORON ++#define IRQ_NO_SOFTIRQ_CALL GOT_YOU_MORON + #undef IRQF_MODIFY_MASK + #define IRQF_MODIFY_MASK GOT_YOU_MORON + +@@ -38,6 +40,16 @@ + desc->status_use_accessors |= (set & _IRQF_MODIFY_MASK); + } + ++static inline bool irq_settings_no_softirq_call(struct irq_desc *desc) ++{ ++ return desc->status_use_accessors & _IRQ_NO_SOFTIRQ_CALL; ++} ++ ++static inline void irq_settings_set_no_softirq_call(struct irq_desc *desc) ++{ ++ desc->status_use_accessors |= _IRQ_NO_SOFTIRQ_CALL; ++} ++ + static inline bool irq_settings_is_per_cpu(struct irq_desc *desc) + { + return desc->status_use_accessors & _IRQ_PER_CPU; +diff -Nur linux-4.1.10.orig/kernel/irq/spurious.c linux-4.1.10/kernel/irq/spurious.c +--- linux-4.1.10.orig/kernel/irq/spurious.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/irq/spurious.c 2015-10-07 18:00:08.000000000 +0200 +@@ -444,6 +444,10 @@ + + static int __init irqfixup_setup(char *str) + { ++#ifdef CONFIG_PREEMPT_RT_BASE ++ pr_warn("irqfixup boot option not supported w/ CONFIG_PREEMPT_RT_BASE\n"); ++ return 1; ++#endif + irqfixup = 1; + printk(KERN_WARNING "Misrouted IRQ fixup support enabled.\n"); + printk(KERN_WARNING "This may impact system performance.\n"); +@@ -456,6 +460,10 @@ + + static int __init irqpoll_setup(char *str) + { ++#ifdef CONFIG_PREEMPT_RT_BASE ++ pr_warn("irqpoll boot option not supported w/ CONFIG_PREEMPT_RT_BASE\n"); ++ return 1; ++#endif + irqfixup = 2; + printk(KERN_WARNING "Misrouted IRQ fixup and polling support " + "enabled\n"); +diff -Nur linux-4.1.10.orig/kernel/irq_work.c linux-4.1.10/kernel/irq_work.c +--- linux-4.1.10.orig/kernel/irq_work.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/irq_work.c 2015-10-07 18:00:08.000000000 +0200 +@@ -17,6 +17,7 @@ + #include + #include + #include ++#include + #include + + +@@ -65,6 +66,8 @@ + */ + bool irq_work_queue_on(struct irq_work *work, int cpu) + { ++ struct llist_head *list; ++ + /* All work should have been flushed before going offline */ + WARN_ON_ONCE(cpu_is_offline(cpu)); + +@@ -75,7 +78,12 @@ + if (!irq_work_claim(work)) + return false; + +- if (llist_add(&work->llnode, &per_cpu(raised_list, cpu))) ++ if (IS_ENABLED(CONFIG_PREEMPT_RT_FULL) && !(work->flags & IRQ_WORK_HARD_IRQ)) ++ list = &per_cpu(lazy_list, cpu); ++ else ++ list = &per_cpu(raised_list, cpu); ++ ++ if (llist_add(&work->llnode, list)) + arch_send_call_function_single_ipi(cpu); + + return true; +@@ -86,6 +94,9 @@ + /* Enqueue the irq work @work on the current CPU */ + bool irq_work_queue(struct irq_work *work) + { ++ struct llist_head *list; ++ bool lazy_work, realtime = IS_ENABLED(CONFIG_PREEMPT_RT_FULL); ++ + /* Only queue if not already pending */ + if (!irq_work_claim(work)) + return false; +@@ -93,13 +104,15 @@ + /* Queue the entry and raise the IPI if needed. */ + preempt_disable(); + +- /* If the work is "lazy", handle it from next tick if any */ +- if (work->flags & IRQ_WORK_LAZY) { +- if (llist_add(&work->llnode, this_cpu_ptr(&lazy_list)) && +- tick_nohz_tick_stopped()) +- arch_irq_work_raise(); +- } else { +- if (llist_add(&work->llnode, this_cpu_ptr(&raised_list))) ++ lazy_work = work->flags & IRQ_WORK_LAZY; ++ ++ if (lazy_work || (realtime && !(work->flags & IRQ_WORK_HARD_IRQ))) ++ list = this_cpu_ptr(&lazy_list); ++ else ++ list = this_cpu_ptr(&raised_list); ++ ++ if (llist_add(&work->llnode, list)) { ++ if (!lazy_work || tick_nohz_tick_stopped()) + arch_irq_work_raise(); + } + +@@ -116,9 +129,8 @@ + raised = this_cpu_ptr(&raised_list); + lazy = this_cpu_ptr(&lazy_list); + +- if (llist_empty(raised) || arch_irq_work_has_interrupt()) +- if (llist_empty(lazy)) +- return false; ++ if (llist_empty(raised) && llist_empty(lazy)) ++ return false; + + /* All work should have been flushed before going offline */ + WARN_ON_ONCE(cpu_is_offline(smp_processor_id())); +@@ -132,7 +144,7 @@ + struct irq_work *work; + struct llist_node *llnode; + +- BUG_ON(!irqs_disabled()); ++ BUG_ON_NONRT(!irqs_disabled()); + + if (llist_empty(list)) + return; +@@ -169,7 +181,16 @@ + void irq_work_run(void) + { + irq_work_run_list(this_cpu_ptr(&raised_list)); +- irq_work_run_list(this_cpu_ptr(&lazy_list)); ++ if (IS_ENABLED(CONFIG_PREEMPT_RT_FULL)) { ++ /* ++ * NOTE: we raise softirq via IPI for safety, ++ * and execute in irq_work_tick() to move the ++ * overhead from hard to soft irq context. ++ */ ++ if (!llist_empty(this_cpu_ptr(&lazy_list))) ++ raise_softirq(TIMER_SOFTIRQ); ++ } else ++ irq_work_run_list(this_cpu_ptr(&lazy_list)); + } + EXPORT_SYMBOL_GPL(irq_work_run); + +diff -Nur linux-4.1.10.orig/kernel/Kconfig.locks linux-4.1.10/kernel/Kconfig.locks +--- linux-4.1.10.orig/kernel/Kconfig.locks 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/Kconfig.locks 2015-10-07 18:00:08.000000000 +0200 +@@ -225,11 +225,11 @@ + + config MUTEX_SPIN_ON_OWNER + def_bool y +- depends on SMP && !DEBUG_MUTEXES && ARCH_SUPPORTS_ATOMIC_RMW ++ depends on SMP && !DEBUG_MUTEXES && ARCH_SUPPORTS_ATOMIC_RMW && !PREEMPT_RT_FULL + + config RWSEM_SPIN_ON_OWNER + def_bool y +- depends on SMP && RWSEM_XCHGADD_ALGORITHM && ARCH_SUPPORTS_ATOMIC_RMW ++ depends on SMP && RWSEM_XCHGADD_ALGORITHM && ARCH_SUPPORTS_ATOMIC_RMW && !PREEMPT_RT_FULL + + config LOCK_SPIN_ON_OWNER + def_bool y +diff -Nur linux-4.1.10.orig/kernel/Kconfig.preempt linux-4.1.10/kernel/Kconfig.preempt +--- linux-4.1.10.orig/kernel/Kconfig.preempt 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/Kconfig.preempt 2015-10-07 18:00:08.000000000 +0200 +@@ -1,3 +1,16 @@ ++config PREEMPT ++ bool ++ select PREEMPT_COUNT ++ ++config PREEMPT_RT_BASE ++ bool ++ select PREEMPT ++ ++config HAVE_PREEMPT_LAZY ++ bool ++ ++config PREEMPT_LAZY ++ def_bool y if HAVE_PREEMPT_LAZY && PREEMPT_RT_FULL + + choice + prompt "Preemption Model" +@@ -33,9 +46,9 @@ + + Select this if you are building a kernel for a desktop system. + +-config PREEMPT ++config PREEMPT__LL + bool "Preemptible Kernel (Low-Latency Desktop)" +- select PREEMPT_COUNT ++ select PREEMPT + select UNINLINE_SPIN_UNLOCK if !ARCH_INLINE_SPIN_UNLOCK + help + This option reduces the latency of the kernel by making +@@ -52,6 +65,22 @@ + embedded system with latency requirements in the milliseconds + range. + ++config PREEMPT_RTB ++ bool "Preemptible Kernel (Basic RT)" ++ select PREEMPT_RT_BASE ++ help ++ This option is basically the same as (Low-Latency Desktop) but ++ enables changes which are preliminary for the full preemptible ++ RT kernel. ++ ++config PREEMPT_RT_FULL ++ bool "Fully Preemptible Kernel (RT)" ++ depends on IRQ_FORCED_THREADING ++ select PREEMPT_RT_BASE ++ select PREEMPT_RCU ++ help ++ All and everything ++ + endchoice + + config PREEMPT_COUNT +diff -Nur linux-4.1.10.orig/kernel/ksysfs.c linux-4.1.10/kernel/ksysfs.c +--- linux-4.1.10.orig/kernel/ksysfs.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/ksysfs.c 2015-10-07 18:00:08.000000000 +0200 +@@ -136,6 +136,15 @@ + + #endif /* CONFIG_KEXEC */ + ++#if defined(CONFIG_PREEMPT_RT_FULL) ++static ssize_t realtime_show(struct kobject *kobj, ++ struct kobj_attribute *attr, char *buf) ++{ ++ return sprintf(buf, "%d\n", 1); ++} ++KERNEL_ATTR_RO(realtime); ++#endif ++ + /* whether file capabilities are enabled */ + static ssize_t fscaps_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +@@ -203,6 +212,9 @@ + &vmcoreinfo_attr.attr, + #endif + &rcu_expedited_attr.attr, ++#ifdef CONFIG_PREEMPT_RT_FULL ++ &realtime_attr.attr, ++#endif + NULL + }; + +diff -Nur linux-4.1.10.orig/kernel/locking/lglock.c linux-4.1.10/kernel/locking/lglock.c +--- linux-4.1.10.orig/kernel/locking/lglock.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/locking/lglock.c 2015-10-07 18:00:08.000000000 +0200 +@@ -4,6 +4,15 @@ + #include + #include + ++#ifndef CONFIG_PREEMPT_RT_FULL ++# define lg_lock_ptr arch_spinlock_t ++# define lg_do_lock(l) arch_spin_lock(l) ++# define lg_do_unlock(l) arch_spin_unlock(l) ++#else ++# define lg_lock_ptr struct rt_mutex ++# define lg_do_lock(l) __rt_spin_lock(l) ++# define lg_do_unlock(l) __rt_spin_unlock(l) ++#endif + /* + * Note there is no uninit, so lglocks cannot be defined in + * modules (but it's fine to use them from there) +@@ -12,51 +21,60 @@ + + void lg_lock_init(struct lglock *lg, char *name) + { ++#ifdef CONFIG_PREEMPT_RT_FULL ++ int i; ++ ++ for_each_possible_cpu(i) { ++ struct rt_mutex *lock = per_cpu_ptr(lg->lock, i); ++ ++ rt_mutex_init(lock); ++ } ++#endif + LOCKDEP_INIT_MAP(&lg->lock_dep_map, name, &lg->lock_key, 0); + } + EXPORT_SYMBOL(lg_lock_init); + + void lg_local_lock(struct lglock *lg) + { +- arch_spinlock_t *lock; ++ lg_lock_ptr *lock; + +- preempt_disable(); ++ migrate_disable(); + lock_acquire_shared(&lg->lock_dep_map, 0, 0, NULL, _RET_IP_); + lock = this_cpu_ptr(lg->lock); +- arch_spin_lock(lock); ++ lg_do_lock(lock); + } + EXPORT_SYMBOL(lg_local_lock); + + void lg_local_unlock(struct lglock *lg) + { +- arch_spinlock_t *lock; ++ lg_lock_ptr *lock; + + lock_release(&lg->lock_dep_map, 1, _RET_IP_); + lock = this_cpu_ptr(lg->lock); +- arch_spin_unlock(lock); +- preempt_enable(); ++ lg_do_unlock(lock); ++ migrate_enable(); + } + EXPORT_SYMBOL(lg_local_unlock); + + void lg_local_lock_cpu(struct lglock *lg, int cpu) + { +- arch_spinlock_t *lock; ++ lg_lock_ptr *lock; + +- preempt_disable(); ++ preempt_disable_nort(); + lock_acquire_shared(&lg->lock_dep_map, 0, 0, NULL, _RET_IP_); + lock = per_cpu_ptr(lg->lock, cpu); +- arch_spin_lock(lock); ++ lg_do_lock(lock); + } + EXPORT_SYMBOL(lg_local_lock_cpu); + + void lg_local_unlock_cpu(struct lglock *lg, int cpu) + { +- arch_spinlock_t *lock; ++ lg_lock_ptr *lock; + + lock_release(&lg->lock_dep_map, 1, _RET_IP_); + lock = per_cpu_ptr(lg->lock, cpu); +- arch_spin_unlock(lock); +- preempt_enable(); ++ lg_do_unlock(lock); ++ preempt_enable_nort(); + } + EXPORT_SYMBOL(lg_local_unlock_cpu); + +@@ -64,12 +82,12 @@ + { + int i; + +- preempt_disable(); ++ preempt_disable_nort(); + lock_acquire_exclusive(&lg->lock_dep_map, 0, 0, NULL, _RET_IP_); + for_each_possible_cpu(i) { +- arch_spinlock_t *lock; ++ lg_lock_ptr *lock; + lock = per_cpu_ptr(lg->lock, i); +- arch_spin_lock(lock); ++ lg_do_lock(lock); + } + } + EXPORT_SYMBOL(lg_global_lock); +@@ -80,10 +98,35 @@ + + lock_release(&lg->lock_dep_map, 1, _RET_IP_); + for_each_possible_cpu(i) { +- arch_spinlock_t *lock; ++ lg_lock_ptr *lock; + lock = per_cpu_ptr(lg->lock, i); +- arch_spin_unlock(lock); ++ lg_do_unlock(lock); + } +- preempt_enable(); ++ preempt_enable_nort(); + } + EXPORT_SYMBOL(lg_global_unlock); ++ ++#ifdef CONFIG_PREEMPT_RT_FULL ++/* ++ * HACK: If you use this, you get to keep the pieces. ++ * Used in queue_stop_cpus_work() when stop machinery ++ * is called from inactive CPU, so we can't schedule. ++ */ ++# define lg_do_trylock_relax(l) \ ++ do { \ ++ while (!__rt_spin_trylock(l)) \ ++ cpu_relax(); \ ++ } while (0) ++ ++void lg_global_trylock_relax(struct lglock *lg) ++{ ++ int i; ++ ++ lock_acquire_exclusive(&lg->lock_dep_map, 0, 0, NULL, _RET_IP_); ++ for_each_possible_cpu(i) { ++ lg_lock_ptr *lock; ++ lock = per_cpu_ptr(lg->lock, i); ++ lg_do_trylock_relax(lock); ++ } ++} ++#endif +diff -Nur linux-4.1.10.orig/kernel/locking/lockdep.c linux-4.1.10/kernel/locking/lockdep.c +--- linux-4.1.10.orig/kernel/locking/lockdep.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/locking/lockdep.c 2015-10-07 18:00:08.000000000 +0200 +@@ -3563,6 +3563,7 @@ + } + } + ++#ifndef CONFIG_PREEMPT_RT_FULL + /* + * We dont accurately track softirq state in e.g. + * hardirq contexts (such as on 4KSTACKS), so only +@@ -3577,6 +3578,7 @@ + DEBUG_LOCKS_WARN_ON(!current->softirqs_enabled); + } + } ++#endif + + if (!debug_locks) + print_irqtrace_events(current); +diff -Nur linux-4.1.10.orig/kernel/locking/locktorture.c linux-4.1.10/kernel/locking/locktorture.c +--- linux-4.1.10.orig/kernel/locking/locktorture.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/locking/locktorture.c 2015-10-07 18:00:08.000000000 +0200 +@@ -24,7 +24,6 @@ + #include + #include + #include +-#include + #include + #include + #include +diff -Nur linux-4.1.10.orig/kernel/locking/Makefile linux-4.1.10/kernel/locking/Makefile +--- linux-4.1.10.orig/kernel/locking/Makefile 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/locking/Makefile 2015-10-07 18:00:08.000000000 +0200 +@@ -1,5 +1,5 @@ + +-obj-y += mutex.o semaphore.o rwsem.o ++obj-y += semaphore.o + + ifdef CONFIG_FUNCTION_TRACER + CFLAGS_REMOVE_lockdep.o = $(CC_FLAGS_FTRACE) +@@ -8,7 +8,11 @@ + CFLAGS_REMOVE_rtmutex-debug.o = $(CC_FLAGS_FTRACE) + endif + ++ifneq ($(CONFIG_PREEMPT_RT_FULL),y) ++obj-y += mutex.o + obj-$(CONFIG_DEBUG_MUTEXES) += mutex-debug.o ++obj-y += rwsem.o ++endif + obj-$(CONFIG_LOCKDEP) += lockdep.o + ifeq ($(CONFIG_PROC_FS),y) + obj-$(CONFIG_LOCKDEP) += lockdep_proc.o +@@ -22,8 +26,11 @@ + obj-$(CONFIG_RT_MUTEX_TESTER) += rtmutex-tester.o + obj-$(CONFIG_DEBUG_SPINLOCK) += spinlock.o + obj-$(CONFIG_DEBUG_SPINLOCK) += spinlock_debug.o ++ifneq ($(CONFIG_PREEMPT_RT_FULL),y) + obj-$(CONFIG_RWSEM_GENERIC_SPINLOCK) += rwsem-spinlock.o + obj-$(CONFIG_RWSEM_XCHGADD_ALGORITHM) += rwsem-xadd.o ++endif + obj-$(CONFIG_PERCPU_RWSEM) += percpu-rwsem.o ++obj-$(CONFIG_PREEMPT_RT_FULL) += rt.o + obj-$(CONFIG_QUEUE_RWLOCK) += qrwlock.o + obj-$(CONFIG_LOCK_TORTURE_TEST) += locktorture.o +diff -Nur linux-4.1.10.orig/kernel/locking/rt.c linux-4.1.10/kernel/locking/rt.c +--- linux-4.1.10.orig/kernel/locking/rt.c 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/kernel/locking/rt.c 2015-10-07 18:00:08.000000000 +0200 +@@ -0,0 +1,461 @@ ++/* ++ * kernel/rt.c ++ * ++ * Real-Time Preemption Support ++ * ++ * started by Ingo Molnar: ++ * ++ * Copyright (C) 2004-2006 Red Hat, Inc., Ingo Molnar ++ * Copyright (C) 2006, Timesys Corp., Thomas Gleixner ++ * ++ * historic credit for proving that Linux spinlocks can be implemented via ++ * RT-aware mutexes goes to many people: The Pmutex project (Dirk Grambow ++ * and others) who prototyped it on 2.4 and did lots of comparative ++ * research and analysis; TimeSys, for proving that you can implement a ++ * fully preemptible kernel via the use of IRQ threading and mutexes; ++ * Bill Huey for persuasively arguing on lkml that the mutex model is the ++ * right one; and to MontaVista, who ported pmutexes to 2.6. ++ * ++ * This code is a from-scratch implementation and is not based on pmutexes, ++ * but the idea of converting spinlocks to mutexes is used here too. ++ * ++ * lock debugging, locking tree, deadlock detection: ++ * ++ * Copyright (C) 2004, LynuxWorks, Inc., Igor Manyilov, Bill Huey ++ * Released under the General Public License (GPL). ++ * ++ * Includes portions of the generic R/W semaphore implementation from: ++ * ++ * Copyright (c) 2001 David Howells (dhowells@redhat.com). ++ * - Derived partially from idea by Andrea Arcangeli ++ * - Derived also from comments by Linus ++ * ++ * Pending ownership of locks and ownership stealing: ++ * ++ * Copyright (C) 2005, Kihon Technologies Inc., Steven Rostedt ++ * ++ * (also by Steven Rostedt) ++ * - Converted single pi_lock to individual task locks. ++ * ++ * By Esben Nielsen: ++ * Doing priority inheritance with help of the scheduler. ++ * ++ * Copyright (C) 2006, Timesys Corp., Thomas Gleixner ++ * - major rework based on Esben Nielsens initial patch ++ * - replaced thread_info references by task_struct refs ++ * - removed task->pending_owner dependency ++ * - BKL drop/reacquire for semaphore style locks to avoid deadlocks ++ * in the scheduler return path as discussed with Steven Rostedt ++ * ++ * Copyright (C) 2006, Kihon Technologies Inc. ++ * Steven Rostedt ++ * - debugged and patched Thomas Gleixner's rework. ++ * - added back the cmpxchg to the rework. ++ * - turned atomic require back on for SMP. ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "rtmutex_common.h" ++ ++/* ++ * struct mutex functions ++ */ ++void __mutex_do_init(struct mutex *mutex, const char *name, ++ struct lock_class_key *key) ++{ ++#ifdef CONFIG_DEBUG_LOCK_ALLOC ++ /* ++ * Make sure we are not reinitializing a held lock: ++ */ ++ debug_check_no_locks_freed((void *)mutex, sizeof(*mutex)); ++ lockdep_init_map(&mutex->dep_map, name, key, 0); ++#endif ++ mutex->lock.save_state = 0; ++} ++EXPORT_SYMBOL(__mutex_do_init); ++ ++void __lockfunc _mutex_lock(struct mutex *lock) ++{ ++ mutex_acquire(&lock->dep_map, 0, 0, _RET_IP_); ++ rt_mutex_lock(&lock->lock); ++} ++EXPORT_SYMBOL(_mutex_lock); ++ ++int __lockfunc _mutex_lock_interruptible(struct mutex *lock) ++{ ++ int ret; ++ ++ mutex_acquire(&lock->dep_map, 0, 0, _RET_IP_); ++ ret = rt_mutex_lock_interruptible(&lock->lock); ++ if (ret) ++ mutex_release(&lock->dep_map, 1, _RET_IP_); ++ return ret; ++} ++EXPORT_SYMBOL(_mutex_lock_interruptible); ++ ++int __lockfunc _mutex_lock_killable(struct mutex *lock) ++{ ++ int ret; ++ ++ mutex_acquire(&lock->dep_map, 0, 0, _RET_IP_); ++ ret = rt_mutex_lock_killable(&lock->lock); ++ if (ret) ++ mutex_release(&lock->dep_map, 1, _RET_IP_); ++ return ret; ++} ++EXPORT_SYMBOL(_mutex_lock_killable); ++ ++#ifdef CONFIG_DEBUG_LOCK_ALLOC ++void __lockfunc _mutex_lock_nested(struct mutex *lock, int subclass) ++{ ++ mutex_acquire_nest(&lock->dep_map, subclass, 0, NULL, _RET_IP_); ++ rt_mutex_lock(&lock->lock); ++} ++EXPORT_SYMBOL(_mutex_lock_nested); ++ ++void __lockfunc _mutex_lock_nest_lock(struct mutex *lock, struct lockdep_map *nest) ++{ ++ mutex_acquire_nest(&lock->dep_map, 0, 0, nest, _RET_IP_); ++ rt_mutex_lock(&lock->lock); ++} ++EXPORT_SYMBOL(_mutex_lock_nest_lock); ++ ++int __lockfunc _mutex_lock_interruptible_nested(struct mutex *lock, int subclass) ++{ ++ int ret; ++ ++ mutex_acquire_nest(&lock->dep_map, subclass, 0, NULL, _RET_IP_); ++ ret = rt_mutex_lock_interruptible(&lock->lock); ++ if (ret) ++ mutex_release(&lock->dep_map, 1, _RET_IP_); ++ return ret; ++} ++EXPORT_SYMBOL(_mutex_lock_interruptible_nested); ++ ++int __lockfunc _mutex_lock_killable_nested(struct mutex *lock, int subclass) ++{ ++ int ret; ++ ++ mutex_acquire(&lock->dep_map, subclass, 0, _RET_IP_); ++ ret = rt_mutex_lock_killable(&lock->lock); ++ if (ret) ++ mutex_release(&lock->dep_map, 1, _RET_IP_); ++ return ret; ++} ++EXPORT_SYMBOL(_mutex_lock_killable_nested); ++#endif ++ ++int __lockfunc _mutex_trylock(struct mutex *lock) ++{ ++ int ret = rt_mutex_trylock(&lock->lock); ++ ++ if (ret) ++ mutex_acquire(&lock->dep_map, 0, 1, _RET_IP_); ++ ++ return ret; ++} ++EXPORT_SYMBOL(_mutex_trylock); ++ ++void __lockfunc _mutex_unlock(struct mutex *lock) ++{ ++ mutex_release(&lock->dep_map, 1, _RET_IP_); ++ rt_mutex_unlock(&lock->lock); ++} ++EXPORT_SYMBOL(_mutex_unlock); ++ ++/* ++ * rwlock_t functions ++ */ ++int __lockfunc rt_write_trylock(rwlock_t *rwlock) ++{ ++ int ret; ++ ++ migrate_disable(); ++ ret = rt_mutex_trylock(&rwlock->lock); ++ if (ret) ++ rwlock_acquire(&rwlock->dep_map, 0, 1, _RET_IP_); ++ else ++ migrate_enable(); ++ ++ return ret; ++} ++EXPORT_SYMBOL(rt_write_trylock); ++ ++int __lockfunc rt_write_trylock_irqsave(rwlock_t *rwlock, unsigned long *flags) ++{ ++ int ret; ++ ++ *flags = 0; ++ ret = rt_write_trylock(rwlock); ++ return ret; ++} ++EXPORT_SYMBOL(rt_write_trylock_irqsave); ++ ++int __lockfunc rt_read_trylock(rwlock_t *rwlock) ++{ ++ struct rt_mutex *lock = &rwlock->lock; ++ int ret = 1; ++ ++ /* ++ * recursive read locks succeed when current owns the lock, ++ * but not when read_depth == 0 which means that the lock is ++ * write locked. ++ */ ++ if (rt_mutex_owner(lock) != current) { ++ migrate_disable(); ++ ret = rt_mutex_trylock(lock); ++ if (ret) ++ rwlock_acquire(&rwlock->dep_map, 0, 1, _RET_IP_); ++ else ++ migrate_enable(); ++ ++ } else if (!rwlock->read_depth) { ++ ret = 0; ++ } ++ ++ if (ret) ++ rwlock->read_depth++; ++ ++ return ret; ++} ++EXPORT_SYMBOL(rt_read_trylock); ++ ++void __lockfunc rt_write_lock(rwlock_t *rwlock) ++{ ++ rwlock_acquire(&rwlock->dep_map, 0, 0, _RET_IP_); ++ migrate_disable(); ++ __rt_spin_lock(&rwlock->lock); ++} ++EXPORT_SYMBOL(rt_write_lock); ++ ++void __lockfunc rt_read_lock(rwlock_t *rwlock) ++{ ++ struct rt_mutex *lock = &rwlock->lock; ++ ++ ++ /* ++ * recursive read locks succeed when current owns the lock ++ */ ++ if (rt_mutex_owner(lock) != current) { ++ migrate_disable(); ++ rwlock_acquire(&rwlock->dep_map, 0, 0, _RET_IP_); ++ __rt_spin_lock(lock); ++ } ++ rwlock->read_depth++; ++} ++ ++EXPORT_SYMBOL(rt_read_lock); ++ ++void __lockfunc rt_write_unlock(rwlock_t *rwlock) ++{ ++ /* NOTE: we always pass in '1' for nested, for simplicity */ ++ rwlock_release(&rwlock->dep_map, 1, _RET_IP_); ++ __rt_spin_unlock(&rwlock->lock); ++ migrate_enable(); ++} ++EXPORT_SYMBOL(rt_write_unlock); ++ ++void __lockfunc rt_read_unlock(rwlock_t *rwlock) ++{ ++ /* Release the lock only when read_depth is down to 0 */ ++ if (--rwlock->read_depth == 0) { ++ rwlock_release(&rwlock->dep_map, 1, _RET_IP_); ++ __rt_spin_unlock(&rwlock->lock); ++ migrate_enable(); ++ } ++} ++EXPORT_SYMBOL(rt_read_unlock); ++ ++unsigned long __lockfunc rt_write_lock_irqsave(rwlock_t *rwlock) ++{ ++ rt_write_lock(rwlock); ++ ++ return 0; ++} ++EXPORT_SYMBOL(rt_write_lock_irqsave); ++ ++unsigned long __lockfunc rt_read_lock_irqsave(rwlock_t *rwlock) ++{ ++ rt_read_lock(rwlock); ++ ++ return 0; ++} ++EXPORT_SYMBOL(rt_read_lock_irqsave); ++ ++void __rt_rwlock_init(rwlock_t *rwlock, char *name, struct lock_class_key *key) ++{ ++#ifdef CONFIG_DEBUG_LOCK_ALLOC ++ /* ++ * Make sure we are not reinitializing a held lock: ++ */ ++ debug_check_no_locks_freed((void *)rwlock, sizeof(*rwlock)); ++ lockdep_init_map(&rwlock->dep_map, name, key, 0); ++#endif ++ rwlock->lock.save_state = 1; ++ rwlock->read_depth = 0; ++} ++EXPORT_SYMBOL(__rt_rwlock_init); ++ ++/* ++ * rw_semaphores ++ */ ++ ++void rt_up_write(struct rw_semaphore *rwsem) ++{ ++ rwsem_release(&rwsem->dep_map, 1, _RET_IP_); ++ rt_mutex_unlock(&rwsem->lock); ++} ++EXPORT_SYMBOL(rt_up_write); ++ ++void __rt_up_read(struct rw_semaphore *rwsem) ++{ ++ if (--rwsem->read_depth == 0) ++ rt_mutex_unlock(&rwsem->lock); ++} ++ ++void rt_up_read(struct rw_semaphore *rwsem) ++{ ++ rwsem_release(&rwsem->dep_map, 1, _RET_IP_); ++ __rt_up_read(rwsem); ++} ++EXPORT_SYMBOL(rt_up_read); ++ ++/* ++ * downgrade a write lock into a read lock ++ * - just wake up any readers at the front of the queue ++ */ ++void rt_downgrade_write(struct rw_semaphore *rwsem) ++{ ++ BUG_ON(rt_mutex_owner(&rwsem->lock) != current); ++ rwsem->read_depth = 1; ++} ++EXPORT_SYMBOL(rt_downgrade_write); ++ ++int rt_down_write_trylock(struct rw_semaphore *rwsem) ++{ ++ int ret = rt_mutex_trylock(&rwsem->lock); ++ ++ if (ret) ++ rwsem_acquire(&rwsem->dep_map, 0, 1, _RET_IP_); ++ return ret; ++} ++EXPORT_SYMBOL(rt_down_write_trylock); ++ ++void rt_down_write(struct rw_semaphore *rwsem) ++{ ++ rwsem_acquire(&rwsem->dep_map, 0, 0, _RET_IP_); ++ rt_mutex_lock(&rwsem->lock); ++} ++EXPORT_SYMBOL(rt_down_write); ++ ++void rt_down_write_nested(struct rw_semaphore *rwsem, int subclass) ++{ ++ rwsem_acquire(&rwsem->dep_map, subclass, 0, _RET_IP_); ++ rt_mutex_lock(&rwsem->lock); ++} ++EXPORT_SYMBOL(rt_down_write_nested); ++ ++void rt_down_write_nested_lock(struct rw_semaphore *rwsem, ++ struct lockdep_map *nest) ++{ ++ rwsem_acquire_nest(&rwsem->dep_map, 0, 0, nest, _RET_IP_); ++ rt_mutex_lock(&rwsem->lock); ++} ++EXPORT_SYMBOL(rt_down_write_nested_lock); ++ ++int rt_down_read_trylock(struct rw_semaphore *rwsem) ++{ ++ struct rt_mutex *lock = &rwsem->lock; ++ int ret = 1; ++ ++ /* ++ * recursive read locks succeed when current owns the rwsem, ++ * but not when read_depth == 0 which means that the rwsem is ++ * write locked. ++ */ ++ if (rt_mutex_owner(lock) != current) ++ ret = rt_mutex_trylock(&rwsem->lock); ++ else if (!rwsem->read_depth) ++ ret = 0; ++ ++ if (ret) { ++ rwsem->read_depth++; ++ rwsem_acquire(&rwsem->dep_map, 0, 1, _RET_IP_); ++ } ++ return ret; ++} ++EXPORT_SYMBOL(rt_down_read_trylock); ++ ++static void __rt_down_read(struct rw_semaphore *rwsem, int subclass) ++{ ++ struct rt_mutex *lock = &rwsem->lock; ++ ++ rwsem_acquire_read(&rwsem->dep_map, subclass, 0, _RET_IP_); ++ ++ if (rt_mutex_owner(lock) != current) ++ rt_mutex_lock(&rwsem->lock); ++ rwsem->read_depth++; ++} ++ ++void rt_down_read(struct rw_semaphore *rwsem) ++{ ++ __rt_down_read(rwsem, 0); ++} ++EXPORT_SYMBOL(rt_down_read); ++ ++void rt_down_read_nested(struct rw_semaphore *rwsem, int subclass) ++{ ++ __rt_down_read(rwsem, subclass); ++} ++EXPORT_SYMBOL(rt_down_read_nested); ++ ++void __rt_rwsem_init(struct rw_semaphore *rwsem, const char *name, ++ struct lock_class_key *key) ++{ ++#ifdef CONFIG_DEBUG_LOCK_ALLOC ++ /* ++ * Make sure we are not reinitializing a held lock: ++ */ ++ debug_check_no_locks_freed((void *)rwsem, sizeof(*rwsem)); ++ lockdep_init_map(&rwsem->dep_map, name, key, 0); ++#endif ++ rwsem->read_depth = 0; ++ rwsem->lock.save_state = 0; ++} ++EXPORT_SYMBOL(__rt_rwsem_init); ++ ++/** ++ * atomic_dec_and_mutex_lock - return holding mutex if we dec to 0 ++ * @cnt: the atomic which we are to dec ++ * @lock: the mutex to return holding if we dec to 0 ++ * ++ * return true and hold lock if we dec to 0, return false otherwise ++ */ ++int atomic_dec_and_mutex_lock(atomic_t *cnt, struct mutex *lock) ++{ ++ /* dec if we can't possibly hit 0 */ ++ if (atomic_add_unless(cnt, -1, 1)) ++ return 0; ++ /* we might hit 0, so take the lock */ ++ mutex_lock(lock); ++ if (!atomic_dec_and_test(cnt)) { ++ /* when we actually did the dec, we didn't hit 0 */ ++ mutex_unlock(lock); ++ return 0; ++ } ++ /* we hit 0, and we hold the lock */ ++ return 1; ++} ++EXPORT_SYMBOL(atomic_dec_and_mutex_lock); +diff -Nur linux-4.1.10.orig/kernel/locking/rtmutex.c linux-4.1.10/kernel/locking/rtmutex.c +--- linux-4.1.10.orig/kernel/locking/rtmutex.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/locking/rtmutex.c 2015-10-07 18:00:08.000000000 +0200 +@@ -7,6 +7,11 @@ + * Copyright (C) 2005-2006 Timesys Corp., Thomas Gleixner + * Copyright (C) 2005 Kihon Technologies Inc., Steven Rostedt + * Copyright (C) 2006 Esben Nielsen ++ * Adaptive Spinlocks: ++ * Copyright (C) 2008 Novell, Inc., Gregory Haskins, Sven Dietrich, ++ * and Peter Morreale, ++ * Adaptive Spinlocks simplification: ++ * Copyright (C) 2008 Red Hat, Inc., Steven Rostedt + * + * See Documentation/locking/rt-mutex-design.txt for details. + */ +@@ -16,6 +21,7 @@ + #include + #include + #include ++#include + + #include "rtmutex_common.h" + +@@ -69,6 +75,12 @@ + clear_rt_mutex_waiters(lock); + } + ++static int rt_mutex_real_waiter(struct rt_mutex_waiter *waiter) ++{ ++ return waiter && waiter != PI_WAKEUP_INPROGRESS && ++ waiter != PI_REQUEUE_INPROGRESS; ++} ++ + /* + * We can speed up the acquire/release, if the architecture + * supports cmpxchg and if there's no debugging state to be set up +@@ -300,7 +312,7 @@ + * of task. We do not use the spin_xx_mutex() variants here as we are + * outside of the debug path.) + */ +-static void rt_mutex_adjust_prio(struct task_struct *task) ++void rt_mutex_adjust_prio(struct task_struct *task) + { + unsigned long flags; + +@@ -335,6 +347,14 @@ + return debug_rt_mutex_detect_deadlock(waiter, chwalk); + } + ++static void rt_mutex_wake_waiter(struct rt_mutex_waiter *waiter) ++{ ++ if (waiter->savestate) ++ wake_up_lock_sleeper(waiter->task); ++ else ++ wake_up_process(waiter->task); ++} ++ + /* + * Max number of times we'll walk the boosting chain: + */ +@@ -342,7 +362,8 @@ + + static inline struct rt_mutex *task_blocked_on_lock(struct task_struct *p) + { +- return p->pi_blocked_on ? p->pi_blocked_on->lock : NULL; ++ return rt_mutex_real_waiter(p->pi_blocked_on) ? ++ p->pi_blocked_on->lock : NULL; + } + + /* +@@ -479,7 +500,7 @@ + * reached or the state of the chain has changed while we + * dropped the locks. + */ +- if (!waiter) ++ if (!rt_mutex_real_waiter(waiter)) + goto out_unlock_pi; + + /* +@@ -641,13 +662,16 @@ + * follow here. This is the end of the chain we are walking. + */ + if (!rt_mutex_owner(lock)) { ++ struct rt_mutex_waiter *lock_top_waiter; ++ + /* + * If the requeue [7] above changed the top waiter, + * then we need to wake the new top waiter up to try + * to get the lock. + */ +- if (prerequeue_top_waiter != rt_mutex_top_waiter(lock)) +- wake_up_process(rt_mutex_top_waiter(lock)->task); ++ lock_top_waiter = rt_mutex_top_waiter(lock); ++ if (prerequeue_top_waiter != lock_top_waiter) ++ rt_mutex_wake_waiter(lock_top_waiter); + raw_spin_unlock(&lock->wait_lock); + return 0; + } +@@ -740,6 +764,25 @@ + return ret; + } + ++ ++#define STEAL_NORMAL 0 ++#define STEAL_LATERAL 1 ++ ++/* ++ * Note that RT tasks are excluded from lateral-steals to prevent the ++ * introduction of an unbounded latency ++ */ ++static inline int lock_is_stealable(struct task_struct *task, ++ struct task_struct *pendowner, int mode) ++{ ++ if (mode == STEAL_NORMAL || rt_task(task)) { ++ if (task->prio >= pendowner->prio) ++ return 0; ++ } else if (task->prio > pendowner->prio) ++ return 0; ++ return 1; ++} ++ + /* + * Try to take an rt-mutex + * +@@ -750,8 +793,9 @@ + * @waiter: The waiter that is queued to the lock's wait list if the + * callsite called task_blocked_on_lock(), otherwise NULL + */ +-static int try_to_take_rt_mutex(struct rt_mutex *lock, struct task_struct *task, +- struct rt_mutex_waiter *waiter) ++static int __try_to_take_rt_mutex(struct rt_mutex *lock, ++ struct task_struct *task, ++ struct rt_mutex_waiter *waiter, int mode) + { + unsigned long flags; + +@@ -790,8 +834,10 @@ + * If waiter is not the highest priority waiter of + * @lock, give up. + */ +- if (waiter != rt_mutex_top_waiter(lock)) ++ if (waiter != rt_mutex_top_waiter(lock)) { ++ /* XXX lock_is_stealable() ? */ + return 0; ++ } + + /* + * We can acquire the lock. Remove the waiter from the +@@ -809,14 +855,10 @@ + * not need to be dequeued. + */ + if (rt_mutex_has_waiters(lock)) { +- /* +- * If @task->prio is greater than or equal to +- * the top waiter priority (kernel view), +- * @task lost. +- */ +- if (task->prio >= rt_mutex_top_waiter(lock)->prio) +- return 0; ++ struct task_struct *pown = rt_mutex_top_waiter(lock)->task; + ++ if (task != pown && !lock_is_stealable(task, pown, mode)) ++ return 0; + /* + * The current top waiter stays enqueued. We + * don't have to change anything in the lock +@@ -865,6 +907,347 @@ + return 1; + } + ++#ifdef CONFIG_PREEMPT_RT_FULL ++/* ++ * preemptible spin_lock functions: ++ */ ++static inline void rt_spin_lock_fastlock(struct rt_mutex *lock, ++ void (*slowfn)(struct rt_mutex *lock)) ++{ ++ might_sleep_no_state_check(); ++ ++ if (likely(rt_mutex_cmpxchg(lock, NULL, current))) ++ rt_mutex_deadlock_account_lock(lock, current); ++ else ++ slowfn(lock); ++} ++ ++static inline void rt_spin_lock_fastunlock(struct rt_mutex *lock, ++ void (*slowfn)(struct rt_mutex *lock)) ++{ ++ if (likely(rt_mutex_cmpxchg(lock, current, NULL))) ++ rt_mutex_deadlock_account_unlock(current); ++ else ++ slowfn(lock); ++} ++#ifdef CONFIG_SMP ++/* ++ * Note that owner is a speculative pointer and dereferencing relies ++ * on rcu_read_lock() and the check against the lock owner. ++ */ ++static int adaptive_wait(struct rt_mutex *lock, ++ struct task_struct *owner) ++{ ++ int res = 0; ++ ++ rcu_read_lock(); ++ for (;;) { ++ if (owner != rt_mutex_owner(lock)) ++ break; ++ /* ++ * Ensure that owner->on_cpu is dereferenced _after_ ++ * checking the above to be valid. ++ */ ++ barrier(); ++ if (!owner->on_cpu) { ++ res = 1; ++ break; ++ } ++ cpu_relax(); ++ } ++ rcu_read_unlock(); ++ return res; ++} ++#else ++static int adaptive_wait(struct rt_mutex *lock, ++ struct task_struct *orig_owner) ++{ ++ return 1; ++} ++#endif ++ ++# define pi_lock(lock) raw_spin_lock_irq(lock) ++# define pi_unlock(lock) raw_spin_unlock_irq(lock) ++ ++static int task_blocks_on_rt_mutex(struct rt_mutex *lock, ++ struct rt_mutex_waiter *waiter, ++ struct task_struct *task, ++ enum rtmutex_chainwalk chwalk); ++/* ++ * Slow path lock function spin_lock style: this variant is very ++ * careful not to miss any non-lock wakeups. ++ * ++ * We store the current state under p->pi_lock in p->saved_state and ++ * the try_to_wake_up() code handles this accordingly. ++ */ ++static void noinline __sched rt_spin_lock_slowlock(struct rt_mutex *lock) ++{ ++ struct task_struct *lock_owner, *self = current; ++ struct rt_mutex_waiter waiter, *top_waiter; ++ int ret; ++ ++ rt_mutex_init_waiter(&waiter, true); ++ ++ raw_spin_lock(&lock->wait_lock); ++ ++ if (__try_to_take_rt_mutex(lock, self, NULL, STEAL_LATERAL)) { ++ raw_spin_unlock(&lock->wait_lock); ++ return; ++ } ++ ++ BUG_ON(rt_mutex_owner(lock) == self); ++ ++ /* ++ * We save whatever state the task is in and we'll restore it ++ * after acquiring the lock taking real wakeups into account ++ * as well. We are serialized via pi_lock against wakeups. See ++ * try_to_wake_up(). ++ */ ++ pi_lock(&self->pi_lock); ++ self->saved_state = self->state; ++ __set_current_state_no_track(TASK_UNINTERRUPTIBLE); ++ pi_unlock(&self->pi_lock); ++ ++ ret = task_blocks_on_rt_mutex(lock, &waiter, self, 0); ++ BUG_ON(ret); ++ ++ for (;;) { ++ /* Try to acquire the lock again. */ ++ if (__try_to_take_rt_mutex(lock, self, &waiter, STEAL_LATERAL)) ++ break; ++ ++ top_waiter = rt_mutex_top_waiter(lock); ++ lock_owner = rt_mutex_owner(lock); ++ ++ raw_spin_unlock(&lock->wait_lock); ++ ++ debug_rt_mutex_print_deadlock(&waiter); ++ ++ if (top_waiter != &waiter || adaptive_wait(lock, lock_owner)) ++ schedule_rt_mutex(lock); ++ ++ raw_spin_lock(&lock->wait_lock); ++ ++ pi_lock(&self->pi_lock); ++ __set_current_state_no_track(TASK_UNINTERRUPTIBLE); ++ pi_unlock(&self->pi_lock); ++ } ++ ++ /* ++ * Restore the task state to current->saved_state. We set it ++ * to the original state above and the try_to_wake_up() code ++ * has possibly updated it when a real (non-rtmutex) wakeup ++ * happened while we were blocked. Clear saved_state so ++ * try_to_wakeup() does not get confused. ++ */ ++ pi_lock(&self->pi_lock); ++ __set_current_state_no_track(self->saved_state); ++ self->saved_state = TASK_RUNNING; ++ pi_unlock(&self->pi_lock); ++ ++ /* ++ * try_to_take_rt_mutex() sets the waiter bit ++ * unconditionally. We might have to fix that up: ++ */ ++ fixup_rt_mutex_waiters(lock); ++ ++ BUG_ON(rt_mutex_has_waiters(lock) && &waiter == rt_mutex_top_waiter(lock)); ++ BUG_ON(!RB_EMPTY_NODE(&waiter.tree_entry)); ++ ++ raw_spin_unlock(&lock->wait_lock); ++ ++ debug_rt_mutex_free_waiter(&waiter); ++} ++ ++static void wakeup_next_waiter(struct rt_mutex *lock); ++/* ++ * Slow path to release a rt_mutex spin_lock style ++ */ ++static void noinline __sched rt_spin_lock_slowunlock(struct rt_mutex *lock) ++{ ++ raw_spin_lock(&lock->wait_lock); ++ ++ debug_rt_mutex_unlock(lock); ++ ++ rt_mutex_deadlock_account_unlock(current); ++ ++ if (!rt_mutex_has_waiters(lock)) { ++ lock->owner = NULL; ++ raw_spin_unlock(&lock->wait_lock); ++ return; ++ } ++ ++ wakeup_next_waiter(lock); ++ ++ raw_spin_unlock(&lock->wait_lock); ++ ++ /* Undo pi boosting.when necessary */ ++ rt_mutex_adjust_prio(current); ++} ++ ++void __lockfunc rt_spin_lock(spinlock_t *lock) ++{ ++ rt_spin_lock_fastlock(&lock->lock, rt_spin_lock_slowlock); ++ spin_acquire(&lock->dep_map, 0, 0, _RET_IP_); ++} ++EXPORT_SYMBOL(rt_spin_lock); ++ ++void __lockfunc __rt_spin_lock(struct rt_mutex *lock) ++{ ++ rt_spin_lock_fastlock(lock, rt_spin_lock_slowlock); ++} ++EXPORT_SYMBOL(__rt_spin_lock); ++ ++#ifdef CONFIG_DEBUG_LOCK_ALLOC ++void __lockfunc rt_spin_lock_nested(spinlock_t *lock, int subclass) ++{ ++ rt_spin_lock_fastlock(&lock->lock, rt_spin_lock_slowlock); ++ spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_); ++} ++EXPORT_SYMBOL(rt_spin_lock_nested); ++#endif ++ ++void __lockfunc rt_spin_unlock(spinlock_t *lock) ++{ ++ /* NOTE: we always pass in '1' for nested, for simplicity */ ++ spin_release(&lock->dep_map, 1, _RET_IP_); ++ rt_spin_lock_fastunlock(&lock->lock, rt_spin_lock_slowunlock); ++} ++EXPORT_SYMBOL(rt_spin_unlock); ++ ++void __lockfunc __rt_spin_unlock(struct rt_mutex *lock) ++{ ++ rt_spin_lock_fastunlock(lock, rt_spin_lock_slowunlock); ++} ++EXPORT_SYMBOL(__rt_spin_unlock); ++ ++/* ++ * Wait for the lock to get unlocked: instead of polling for an unlock ++ * (like raw spinlocks do), we lock and unlock, to force the kernel to ++ * schedule if there's contention: ++ */ ++void __lockfunc rt_spin_unlock_wait(spinlock_t *lock) ++{ ++ spin_lock(lock); ++ spin_unlock(lock); ++} ++EXPORT_SYMBOL(rt_spin_unlock_wait); ++ ++int __lockfunc __rt_spin_trylock(struct rt_mutex *lock) ++{ ++ return rt_mutex_trylock(lock); ++} ++ ++int __lockfunc rt_spin_trylock(spinlock_t *lock) ++{ ++ int ret = rt_mutex_trylock(&lock->lock); ++ ++ if (ret) ++ spin_acquire(&lock->dep_map, 0, 1, _RET_IP_); ++ return ret; ++} ++EXPORT_SYMBOL(rt_spin_trylock); ++ ++int __lockfunc rt_spin_trylock_bh(spinlock_t *lock) ++{ ++ int ret; ++ ++ local_bh_disable(); ++ ret = rt_mutex_trylock(&lock->lock); ++ if (ret) { ++ migrate_disable(); ++ spin_acquire(&lock->dep_map, 0, 1, _RET_IP_); ++ } else ++ local_bh_enable(); ++ return ret; ++} ++EXPORT_SYMBOL(rt_spin_trylock_bh); ++ ++int __lockfunc rt_spin_trylock_irqsave(spinlock_t *lock, unsigned long *flags) ++{ ++ int ret; ++ ++ *flags = 0; ++ ret = rt_mutex_trylock(&lock->lock); ++ if (ret) { ++ migrate_disable(); ++ spin_acquire(&lock->dep_map, 0, 1, _RET_IP_); ++ } ++ return ret; ++} ++EXPORT_SYMBOL(rt_spin_trylock_irqsave); ++ ++int atomic_dec_and_spin_lock(atomic_t *atomic, spinlock_t *lock) ++{ ++ /* Subtract 1 from counter unless that drops it to 0 (ie. it was 1) */ ++ if (atomic_add_unless(atomic, -1, 1)) ++ return 0; ++ migrate_disable(); ++ rt_spin_lock(lock); ++ if (atomic_dec_and_test(atomic)) ++ return 1; ++ rt_spin_unlock(lock); ++ migrate_enable(); ++ return 0; ++} ++EXPORT_SYMBOL(atomic_dec_and_spin_lock); ++ ++ void ++__rt_spin_lock_init(spinlock_t *lock, char *name, struct lock_class_key *key) ++{ ++#ifdef CONFIG_DEBUG_LOCK_ALLOC ++ /* ++ * Make sure we are not reinitializing a held lock: ++ */ ++ debug_check_no_locks_freed((void *)lock, sizeof(*lock)); ++ lockdep_init_map(&lock->dep_map, name, key, 0); ++#endif ++} ++EXPORT_SYMBOL(__rt_spin_lock_init); ++ ++#endif /* PREEMPT_RT_FULL */ ++ ++#ifdef CONFIG_PREEMPT_RT_FULL ++ static inline int __sched ++__mutex_lock_check_stamp(struct rt_mutex *lock, struct ww_acquire_ctx *ctx) ++{ ++ struct ww_mutex *ww = container_of(lock, struct ww_mutex, base.lock); ++ struct ww_acquire_ctx *hold_ctx = ACCESS_ONCE(ww->ctx); ++ ++ if (!hold_ctx) ++ return 0; ++ ++ if (unlikely(ctx == hold_ctx)) ++ return -EALREADY; ++ ++ if (ctx->stamp - hold_ctx->stamp <= LONG_MAX && ++ (ctx->stamp != hold_ctx->stamp || ctx > hold_ctx)) { ++#ifdef CONFIG_DEBUG_MUTEXES ++ DEBUG_LOCKS_WARN_ON(ctx->contending_lock); ++ ctx->contending_lock = ww; ++#endif ++ return -EDEADLK; ++ } ++ ++ return 0; ++} ++#else ++ static inline int __sched ++__mutex_lock_check_stamp(struct rt_mutex *lock, struct ww_acquire_ctx *ctx) ++{ ++ BUG(); ++ return 0; ++} ++ ++#endif ++ ++static inline int ++try_to_take_rt_mutex(struct rt_mutex *lock, struct task_struct *task, ++ struct rt_mutex_waiter *waiter) ++{ ++ return __try_to_take_rt_mutex(lock, task, waiter, STEAL_NORMAL); ++} ++ + /* + * Task blocks on lock. + * +@@ -896,6 +1279,23 @@ + return -EDEADLK; + + raw_spin_lock_irqsave(&task->pi_lock, flags); ++ ++ /* ++ * In the case of futex requeue PI, this will be a proxy ++ * lock. The task will wake unaware that it is enqueueed on ++ * this lock. Avoid blocking on two locks and corrupting ++ * pi_blocked_on via the PI_WAKEUP_INPROGRESS ++ * flag. futex_wait_requeue_pi() sets this when it wakes up ++ * before requeue (due to a signal or timeout). Do not enqueue ++ * the task if PI_WAKEUP_INPROGRESS is set. ++ */ ++ if (task != current && task->pi_blocked_on == PI_WAKEUP_INPROGRESS) { ++ raw_spin_unlock_irqrestore(&task->pi_lock, flags); ++ return -EAGAIN; ++ } ++ ++ BUG_ON(rt_mutex_real_waiter(task->pi_blocked_on)); ++ + __rt_mutex_adjust_prio(task); + waiter->task = task; + waiter->lock = lock; +@@ -919,7 +1319,7 @@ + rt_mutex_enqueue_pi(owner, waiter); + + __rt_mutex_adjust_prio(owner); +- if (owner->pi_blocked_on) ++ if (rt_mutex_real_waiter(owner->pi_blocked_on)) + chain_walk = 1; + } else if (rt_mutex_cond_detect_deadlock(waiter, chwalk)) { + chain_walk = 1; +@@ -957,8 +1357,9 @@ + /* + * Wake up the next waiter on the lock. + * +- * Remove the top waiter from the current tasks pi waiter list and +- * wake it up. ++ * Remove the top waiter from the current tasks pi waiter list, ++ * wake it up and return whether the current task needs to undo ++ * a potential priority boosting. + * + * Called with lock->wait_lock held. + */ +@@ -996,7 +1397,7 @@ + * long as we hold lock->wait_lock. The waiter task needs to + * acquire it in order to dequeue the waiter. + */ +- wake_up_process(waiter->task); ++ rt_mutex_wake_waiter(waiter); + } + + /* +@@ -1010,7 +1411,7 @@ + { + bool is_top_waiter = (waiter == rt_mutex_top_waiter(lock)); + struct task_struct *owner = rt_mutex_owner(lock); +- struct rt_mutex *next_lock; ++ struct rt_mutex *next_lock = NULL; + unsigned long flags; + + raw_spin_lock_irqsave(¤t->pi_lock, flags); +@@ -1035,7 +1436,8 @@ + __rt_mutex_adjust_prio(owner); + + /* Store the lock on which owner is blocked or NULL */ +- next_lock = task_blocked_on_lock(owner); ++ if (rt_mutex_real_waiter(owner->pi_blocked_on)) ++ next_lock = task_blocked_on_lock(owner); + + raw_spin_unlock_irqrestore(&owner->pi_lock, flags); + +@@ -1071,17 +1473,17 @@ + raw_spin_lock_irqsave(&task->pi_lock, flags); + + waiter = task->pi_blocked_on; +- if (!waiter || (waiter->prio == task->prio && ++ if (!rt_mutex_real_waiter(waiter) || (waiter->prio == task->prio && + !dl_prio(task->prio))) { + raw_spin_unlock_irqrestore(&task->pi_lock, flags); + return; + } + next_lock = waiter->lock; +- raw_spin_unlock_irqrestore(&task->pi_lock, flags); + + /* gets dropped in rt_mutex_adjust_prio_chain()! */ + get_task_struct(task); + ++ raw_spin_unlock_irqrestore(&task->pi_lock, flags); + rt_mutex_adjust_prio_chain(task, RT_MUTEX_MIN_CHAINWALK, NULL, + next_lock, NULL, task); + } +@@ -1099,7 +1501,8 @@ + static int __sched + __rt_mutex_slowlock(struct rt_mutex *lock, int state, + struct hrtimer_sleeper *timeout, +- struct rt_mutex_waiter *waiter) ++ struct rt_mutex_waiter *waiter, ++ struct ww_acquire_ctx *ww_ctx) + { + int ret = 0; + +@@ -1122,6 +1525,12 @@ + break; + } + ++ if (ww_ctx && ww_ctx->acquired > 0) { ++ ret = __mutex_lock_check_stamp(lock, ww_ctx); ++ if (ret) ++ break; ++ } ++ + raw_spin_unlock(&lock->wait_lock); + + debug_rt_mutex_print_deadlock(waiter); +@@ -1156,25 +1565,102 @@ + } + } + ++static __always_inline void ww_mutex_lock_acquired(struct ww_mutex *ww, ++ struct ww_acquire_ctx *ww_ctx) ++{ ++#ifdef CONFIG_DEBUG_MUTEXES ++ /* ++ * If this WARN_ON triggers, you used ww_mutex_lock to acquire, ++ * but released with a normal mutex_unlock in this call. ++ * ++ * This should never happen, always use ww_mutex_unlock. ++ */ ++ DEBUG_LOCKS_WARN_ON(ww->ctx); ++ ++ /* ++ * Not quite done after calling ww_acquire_done() ? ++ */ ++ DEBUG_LOCKS_WARN_ON(ww_ctx->done_acquire); ++ ++ if (ww_ctx->contending_lock) { ++ /* ++ * After -EDEADLK you tried to ++ * acquire a different ww_mutex? Bad! ++ */ ++ DEBUG_LOCKS_WARN_ON(ww_ctx->contending_lock != ww); ++ ++ /* ++ * You called ww_mutex_lock after receiving -EDEADLK, ++ * but 'forgot' to unlock everything else first? ++ */ ++ DEBUG_LOCKS_WARN_ON(ww_ctx->acquired > 0); ++ ww_ctx->contending_lock = NULL; ++ } ++ ++ /* ++ * Naughty, using a different class will lead to undefined behavior! ++ */ ++ DEBUG_LOCKS_WARN_ON(ww_ctx->ww_class != ww->ww_class); ++#endif ++ ww_ctx->acquired++; ++} ++ ++#ifdef CONFIG_PREEMPT_RT_FULL ++static void ww_mutex_account_lock(struct rt_mutex *lock, ++ struct ww_acquire_ctx *ww_ctx) ++{ ++ struct ww_mutex *ww = container_of(lock, struct ww_mutex, base.lock); ++ struct rt_mutex_waiter *waiter, *n; ++ ++ /* ++ * This branch gets optimized out for the common case, ++ * and is only important for ww_mutex_lock. ++ */ ++ ww_mutex_lock_acquired(ww, ww_ctx); ++ ww->ctx = ww_ctx; ++ ++ /* ++ * Give any possible sleeping processes the chance to wake up, ++ * so they can recheck if they have to back off. ++ */ ++ rbtree_postorder_for_each_entry_safe(waiter, n, &lock->waiters, ++ tree_entry) { ++ /* XXX debug rt mutex waiter wakeup */ ++ ++ BUG_ON(waiter->lock != lock); ++ rt_mutex_wake_waiter(waiter); ++ } ++} ++ ++#else ++ ++static void ww_mutex_account_lock(struct rt_mutex *lock, ++ struct ww_acquire_ctx *ww_ctx) ++{ ++ BUG(); ++} ++#endif ++ + /* + * Slow path lock function: + */ + static int __sched + rt_mutex_slowlock(struct rt_mutex *lock, int state, + struct hrtimer_sleeper *timeout, +- enum rtmutex_chainwalk chwalk) ++ enum rtmutex_chainwalk chwalk, ++ struct ww_acquire_ctx *ww_ctx) + { + struct rt_mutex_waiter waiter; + int ret = 0; + +- debug_rt_mutex_init_waiter(&waiter); +- RB_CLEAR_NODE(&waiter.pi_tree_entry); +- RB_CLEAR_NODE(&waiter.tree_entry); ++ rt_mutex_init_waiter(&waiter, false); + + raw_spin_lock(&lock->wait_lock); + + /* Try to acquire the lock again: */ + if (try_to_take_rt_mutex(lock, current, NULL)) { ++ if (ww_ctx) ++ ww_mutex_account_lock(lock, ww_ctx); + raw_spin_unlock(&lock->wait_lock); + return 0; + } +@@ -1192,13 +1678,23 @@ + + if (likely(!ret)) + /* sleep on the mutex */ +- ret = __rt_mutex_slowlock(lock, state, timeout, &waiter); ++ ret = __rt_mutex_slowlock(lock, state, timeout, &waiter, ++ ww_ctx); ++ else if (ww_ctx) { ++ /* ww_mutex received EDEADLK, let it become EALREADY */ ++ ret = __mutex_lock_check_stamp(lock, ww_ctx); ++ BUG_ON(!ret); ++ } + + if (unlikely(ret)) { + __set_current_state(TASK_RUNNING); + if (rt_mutex_has_waiters(lock)) + remove_waiter(lock, &waiter); +- rt_mutex_handle_deadlock(ret, chwalk, &waiter); ++ /* ww_mutex want to report EDEADLK/EALREADY, let them */ ++ if (!ww_ctx) ++ rt_mutex_handle_deadlock(ret, chwalk, &waiter); ++ } else if (ww_ctx) { ++ ww_mutex_account_lock(lock, ww_ctx); + } + + /* +@@ -1255,7 +1751,7 @@ + /* + * Slow path to release a rt-mutex: + */ +-static void __sched ++static bool __sched + rt_mutex_slowunlock(struct rt_mutex *lock) + { + raw_spin_lock(&lock->wait_lock); +@@ -1298,7 +1794,7 @@ + while (!rt_mutex_has_waiters(lock)) { + /* Drops lock->wait_lock ! */ + if (unlock_rt_mutex_safe(lock) == true) +- return; ++ return false; + /* Relock the rtmutex and try again */ + raw_spin_lock(&lock->wait_lock); + } +@@ -1311,8 +1807,7 @@ + + raw_spin_unlock(&lock->wait_lock); + +- /* Undo pi boosting if necessary: */ +- rt_mutex_adjust_prio(current); ++ return true; + } + + /* +@@ -1323,31 +1818,36 @@ + */ + static inline int + rt_mutex_fastlock(struct rt_mutex *lock, int state, ++ struct ww_acquire_ctx *ww_ctx, + int (*slowfn)(struct rt_mutex *lock, int state, + struct hrtimer_sleeper *timeout, +- enum rtmutex_chainwalk chwalk)) ++ enum rtmutex_chainwalk chwalk, ++ struct ww_acquire_ctx *ww_ctx)) + { + if (likely(rt_mutex_cmpxchg(lock, NULL, current))) { + rt_mutex_deadlock_account_lock(lock, current); + return 0; + } else +- return slowfn(lock, state, NULL, RT_MUTEX_MIN_CHAINWALK); ++ return slowfn(lock, state, NULL, RT_MUTEX_MIN_CHAINWALK, ++ ww_ctx); + } + + static inline int + rt_mutex_timed_fastlock(struct rt_mutex *lock, int state, + struct hrtimer_sleeper *timeout, + enum rtmutex_chainwalk chwalk, ++ struct ww_acquire_ctx *ww_ctx, + int (*slowfn)(struct rt_mutex *lock, int state, + struct hrtimer_sleeper *timeout, +- enum rtmutex_chainwalk chwalk)) ++ enum rtmutex_chainwalk chwalk, ++ struct ww_acquire_ctx *ww_ctx)) + { + if (chwalk == RT_MUTEX_MIN_CHAINWALK && + likely(rt_mutex_cmpxchg(lock, NULL, current))) { + rt_mutex_deadlock_account_lock(lock, current); + return 0; + } else +- return slowfn(lock, state, timeout, chwalk); ++ return slowfn(lock, state, timeout, chwalk, ww_ctx); + } + + static inline int +@@ -1363,12 +1863,14 @@ + + static inline void + rt_mutex_fastunlock(struct rt_mutex *lock, +- void (*slowfn)(struct rt_mutex *lock)) ++ bool (*slowfn)(struct rt_mutex *lock)) + { +- if (likely(rt_mutex_cmpxchg(lock, current, NULL))) ++ if (likely(rt_mutex_cmpxchg(lock, current, NULL))) { + rt_mutex_deadlock_account_unlock(current); +- else +- slowfn(lock); ++ } else if (slowfn(lock)) { ++ /* Undo pi boosting if necessary: */ ++ rt_mutex_adjust_prio(current); ++ } + } + + /** +@@ -1380,7 +1882,7 @@ + { + might_sleep(); + +- rt_mutex_fastlock(lock, TASK_UNINTERRUPTIBLE, rt_mutex_slowlock); ++ rt_mutex_fastlock(lock, TASK_UNINTERRUPTIBLE, NULL, rt_mutex_slowlock); + } + EXPORT_SYMBOL_GPL(rt_mutex_lock); + +@@ -1397,7 +1899,7 @@ + { + might_sleep(); + +- return rt_mutex_fastlock(lock, TASK_INTERRUPTIBLE, rt_mutex_slowlock); ++ return rt_mutex_fastlock(lock, TASK_INTERRUPTIBLE, NULL, rt_mutex_slowlock); + } + EXPORT_SYMBOL_GPL(rt_mutex_lock_interruptible); + +@@ -1410,11 +1912,30 @@ + might_sleep(); + + return rt_mutex_timed_fastlock(lock, TASK_INTERRUPTIBLE, timeout, +- RT_MUTEX_FULL_CHAINWALK, ++ RT_MUTEX_FULL_CHAINWALK, NULL, + rt_mutex_slowlock); + } + + /** ++ * rt_mutex_lock_killable - lock a rt_mutex killable ++ * ++ * @lock: the rt_mutex to be locked ++ * @detect_deadlock: deadlock detection on/off ++ * ++ * Returns: ++ * 0 on success ++ * -EINTR when interrupted by a signal ++ * -EDEADLK when the lock would deadlock (when deadlock detection is on) ++ */ ++int __sched rt_mutex_lock_killable(struct rt_mutex *lock) ++{ ++ might_sleep(); ++ ++ return rt_mutex_fastlock(lock, TASK_KILLABLE, NULL, rt_mutex_slowlock); ++} ++EXPORT_SYMBOL_GPL(rt_mutex_lock_killable); ++ ++/** + * rt_mutex_timed_lock - lock a rt_mutex interruptible + * the timeout structure is provided + * by the caller +@@ -1434,6 +1955,7 @@ + + return rt_mutex_timed_fastlock(lock, TASK_INTERRUPTIBLE, timeout, + RT_MUTEX_MIN_CHAINWALK, ++ NULL, + rt_mutex_slowlock); + } + EXPORT_SYMBOL_GPL(rt_mutex_timed_lock); +@@ -1463,6 +1985,22 @@ + EXPORT_SYMBOL_GPL(rt_mutex_unlock); + + /** ++ * rt_mutex_futex_unlock - Futex variant of rt_mutex_unlock ++ * @lock: the rt_mutex to be unlocked ++ * ++ * Returns: true/false indicating whether priority adjustment is ++ * required or not. ++ */ ++bool __sched rt_mutex_futex_unlock(struct rt_mutex *lock) ++{ ++ if (likely(rt_mutex_cmpxchg(lock, current, NULL))) { ++ rt_mutex_deadlock_account_unlock(current); ++ return false; ++ } ++ return rt_mutex_slowunlock(lock); ++} ++ ++/** + * rt_mutex_destroy - mark a mutex unusable + * @lock: the mutex to be destroyed + * +@@ -1492,13 +2030,12 @@ + void __rt_mutex_init(struct rt_mutex *lock, const char *name) + { + lock->owner = NULL; +- raw_spin_lock_init(&lock->wait_lock); + lock->waiters = RB_ROOT; + lock->waiters_leftmost = NULL; + + debug_rt_mutex_init(lock, name); + } +-EXPORT_SYMBOL_GPL(__rt_mutex_init); ++EXPORT_SYMBOL(__rt_mutex_init); + + /** + * rt_mutex_init_proxy_locked - initialize and lock a rt_mutex on behalf of a +@@ -1513,7 +2050,7 @@ + void rt_mutex_init_proxy_locked(struct rt_mutex *lock, + struct task_struct *proxy_owner) + { +- __rt_mutex_init(lock, NULL); ++ rt_mutex_init(lock); + debug_rt_mutex_proxy_lock(lock, proxy_owner); + rt_mutex_set_owner(lock, proxy_owner); + rt_mutex_deadlock_account_lock(lock, proxy_owner); +@@ -1561,6 +2098,35 @@ + return 1; + } + ++#ifdef CONFIG_PREEMPT_RT_FULL ++ /* ++ * In PREEMPT_RT there's an added race. ++ * If the task, that we are about to requeue, times out, ++ * it can set the PI_WAKEUP_INPROGRESS. This tells the requeue ++ * to skip this task. But right after the task sets ++ * its pi_blocked_on to PI_WAKEUP_INPROGRESS it can then ++ * block on the spin_lock(&hb->lock), which in RT is an rtmutex. ++ * This will replace the PI_WAKEUP_INPROGRESS with the actual ++ * lock that it blocks on. We *must not* place this task ++ * on this proxy lock in that case. ++ * ++ * To prevent this race, we first take the task's pi_lock ++ * and check if it has updated its pi_blocked_on. If it has, ++ * we assume that it woke up and we return -EAGAIN. ++ * Otherwise, we set the task's pi_blocked_on to ++ * PI_REQUEUE_INPROGRESS, so that if the task is waking up ++ * it will know that we are in the process of requeuing it. ++ */ ++ raw_spin_lock_irq(&task->pi_lock); ++ if (task->pi_blocked_on) { ++ raw_spin_unlock_irq(&task->pi_lock); ++ raw_spin_unlock(&lock->wait_lock); ++ return -EAGAIN; ++ } ++ task->pi_blocked_on = PI_REQUEUE_INPROGRESS; ++ raw_spin_unlock_irq(&task->pi_lock); ++#endif ++ + /* We enforce deadlock detection for futexes */ + ret = task_blocks_on_rt_mutex(lock, waiter, task, + RT_MUTEX_FULL_CHAINWALK); +@@ -1631,7 +2197,7 @@ + set_current_state(TASK_INTERRUPTIBLE); + + /* sleep on the mutex */ +- ret = __rt_mutex_slowlock(lock, TASK_INTERRUPTIBLE, to, waiter); ++ ret = __rt_mutex_slowlock(lock, TASK_INTERRUPTIBLE, to, waiter, NULL); + + if (unlikely(ret)) + remove_waiter(lock, waiter); +@@ -1646,3 +2212,89 @@ + + return ret; + } ++ ++static inline int ++ww_mutex_deadlock_injection(struct ww_mutex *lock, struct ww_acquire_ctx *ctx) ++{ ++#ifdef CONFIG_DEBUG_WW_MUTEX_SLOWPATH ++ unsigned tmp; ++ ++ if (ctx->deadlock_inject_countdown-- == 0) { ++ tmp = ctx->deadlock_inject_interval; ++ if (tmp > UINT_MAX/4) ++ tmp = UINT_MAX; ++ else ++ tmp = tmp*2 + tmp + tmp/2; ++ ++ ctx->deadlock_inject_interval = tmp; ++ ctx->deadlock_inject_countdown = tmp; ++ ctx->contending_lock = lock; ++ ++ ww_mutex_unlock(lock); ++ ++ return -EDEADLK; ++ } ++#endif ++ ++ return 0; ++} ++ ++#ifdef CONFIG_PREEMPT_RT_FULL ++int __sched ++__ww_mutex_lock_interruptible(struct ww_mutex *lock, struct ww_acquire_ctx *ww_ctx) ++{ ++ int ret; ++ ++ might_sleep(); ++ ++ mutex_acquire_nest(&lock->base.dep_map, 0, 0, &ww_ctx->dep_map, _RET_IP_); ++ ret = rt_mutex_slowlock(&lock->base.lock, TASK_INTERRUPTIBLE, NULL, 0, ww_ctx); ++ if (ret) ++ mutex_release(&lock->base.dep_map, 1, _RET_IP_); ++ else if (!ret && ww_ctx->acquired > 1) ++ return ww_mutex_deadlock_injection(lock, ww_ctx); ++ ++ return ret; ++} ++EXPORT_SYMBOL_GPL(__ww_mutex_lock_interruptible); ++ ++int __sched ++__ww_mutex_lock(struct ww_mutex *lock, struct ww_acquire_ctx *ww_ctx) ++{ ++ int ret; ++ ++ might_sleep(); ++ ++ mutex_acquire_nest(&lock->base.dep_map, 0, 0, &ww_ctx->dep_map, _RET_IP_); ++ ret = rt_mutex_slowlock(&lock->base.lock, TASK_UNINTERRUPTIBLE, NULL, 0, ww_ctx); ++ if (ret) ++ mutex_release(&lock->base.dep_map, 1, _RET_IP_); ++ else if (!ret && ww_ctx->acquired > 1) ++ return ww_mutex_deadlock_injection(lock, ww_ctx); ++ ++ return ret; ++} ++EXPORT_SYMBOL_GPL(__ww_mutex_lock); ++ ++void __sched ww_mutex_unlock(struct ww_mutex *lock) ++{ ++ int nest = !!lock->ctx; ++ ++ /* ++ * The unlocking fastpath is the 0->1 transition from 'locked' ++ * into 'unlocked' state: ++ */ ++ if (nest) { ++#ifdef CONFIG_DEBUG_MUTEXES ++ DEBUG_LOCKS_WARN_ON(!lock->ctx->acquired); ++#endif ++ if (lock->ctx->acquired > 0) ++ lock->ctx->acquired--; ++ lock->ctx = NULL; ++ } ++ ++ mutex_release(&lock->base.dep_map, nest, _RET_IP_); ++ rt_mutex_unlock(&lock->base.lock); ++} ++EXPORT_SYMBOL(ww_mutex_unlock); ++#endif +diff -Nur linux-4.1.10.orig/kernel/locking/rtmutex_common.h linux-4.1.10/kernel/locking/rtmutex_common.h +--- linux-4.1.10.orig/kernel/locking/rtmutex_common.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/locking/rtmutex_common.h 2015-10-07 18:00:08.000000000 +0200 +@@ -49,6 +49,7 @@ + struct rb_node pi_tree_entry; + struct task_struct *task; + struct rt_mutex *lock; ++ bool savestate; + #ifdef CONFIG_DEBUG_RT_MUTEXES + unsigned long ip; + struct pid *deadlock_task_pid; +@@ -119,6 +120,9 @@ + /* + * PI-futex support (proxy locking functions, etc.): + */ ++#define PI_WAKEUP_INPROGRESS ((struct rt_mutex_waiter *) 1) ++#define PI_REQUEUE_INPROGRESS ((struct rt_mutex_waiter *) 2) ++ + extern struct task_struct *rt_mutex_next_owner(struct rt_mutex *lock); + extern void rt_mutex_init_proxy_locked(struct rt_mutex *lock, + struct task_struct *proxy_owner); +@@ -132,10 +136,24 @@ + struct rt_mutex_waiter *waiter); + extern int rt_mutex_timed_futex_lock(struct rt_mutex *l, struct hrtimer_sleeper *to); + ++extern bool rt_mutex_futex_unlock(struct rt_mutex *lock); ++ ++extern void rt_mutex_adjust_prio(struct task_struct *task); ++ + #ifdef CONFIG_DEBUG_RT_MUTEXES + # include "rtmutex-debug.h" + #else + # include "rtmutex.h" + #endif + ++static inline void ++rt_mutex_init_waiter(struct rt_mutex_waiter *waiter, bool savestate) ++{ ++ debug_rt_mutex_init_waiter(waiter); ++ waiter->task = NULL; ++ waiter->savestate = savestate; ++ RB_CLEAR_NODE(&waiter->pi_tree_entry); ++ RB_CLEAR_NODE(&waiter->tree_entry); ++} ++ + #endif +diff -Nur linux-4.1.10.orig/kernel/locking/spinlock.c linux-4.1.10/kernel/locking/spinlock.c +--- linux-4.1.10.orig/kernel/locking/spinlock.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/locking/spinlock.c 2015-10-07 18:00:08.000000000 +0200 +@@ -124,8 +124,11 @@ + * __[spin|read|write]_lock_bh() + */ + BUILD_LOCK_OPS(spin, raw_spinlock); ++ ++#ifndef CONFIG_PREEMPT_RT_FULL + BUILD_LOCK_OPS(read, rwlock); + BUILD_LOCK_OPS(write, rwlock); ++#endif + + #endif + +@@ -209,6 +212,8 @@ + EXPORT_SYMBOL(_raw_spin_unlock_bh); + #endif + ++#ifndef CONFIG_PREEMPT_RT_FULL ++ + #ifndef CONFIG_INLINE_READ_TRYLOCK + int __lockfunc _raw_read_trylock(rwlock_t *lock) + { +@@ -353,6 +358,8 @@ + EXPORT_SYMBOL(_raw_write_unlock_bh); + #endif + ++#endif /* !PREEMPT_RT_FULL */ ++ + #ifdef CONFIG_DEBUG_LOCK_ALLOC + + void __lockfunc _raw_spin_lock_nested(raw_spinlock_t *lock, int subclass) +diff -Nur linux-4.1.10.orig/kernel/locking/spinlock_debug.c linux-4.1.10/kernel/locking/spinlock_debug.c +--- linux-4.1.10.orig/kernel/locking/spinlock_debug.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/locking/spinlock_debug.c 2015-10-07 18:00:08.000000000 +0200 +@@ -31,6 +31,7 @@ + + EXPORT_SYMBOL(__raw_spin_lock_init); + ++#ifndef CONFIG_PREEMPT_RT_FULL + void __rwlock_init(rwlock_t *lock, const char *name, + struct lock_class_key *key) + { +@@ -48,6 +49,7 @@ + } + + EXPORT_SYMBOL(__rwlock_init); ++#endif + + static void spin_dump(raw_spinlock_t *lock, const char *msg) + { +@@ -159,6 +161,7 @@ + arch_spin_unlock(&lock->raw_lock); + } + ++#ifndef CONFIG_PREEMPT_RT_FULL + static void rwlock_bug(rwlock_t *lock, const char *msg) + { + if (!debug_locks_off()) +@@ -300,3 +303,5 @@ + debug_write_unlock(lock); + arch_write_unlock(&lock->raw_lock); + } ++ ++#endif +diff -Nur linux-4.1.10.orig/kernel/panic.c linux-4.1.10/kernel/panic.c +--- linux-4.1.10.orig/kernel/panic.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/panic.c 2015-10-07 18:00:08.000000000 +0200 +@@ -387,9 +387,11 @@ + + static int init_oops_id(void) + { ++#ifndef CONFIG_PREEMPT_RT_FULL + if (!oops_id) + get_random_bytes(&oops_id, sizeof(oops_id)); + else ++#endif + oops_id++; + + return 0; +diff -Nur linux-4.1.10.orig/kernel/power/hibernate.c linux-4.1.10/kernel/power/hibernate.c +--- linux-4.1.10.orig/kernel/power/hibernate.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/power/hibernate.c 2015-10-07 18:00:08.000000000 +0200 +@@ -285,6 +285,8 @@ + + local_irq_disable(); + ++ system_state = SYSTEM_SUSPEND; ++ + error = syscore_suspend(); + if (error) { + printk(KERN_ERR "PM: Some system devices failed to power down, " +@@ -314,6 +316,7 @@ + syscore_resume(); + + Enable_irqs: ++ system_state = SYSTEM_RUNNING; + local_irq_enable(); + + Enable_cpus: +@@ -437,6 +440,7 @@ + goto Enable_cpus; + + local_irq_disable(); ++ system_state = SYSTEM_SUSPEND; + + error = syscore_suspend(); + if (error) +@@ -470,6 +474,7 @@ + syscore_resume(); + + Enable_irqs: ++ system_state = SYSTEM_RUNNING; + local_irq_enable(); + + Enable_cpus: +@@ -555,6 +560,7 @@ + goto Platform_finish; + + local_irq_disable(); ++ system_state = SYSTEM_SUSPEND; + syscore_suspend(); + if (pm_wakeup_pending()) { + error = -EAGAIN; +@@ -567,6 +573,7 @@ + + Power_up: + syscore_resume(); ++ system_state = SYSTEM_RUNNING; + local_irq_enable(); + enable_nonboot_cpus(); + +diff -Nur linux-4.1.10.orig/kernel/power/suspend.c linux-4.1.10/kernel/power/suspend.c +--- linux-4.1.10.orig/kernel/power/suspend.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/power/suspend.c 2015-10-07 18:00:08.000000000 +0200 +@@ -356,6 +356,8 @@ + arch_suspend_disable_irqs(); + BUG_ON(!irqs_disabled()); + ++ system_state = SYSTEM_SUSPEND; ++ + error = syscore_suspend(); + if (!error) { + *wakeup = pm_wakeup_pending(); +@@ -370,6 +372,8 @@ + syscore_resume(); + } + ++ system_state = SYSTEM_RUNNING; ++ + arch_suspend_enable_irqs(); + BUG_ON(irqs_disabled()); + +diff -Nur linux-4.1.10.orig/kernel/printk/printk.c linux-4.1.10/kernel/printk/printk.c +--- linux-4.1.10.orig/kernel/printk/printk.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/printk/printk.c 2015-10-07 18:00:08.000000000 +0200 +@@ -1163,6 +1163,7 @@ + { + char *text; + int len = 0; ++ int attempts = 0; + + text = kmalloc(LOG_LINE_MAX + PREFIX_MAX, GFP_KERNEL); + if (!text) +@@ -1174,7 +1175,14 @@ + u64 seq; + u32 idx; + enum log_flags prev; +- ++ int num_msg; ++try_again: ++ attempts++; ++ if (attempts > 10) { ++ len = -EBUSY; ++ goto out; ++ } ++ num_msg = 0; + if (clear_seq < log_first_seq) { + /* messages are gone, move to first available one */ + clear_seq = log_first_seq; +@@ -1195,6 +1203,14 @@ + prev = msg->flags; + idx = log_next(idx); + seq++; ++ num_msg++; ++ if (num_msg > 5) { ++ num_msg = 0; ++ raw_spin_unlock_irq(&logbuf_lock); ++ raw_spin_lock_irq(&logbuf_lock); ++ if (clear_seq < log_first_seq) ++ goto try_again; ++ } + } + + /* move first record forward until length fits into the buffer */ +@@ -1208,6 +1224,14 @@ + prev = msg->flags; + idx = log_next(idx); + seq++; ++ num_msg++; ++ if (num_msg > 5) { ++ num_msg = 0; ++ raw_spin_unlock_irq(&logbuf_lock); ++ raw_spin_lock_irq(&logbuf_lock); ++ if (clear_seq < log_first_seq) ++ goto try_again; ++ } + } + + /* last message fitting into this dump */ +@@ -1248,6 +1272,7 @@ + clear_seq = log_next_seq; + clear_idx = log_next_idx; + } ++out: + raw_spin_unlock_irq(&logbuf_lock); + + kfree(text); +@@ -1401,6 +1426,7 @@ + if (!console_drivers) + return; + ++ migrate_disable(); + for_each_console(con) { + if (exclusive_console && con != exclusive_console) + continue; +@@ -1413,6 +1439,7 @@ + continue; + con->write(con, text, len); + } ++ migrate_enable(); + } + + /* +@@ -1473,6 +1500,15 @@ + static int console_trylock_for_printk(void) + { + unsigned int cpu = smp_processor_id(); ++#ifdef CONFIG_PREEMPT_RT_FULL ++ int lock = !early_boot_irqs_disabled && (preempt_count() == 0) && ++ !irqs_disabled(); ++#else ++ int lock = 1; ++#endif ++ ++ if (!lock) ++ return 0; + + if (!console_trylock()) + return 0; +@@ -1607,6 +1643,62 @@ + return textlen; + } + ++#ifdef CONFIG_EARLY_PRINTK ++struct console *early_console; ++ ++static void early_vprintk(const char *fmt, va_list ap) ++{ ++ if (early_console) { ++ char buf[512]; ++ int n = vscnprintf(buf, sizeof(buf), fmt, ap); ++ ++ early_console->write(early_console, buf, n); ++ } ++} ++ ++asmlinkage void early_printk(const char *fmt, ...) ++{ ++ va_list ap; ++ ++ va_start(ap, fmt); ++ early_vprintk(fmt, ap); ++ va_end(ap); ++} ++ ++/* ++ * This is independent of any log levels - a global ++ * kill switch that turns off all of printk. ++ * ++ * Used by the NMI watchdog if early-printk is enabled. ++ */ ++static bool __read_mostly printk_killswitch; ++ ++static int __init force_early_printk_setup(char *str) ++{ ++ printk_killswitch = true; ++ return 0; ++} ++early_param("force_early_printk", force_early_printk_setup); ++ ++void printk_kill(void) ++{ ++ printk_killswitch = true; ++} ++ ++static int forced_early_printk(const char *fmt, va_list ap) ++{ ++ if (!printk_killswitch) ++ return 0; ++ early_vprintk(fmt, ap); ++ return 1; ++} ++#else ++static inline int forced_early_printk(const char *fmt, va_list ap) ++{ ++ return 0; ++} ++#endif ++ + asmlinkage int vprintk_emit(int facility, int level, + const char *dict, size_t dictlen, + const char *fmt, va_list args) +@@ -1623,6 +1715,13 @@ + /* cpu currently holding logbuf_lock in this function */ + static unsigned int logbuf_cpu = UINT_MAX; + ++ /* ++ * Fall back to early_printk if a debugging subsystem has ++ * killed printk output ++ */ ++ if (unlikely(forced_early_printk(fmt, args))) ++ return 1; ++ + if (level == LOGLEVEL_SCHED) { + level = LOGLEVEL_DEFAULT; + in_sched = true; +@@ -1764,8 +1863,7 @@ + * console_sem which would prevent anyone from printing to + * console + */ +- preempt_disable(); +- ++ migrate_disable(); + /* + * Try to acquire and then immediately release the console + * semaphore. The release will print out buffers and wake up +@@ -1773,7 +1871,7 @@ + */ + if (console_trylock_for_printk()) + console_unlock(); +- preempt_enable(); ++ migrate_enable(); + lockdep_on(); + } + +@@ -1902,26 +2000,6 @@ + + #endif /* CONFIG_PRINTK */ + +-#ifdef CONFIG_EARLY_PRINTK +-struct console *early_console; +- +-asmlinkage __visible void early_printk(const char *fmt, ...) +-{ +- va_list ap; +- char buf[512]; +- int n; +- +- if (!early_console) +- return; +- +- va_start(ap, fmt); +- n = vscnprintf(buf, sizeof(buf), fmt, ap); +- va_end(ap); +- +- early_console->write(early_console, buf, n); +-} +-#endif +- + static int __add_preferred_console(char *name, int idx, char *options, + char *brl_options) + { +@@ -2143,11 +2221,16 @@ + goto out; + + len = cont_print_text(text, size); ++#ifndef CONFIG_PREEMPT_RT_FULL + raw_spin_unlock(&logbuf_lock); + stop_critical_timings(); + call_console_drivers(cont.level, text, len); + start_critical_timings(); + local_irq_restore(flags); ++#else ++ raw_spin_unlock_irqrestore(&logbuf_lock, flags); ++ call_console_drivers(cont.level, text, len); ++#endif + return; + out: + raw_spin_unlock_irqrestore(&logbuf_lock, flags); +@@ -2235,12 +2318,17 @@ + console_idx = log_next(console_idx); + console_seq++; + console_prev = msg->flags; ++#ifdef CONFIG_PREEMPT_RT_FULL ++ raw_spin_unlock_irqrestore(&logbuf_lock, flags); ++ call_console_drivers(level, text, len); ++#else + raw_spin_unlock(&logbuf_lock); + + stop_critical_timings(); /* don't trace print latency */ + call_console_drivers(level, text, len); + start_critical_timings(); + local_irq_restore(flags); ++#endif + } + console_locked = 0; + +diff -Nur linux-4.1.10.orig/kernel/ptrace.c linux-4.1.10/kernel/ptrace.c +--- linux-4.1.10.orig/kernel/ptrace.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/ptrace.c 2015-10-07 18:00:08.000000000 +0200 +@@ -129,7 +129,12 @@ + + spin_lock_irq(&task->sighand->siglock); + if (task_is_traced(task) && !__fatal_signal_pending(task)) { +- task->state = __TASK_TRACED; ++ raw_spin_lock_irq(&task->pi_lock); ++ if (task->state & __TASK_TRACED) ++ task->state = __TASK_TRACED; ++ else ++ task->saved_state = __TASK_TRACED; ++ raw_spin_unlock_irq(&task->pi_lock); + ret = true; + } + spin_unlock_irq(&task->sighand->siglock); +diff -Nur linux-4.1.10.orig/kernel/rcu/tree.c linux-4.1.10/kernel/rcu/tree.c +--- linux-4.1.10.orig/kernel/rcu/tree.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/rcu/tree.c 2015-10-07 18:00:08.000000000 +0200 +@@ -56,6 +56,11 @@ + #include + #include + #include ++#include ++#include ++#include ++#include ++#include "../time/tick-internal.h" + + #include "tree.h" + #include "rcu.h" +@@ -220,6 +225,19 @@ + } + } + ++#ifdef CONFIG_PREEMPT_RT_FULL ++static void rcu_preempt_qs(void); ++ ++void rcu_bh_qs(void) ++{ ++ unsigned long flags; ++ ++ /* Callers to this function, rcu_preempt_qs(), must disable irqs. */ ++ local_irq_save(flags); ++ rcu_preempt_qs(); ++ local_irq_restore(flags); ++} ++#else + void rcu_bh_qs(void) + { + if (!__this_cpu_read(rcu_bh_data.passed_quiesce)) { +@@ -229,6 +247,7 @@ + __this_cpu_write(rcu_bh_data.passed_quiesce, 1); + } + } ++#endif + + static DEFINE_PER_CPU(int, rcu_sched_qs_mask); + +@@ -404,6 +423,7 @@ + } + EXPORT_SYMBOL_GPL(rcu_batches_completed_sched); + ++#ifndef CONFIG_PREEMPT_RT_FULL + /* + * Return the number of RCU BH batches completed thus far for debug & stats. + */ +@@ -431,6 +451,13 @@ + } + EXPORT_SYMBOL_GPL(rcu_bh_force_quiescent_state); + ++#else ++void rcu_force_quiescent_state(void) ++{ ++} ++EXPORT_SYMBOL_GPL(rcu_force_quiescent_state); ++#endif ++ + /* + * Force a quiescent state for RCU-sched. + */ +@@ -1545,7 +1572,7 @@ + !ACCESS_ONCE(rsp->gp_flags) || + !rsp->gp_kthread) + return; +- wake_up(&rsp->gp_wq); ++ swait_wake(&rsp->gp_wq); + } + + /* +@@ -1986,7 +2013,7 @@ + ACCESS_ONCE(rsp->gpnum), + TPS("reqwait")); + rsp->gp_state = RCU_GP_WAIT_GPS; +- wait_event_interruptible(rsp->gp_wq, ++ swait_event_interruptible(rsp->gp_wq, + ACCESS_ONCE(rsp->gp_flags) & + RCU_GP_FLAG_INIT); + /* Locking provides needed memory barrier. */ +@@ -2015,7 +2042,7 @@ + ACCESS_ONCE(rsp->gpnum), + TPS("fqswait")); + rsp->gp_state = RCU_GP_WAIT_FQS; +- ret = wait_event_interruptible_timeout(rsp->gp_wq, ++ ret = swait_event_interruptible_timeout(rsp->gp_wq, + ((gf = ACCESS_ONCE(rsp->gp_flags)) & + RCU_GP_FLAG_FQS) || + (!ACCESS_ONCE(rnp->qsmask) && +@@ -2860,18 +2887,17 @@ + /* + * Do RCU core processing for the current CPU. + */ +-static void rcu_process_callbacks(struct softirq_action *unused) ++static void rcu_process_callbacks(void) + { + struct rcu_state *rsp; + + if (cpu_is_offline(smp_processor_id())) + return; +- trace_rcu_utilization(TPS("Start RCU core")); + for_each_rcu_flavor(rsp) + __rcu_process_callbacks(rsp); +- trace_rcu_utilization(TPS("End RCU core")); + } + ++static DEFINE_PER_CPU(struct task_struct *, rcu_cpu_kthread_task); + /* + * Schedule RCU callback invocation. If the specified type of RCU + * does not support RCU priority boosting, just do a direct call, +@@ -2883,18 +2909,105 @@ + { + if (unlikely(!ACCESS_ONCE(rcu_scheduler_fully_active))) + return; +- if (likely(!rsp->boost)) { +- rcu_do_batch(rsp, rdp); ++ rcu_do_batch(rsp, rdp); ++} ++ ++static void rcu_wake_cond(struct task_struct *t, int status) ++{ ++ /* ++ * If the thread is yielding, only wake it when this ++ * is invoked from idle ++ */ ++ if (t && (status != RCU_KTHREAD_YIELDING || is_idle_task(current))) ++ wake_up_process(t); ++} ++ ++/* ++ * Wake up this CPU's rcuc kthread to do RCU core processing. ++ */ ++static void invoke_rcu_core(void) ++{ ++ unsigned long flags; ++ struct task_struct *t; ++ ++ if (!cpu_online(smp_processor_id())) + return; ++ local_irq_save(flags); ++ __this_cpu_write(rcu_cpu_has_work, 1); ++ t = __this_cpu_read(rcu_cpu_kthread_task); ++ if (t != NULL && current != t) ++ rcu_wake_cond(t, __this_cpu_read(rcu_cpu_kthread_status)); ++ local_irq_restore(flags); ++} ++ ++static void rcu_cpu_kthread_park(unsigned int cpu) ++{ ++ per_cpu(rcu_cpu_kthread_status, cpu) = RCU_KTHREAD_OFFCPU; ++} ++ ++static int rcu_cpu_kthread_should_run(unsigned int cpu) ++{ ++ return __this_cpu_read(rcu_cpu_has_work); ++} ++ ++/* ++ * Per-CPU kernel thread that invokes RCU callbacks. This replaces the ++ * RCU softirq used in flavors and configurations of RCU that do not ++ * support RCU priority boosting. ++ */ ++static void rcu_cpu_kthread(unsigned int cpu) ++{ ++ unsigned int *statusp = this_cpu_ptr(&rcu_cpu_kthread_status); ++ char work, *workp = this_cpu_ptr(&rcu_cpu_has_work); ++ int spincnt; ++ ++ for (spincnt = 0; spincnt < 10; spincnt++) { ++ trace_rcu_utilization(TPS("Start CPU kthread@rcu_wait")); ++ local_bh_disable(); ++ *statusp = RCU_KTHREAD_RUNNING; ++ this_cpu_inc(rcu_cpu_kthread_loops); ++ local_irq_disable(); ++ work = *workp; ++ *workp = 0; ++ local_irq_enable(); ++ if (work) ++ rcu_process_callbacks(); ++ local_bh_enable(); ++ if (*workp == 0) { ++ trace_rcu_utilization(TPS("End CPU kthread@rcu_wait")); ++ *statusp = RCU_KTHREAD_WAITING; ++ return; ++ } + } +- invoke_rcu_callbacks_kthread(); ++ *statusp = RCU_KTHREAD_YIELDING; ++ trace_rcu_utilization(TPS("Start CPU kthread@rcu_yield")); ++ schedule_timeout_interruptible(2); ++ trace_rcu_utilization(TPS("End CPU kthread@rcu_yield")); ++ *statusp = RCU_KTHREAD_WAITING; + } + +-static void invoke_rcu_core(void) ++static struct smp_hotplug_thread rcu_cpu_thread_spec = { ++ .store = &rcu_cpu_kthread_task, ++ .thread_should_run = rcu_cpu_kthread_should_run, ++ .thread_fn = rcu_cpu_kthread, ++ .thread_comm = "rcuc/%u", ++ .setup = rcu_cpu_kthread_setup, ++ .park = rcu_cpu_kthread_park, ++}; ++ ++/* ++ * Spawn per-CPU RCU core processing kthreads. ++ */ ++static int __init rcu_spawn_core_kthreads(void) + { +- if (cpu_online(smp_processor_id())) +- raise_softirq(RCU_SOFTIRQ); ++ int cpu; ++ ++ for_each_possible_cpu(cpu) ++ per_cpu(rcu_cpu_has_work, cpu) = 0; ++ BUG_ON(smpboot_register_percpu_thread(&rcu_cpu_thread_spec)); ++ return 0; + } ++early_initcall(rcu_spawn_core_kthreads); + + /* + * Handle any core-RCU processing required by a call_rcu() invocation. +@@ -3040,6 +3153,7 @@ + } + EXPORT_SYMBOL_GPL(call_rcu_sched); + ++#ifndef CONFIG_PREEMPT_RT_FULL + /* + * Queue an RCU callback for invocation after a quicker grace period. + */ +@@ -3048,6 +3162,7 @@ + __call_rcu(head, func, &rcu_bh_state, -1, 0); + } + EXPORT_SYMBOL_GPL(call_rcu_bh); ++#endif + + /* + * Queue an RCU callback for lazy invocation after a grace period. +@@ -3139,6 +3254,7 @@ + } + EXPORT_SYMBOL_GPL(synchronize_sched); + ++#ifndef CONFIG_PREEMPT_RT_FULL + /** + * synchronize_rcu_bh - wait until an rcu_bh grace period has elapsed. + * +@@ -3165,6 +3281,7 @@ + wait_rcu_gp(call_rcu_bh); + } + EXPORT_SYMBOL_GPL(synchronize_rcu_bh); ++#endif + + /** + * get_state_synchronize_rcu - Snapshot current RCU state +@@ -3677,6 +3794,7 @@ + mutex_unlock(&rsp->barrier_mutex); + } + ++#ifndef CONFIG_PREEMPT_RT_FULL + /** + * rcu_barrier_bh - Wait until all in-flight call_rcu_bh() callbacks complete. + */ +@@ -3685,6 +3803,7 @@ + _rcu_barrier(&rcu_bh_state); + } + EXPORT_SYMBOL_GPL(rcu_barrier_bh); ++#endif + + /** + * rcu_barrier_sched - Wait for in-flight call_rcu_sched() callbacks. +@@ -4021,7 +4140,7 @@ + } + } + +- init_waitqueue_head(&rsp->gp_wq); ++ init_swait_head(&rsp->gp_wq); + rnp = rsp->level[rcu_num_lvls - 1]; + for_each_possible_cpu(i) { + while (i > rnp->grphi) +@@ -4120,7 +4239,6 @@ + rcu_init_one(&rcu_bh_state, &rcu_bh_data); + rcu_init_one(&rcu_sched_state, &rcu_sched_data); + __rcu_init_preempt(); +- open_softirq(RCU_SOFTIRQ, rcu_process_callbacks); + + /* + * We don't need protection against CPU-hotplug here because +diff -Nur linux-4.1.10.orig/kernel/rcu/tree.h linux-4.1.10/kernel/rcu/tree.h +--- linux-4.1.10.orig/kernel/rcu/tree.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/rcu/tree.h 2015-10-07 18:00:08.000000000 +0200 +@@ -27,6 +27,7 @@ + #include + #include + #include ++#include + + /* + * Define shape of hierarchy based on NR_CPUS, CONFIG_RCU_FANOUT, and +@@ -210,7 +211,7 @@ + /* This can happen due to race conditions. */ + #endif /* #ifdef CONFIG_RCU_BOOST */ + #ifdef CONFIG_RCU_NOCB_CPU +- wait_queue_head_t nocb_gp_wq[2]; ++ struct swait_head nocb_gp_wq[2]; + /* Place for rcu_nocb_kthread() to wait GP. */ + #endif /* #ifdef CONFIG_RCU_NOCB_CPU */ + int need_future_gp[2]; +@@ -349,7 +350,7 @@ + atomic_long_t nocb_q_count_lazy; /* invocation (all stages). */ + struct rcu_head *nocb_follower_head; /* CBs ready to invoke. */ + struct rcu_head **nocb_follower_tail; +- wait_queue_head_t nocb_wq; /* For nocb kthreads to sleep on. */ ++ struct swait_head nocb_wq; /* For nocb kthreads to sleep on. */ + struct task_struct *nocb_kthread; + int nocb_defer_wakeup; /* Defer wakeup of nocb_kthread. */ + +@@ -438,7 +439,7 @@ + unsigned long gpnum; /* Current gp number. */ + unsigned long completed; /* # of last completed gp. */ + struct task_struct *gp_kthread; /* Task for grace periods. */ +- wait_queue_head_t gp_wq; /* Where GP task waits. */ ++ struct swait_head gp_wq; /* Where GP task waits. */ + short gp_flags; /* Commands for GP task. */ + short gp_state; /* GP kthread sleep state. */ + +@@ -529,12 +530,10 @@ + DECLARE_PER_CPU(struct rcu_data, rcu_preempt_data); + #endif /* #ifdef CONFIG_PREEMPT_RCU */ + +-#ifdef CONFIG_RCU_BOOST + DECLARE_PER_CPU(unsigned int, rcu_cpu_kthread_status); + DECLARE_PER_CPU(int, rcu_cpu_kthread_cpu); + DECLARE_PER_CPU(unsigned int, rcu_cpu_kthread_loops); + DECLARE_PER_CPU(char, rcu_cpu_has_work); +-#endif /* #ifdef CONFIG_RCU_BOOST */ + + #ifndef RCU_TREE_NONCORE + +@@ -553,10 +552,9 @@ + static void __init __rcu_init_preempt(void); + static void rcu_initiate_boost(struct rcu_node *rnp, unsigned long flags); + static void rcu_preempt_boost_start_gp(struct rcu_node *rnp); +-static void invoke_rcu_callbacks_kthread(void); + static bool rcu_is_callbacks_kthread(void); ++static void rcu_cpu_kthread_setup(unsigned int cpu); + #ifdef CONFIG_RCU_BOOST +-static void rcu_preempt_do_callbacks(void); + static int rcu_spawn_one_boost_kthread(struct rcu_state *rsp, + struct rcu_node *rnp); + #endif /* #ifdef CONFIG_RCU_BOOST */ +diff -Nur linux-4.1.10.orig/kernel/rcu/tree_plugin.h linux-4.1.10/kernel/rcu/tree_plugin.h +--- linux-4.1.10.orig/kernel/rcu/tree_plugin.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/rcu/tree_plugin.h 2015-10-07 18:00:08.000000000 +0200 +@@ -24,27 +24,20 @@ + * Paul E. McKenney + */ + +-#include +-#include +-#include +-#include +-#include "../time/tick-internal.h" +- + #ifdef CONFIG_RCU_BOOST + + #include "../locking/rtmutex_common.h" + ++#endif /* #ifdef CONFIG_RCU_BOOST */ ++ + /* + * Control variables for per-CPU and per-rcu_node kthreads. These + * handle all flavors of RCU. + */ +-static DEFINE_PER_CPU(struct task_struct *, rcu_cpu_kthread_task); + DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_status); + DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_loops); + DEFINE_PER_CPU(char, rcu_cpu_has_work); + +-#endif /* #ifdef CONFIG_RCU_BOOST */ +- + #ifdef CONFIG_RCU_NOCB_CPU + static cpumask_var_t rcu_nocb_mask; /* CPUs to have callbacks offloaded. */ + static bool have_rcu_nocb_mask; /* Was rcu_nocb_mask allocated? */ +@@ -291,7 +284,7 @@ + } + + /* Hardware IRQ handlers cannot block, complain if they get here. */ +- if (in_irq() || in_serving_softirq()) { ++ if (preempt_count() & (HARDIRQ_MASK | SOFTIRQ_OFFSET)) { + lockdep_rcu_suspicious(__FILE__, __LINE__, + "rcu_read_unlock() from irq or softirq with blocking in critical section!!!\n"); + pr_alert("->rcu_read_unlock_special: %#x (b: %d, nq: %d)\n", +@@ -496,15 +489,6 @@ + t->rcu_read_unlock_special.b.need_qs = true; + } + +-#ifdef CONFIG_RCU_BOOST +- +-static void rcu_preempt_do_callbacks(void) +-{ +- rcu_do_batch(&rcu_preempt_state, this_cpu_ptr(&rcu_preempt_data)); +-} +- +-#endif /* #ifdef CONFIG_RCU_BOOST */ +- + /* + * Queue a preemptible-RCU callback for invocation after a grace period. + */ +@@ -939,6 +923,19 @@ + + #endif /* #else #ifdef CONFIG_PREEMPT_RCU */ + ++/* ++ * If boosting, set rcuc kthreads to realtime priority. ++ */ ++static void rcu_cpu_kthread_setup(unsigned int cpu) ++{ ++#ifdef CONFIG_RCU_BOOST ++ struct sched_param sp; ++ ++ sp.sched_priority = kthread_prio; ++ sched_setscheduler_nocheck(current, SCHED_FIFO, &sp); ++#endif /* #ifdef CONFIG_RCU_BOOST */ ++} ++ + #ifdef CONFIG_RCU_BOOST + + #include "../locking/rtmutex_common.h" +@@ -970,16 +967,6 @@ + + #endif /* #else #ifdef CONFIG_RCU_TRACE */ + +-static void rcu_wake_cond(struct task_struct *t, int status) +-{ +- /* +- * If the thread is yielding, only wake it when this +- * is invoked from idle +- */ +- if (status != RCU_KTHREAD_YIELDING || is_idle_task(current)) +- wake_up_process(t); +-} +- + /* + * Carry out RCU priority boosting on the task indicated by ->exp_tasks + * or ->boost_tasks, advancing the pointer to the next task in the +@@ -1125,23 +1112,6 @@ + } + + /* +- * Wake up the per-CPU kthread to invoke RCU callbacks. +- */ +-static void invoke_rcu_callbacks_kthread(void) +-{ +- unsigned long flags; +- +- local_irq_save(flags); +- __this_cpu_write(rcu_cpu_has_work, 1); +- if (__this_cpu_read(rcu_cpu_kthread_task) != NULL && +- current != __this_cpu_read(rcu_cpu_kthread_task)) { +- rcu_wake_cond(__this_cpu_read(rcu_cpu_kthread_task), +- __this_cpu_read(rcu_cpu_kthread_status)); +- } +- local_irq_restore(flags); +-} +- +-/* + * Is the current CPU running the RCU-callbacks kthread? + * Caller must have preemption disabled. + */ +@@ -1196,67 +1166,6 @@ + return 0; + } + +-static void rcu_kthread_do_work(void) +-{ +- rcu_do_batch(&rcu_sched_state, this_cpu_ptr(&rcu_sched_data)); +- rcu_do_batch(&rcu_bh_state, this_cpu_ptr(&rcu_bh_data)); +- rcu_preempt_do_callbacks(); +-} +- +-static void rcu_cpu_kthread_setup(unsigned int cpu) +-{ +- struct sched_param sp; +- +- sp.sched_priority = kthread_prio; +- sched_setscheduler_nocheck(current, SCHED_FIFO, &sp); +-} +- +-static void rcu_cpu_kthread_park(unsigned int cpu) +-{ +- per_cpu(rcu_cpu_kthread_status, cpu) = RCU_KTHREAD_OFFCPU; +-} +- +-static int rcu_cpu_kthread_should_run(unsigned int cpu) +-{ +- return __this_cpu_read(rcu_cpu_has_work); +-} +- +-/* +- * Per-CPU kernel thread that invokes RCU callbacks. This replaces the +- * RCU softirq used in flavors and configurations of RCU that do not +- * support RCU priority boosting. +- */ +-static void rcu_cpu_kthread(unsigned int cpu) +-{ +- unsigned int *statusp = this_cpu_ptr(&rcu_cpu_kthread_status); +- char work, *workp = this_cpu_ptr(&rcu_cpu_has_work); +- int spincnt; +- +- for (spincnt = 0; spincnt < 10; spincnt++) { +- trace_rcu_utilization(TPS("Start CPU kthread@rcu_wait")); +- local_bh_disable(); +- *statusp = RCU_KTHREAD_RUNNING; +- this_cpu_inc(rcu_cpu_kthread_loops); +- local_irq_disable(); +- work = *workp; +- *workp = 0; +- local_irq_enable(); +- if (work) +- rcu_kthread_do_work(); +- local_bh_enable(); +- if (*workp == 0) { +- trace_rcu_utilization(TPS("End CPU kthread@rcu_wait")); +- *statusp = RCU_KTHREAD_WAITING; +- return; +- } +- } +- *statusp = RCU_KTHREAD_YIELDING; +- trace_rcu_utilization(TPS("Start CPU kthread@rcu_yield")); +- schedule_timeout_interruptible(2); +- trace_rcu_utilization(TPS("End CPU kthread@rcu_yield")); +- *statusp = RCU_KTHREAD_WAITING; +-} +- + /* + * Set the per-rcu_node kthread's affinity to cover all CPUs that are + * served by the rcu_node in question. The CPU hotplug lock is still +@@ -1286,26 +1195,12 @@ + free_cpumask_var(cm); + } + +-static struct smp_hotplug_thread rcu_cpu_thread_spec = { +- .store = &rcu_cpu_kthread_task, +- .thread_should_run = rcu_cpu_kthread_should_run, +- .thread_fn = rcu_cpu_kthread, +- .thread_comm = "rcuc/%u", +- .setup = rcu_cpu_kthread_setup, +- .park = rcu_cpu_kthread_park, +-}; +- + /* + * Spawn boost kthreads -- called as soon as the scheduler is running. + */ + static void __init rcu_spawn_boost_kthreads(void) + { + struct rcu_node *rnp; +- int cpu; +- +- for_each_possible_cpu(cpu) +- per_cpu(rcu_cpu_has_work, cpu) = 0; +- BUG_ON(smpboot_register_percpu_thread(&rcu_cpu_thread_spec)); + rcu_for_each_leaf_node(rcu_state_p, rnp) + (void)rcu_spawn_one_boost_kthread(rcu_state_p, rnp); + } +@@ -1328,11 +1223,6 @@ + raw_spin_unlock_irqrestore(&rnp->lock, flags); + } + +-static void invoke_rcu_callbacks_kthread(void) +-{ +- WARN_ON_ONCE(1); +-} +- + static bool rcu_is_callbacks_kthread(void) + { + return false; +@@ -1356,7 +1246,7 @@ + + #endif /* #else #ifdef CONFIG_RCU_BOOST */ + +-#if !defined(CONFIG_RCU_FAST_NO_HZ) ++#if !defined(CONFIG_RCU_FAST_NO_HZ) || defined(CONFIG_PREEMPT_RT_FULL) + + /* + * Check to see if any future RCU-related work will need to be done +@@ -1374,7 +1264,9 @@ + return rcu_cpu_has_callbacks(NULL); + } + #endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */ ++#endif /* !defined(CONFIG_RCU_FAST_NO_HZ) || defined(CONFIG_PREEMPT_RT_FULL) */ + ++#if !defined(CONFIG_RCU_FAST_NO_HZ) + /* + * Because we do not have RCU_FAST_NO_HZ, don't bother cleaning up + * after it. +@@ -1472,6 +1364,8 @@ + return cbs_ready; + } + ++#ifndef CONFIG_PREEMPT_RT_FULL ++ + /* + * Allow the CPU to enter dyntick-idle mode unless it has callbacks ready + * to invoke. If the CPU has callbacks, try to advance them. Tell the +@@ -1512,7 +1406,7 @@ + return 0; + } + #endif /* #ifndef CONFIG_RCU_NOCB_CPU_ALL */ +- ++#endif /* #ifndef CONFIG_PREEMPT_RT_FULL */ + /* + * Prepare a CPU for idle from an RCU perspective. The first major task + * is to sense whether nohz mode has been enabled or disabled via sysfs. +@@ -1859,7 +1753,7 @@ + */ + static void rcu_nocb_gp_cleanup(struct rcu_state *rsp, struct rcu_node *rnp) + { +- wake_up_all(&rnp->nocb_gp_wq[rnp->completed & 0x1]); ++ swait_wake_all(&rnp->nocb_gp_wq[rnp->completed & 0x1]); + } + + /* +@@ -1877,8 +1771,8 @@ + + static void rcu_init_one_nocb(struct rcu_node *rnp) + { +- init_waitqueue_head(&rnp->nocb_gp_wq[0]); +- init_waitqueue_head(&rnp->nocb_gp_wq[1]); ++ init_swait_head(&rnp->nocb_gp_wq[0]); ++ init_swait_head(&rnp->nocb_gp_wq[1]); + } + + #ifndef CONFIG_RCU_NOCB_CPU_ALL +@@ -1903,7 +1797,7 @@ + if (ACCESS_ONCE(rdp_leader->nocb_leader_sleep) || force) { + /* Prior smp_mb__after_atomic() orders against prior enqueue. */ + ACCESS_ONCE(rdp_leader->nocb_leader_sleep) = false; +- wake_up(&rdp_leader->nocb_wq); ++ swait_wake(&rdp_leader->nocb_wq); + } + } + +@@ -2116,7 +2010,7 @@ + */ + trace_rcu_future_gp(rnp, rdp, c, TPS("StartWait")); + for (;;) { +- wait_event_interruptible( ++ swait_event_interruptible( + rnp->nocb_gp_wq[c & 0x1], + (d = ULONG_CMP_GE(ACCESS_ONCE(rnp->completed), c))); + if (likely(d)) +@@ -2144,7 +2038,7 @@ + /* Wait for callbacks to appear. */ + if (!rcu_nocb_poll) { + trace_rcu_nocb_wake(my_rdp->rsp->name, my_rdp->cpu, "Sleep"); +- wait_event_interruptible(my_rdp->nocb_wq, ++ swait_event_interruptible(my_rdp->nocb_wq, + !ACCESS_ONCE(my_rdp->nocb_leader_sleep)); + /* Memory barrier handled by smp_mb() calls below and repoll. */ + } else if (firsttime) { +@@ -2219,7 +2113,7 @@ + * List was empty, wake up the follower. + * Memory barriers supplied by atomic_long_add(). + */ +- wake_up(&rdp->nocb_wq); ++ swait_wake(&rdp->nocb_wq); + } + } + +@@ -2240,7 +2134,7 @@ + if (!rcu_nocb_poll) { + trace_rcu_nocb_wake(rdp->rsp->name, rdp->cpu, + "FollowerSleep"); +- wait_event_interruptible(rdp->nocb_wq, ++ swait_event_interruptible(rdp->nocb_wq, + ACCESS_ONCE(rdp->nocb_follower_head)); + } else if (firsttime) { + /* Don't drown trace log with "Poll"! */ +@@ -2399,7 +2293,7 @@ + static void __init rcu_boot_init_nocb_percpu_data(struct rcu_data *rdp) + { + rdp->nocb_tail = &rdp->nocb_head; +- init_waitqueue_head(&rdp->nocb_wq); ++ init_swait_head(&rdp->nocb_wq); + rdp->nocb_follower_tail = &rdp->nocb_follower_head; + } + +diff -Nur linux-4.1.10.orig/kernel/rcu/update.c linux-4.1.10/kernel/rcu/update.c +--- linux-4.1.10.orig/kernel/rcu/update.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/rcu/update.c 2015-10-07 18:00:08.000000000 +0200 +@@ -227,6 +227,7 @@ + } + EXPORT_SYMBOL_GPL(rcu_read_lock_held); + ++#ifndef CONFIG_PREEMPT_RT_FULL + /** + * rcu_read_lock_bh_held() - might we be in RCU-bh read-side critical section? + * +@@ -253,6 +254,7 @@ + return in_softirq() || irqs_disabled(); + } + EXPORT_SYMBOL_GPL(rcu_read_lock_bh_held); ++#endif + + #endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */ + +diff -Nur linux-4.1.10.orig/kernel/relay.c linux-4.1.10/kernel/relay.c +--- linux-4.1.10.orig/kernel/relay.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/relay.c 2015-10-07 18:00:08.000000000 +0200 +@@ -339,6 +339,10 @@ + { + struct rchan_buf *buf = (struct rchan_buf *)data; + wake_up_interruptible(&buf->read_wait); ++ /* ++ * Stupid polling for now: ++ */ ++ mod_timer(&buf->timer, jiffies + 1); + } + + /** +@@ -356,6 +360,7 @@ + init_waitqueue_head(&buf->read_wait); + kref_init(&buf->kref); + setup_timer(&buf->timer, wakeup_readers, (unsigned long)buf); ++ mod_timer(&buf->timer, jiffies + 1); + } else + del_timer_sync(&buf->timer); + +@@ -739,15 +744,6 @@ + else + buf->early_bytes += buf->chan->subbuf_size - + buf->padding[old_subbuf]; +- smp_mb(); +- if (waitqueue_active(&buf->read_wait)) +- /* +- * Calling wake_up_interruptible() from here +- * will deadlock if we happen to be logging +- * from the scheduler (trying to re-grab +- * rq->lock), so defer it. +- */ +- mod_timer(&buf->timer, jiffies + 1); + } + + old = buf->data; +diff -Nur linux-4.1.10.orig/kernel/sched/completion.c linux-4.1.10/kernel/sched/completion.c +--- linux-4.1.10.orig/kernel/sched/completion.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/sched/completion.c 2015-10-07 18:00:08.000000000 +0200 +@@ -30,10 +30,10 @@ + { + unsigned long flags; + +- spin_lock_irqsave(&x->wait.lock, flags); ++ raw_spin_lock_irqsave(&x->wait.lock, flags); + x->done++; +- __wake_up_locked(&x->wait, TASK_NORMAL, 1); +- spin_unlock_irqrestore(&x->wait.lock, flags); ++ __swait_wake_locked(&x->wait, TASK_NORMAL, 1); ++ raw_spin_unlock_irqrestore(&x->wait.lock, flags); + } + EXPORT_SYMBOL(complete); + +@@ -50,10 +50,10 @@ + { + unsigned long flags; + +- spin_lock_irqsave(&x->wait.lock, flags); ++ raw_spin_lock_irqsave(&x->wait.lock, flags); + x->done += UINT_MAX/2; +- __wake_up_locked(&x->wait, TASK_NORMAL, 0); +- spin_unlock_irqrestore(&x->wait.lock, flags); ++ __swait_wake_locked(&x->wait, TASK_NORMAL, 0); ++ raw_spin_unlock_irqrestore(&x->wait.lock, flags); + } + EXPORT_SYMBOL(complete_all); + +@@ -62,20 +62,20 @@ + long (*action)(long), long timeout, int state) + { + if (!x->done) { +- DECLARE_WAITQUEUE(wait, current); ++ DEFINE_SWAITER(wait); + +- __add_wait_queue_tail_exclusive(&x->wait, &wait); ++ swait_prepare_locked(&x->wait, &wait); + do { + if (signal_pending_state(state, current)) { + timeout = -ERESTARTSYS; + break; + } + __set_current_state(state); +- spin_unlock_irq(&x->wait.lock); ++ raw_spin_unlock_irq(&x->wait.lock); + timeout = action(timeout); +- spin_lock_irq(&x->wait.lock); ++ raw_spin_lock_irq(&x->wait.lock); + } while (!x->done && timeout); +- __remove_wait_queue(&x->wait, &wait); ++ swait_finish_locked(&x->wait, &wait); + if (!x->done) + return timeout; + } +@@ -89,9 +89,9 @@ + { + might_sleep(); + +- spin_lock_irq(&x->wait.lock); ++ raw_spin_lock_irq(&x->wait.lock); + timeout = do_wait_for_common(x, action, timeout, state); +- spin_unlock_irq(&x->wait.lock); ++ raw_spin_unlock_irq(&x->wait.lock); + return timeout; + } + +@@ -277,12 +277,12 @@ + if (!READ_ONCE(x->done)) + return 0; + +- spin_lock_irqsave(&x->wait.lock, flags); ++ raw_spin_lock_irqsave(&x->wait.lock, flags); + if (!x->done) + ret = 0; + else + x->done--; +- spin_unlock_irqrestore(&x->wait.lock, flags); ++ raw_spin_unlock_irqrestore(&x->wait.lock, flags); + return ret; + } + EXPORT_SYMBOL(try_wait_for_completion); +@@ -311,7 +311,7 @@ + * after it's acquired the lock. + */ + smp_rmb(); +- spin_unlock_wait(&x->wait.lock); ++ raw_spin_unlock_wait(&x->wait.lock); + return true; + } + EXPORT_SYMBOL(completion_done); +diff -Nur linux-4.1.10.orig/kernel/sched/core.c linux-4.1.10/kernel/sched/core.c +--- linux-4.1.10.orig/kernel/sched/core.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/sched/core.c 2015-10-07 18:00:08.000000000 +0200 +@@ -282,7 +282,11 @@ + * Number of tasks to iterate in a single balance run. + * Limited because this is done with IRQs disabled. + */ ++#ifndef CONFIG_PREEMPT_RT_FULL + const_debug unsigned int sysctl_sched_nr_migrate = 32; ++#else ++const_debug unsigned int sysctl_sched_nr_migrate = 8; ++#endif + + /* + * period over which we average the RT time consumption, measured +@@ -461,6 +465,7 @@ + + hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + rq->hrtick_timer.function = hrtick; ++ rq->hrtick_timer.irqsafe = 1; + } + #else /* CONFIG_SCHED_HRTICK */ + static inline void hrtick_clear(struct rq *rq) +@@ -541,6 +546,52 @@ + #endif + #endif + ++void wake_q_add(struct wake_q_head *head, struct task_struct *task) ++{ ++ struct wake_q_node *node = &task->wake_q; ++ ++ /* ++ * Atomically grab the task, if ->wake_q is !nil already it means ++ * its already queued (either by us or someone else) and will get the ++ * wakeup due to that. ++ * ++ * This cmpxchg() implies a full barrier, which pairs with the write ++ * barrier implied by the wakeup in wake_up_list(). ++ */ ++ if (cmpxchg(&node->next, NULL, WAKE_Q_TAIL)) ++ return; ++ ++ get_task_struct(task); ++ ++ /* ++ * The head is context local, there can be no concurrency. ++ */ ++ *head->lastp = node; ++ head->lastp = &node->next; ++} ++ ++void wake_up_q(struct wake_q_head *head) ++{ ++ struct wake_q_node *node = head->first; ++ ++ while (node != WAKE_Q_TAIL) { ++ struct task_struct *task; ++ ++ task = container_of(node, struct task_struct, wake_q); ++ BUG_ON(!task); ++ /* task can safely be re-inserted now */ ++ node = node->next; ++ task->wake_q.next = NULL; ++ ++ /* ++ * wake_up_process() implies a wmb() to pair with the queueing ++ * in wake_q_add() so as not to miss wakeups. ++ */ ++ wake_up_process(task); ++ put_task_struct(task); ++ } ++} ++ + /* + * resched_curr - mark rq's current task 'to be rescheduled now'. + * +@@ -572,6 +623,38 @@ + trace_sched_wake_idle_without_ipi(cpu); + } + ++#ifdef CONFIG_PREEMPT_LAZY ++void resched_curr_lazy(struct rq *rq) ++{ ++ struct task_struct *curr = rq->curr; ++ int cpu; ++ ++ if (!sched_feat(PREEMPT_LAZY)) { ++ resched_curr(rq); ++ return; ++ } ++ ++ lockdep_assert_held(&rq->lock); ++ ++ if (test_tsk_need_resched(curr)) ++ return; ++ ++ if (test_tsk_need_resched_lazy(curr)) ++ return; ++ ++ set_tsk_need_resched_lazy(curr); ++ ++ cpu = cpu_of(rq); ++ if (cpu == smp_processor_id()) ++ return; ++ ++ /* NEED_RESCHED_LAZY must be visible before we test polling */ ++ smp_mb(); ++ if (!tsk_is_polling(curr)) ++ smp_send_reschedule(cpu); ++} ++#endif ++ + void resched_cpu(int cpu) + { + struct rq *rq = cpu_rq(cpu); +@@ -595,12 +678,14 @@ + */ + int get_nohz_timer_target(int pinned) + { +- int cpu = smp_processor_id(); ++ int cpu; + int i; + struct sched_domain *sd; + ++ preempt_disable_rt(); ++ cpu = smp_processor_id(); + if (pinned || !get_sysctl_timer_migration() || !idle_cpu(cpu)) +- return cpu; ++ goto preempt_en_rt; + + rcu_read_lock(); + for_each_domain(cpu, sd) { +@@ -613,6 +698,8 @@ + } + unlock: + rcu_read_unlock(); ++preempt_en_rt: ++ preempt_enable_rt(); + return cpu; + } + /* +@@ -1164,6 +1251,18 @@ + + static int migration_cpu_stop(void *data); + ++static bool check_task_state(struct task_struct *p, long match_state) ++{ ++ bool match = false; ++ ++ raw_spin_lock_irq(&p->pi_lock); ++ if (p->state == match_state || p->saved_state == match_state) ++ match = true; ++ raw_spin_unlock_irq(&p->pi_lock); ++ ++ return match; ++} ++ + /* + * wait_task_inactive - wait for a thread to unschedule. + * +@@ -1208,7 +1307,7 @@ + * is actually now running somewhere else! + */ + while (task_running(rq, p)) { +- if (match_state && unlikely(p->state != match_state)) ++ if (match_state && !check_task_state(p, match_state)) + return 0; + cpu_relax(); + } +@@ -1223,7 +1322,8 @@ + running = task_running(rq, p); + queued = task_on_rq_queued(p); + ncsw = 0; +- if (!match_state || p->state == match_state) ++ if (!match_state || p->state == match_state || ++ p->saved_state == match_state) + ncsw = p->nvcsw | LONG_MIN; /* sets MSB */ + task_rq_unlock(rq, p, &flags); + +@@ -1449,10 +1549,6 @@ + { + activate_task(rq, p, en_flags); + p->on_rq = TASK_ON_RQ_QUEUED; +- +- /* if a worker is waking up, notify workqueue */ +- if (p->flags & PF_WQ_WORKER) +- wq_worker_waking_up(p, cpu_of(rq)); + } + + /* +@@ -1666,8 +1762,27 @@ + */ + smp_mb__before_spinlock(); + raw_spin_lock_irqsave(&p->pi_lock, flags); +- if (!(p->state & state)) ++ if (!(p->state & state)) { ++ /* ++ * The task might be running due to a spinlock sleeper ++ * wakeup. Check the saved state and set it to running ++ * if the wakeup condition is true. ++ */ ++ if (!(wake_flags & WF_LOCK_SLEEPER)) { ++ if (p->saved_state & state) { ++ p->saved_state = TASK_RUNNING; ++ success = 1; ++ } ++ } + goto out; ++ } ++ ++ /* ++ * If this is a regular wakeup, then we can unconditionally ++ * clear the saved state of a "lock sleeper". ++ */ ++ if (!(wake_flags & WF_LOCK_SLEEPER)) ++ p->saved_state = TASK_RUNNING; + + success = 1; /* we're going to change ->state */ + cpu = task_cpu(p); +@@ -1710,42 +1825,6 @@ + } + + /** +- * try_to_wake_up_local - try to wake up a local task with rq lock held +- * @p: the thread to be awakened +- * +- * Put @p on the run-queue if it's not already there. The caller must +- * ensure that this_rq() is locked, @p is bound to this_rq() and not +- * the current task. +- */ +-static void try_to_wake_up_local(struct task_struct *p) +-{ +- struct rq *rq = task_rq(p); +- +- if (WARN_ON_ONCE(rq != this_rq()) || +- WARN_ON_ONCE(p == current)) +- return; +- +- lockdep_assert_held(&rq->lock); +- +- if (!raw_spin_trylock(&p->pi_lock)) { +- raw_spin_unlock(&rq->lock); +- raw_spin_lock(&p->pi_lock); +- raw_spin_lock(&rq->lock); +- } +- +- if (!(p->state & TASK_NORMAL)) +- goto out; +- +- if (!task_on_rq_queued(p)) +- ttwu_activate(rq, p, ENQUEUE_WAKEUP); +- +- ttwu_do_wakeup(rq, p, 0); +- ttwu_stat(p, smp_processor_id(), 0); +-out: +- raw_spin_unlock(&p->pi_lock); +-} +- +-/** + * wake_up_process - Wake up a specific process + * @p: The process to be woken up. + * +@@ -1759,11 +1838,23 @@ + */ + int wake_up_process(struct task_struct *p) + { +- WARN_ON(task_is_stopped_or_traced(p)); ++ WARN_ON(__task_is_stopped_or_traced(p)); + return try_to_wake_up(p, TASK_NORMAL, 0); + } + EXPORT_SYMBOL(wake_up_process); + ++/** ++ * wake_up_lock_sleeper - Wake up a specific process blocked on a "sleeping lock" ++ * @p: The process to be woken up. ++ * ++ * Same as wake_up_process() above, but wake_flags=WF_LOCK_SLEEPER to indicate ++ * the nature of the wakeup. ++ */ ++int wake_up_lock_sleeper(struct task_struct *p) ++{ ++ return try_to_wake_up(p, TASK_ALL, WF_LOCK_SLEEPER); ++} ++ + int wake_up_state(struct task_struct *p, unsigned int state) + { + return try_to_wake_up(p, state, 0); +@@ -1959,6 +2050,9 @@ + p->on_cpu = 0; + #endif + init_task_preempt_count(p); ++#ifdef CONFIG_HAVE_PREEMPT_LAZY ++ task_thread_info(p)->preempt_lazy_count = 0; ++#endif + #ifdef CONFIG_SMP + plist_node_init(&p->pushable_tasks, MAX_PRIO); + RB_CLEAR_NODE(&p->pushable_dl_tasks); +@@ -2231,8 +2325,12 @@ + finish_arch_post_lock_switch(); + + fire_sched_in_preempt_notifiers(current); ++ /* ++ * We use mmdrop_delayed() here so we don't have to do the ++ * full __mmdrop() when we are the last user. ++ */ + if (mm) +- mmdrop(mm); ++ mmdrop_delayed(mm); + if (unlikely(prev_state == TASK_DEAD)) { + if (prev->sched_class->task_dead) + prev->sched_class->task_dead(prev); +@@ -2650,6 +2748,133 @@ + schedstat_inc(this_rq(), sched_count); + } + ++#if defined(CONFIG_PREEMPT_RT_FULL) && defined(CONFIG_SMP) ++#define MIGRATE_DISABLE_SET_AFFIN (1<<30) /* Can't make a negative */ ++#define migrate_disabled_updated(p) ((p)->migrate_disable & MIGRATE_DISABLE_SET_AFFIN) ++#define migrate_disable_count(p) ((p)->migrate_disable & ~MIGRATE_DISABLE_SET_AFFIN) ++ ++static inline void update_migrate_disable(struct task_struct *p) ++{ ++ const struct cpumask *mask; ++ ++ if (likely(!p->migrate_disable)) ++ return; ++ ++ /* Did we already update affinity? */ ++ if (unlikely(migrate_disabled_updated(p))) ++ return; ++ ++ /* ++ * Since this is always current we can get away with only locking ++ * rq->lock, the ->cpus_allowed value can normally only be changed ++ * while holding both p->pi_lock and rq->lock, but seeing that this ++ * is current, we cannot actually be waking up, so all code that ++ * relies on serialization against p->pi_lock is out of scope. ++ * ++ * Having rq->lock serializes us against things like ++ * set_cpus_allowed_ptr() that can still happen concurrently. ++ */ ++ mask = tsk_cpus_allowed(p); ++ ++ if (p->sched_class->set_cpus_allowed) ++ p->sched_class->set_cpus_allowed(p, mask); ++ /* mask==cpumask_of(task_cpu(p)) which has a cpumask_weight==1 */ ++ p->nr_cpus_allowed = 1; ++ ++ /* Let migrate_enable know to fix things back up */ ++ p->migrate_disable |= MIGRATE_DISABLE_SET_AFFIN; ++} ++ ++void migrate_disable(void) ++{ ++ struct task_struct *p = current; ++ ++ if (in_atomic()) { ++#ifdef CONFIG_SCHED_DEBUG ++ p->migrate_disable_atomic++; ++#endif ++ return; ++ } ++ ++#ifdef CONFIG_SCHED_DEBUG ++ if (unlikely(p->migrate_disable_atomic)) { ++ tracing_off(); ++ WARN_ON_ONCE(1); ++ } ++#endif ++ ++ if (p->migrate_disable) { ++ p->migrate_disable++; ++ return; ++ } ++ ++ preempt_disable(); ++ preempt_lazy_disable(); ++ pin_current_cpu(); ++ p->migrate_disable = 1; ++ preempt_enable(); ++} ++EXPORT_SYMBOL(migrate_disable); ++ ++void migrate_enable(void) ++{ ++ struct task_struct *p = current; ++ const struct cpumask *mask; ++ unsigned long flags; ++ struct rq *rq; ++ ++ if (in_atomic()) { ++#ifdef CONFIG_SCHED_DEBUG ++ p->migrate_disable_atomic--; ++#endif ++ return; ++ } ++ ++#ifdef CONFIG_SCHED_DEBUG ++ if (unlikely(p->migrate_disable_atomic)) { ++ tracing_off(); ++ WARN_ON_ONCE(1); ++ } ++#endif ++ WARN_ON_ONCE(p->migrate_disable <= 0); ++ ++ if (migrate_disable_count(p) > 1) { ++ p->migrate_disable--; ++ return; ++ } ++ ++ preempt_disable(); ++ if (unlikely(migrate_disabled_updated(p))) { ++ /* ++ * Undo whatever update_migrate_disable() did, also see there ++ * about locking. ++ */ ++ rq = this_rq(); ++ raw_spin_lock_irqsave(&rq->lock, flags); ++ ++ /* ++ * Clearing migrate_disable causes tsk_cpus_allowed to ++ * show the tasks original cpu affinity. ++ */ ++ p->migrate_disable = 0; ++ mask = tsk_cpus_allowed(p); ++ if (p->sched_class->set_cpus_allowed) ++ p->sched_class->set_cpus_allowed(p, mask); ++ p->nr_cpus_allowed = cpumask_weight(mask); ++ raw_spin_unlock_irqrestore(&rq->lock, flags); ++ } else ++ p->migrate_disable = 0; ++ ++ unpin_current_cpu(); ++ preempt_enable(); ++ preempt_lazy_enable(); ++} ++EXPORT_SYMBOL(migrate_enable); ++#else ++static inline void update_migrate_disable(struct task_struct *p) { } ++#define migrate_disabled_updated(p) 0 ++#endif ++ + /* + * Pick up the highest-prio task: + */ +@@ -2756,6 +2981,8 @@ + smp_mb__before_spinlock(); + raw_spin_lock_irq(&rq->lock); + ++ update_migrate_disable(prev); ++ + rq->clock_skip_update <<= 1; /* promote REQ to ACT */ + + switch_count = &prev->nivcsw; +@@ -2765,19 +2992,6 @@ + } else { + deactivate_task(rq, prev, DEQUEUE_SLEEP); + prev->on_rq = 0; +- +- /* +- * If a worker went to sleep, notify and ask workqueue +- * whether it wants to wake up a task to maintain +- * concurrency. +- */ +- if (prev->flags & PF_WQ_WORKER) { +- struct task_struct *to_wakeup; +- +- to_wakeup = wq_worker_sleeping(prev, cpu); +- if (to_wakeup) +- try_to_wake_up_local(to_wakeup); +- } + } + switch_count = &prev->nvcsw; + } +@@ -2787,6 +3001,7 @@ + + next = pick_next_task(rq, prev); + clear_tsk_need_resched(prev); ++ clear_tsk_need_resched_lazy(prev); + clear_preempt_need_resched(); + rq->clock_skip_update = 0; + +@@ -2807,8 +3022,19 @@ + + static inline void sched_submit_work(struct task_struct *tsk) + { +- if (!tsk->state || tsk_is_pi_blocked(tsk)) ++ if (!tsk->state) ++ return; ++ /* ++ * If a worker went to sleep, notify and ask workqueue whether ++ * it wants to wake up a task to maintain concurrency. ++ */ ++ if (tsk->flags & PF_WQ_WORKER) ++ wq_worker_sleeping(tsk); ++ ++ ++ if (tsk_is_pi_blocked(tsk)) + return; ++ + /* + * If we are going to sleep and we have plugged IO queued, + * make sure to submit it to avoid deadlocks. +@@ -2817,6 +3043,12 @@ + blk_schedule_flush_plug(tsk); + } + ++static void sched_update_worker(struct task_struct *tsk) ++{ ++ if (tsk->flags & PF_WQ_WORKER) ++ wq_worker_running(tsk); ++} ++ + asmlinkage __visible void __sched schedule(void) + { + struct task_struct *tsk = current; +@@ -2825,6 +3057,7 @@ + do { + __schedule(); + } while (need_resched()); ++ sched_update_worker(tsk); + } + EXPORT_SYMBOL(schedule); + +@@ -2916,6 +3149,14 @@ + if (likely(!preemptible())) + return; + ++#ifdef CONFIG_PREEMPT_LAZY ++ /* ++ * Check for lazy preemption ++ */ ++ if (current_thread_info()->preempt_lazy_count && ++ !test_thread_flag(TIF_NEED_RESCHED)) ++ return; ++#endif + do { + __preempt_count_add(PREEMPT_ACTIVE); + /* +@@ -2924,7 +3165,16 @@ + * an infinite recursion. + */ + prev_ctx = exception_enter(); ++ /* ++ * The add/subtract must not be traced by the function ++ * tracer. But we still want to account for the ++ * preempt off latency tracer. Since the _notrace versions ++ * of add/subtract skip the accounting for latency tracer ++ * we must force it manually. ++ */ ++ start_critical_timings(); + __schedule(); ++ stop_critical_timings(); + exception_exit(prev_ctx); + + __preempt_count_sub(PREEMPT_ACTIVE); +@@ -4261,6 +4511,7 @@ + } + EXPORT_SYMBOL(__cond_resched_lock); + ++#ifndef CONFIG_PREEMPT_RT_FULL + int __sched __cond_resched_softirq(void) + { + BUG_ON(!in_softirq()); +@@ -4274,6 +4525,7 @@ + return 0; + } + EXPORT_SYMBOL(__cond_resched_softirq); ++#endif + + /** + * yield - yield the current processor to other threads. +@@ -4628,7 +4880,9 @@ + + /* Set the preempt count _outside_ the spinlocks! */ + init_idle_preempt_count(idle, cpu); +- ++#ifdef CONFIG_HAVE_PREEMPT_LAZY ++ task_thread_info(idle)->preempt_lazy_count = 0; ++#endif + /* + * The idle tasks have their own, simple scheduling class: + */ +@@ -4748,11 +5002,91 @@ + + void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask) + { +- if (p->sched_class->set_cpus_allowed) +- p->sched_class->set_cpus_allowed(p, new_mask); ++ if (!migrate_disabled_updated(p)) { ++ if (p->sched_class->set_cpus_allowed) ++ p->sched_class->set_cpus_allowed(p, new_mask); ++ p->nr_cpus_allowed = cpumask_weight(new_mask); ++ } + + cpumask_copy(&p->cpus_allowed, new_mask); +- p->nr_cpus_allowed = cpumask_weight(new_mask); ++} ++ ++static DEFINE_PER_CPU(struct cpumask, sched_cpumasks); ++static DEFINE_MUTEX(sched_down_mutex); ++static cpumask_t sched_down_cpumask; ++ ++void tell_sched_cpu_down_begin(int cpu) ++{ ++ mutex_lock(&sched_down_mutex); ++ cpumask_set_cpu(cpu, &sched_down_cpumask); ++ mutex_unlock(&sched_down_mutex); ++} ++ ++void tell_sched_cpu_down_done(int cpu) ++{ ++ mutex_lock(&sched_down_mutex); ++ cpumask_clear_cpu(cpu, &sched_down_cpumask); ++ mutex_unlock(&sched_down_mutex); ++} ++ ++/** ++ * migrate_me - try to move the current task off this cpu ++ * ++ * Used by the pin_current_cpu() code to try to get tasks ++ * to move off the current CPU as it is going down. ++ * It will only move the task if the task isn't pinned to ++ * the CPU (with migrate_disable, affinity or NO_SETAFFINITY) ++ * and the task has to be in a RUNNING state. Otherwise the ++ * movement of the task will wake it up (change its state ++ * to running) when the task did not expect it. ++ * ++ * Returns 1 if it succeeded in moving the current task ++ * 0 otherwise. ++ */ ++int migrate_me(void) ++{ ++ struct task_struct *p = current; ++ struct migration_arg arg; ++ struct cpumask *cpumask; ++ struct cpumask *mask; ++ unsigned long flags; ++ unsigned int dest_cpu; ++ struct rq *rq; ++ ++ /* ++ * We can not migrate tasks bounded to a CPU or tasks not ++ * running. The movement of the task will wake it up. ++ */ ++ if (p->flags & PF_NO_SETAFFINITY || p->state) ++ return 0; ++ ++ mutex_lock(&sched_down_mutex); ++ rq = task_rq_lock(p, &flags); ++ ++ cpumask = this_cpu_ptr(&sched_cpumasks); ++ mask = &p->cpus_allowed; ++ ++ cpumask_andnot(cpumask, mask, &sched_down_cpumask); ++ ++ if (!cpumask_weight(cpumask)) { ++ /* It's only on this CPU? */ ++ task_rq_unlock(rq, p, &flags); ++ mutex_unlock(&sched_down_mutex); ++ return 0; ++ } ++ ++ dest_cpu = cpumask_any_and(cpu_active_mask, cpumask); ++ ++ arg.task = p; ++ arg.dest_cpu = dest_cpu; ++ ++ task_rq_unlock(rq, p, &flags); ++ ++ stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg); ++ tlb_migrate_finish(p->mm); ++ mutex_unlock(&sched_down_mutex); ++ ++ return 1; + } + + /* +@@ -4798,7 +5132,7 @@ + do_set_cpus_allowed(p, new_mask); + + /* Can the task run on the task's current CPU? If so, we're done */ +- if (cpumask_test_cpu(task_cpu(p), new_mask)) ++ if (cpumask_test_cpu(task_cpu(p), new_mask) || __migrate_disabled(p)) + goto out; + + dest_cpu = cpumask_any_and(cpu_active_mask, new_mask); +@@ -4938,6 +5272,8 @@ + + #ifdef CONFIG_HOTPLUG_CPU + ++static DEFINE_PER_CPU(struct mm_struct *, idle_last_mm); ++ + /* + * Ensures that the idle task is using init_mm right before its cpu goes + * offline. +@@ -4952,7 +5288,11 @@ + switch_mm(mm, &init_mm, current); + finish_arch_post_lock_switch(); + } +- mmdrop(mm); ++ /* ++ * Defer the cleanup to an alive cpu. On RT we can neither ++ * call mmdrop() nor mmdrop_delayed() from here. ++ */ ++ per_cpu(idle_last_mm, smp_processor_id()) = mm; + } + + /* +@@ -5295,6 +5635,10 @@ + + case CPU_DEAD: + calc_load_migrate(rq); ++ if (per_cpu(idle_last_mm, cpu)) { ++ mmdrop(per_cpu(idle_last_mm, cpu)); ++ per_cpu(idle_last_mm, cpu) = NULL; ++ } + break; + #endif + } +@@ -7274,7 +7618,8 @@ + #ifdef CONFIG_DEBUG_ATOMIC_SLEEP + static inline int preempt_count_equals(int preempt_offset) + { +- int nested = (preempt_count() & ~PREEMPT_ACTIVE) + rcu_preempt_depth(); ++ int nested = (preempt_count() & ~PREEMPT_ACTIVE) + ++ sched_rcu_preempt_depth(); + + return (nested == preempt_offset); + } +diff -Nur linux-4.1.10.orig/kernel/sched/core.c.orig linux-4.1.10/kernel/sched/core.c.orig +--- linux-4.1.10.orig/kernel/sched/core.c.orig 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/kernel/sched/core.c.orig 2015-10-03 13:49:38.000000000 +0200 +@@ -0,0 +1,8389 @@ ++/* ++ * kernel/sched/core.c ++ * ++ * Kernel scheduler and related syscalls ++ * ++ * Copyright (C) 1991-2002 Linus Torvalds ++ * ++ * 1996-12-23 Modified by Dave Grothe to fix bugs in semaphores and ++ * make semaphores SMP safe ++ * 1998-11-19 Implemented schedule_timeout() and related stuff ++ * by Andrea Arcangeli ++ * 2002-01-04 New ultra-scalable O(1) scheduler by Ingo Molnar: ++ * hybrid priority-list and round-robin design with ++ * an array-switch method of distributing timeslices ++ * and per-CPU runqueues. Cleanups and useful suggestions ++ * by Davide Libenzi, preemptible kernel bits by Robert Love. ++ * 2003-09-03 Interactivity tuning by Con Kolivas. ++ * 2004-04-02 Scheduler domains code by Nick Piggin ++ * 2007-04-15 Work begun on replacing all interactivity tuning with a ++ * fair scheduling design by Con Kolivas. ++ * 2007-05-05 Load balancing (smp-nice) and other improvements ++ * by Peter Williams ++ * 2007-05-06 Interactivity improvements to CFS by Mike Galbraith ++ * 2007-07-01 Group scheduling enhancements by Srivatsa Vaddagiri ++ * 2007-11-29 RT balancing improvements by Steven Rostedt, Gregory Haskins, ++ * Thomas Gleixner, Mike Kravetz ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++#include ++#include ++#include ++#ifdef CONFIG_PARAVIRT ++#include ++#endif ++ ++#include "sched.h" ++#include "../workqueue_internal.h" ++#include "../smpboot.h" ++ ++#define CREATE_TRACE_POINTS ++#include ++ ++void start_bandwidth_timer(struct hrtimer *period_timer, ktime_t period) ++{ ++ unsigned long delta; ++ ktime_t soft, hard, now; ++ ++ for (;;) { ++ if (hrtimer_active(period_timer)) ++ break; ++ ++ now = hrtimer_cb_get_time(period_timer); ++ hrtimer_forward(period_timer, now, period); ++ ++ soft = hrtimer_get_softexpires(period_timer); ++ hard = hrtimer_get_expires(period_timer); ++ delta = ktime_to_ns(ktime_sub(hard, soft)); ++ __hrtimer_start_range_ns(period_timer, soft, delta, ++ HRTIMER_MODE_ABS_PINNED, 0); ++ } ++} ++ ++DEFINE_MUTEX(sched_domains_mutex); ++DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues); ++ ++static void update_rq_clock_task(struct rq *rq, s64 delta); ++ ++void update_rq_clock(struct rq *rq) ++{ ++ s64 delta; ++ ++ lockdep_assert_held(&rq->lock); ++ ++ if (rq->clock_skip_update & RQCF_ACT_SKIP) ++ return; ++ ++ delta = sched_clock_cpu(cpu_of(rq)) - rq->clock; ++ if (delta < 0) ++ return; ++ rq->clock += delta; ++ update_rq_clock_task(rq, delta); ++} ++ ++/* ++ * Debugging: various feature bits ++ */ ++ ++#define SCHED_FEAT(name, enabled) \ ++ (1UL << __SCHED_FEAT_##name) * enabled | ++ ++const_debug unsigned int sysctl_sched_features = ++#include "features.h" ++ 0; ++ ++#undef SCHED_FEAT ++ ++#ifdef CONFIG_SCHED_DEBUG ++#define SCHED_FEAT(name, enabled) \ ++ #name , ++ ++static const char * const sched_feat_names[] = { ++#include "features.h" ++}; ++ ++#undef SCHED_FEAT ++ ++static int sched_feat_show(struct seq_file *m, void *v) ++{ ++ int i; ++ ++ for (i = 0; i < __SCHED_FEAT_NR; i++) { ++ if (!(sysctl_sched_features & (1UL << i))) ++ seq_puts(m, "NO_"); ++ seq_printf(m, "%s ", sched_feat_names[i]); ++ } ++ seq_puts(m, "\n"); ++ ++ return 0; ++} ++ ++#ifdef HAVE_JUMP_LABEL ++ ++#define jump_label_key__true STATIC_KEY_INIT_TRUE ++#define jump_label_key__false STATIC_KEY_INIT_FALSE ++ ++#define SCHED_FEAT(name, enabled) \ ++ jump_label_key__##enabled , ++ ++struct static_key sched_feat_keys[__SCHED_FEAT_NR] = { ++#include "features.h" ++}; ++ ++#undef SCHED_FEAT ++ ++static void sched_feat_disable(int i) ++{ ++ if (static_key_enabled(&sched_feat_keys[i])) ++ static_key_slow_dec(&sched_feat_keys[i]); ++} ++ ++static void sched_feat_enable(int i) ++{ ++ if (!static_key_enabled(&sched_feat_keys[i])) ++ static_key_slow_inc(&sched_feat_keys[i]); ++} ++#else ++static void sched_feat_disable(int i) { }; ++static void sched_feat_enable(int i) { }; ++#endif /* HAVE_JUMP_LABEL */ ++ ++static int sched_feat_set(char *cmp) ++{ ++ int i; ++ int neg = 0; ++ ++ if (strncmp(cmp, "NO_", 3) == 0) { ++ neg = 1; ++ cmp += 3; ++ } ++ ++ for (i = 0; i < __SCHED_FEAT_NR; i++) { ++ if (strcmp(cmp, sched_feat_names[i]) == 0) { ++ if (neg) { ++ sysctl_sched_features &= ~(1UL << i); ++ sched_feat_disable(i); ++ } else { ++ sysctl_sched_features |= (1UL << i); ++ sched_feat_enable(i); ++ } ++ break; ++ } ++ } ++ ++ return i; ++} ++ ++static ssize_t ++sched_feat_write(struct file *filp, const char __user *ubuf, ++ size_t cnt, loff_t *ppos) ++{ ++ char buf[64]; ++ char *cmp; ++ int i; ++ struct inode *inode; ++ ++ if (cnt > 63) ++ cnt = 63; ++ ++ if (copy_from_user(&buf, ubuf, cnt)) ++ return -EFAULT; ++ ++ buf[cnt] = 0; ++ cmp = strstrip(buf); ++ ++ /* Ensure the static_key remains in a consistent state */ ++ inode = file_inode(filp); ++ mutex_lock(&inode->i_mutex); ++ i = sched_feat_set(cmp); ++ mutex_unlock(&inode->i_mutex); ++ if (i == __SCHED_FEAT_NR) ++ return -EINVAL; ++ ++ *ppos += cnt; ++ ++ return cnt; ++} ++ ++static int sched_feat_open(struct inode *inode, struct file *filp) ++{ ++ return single_open(filp, sched_feat_show, NULL); ++} ++ ++static const struct file_operations sched_feat_fops = { ++ .open = sched_feat_open, ++ .write = sched_feat_write, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = single_release, ++}; ++ ++static __init int sched_init_debug(void) ++{ ++ debugfs_create_file("sched_features", 0644, NULL, NULL, ++ &sched_feat_fops); ++ ++ return 0; ++} ++late_initcall(sched_init_debug); ++#endif /* CONFIG_SCHED_DEBUG */ ++ ++/* ++ * Number of tasks to iterate in a single balance run. ++ * Limited because this is done with IRQs disabled. ++ */ ++const_debug unsigned int sysctl_sched_nr_migrate = 32; ++ ++/* ++ * period over which we average the RT time consumption, measured ++ * in ms. ++ * ++ * default: 1s ++ */ ++const_debug unsigned int sysctl_sched_time_avg = MSEC_PER_SEC; ++ ++/* ++ * period over which we measure -rt task cpu usage in us. ++ * default: 1s ++ */ ++unsigned int sysctl_sched_rt_period = 1000000; ++ ++__read_mostly int scheduler_running; ++ ++/* ++ * part of the period that we allow rt tasks to run in us. ++ * default: 0.95s ++ */ ++int sysctl_sched_rt_runtime = 950000; ++ ++/* cpus with isolated domains */ ++cpumask_var_t cpu_isolated_map; ++ ++/* ++ * this_rq_lock - lock this runqueue and disable interrupts. ++ */ ++static struct rq *this_rq_lock(void) ++ __acquires(rq->lock) ++{ ++ struct rq *rq; ++ ++ local_irq_disable(); ++ rq = this_rq(); ++ raw_spin_lock(&rq->lock); ++ ++ return rq; ++} ++ ++#ifdef CONFIG_SCHED_HRTICK ++/* ++ * Use HR-timers to deliver accurate preemption points. ++ */ ++ ++static void hrtick_clear(struct rq *rq) ++{ ++ if (hrtimer_active(&rq->hrtick_timer)) ++ hrtimer_cancel(&rq->hrtick_timer); ++} ++ ++/* ++ * High-resolution timer tick. ++ * Runs from hardirq context with interrupts disabled. ++ */ ++static enum hrtimer_restart hrtick(struct hrtimer *timer) ++{ ++ struct rq *rq = container_of(timer, struct rq, hrtick_timer); ++ ++ WARN_ON_ONCE(cpu_of(rq) != smp_processor_id()); ++ ++ raw_spin_lock(&rq->lock); ++ update_rq_clock(rq); ++ rq->curr->sched_class->task_tick(rq, rq->curr, 1); ++ raw_spin_unlock(&rq->lock); ++ ++ return HRTIMER_NORESTART; ++} ++ ++#ifdef CONFIG_SMP ++ ++static int __hrtick_restart(struct rq *rq) ++{ ++ struct hrtimer *timer = &rq->hrtick_timer; ++ ktime_t time = hrtimer_get_softexpires(timer); ++ ++ return __hrtimer_start_range_ns(timer, time, 0, HRTIMER_MODE_ABS_PINNED, 0); ++} ++ ++/* ++ * called from hardirq (IPI) context ++ */ ++static void __hrtick_start(void *arg) ++{ ++ struct rq *rq = arg; ++ ++ raw_spin_lock(&rq->lock); ++ __hrtick_restart(rq); ++ rq->hrtick_csd_pending = 0; ++ raw_spin_unlock(&rq->lock); ++} ++ ++/* ++ * Called to set the hrtick timer state. ++ * ++ * called with rq->lock held and irqs disabled ++ */ ++void hrtick_start(struct rq *rq, u64 delay) ++{ ++ struct hrtimer *timer = &rq->hrtick_timer; ++ ktime_t time; ++ s64 delta; ++ ++ /* ++ * Don't schedule slices shorter than 10000ns, that just ++ * doesn't make sense and can cause timer DoS. ++ */ ++ delta = max_t(s64, delay, 10000LL); ++ time = ktime_add_ns(timer->base->get_time(), delta); ++ ++ hrtimer_set_expires(timer, time); ++ ++ if (rq == this_rq()) { ++ __hrtick_restart(rq); ++ } else if (!rq->hrtick_csd_pending) { ++ smp_call_function_single_async(cpu_of(rq), &rq->hrtick_csd); ++ rq->hrtick_csd_pending = 1; ++ } ++} ++ ++static int ++hotplug_hrtick(struct notifier_block *nfb, unsigned long action, void *hcpu) ++{ ++ int cpu = (int)(long)hcpu; ++ ++ switch (action) { ++ case CPU_UP_CANCELED: ++ case CPU_UP_CANCELED_FROZEN: ++ case CPU_DOWN_PREPARE: ++ case CPU_DOWN_PREPARE_FROZEN: ++ case CPU_DEAD: ++ case CPU_DEAD_FROZEN: ++ hrtick_clear(cpu_rq(cpu)); ++ return NOTIFY_OK; ++ } ++ ++ return NOTIFY_DONE; ++} ++ ++static __init void init_hrtick(void) ++{ ++ hotcpu_notifier(hotplug_hrtick, 0); ++} ++#else ++/* ++ * Called to set the hrtick timer state. ++ * ++ * called with rq->lock held and irqs disabled ++ */ ++void hrtick_start(struct rq *rq, u64 delay) ++{ ++ /* ++ * Don't schedule slices shorter than 10000ns, that just ++ * doesn't make sense. Rely on vruntime for fairness. ++ */ ++ delay = max_t(u64, delay, 10000LL); ++ __hrtimer_start_range_ns(&rq->hrtick_timer, ns_to_ktime(delay), 0, ++ HRTIMER_MODE_REL_PINNED, 0); ++} ++ ++static inline void init_hrtick(void) ++{ ++} ++#endif /* CONFIG_SMP */ ++ ++static void init_rq_hrtick(struct rq *rq) ++{ ++#ifdef CONFIG_SMP ++ rq->hrtick_csd_pending = 0; ++ ++ rq->hrtick_csd.flags = 0; ++ rq->hrtick_csd.func = __hrtick_start; ++ rq->hrtick_csd.info = rq; ++#endif ++ ++ hrtimer_init(&rq->hrtick_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); ++ rq->hrtick_timer.function = hrtick; ++} ++#else /* CONFIG_SCHED_HRTICK */ ++static inline void hrtick_clear(struct rq *rq) ++{ ++} ++ ++static inline void init_rq_hrtick(struct rq *rq) ++{ ++} ++ ++static inline void init_hrtick(void) ++{ ++} ++#endif /* CONFIG_SCHED_HRTICK */ ++ ++/* ++ * cmpxchg based fetch_or, macro so it works for different integer types ++ */ ++#define fetch_or(ptr, val) \ ++({ typeof(*(ptr)) __old, __val = *(ptr); \ ++ for (;;) { \ ++ __old = cmpxchg((ptr), __val, __val | (val)); \ ++ if (__old == __val) \ ++ break; \ ++ __val = __old; \ ++ } \ ++ __old; \ ++}) ++ ++#if defined(CONFIG_SMP) && defined(TIF_POLLING_NRFLAG) ++/* ++ * Atomically set TIF_NEED_RESCHED and test for TIF_POLLING_NRFLAG, ++ * this avoids any races wrt polling state changes and thereby avoids ++ * spurious IPIs. ++ */ ++static bool set_nr_and_not_polling(struct task_struct *p) ++{ ++ struct thread_info *ti = task_thread_info(p); ++ return !(fetch_or(&ti->flags, _TIF_NEED_RESCHED) & _TIF_POLLING_NRFLAG); ++} ++ ++/* ++ * Atomically set TIF_NEED_RESCHED if TIF_POLLING_NRFLAG is set. ++ * ++ * If this returns true, then the idle task promises to call ++ * sched_ttwu_pending() and reschedule soon. ++ */ ++static bool set_nr_if_polling(struct task_struct *p) ++{ ++ struct thread_info *ti = task_thread_info(p); ++ typeof(ti->flags) old, val = ACCESS_ONCE(ti->flags); ++ ++ for (;;) { ++ if (!(val & _TIF_POLLING_NRFLAG)) ++ return false; ++ if (val & _TIF_NEED_RESCHED) ++ return true; ++ old = cmpxchg(&ti->flags, val, val | _TIF_NEED_RESCHED); ++ if (old == val) ++ break; ++ val = old; ++ } ++ return true; ++} ++ ++#else ++static bool set_nr_and_not_polling(struct task_struct *p) ++{ ++ set_tsk_need_resched(p); ++ return true; ++} ++ ++#ifdef CONFIG_SMP ++static bool set_nr_if_polling(struct task_struct *p) ++{ ++ return false; ++} ++#endif ++#endif ++ ++/* ++ * resched_curr - mark rq's current task 'to be rescheduled now'. ++ * ++ * On UP this means the setting of the need_resched flag, on SMP it ++ * might also involve a cross-CPU call to trigger the scheduler on ++ * the target CPU. ++ */ ++void resched_curr(struct rq *rq) ++{ ++ struct task_struct *curr = rq->curr; ++ int cpu; ++ ++ lockdep_assert_held(&rq->lock); ++ ++ if (test_tsk_need_resched(curr)) ++ return; ++ ++ cpu = cpu_of(rq); ++ ++ if (cpu == smp_processor_id()) { ++ set_tsk_need_resched(curr); ++ set_preempt_need_resched(); ++ return; ++ } ++ ++ if (set_nr_and_not_polling(curr)) ++ smp_send_reschedule(cpu); ++ else ++ trace_sched_wake_idle_without_ipi(cpu); ++} ++ ++void resched_cpu(int cpu) ++{ ++ struct rq *rq = cpu_rq(cpu); ++ unsigned long flags; ++ ++ if (!raw_spin_trylock_irqsave(&rq->lock, flags)) ++ return; ++ resched_curr(rq); ++ raw_spin_unlock_irqrestore(&rq->lock, flags); ++} ++ ++#ifdef CONFIG_SMP ++#ifdef CONFIG_NO_HZ_COMMON ++/* ++ * In the semi idle case, use the nearest busy cpu for migrating timers ++ * from an idle cpu. This is good for power-savings. ++ * ++ * We don't do similar optimization for completely idle system, as ++ * selecting an idle cpu will add more delays to the timers than intended ++ * (as that cpu's timer base may not be uptodate wrt jiffies etc). ++ */ ++int get_nohz_timer_target(int pinned) ++{ ++ int cpu = smp_processor_id(); ++ int i; ++ struct sched_domain *sd; ++ ++ if (pinned || !get_sysctl_timer_migration() || !idle_cpu(cpu)) ++ return cpu; ++ ++ rcu_read_lock(); ++ for_each_domain(cpu, sd) { ++ for_each_cpu(i, sched_domain_span(sd)) { ++ if (!idle_cpu(i)) { ++ cpu = i; ++ goto unlock; ++ } ++ } ++ } ++unlock: ++ rcu_read_unlock(); ++ return cpu; ++} ++/* ++ * When add_timer_on() enqueues a timer into the timer wheel of an ++ * idle CPU then this timer might expire before the next timer event ++ * which is scheduled to wake up that CPU. In case of a completely ++ * idle system the next event might even be infinite time into the ++ * future. wake_up_idle_cpu() ensures that the CPU is woken up and ++ * leaves the inner idle loop so the newly added timer is taken into ++ * account when the CPU goes back to idle and evaluates the timer ++ * wheel for the next timer event. ++ */ ++static void wake_up_idle_cpu(int cpu) ++{ ++ struct rq *rq = cpu_rq(cpu); ++ ++ if (cpu == smp_processor_id()) ++ return; ++ ++ if (set_nr_and_not_polling(rq->idle)) ++ smp_send_reschedule(cpu); ++ else ++ trace_sched_wake_idle_without_ipi(cpu); ++} ++ ++static bool wake_up_full_nohz_cpu(int cpu) ++{ ++ /* ++ * We just need the target to call irq_exit() and re-evaluate ++ * the next tick. The nohz full kick at least implies that. ++ * If needed we can still optimize that later with an ++ * empty IRQ. ++ */ ++ if (tick_nohz_full_cpu(cpu)) { ++ if (cpu != smp_processor_id() || ++ tick_nohz_tick_stopped()) ++ tick_nohz_full_kick_cpu(cpu); ++ return true; ++ } ++ ++ return false; ++} ++ ++void wake_up_nohz_cpu(int cpu) ++{ ++ if (!wake_up_full_nohz_cpu(cpu)) ++ wake_up_idle_cpu(cpu); ++} ++ ++static inline bool got_nohz_idle_kick(void) ++{ ++ int cpu = smp_processor_id(); ++ ++ if (!test_bit(NOHZ_BALANCE_KICK, nohz_flags(cpu))) ++ return false; ++ ++ if (idle_cpu(cpu) && !need_resched()) ++ return true; ++ ++ /* ++ * We can't run Idle Load Balance on this CPU for this time so we ++ * cancel it and clear NOHZ_BALANCE_KICK ++ */ ++ clear_bit(NOHZ_BALANCE_KICK, nohz_flags(cpu)); ++ return false; ++} ++ ++#else /* CONFIG_NO_HZ_COMMON */ ++ ++static inline bool got_nohz_idle_kick(void) ++{ ++ return false; ++} ++ ++#endif /* CONFIG_NO_HZ_COMMON */ ++ ++#ifdef CONFIG_NO_HZ_FULL ++bool sched_can_stop_tick(void) ++{ ++ /* ++ * FIFO realtime policy runs the highest priority task. Other runnable ++ * tasks are of a lower priority. The scheduler tick does nothing. ++ */ ++ if (current->policy == SCHED_FIFO) ++ return true; ++ ++ /* ++ * Round-robin realtime tasks time slice with other tasks at the same ++ * realtime priority. Is this task the only one at this priority? ++ */ ++ if (current->policy == SCHED_RR) { ++ struct sched_rt_entity *rt_se = ¤t->rt; ++ ++ return rt_se->run_list.prev == rt_se->run_list.next; ++ } ++ ++ /* ++ * More than one running task need preemption. ++ * nr_running update is assumed to be visible ++ * after IPI is sent from wakers. ++ */ ++ if (this_rq()->nr_running > 1) ++ return false; ++ ++ return true; ++} ++#endif /* CONFIG_NO_HZ_FULL */ ++ ++void sched_avg_update(struct rq *rq) ++{ ++ s64 period = sched_avg_period(); ++ ++ while ((s64)(rq_clock(rq) - rq->age_stamp) > period) { ++ /* ++ * Inline assembly required to prevent the compiler ++ * optimising this loop into a divmod call. ++ * See __iter_div_u64_rem() for another example of this. ++ */ ++ asm("" : "+rm" (rq->age_stamp)); ++ rq->age_stamp += period; ++ rq->rt_avg /= 2; ++ } ++} ++ ++#endif /* CONFIG_SMP */ ++ ++#if defined(CONFIG_RT_GROUP_SCHED) || (defined(CONFIG_FAIR_GROUP_SCHED) && \ ++ (defined(CONFIG_SMP) || defined(CONFIG_CFS_BANDWIDTH))) ++/* ++ * Iterate task_group tree rooted at *from, calling @down when first entering a ++ * node and @up when leaving it for the final time. ++ * ++ * Caller must hold rcu_lock or sufficient equivalent. ++ */ ++int walk_tg_tree_from(struct task_group *from, ++ tg_visitor down, tg_visitor up, void *data) ++{ ++ struct task_group *parent, *child; ++ int ret; ++ ++ parent = from; ++ ++down: ++ ret = (*down)(parent, data); ++ if (ret) ++ goto out; ++ list_for_each_entry_rcu(child, &parent->children, siblings) { ++ parent = child; ++ goto down; ++ ++up: ++ continue; ++ } ++ ret = (*up)(parent, data); ++ if (ret || parent == from) ++ goto out; ++ ++ child = parent; ++ parent = parent->parent; ++ if (parent) ++ goto up; ++out: ++ return ret; ++} ++ ++int tg_nop(struct task_group *tg, void *data) ++{ ++ return 0; ++} ++#endif ++ ++static void set_load_weight(struct task_struct *p) ++{ ++ int prio = p->static_prio - MAX_RT_PRIO; ++ struct load_weight *load = &p->se.load; ++ ++ /* ++ * SCHED_IDLE tasks get minimal weight: ++ */ ++ if (p->policy == SCHED_IDLE) { ++ load->weight = scale_load(WEIGHT_IDLEPRIO); ++ load->inv_weight = WMULT_IDLEPRIO; ++ return; ++ } ++ ++ load->weight = scale_load(prio_to_weight[prio]); ++ load->inv_weight = prio_to_wmult[prio]; ++} ++ ++static void enqueue_task(struct rq *rq, struct task_struct *p, int flags) ++{ ++ update_rq_clock(rq); ++ sched_info_queued(rq, p); ++ p->sched_class->enqueue_task(rq, p, flags); ++} ++ ++static void dequeue_task(struct rq *rq, struct task_struct *p, int flags) ++{ ++ update_rq_clock(rq); ++ sched_info_dequeued(rq, p); ++ p->sched_class->dequeue_task(rq, p, flags); ++} ++ ++void activate_task(struct rq *rq, struct task_struct *p, int flags) ++{ ++ if (task_contributes_to_load(p)) ++ rq->nr_uninterruptible--; ++ ++ enqueue_task(rq, p, flags); ++} ++ ++void deactivate_task(struct rq *rq, struct task_struct *p, int flags) ++{ ++ if (task_contributes_to_load(p)) ++ rq->nr_uninterruptible++; ++ ++ dequeue_task(rq, p, flags); ++} ++ ++static void update_rq_clock_task(struct rq *rq, s64 delta) ++{ ++/* ++ * In theory, the compile should just see 0 here, and optimize out the call ++ * to sched_rt_avg_update. But I don't trust it... ++ */ ++#if defined(CONFIG_IRQ_TIME_ACCOUNTING) || defined(CONFIG_PARAVIRT_TIME_ACCOUNTING) ++ s64 steal = 0, irq_delta = 0; ++#endif ++#ifdef CONFIG_IRQ_TIME_ACCOUNTING ++ irq_delta = irq_time_read(cpu_of(rq)) - rq->prev_irq_time; ++ ++ /* ++ * Since irq_time is only updated on {soft,}irq_exit, we might run into ++ * this case when a previous update_rq_clock() happened inside a ++ * {soft,}irq region. ++ * ++ * When this happens, we stop ->clock_task and only update the ++ * prev_irq_time stamp to account for the part that fit, so that a next ++ * update will consume the rest. This ensures ->clock_task is ++ * monotonic. ++ * ++ * It does however cause some slight miss-attribution of {soft,}irq ++ * time, a more accurate solution would be to update the irq_time using ++ * the current rq->clock timestamp, except that would require using ++ * atomic ops. ++ */ ++ if (irq_delta > delta) ++ irq_delta = delta; ++ ++ rq->prev_irq_time += irq_delta; ++ delta -= irq_delta; ++#endif ++#ifdef CONFIG_PARAVIRT_TIME_ACCOUNTING ++ if (static_key_false((¶virt_steal_rq_enabled))) { ++ steal = paravirt_steal_clock(cpu_of(rq)); ++ steal -= rq->prev_steal_time_rq; ++ ++ if (unlikely(steal > delta)) ++ steal = delta; ++ ++ rq->prev_steal_time_rq += steal; ++ delta -= steal; ++ } ++#endif ++ ++ rq->clock_task += delta; ++ ++#if defined(CONFIG_IRQ_TIME_ACCOUNTING) || defined(CONFIG_PARAVIRT_TIME_ACCOUNTING) ++ if ((irq_delta + steal) && sched_feat(NONTASK_CAPACITY)) ++ sched_rt_avg_update(rq, irq_delta + steal); ++#endif ++} ++ ++void sched_set_stop_task(int cpu, struct task_struct *stop) ++{ ++ struct sched_param param = { .sched_priority = MAX_RT_PRIO - 1 }; ++ struct task_struct *old_stop = cpu_rq(cpu)->stop; ++ ++ if (stop) { ++ /* ++ * Make it appear like a SCHED_FIFO task, its something ++ * userspace knows about and won't get confused about. ++ * ++ * Also, it will make PI more or less work without too ++ * much confusion -- but then, stop work should not ++ * rely on PI working anyway. ++ */ ++ sched_setscheduler_nocheck(stop, SCHED_FIFO, ¶m); ++ ++ stop->sched_class = &stop_sched_class; ++ } ++ ++ cpu_rq(cpu)->stop = stop; ++ ++ if (old_stop) { ++ /* ++ * Reset it back to a normal scheduling class so that ++ * it can die in pieces. ++ */ ++ old_stop->sched_class = &rt_sched_class; ++ } ++} ++ ++/* ++ * __normal_prio - return the priority that is based on the static prio ++ */ ++static inline int __normal_prio(struct task_struct *p) ++{ ++ return p->static_prio; ++} ++ ++/* ++ * Calculate the expected normal priority: i.e. priority ++ * without taking RT-inheritance into account. Might be ++ * boosted by interactivity modifiers. Changes upon fork, ++ * setprio syscalls, and whenever the interactivity ++ * estimator recalculates. ++ */ ++static inline int normal_prio(struct task_struct *p) ++{ ++ int prio; ++ ++ if (task_has_dl_policy(p)) ++ prio = MAX_DL_PRIO-1; ++ else if (task_has_rt_policy(p)) ++ prio = MAX_RT_PRIO-1 - p->rt_priority; ++ else ++ prio = __normal_prio(p); ++ return prio; ++} ++ ++/* ++ * Calculate the current priority, i.e. the priority ++ * taken into account by the scheduler. This value might ++ * be boosted by RT tasks, or might be boosted by ++ * interactivity modifiers. Will be RT if the task got ++ * RT-boosted. If not then it returns p->normal_prio. ++ */ ++static int effective_prio(struct task_struct *p) ++{ ++ p->normal_prio = normal_prio(p); ++ /* ++ * If we are RT tasks or we were boosted to RT priority, ++ * keep the priority unchanged. Otherwise, update priority ++ * to the normal priority: ++ */ ++ if (!rt_prio(p->prio)) ++ return p->normal_prio; ++ return p->prio; ++} ++ ++/** ++ * task_curr - is this task currently executing on a CPU? ++ * @p: the task in question. ++ * ++ * Return: 1 if the task is currently executing. 0 otherwise. ++ */ ++inline int task_curr(const struct task_struct *p) ++{ ++ return cpu_curr(task_cpu(p)) == p; ++} ++ ++/* ++ * Can drop rq->lock because from sched_class::switched_from() methods drop it. ++ */ ++static inline void check_class_changed(struct rq *rq, struct task_struct *p, ++ const struct sched_class *prev_class, ++ int oldprio) ++{ ++ if (prev_class != p->sched_class) { ++ if (prev_class->switched_from) ++ prev_class->switched_from(rq, p); ++ /* Possble rq->lock 'hole'. */ ++ p->sched_class->switched_to(rq, p); ++ } else if (oldprio != p->prio || dl_task(p)) ++ p->sched_class->prio_changed(rq, p, oldprio); ++} ++ ++void check_preempt_curr(struct rq *rq, struct task_struct *p, int flags) ++{ ++ const struct sched_class *class; ++ ++ if (p->sched_class == rq->curr->sched_class) { ++ rq->curr->sched_class->check_preempt_curr(rq, p, flags); ++ } else { ++ for_each_class(class) { ++ if (class == rq->curr->sched_class) ++ break; ++ if (class == p->sched_class) { ++ resched_curr(rq); ++ break; ++ } ++ } ++ } ++ ++ /* ++ * A queue event has occurred, and we're going to schedule. In ++ * this case, we can save a useless back to back clock update. ++ */ ++ if (task_on_rq_queued(rq->curr) && test_tsk_need_resched(rq->curr)) ++ rq_clock_skip_update(rq, true); ++} ++ ++#ifdef CONFIG_SMP ++void set_task_cpu(struct task_struct *p, unsigned int new_cpu) ++{ ++#ifdef CONFIG_SCHED_DEBUG ++ /* ++ * We should never call set_task_cpu() on a blocked task, ++ * ttwu() will sort out the placement. ++ */ ++ WARN_ON_ONCE(p->state != TASK_RUNNING && p->state != TASK_WAKING && ++ !p->on_rq); ++ ++#ifdef CONFIG_LOCKDEP ++ /* ++ * The caller should hold either p->pi_lock or rq->lock, when changing ++ * a task's CPU. ->pi_lock for waking tasks, rq->lock for runnable tasks. ++ * ++ * sched_move_task() holds both and thus holding either pins the cgroup, ++ * see task_group(). ++ * ++ * Furthermore, all task_rq users should acquire both locks, see ++ * task_rq_lock(). ++ */ ++ WARN_ON_ONCE(debug_locks && !(lockdep_is_held(&p->pi_lock) || ++ lockdep_is_held(&task_rq(p)->lock))); ++#endif ++#endif ++ ++ trace_sched_migrate_task(p, new_cpu); ++ ++ if (task_cpu(p) != new_cpu) { ++ if (p->sched_class->migrate_task_rq) ++ p->sched_class->migrate_task_rq(p, new_cpu); ++ p->se.nr_migrations++; ++ perf_sw_event_sched(PERF_COUNT_SW_CPU_MIGRATIONS, 1, 0); ++ } ++ ++ __set_task_cpu(p, new_cpu); ++} ++ ++static void __migrate_swap_task(struct task_struct *p, int cpu) ++{ ++ if (task_on_rq_queued(p)) { ++ struct rq *src_rq, *dst_rq; ++ ++ src_rq = task_rq(p); ++ dst_rq = cpu_rq(cpu); ++ ++ deactivate_task(src_rq, p, 0); ++ set_task_cpu(p, cpu); ++ activate_task(dst_rq, p, 0); ++ check_preempt_curr(dst_rq, p, 0); ++ } else { ++ /* ++ * Task isn't running anymore; make it appear like we migrated ++ * it before it went to sleep. This means on wakeup we make the ++ * previous cpu our targer instead of where it really is. ++ */ ++ p->wake_cpu = cpu; ++ } ++} ++ ++struct migration_swap_arg { ++ struct task_struct *src_task, *dst_task; ++ int src_cpu, dst_cpu; ++}; ++ ++static int migrate_swap_stop(void *data) ++{ ++ struct migration_swap_arg *arg = data; ++ struct rq *src_rq, *dst_rq; ++ int ret = -EAGAIN; ++ ++ src_rq = cpu_rq(arg->src_cpu); ++ dst_rq = cpu_rq(arg->dst_cpu); ++ ++ double_raw_lock(&arg->src_task->pi_lock, ++ &arg->dst_task->pi_lock); ++ double_rq_lock(src_rq, dst_rq); ++ if (task_cpu(arg->dst_task) != arg->dst_cpu) ++ goto unlock; ++ ++ if (task_cpu(arg->src_task) != arg->src_cpu) ++ goto unlock; ++ ++ if (!cpumask_test_cpu(arg->dst_cpu, tsk_cpus_allowed(arg->src_task))) ++ goto unlock; ++ ++ if (!cpumask_test_cpu(arg->src_cpu, tsk_cpus_allowed(arg->dst_task))) ++ goto unlock; ++ ++ __migrate_swap_task(arg->src_task, arg->dst_cpu); ++ __migrate_swap_task(arg->dst_task, arg->src_cpu); ++ ++ ret = 0; ++ ++unlock: ++ double_rq_unlock(src_rq, dst_rq); ++ raw_spin_unlock(&arg->dst_task->pi_lock); ++ raw_spin_unlock(&arg->src_task->pi_lock); ++ ++ return ret; ++} ++ ++/* ++ * Cross migrate two tasks ++ */ ++int migrate_swap(struct task_struct *cur, struct task_struct *p) ++{ ++ struct migration_swap_arg arg; ++ int ret = -EINVAL; ++ ++ arg = (struct migration_swap_arg){ ++ .src_task = cur, ++ .src_cpu = task_cpu(cur), ++ .dst_task = p, ++ .dst_cpu = task_cpu(p), ++ }; ++ ++ if (arg.src_cpu == arg.dst_cpu) ++ goto out; ++ ++ /* ++ * These three tests are all lockless; this is OK since all of them ++ * will be re-checked with proper locks held further down the line. ++ */ ++ if (!cpu_active(arg.src_cpu) || !cpu_active(arg.dst_cpu)) ++ goto out; ++ ++ if (!cpumask_test_cpu(arg.dst_cpu, tsk_cpus_allowed(arg.src_task))) ++ goto out; ++ ++ if (!cpumask_test_cpu(arg.src_cpu, tsk_cpus_allowed(arg.dst_task))) ++ goto out; ++ ++ trace_sched_swap_numa(cur, arg.src_cpu, p, arg.dst_cpu); ++ ret = stop_two_cpus(arg.dst_cpu, arg.src_cpu, migrate_swap_stop, &arg); ++ ++out: ++ return ret; ++} ++ ++struct migration_arg { ++ struct task_struct *task; ++ int dest_cpu; ++}; ++ ++static int migration_cpu_stop(void *data); ++ ++/* ++ * wait_task_inactive - wait for a thread to unschedule. ++ * ++ * If @match_state is nonzero, it's the @p->state value just checked and ++ * not expected to change. If it changes, i.e. @p might have woken up, ++ * then return zero. When we succeed in waiting for @p to be off its CPU, ++ * we return a positive number (its total switch count). If a second call ++ * a short while later returns the same number, the caller can be sure that ++ * @p has remained unscheduled the whole time. ++ * ++ * The caller must ensure that the task *will* unschedule sometime soon, ++ * else this function might spin for a *long* time. This function can't ++ * be called with interrupts off, or it may introduce deadlock with ++ * smp_call_function() if an IPI is sent by the same process we are ++ * waiting to become inactive. ++ */ ++unsigned long wait_task_inactive(struct task_struct *p, long match_state) ++{ ++ unsigned long flags; ++ int running, queued; ++ unsigned long ncsw; ++ struct rq *rq; ++ ++ for (;;) { ++ /* ++ * We do the initial early heuristics without holding ++ * any task-queue locks at all. We'll only try to get ++ * the runqueue lock when things look like they will ++ * work out! ++ */ ++ rq = task_rq(p); ++ ++ /* ++ * If the task is actively running on another CPU ++ * still, just relax and busy-wait without holding ++ * any locks. ++ * ++ * NOTE! Since we don't hold any locks, it's not ++ * even sure that "rq" stays as the right runqueue! ++ * But we don't care, since "task_running()" will ++ * return false if the runqueue has changed and p ++ * is actually now running somewhere else! ++ */ ++ while (task_running(rq, p)) { ++ if (match_state && unlikely(p->state != match_state)) ++ return 0; ++ cpu_relax(); ++ } ++ ++ /* ++ * Ok, time to look more closely! We need the rq ++ * lock now, to be *sure*. If we're wrong, we'll ++ * just go back and repeat. ++ */ ++ rq = task_rq_lock(p, &flags); ++ trace_sched_wait_task(p); ++ running = task_running(rq, p); ++ queued = task_on_rq_queued(p); ++ ncsw = 0; ++ if (!match_state || p->state == match_state) ++ ncsw = p->nvcsw | LONG_MIN; /* sets MSB */ ++ task_rq_unlock(rq, p, &flags); ++ ++ /* ++ * If it changed from the expected state, bail out now. ++ */ ++ if (unlikely(!ncsw)) ++ break; ++ ++ /* ++ * Was it really running after all now that we ++ * checked with the proper locks actually held? ++ * ++ * Oops. Go back and try again.. ++ */ ++ if (unlikely(running)) { ++ cpu_relax(); ++ continue; ++ } ++ ++ /* ++ * It's not enough that it's not actively running, ++ * it must be off the runqueue _entirely_, and not ++ * preempted! ++ * ++ * So if it was still runnable (but just not actively ++ * running right now), it's preempted, and we should ++ * yield - it could be a while. ++ */ ++ if (unlikely(queued)) { ++ ktime_t to = ktime_set(0, NSEC_PER_SEC/HZ); ++ ++ set_current_state(TASK_UNINTERRUPTIBLE); ++ schedule_hrtimeout(&to, HRTIMER_MODE_REL); ++ continue; ++ } ++ ++ /* ++ * Ahh, all good. It wasn't running, and it wasn't ++ * runnable, which means that it will never become ++ * running in the future either. We're all done! ++ */ ++ break; ++ } ++ ++ return ncsw; ++} ++ ++/*** ++ * kick_process - kick a running thread to enter/exit the kernel ++ * @p: the to-be-kicked thread ++ * ++ * Cause a process which is running on another CPU to enter ++ * kernel-mode, without any delay. (to get signals handled.) ++ * ++ * NOTE: this function doesn't have to take the runqueue lock, ++ * because all it wants to ensure is that the remote task enters ++ * the kernel. If the IPI races and the task has been migrated ++ * to another CPU then no harm is done and the purpose has been ++ * achieved as well. ++ */ ++void kick_process(struct task_struct *p) ++{ ++ int cpu; ++ ++ preempt_disable(); ++ cpu = task_cpu(p); ++ if ((cpu != smp_processor_id()) && task_curr(p)) ++ smp_send_reschedule(cpu); ++ preempt_enable(); ++} ++EXPORT_SYMBOL_GPL(kick_process); ++#endif /* CONFIG_SMP */ ++ ++#ifdef CONFIG_SMP ++/* ++ * ->cpus_allowed is protected by both rq->lock and p->pi_lock ++ */ ++static int select_fallback_rq(int cpu, struct task_struct *p) ++{ ++ int nid = cpu_to_node(cpu); ++ const struct cpumask *nodemask = NULL; ++ enum { cpuset, possible, fail } state = cpuset; ++ int dest_cpu; ++ ++ /* ++ * If the node that the cpu is on has been offlined, cpu_to_node() ++ * will return -1. There is no cpu on the node, and we should ++ * select the cpu on the other node. ++ */ ++ if (nid != -1) { ++ nodemask = cpumask_of_node(nid); ++ ++ /* Look for allowed, online CPU in same node. */ ++ for_each_cpu(dest_cpu, nodemask) { ++ if (!cpu_online(dest_cpu)) ++ continue; ++ if (!cpu_active(dest_cpu)) ++ continue; ++ if (cpumask_test_cpu(dest_cpu, tsk_cpus_allowed(p))) ++ return dest_cpu; ++ } ++ } ++ ++ for (;;) { ++ /* Any allowed, online CPU? */ ++ for_each_cpu(dest_cpu, tsk_cpus_allowed(p)) { ++ if (!cpu_online(dest_cpu)) ++ continue; ++ if (!cpu_active(dest_cpu)) ++ continue; ++ goto out; ++ } ++ ++ switch (state) { ++ case cpuset: ++ /* No more Mr. Nice Guy. */ ++ cpuset_cpus_allowed_fallback(p); ++ state = possible; ++ break; ++ ++ case possible: ++ do_set_cpus_allowed(p, cpu_possible_mask); ++ state = fail; ++ break; ++ ++ case fail: ++ BUG(); ++ break; ++ } ++ } ++ ++out: ++ if (state != cpuset) { ++ /* ++ * Don't tell them about moving exiting tasks or ++ * kernel threads (both mm NULL), since they never ++ * leave kernel. ++ */ ++ if (p->mm && printk_ratelimit()) { ++ printk_deferred("process %d (%s) no longer affine to cpu%d\n", ++ task_pid_nr(p), p->comm, cpu); ++ } ++ } ++ ++ return dest_cpu; ++} ++ ++/* ++ * The caller (fork, wakeup) owns p->pi_lock, ->cpus_allowed is stable. ++ */ ++static inline ++int select_task_rq(struct task_struct *p, int cpu, int sd_flags, int wake_flags) ++{ ++ if (p->nr_cpus_allowed > 1) ++ cpu = p->sched_class->select_task_rq(p, cpu, sd_flags, wake_flags); ++ ++ /* ++ * In order not to call set_task_cpu() on a blocking task we need ++ * to rely on ttwu() to place the task on a valid ->cpus_allowed ++ * cpu. ++ * ++ * Since this is common to all placement strategies, this lives here. ++ * ++ * [ this allows ->select_task() to simply return task_cpu(p) and ++ * not worry about this generic constraint ] ++ */ ++ if (unlikely(!cpumask_test_cpu(cpu, tsk_cpus_allowed(p)) || ++ !cpu_online(cpu))) ++ cpu = select_fallback_rq(task_cpu(p), p); ++ ++ return cpu; ++} ++ ++static void update_avg(u64 *avg, u64 sample) ++{ ++ s64 diff = sample - *avg; ++ *avg += diff >> 3; ++} ++#endif ++ ++static void ++ttwu_stat(struct task_struct *p, int cpu, int wake_flags) ++{ ++#ifdef CONFIG_SCHEDSTATS ++ struct rq *rq = this_rq(); ++ ++#ifdef CONFIG_SMP ++ int this_cpu = smp_processor_id(); ++ ++ if (cpu == this_cpu) { ++ schedstat_inc(rq, ttwu_local); ++ schedstat_inc(p, se.statistics.nr_wakeups_local); ++ } else { ++ struct sched_domain *sd; ++ ++ schedstat_inc(p, se.statistics.nr_wakeups_remote); ++ rcu_read_lock(); ++ for_each_domain(this_cpu, sd) { ++ if (cpumask_test_cpu(cpu, sched_domain_span(sd))) { ++ schedstat_inc(sd, ttwu_wake_remote); ++ break; ++ } ++ } ++ rcu_read_unlock(); ++ } ++ ++ if (wake_flags & WF_MIGRATED) ++ schedstat_inc(p, se.statistics.nr_wakeups_migrate); ++ ++#endif /* CONFIG_SMP */ ++ ++ schedstat_inc(rq, ttwu_count); ++ schedstat_inc(p, se.statistics.nr_wakeups); ++ ++ if (wake_flags & WF_SYNC) ++ schedstat_inc(p, se.statistics.nr_wakeups_sync); ++ ++#endif /* CONFIG_SCHEDSTATS */ ++} ++ ++static void ttwu_activate(struct rq *rq, struct task_struct *p, int en_flags) ++{ ++ activate_task(rq, p, en_flags); ++ p->on_rq = TASK_ON_RQ_QUEUED; ++ ++ /* if a worker is waking up, notify workqueue */ ++ if (p->flags & PF_WQ_WORKER) ++ wq_worker_waking_up(p, cpu_of(rq)); ++} ++ ++/* ++ * Mark the task runnable and perform wakeup-preemption. ++ */ ++static void ++ttwu_do_wakeup(struct rq *rq, struct task_struct *p, int wake_flags) ++{ ++ check_preempt_curr(rq, p, wake_flags); ++ trace_sched_wakeup(p, true); ++ ++ p->state = TASK_RUNNING; ++#ifdef CONFIG_SMP ++ if (p->sched_class->task_woken) ++ p->sched_class->task_woken(rq, p); ++ ++ if (rq->idle_stamp) { ++ u64 delta = rq_clock(rq) - rq->idle_stamp; ++ u64 max = 2*rq->max_idle_balance_cost; ++ ++ update_avg(&rq->avg_idle, delta); ++ ++ if (rq->avg_idle > max) ++ rq->avg_idle = max; ++ ++ rq->idle_stamp = 0; ++ } ++#endif ++} ++ ++static void ++ttwu_do_activate(struct rq *rq, struct task_struct *p, int wake_flags) ++{ ++#ifdef CONFIG_SMP ++ if (p->sched_contributes_to_load) ++ rq->nr_uninterruptible--; ++#endif ++ ++ ttwu_activate(rq, p, ENQUEUE_WAKEUP | ENQUEUE_WAKING); ++ ttwu_do_wakeup(rq, p, wake_flags); ++} ++ ++/* ++ * Called in case the task @p isn't fully descheduled from its runqueue, ++ * in this case we must do a remote wakeup. Its a 'light' wakeup though, ++ * since all we need to do is flip p->state to TASK_RUNNING, since ++ * the task is still ->on_rq. ++ */ ++static int ttwu_remote(struct task_struct *p, int wake_flags) ++{ ++ struct rq *rq; ++ int ret = 0; ++ ++ rq = __task_rq_lock(p); ++ if (task_on_rq_queued(p)) { ++ /* check_preempt_curr() may use rq clock */ ++ update_rq_clock(rq); ++ ttwu_do_wakeup(rq, p, wake_flags); ++ ret = 1; ++ } ++ __task_rq_unlock(rq); ++ ++ return ret; ++} ++ ++#ifdef CONFIG_SMP ++void sched_ttwu_pending(void) ++{ ++ struct rq *rq = this_rq(); ++ struct llist_node *llist = llist_del_all(&rq->wake_list); ++ struct task_struct *p; ++ unsigned long flags; ++ ++ if (!llist) ++ return; ++ ++ raw_spin_lock_irqsave(&rq->lock, flags); ++ ++ while (llist) { ++ p = llist_entry(llist, struct task_struct, wake_entry); ++ llist = llist_next(llist); ++ ttwu_do_activate(rq, p, 0); ++ } ++ ++ raw_spin_unlock_irqrestore(&rq->lock, flags); ++} ++ ++void scheduler_ipi(void) ++{ ++ /* ++ * Fold TIF_NEED_RESCHED into the preempt_count; anybody setting ++ * TIF_NEED_RESCHED remotely (for the first time) will also send ++ * this IPI. ++ */ ++ preempt_fold_need_resched(); ++ ++ if (llist_empty(&this_rq()->wake_list) && !got_nohz_idle_kick()) ++ return; ++ ++ /* ++ * Not all reschedule IPI handlers call irq_enter/irq_exit, since ++ * traditionally all their work was done from the interrupt return ++ * path. Now that we actually do some work, we need to make sure ++ * we do call them. ++ * ++ * Some archs already do call them, luckily irq_enter/exit nest ++ * properly. ++ * ++ * Arguably we should visit all archs and update all handlers, ++ * however a fair share of IPIs are still resched only so this would ++ * somewhat pessimize the simple resched case. ++ */ ++ irq_enter(); ++ sched_ttwu_pending(); ++ ++ /* ++ * Check if someone kicked us for doing the nohz idle load balance. ++ */ ++ if (unlikely(got_nohz_idle_kick())) { ++ this_rq()->idle_balance = 1; ++ raise_softirq_irqoff(SCHED_SOFTIRQ); ++ } ++ irq_exit(); ++} ++ ++static void ttwu_queue_remote(struct task_struct *p, int cpu) ++{ ++ struct rq *rq = cpu_rq(cpu); ++ ++ if (llist_add(&p->wake_entry, &cpu_rq(cpu)->wake_list)) { ++ if (!set_nr_if_polling(rq->idle)) ++ smp_send_reschedule(cpu); ++ else ++ trace_sched_wake_idle_without_ipi(cpu); ++ } ++} ++ ++void wake_up_if_idle(int cpu) ++{ ++ struct rq *rq = cpu_rq(cpu); ++ unsigned long flags; ++ ++ rcu_read_lock(); ++ ++ if (!is_idle_task(rcu_dereference(rq->curr))) ++ goto out; ++ ++ if (set_nr_if_polling(rq->idle)) { ++ trace_sched_wake_idle_without_ipi(cpu); ++ } else { ++ raw_spin_lock_irqsave(&rq->lock, flags); ++ if (is_idle_task(rq->curr)) ++ smp_send_reschedule(cpu); ++ /* Else cpu is not in idle, do nothing here */ ++ raw_spin_unlock_irqrestore(&rq->lock, flags); ++ } ++ ++out: ++ rcu_read_unlock(); ++} ++ ++bool cpus_share_cache(int this_cpu, int that_cpu) ++{ ++ return per_cpu(sd_llc_id, this_cpu) == per_cpu(sd_llc_id, that_cpu); ++} ++#endif /* CONFIG_SMP */ ++ ++static void ttwu_queue(struct task_struct *p, int cpu) ++{ ++ struct rq *rq = cpu_rq(cpu); ++ ++#if defined(CONFIG_SMP) ++ if (sched_feat(TTWU_QUEUE) && !cpus_share_cache(smp_processor_id(), cpu)) { ++ sched_clock_cpu(cpu); /* sync clocks x-cpu */ ++ ttwu_queue_remote(p, cpu); ++ return; ++ } ++#endif ++ ++ raw_spin_lock(&rq->lock); ++ ttwu_do_activate(rq, p, 0); ++ raw_spin_unlock(&rq->lock); ++} ++ ++/** ++ * try_to_wake_up - wake up a thread ++ * @p: the thread to be awakened ++ * @state: the mask of task states that can be woken ++ * @wake_flags: wake modifier flags (WF_*) ++ * ++ * Put it on the run-queue if it's not already there. The "current" ++ * thread is always on the run-queue (except when the actual ++ * re-schedule is in progress), and as such you're allowed to do ++ * the simpler "current->state = TASK_RUNNING" to mark yourself ++ * runnable without the overhead of this. ++ * ++ * Return: %true if @p was woken up, %false if it was already running. ++ * or @state didn't match @p's state. ++ */ ++static int ++try_to_wake_up(struct task_struct *p, unsigned int state, int wake_flags) ++{ ++ unsigned long flags; ++ int cpu, success = 0; ++ ++ /* ++ * If we are going to wake up a thread waiting for CONDITION we ++ * need to ensure that CONDITION=1 done by the caller can not be ++ * reordered with p->state check below. This pairs with mb() in ++ * set_current_state() the waiting thread does. ++ */ ++ smp_mb__before_spinlock(); ++ raw_spin_lock_irqsave(&p->pi_lock, flags); ++ if (!(p->state & state)) ++ goto out; ++ ++ success = 1; /* we're going to change ->state */ ++ cpu = task_cpu(p); ++ ++ if (p->on_rq && ttwu_remote(p, wake_flags)) ++ goto stat; ++ ++#ifdef CONFIG_SMP ++ /* ++ * If the owning (remote) cpu is still in the middle of schedule() with ++ * this task as prev, wait until its done referencing the task. ++ */ ++ while (p->on_cpu) ++ cpu_relax(); ++ /* ++ * Pairs with the smp_wmb() in finish_lock_switch(). ++ */ ++ smp_rmb(); ++ ++ p->sched_contributes_to_load = !!task_contributes_to_load(p); ++ p->state = TASK_WAKING; ++ ++ if (p->sched_class->task_waking) ++ p->sched_class->task_waking(p); ++ ++ cpu = select_task_rq(p, p->wake_cpu, SD_BALANCE_WAKE, wake_flags); ++ if (task_cpu(p) != cpu) { ++ wake_flags |= WF_MIGRATED; ++ set_task_cpu(p, cpu); ++ } ++#endif /* CONFIG_SMP */ ++ ++ ttwu_queue(p, cpu); ++stat: ++ ttwu_stat(p, cpu, wake_flags); ++out: ++ raw_spin_unlock_irqrestore(&p->pi_lock, flags); ++ ++ return success; ++} ++ ++/** ++ * try_to_wake_up_local - try to wake up a local task with rq lock held ++ * @p: the thread to be awakened ++ * ++ * Put @p on the run-queue if it's not already there. The caller must ++ * ensure that this_rq() is locked, @p is bound to this_rq() and not ++ * the current task. ++ */ ++static void try_to_wake_up_local(struct task_struct *p) ++{ ++ struct rq *rq = task_rq(p); ++ ++ if (WARN_ON_ONCE(rq != this_rq()) || ++ WARN_ON_ONCE(p == current)) ++ return; ++ ++ lockdep_assert_held(&rq->lock); ++ ++ if (!raw_spin_trylock(&p->pi_lock)) { ++ raw_spin_unlock(&rq->lock); ++ raw_spin_lock(&p->pi_lock); ++ raw_spin_lock(&rq->lock); ++ } ++ ++ if (!(p->state & TASK_NORMAL)) ++ goto out; ++ ++ if (!task_on_rq_queued(p)) ++ ttwu_activate(rq, p, ENQUEUE_WAKEUP); ++ ++ ttwu_do_wakeup(rq, p, 0); ++ ttwu_stat(p, smp_processor_id(), 0); ++out: ++ raw_spin_unlock(&p->pi_lock); ++} ++ ++/** ++ * wake_up_process - Wake up a specific process ++ * @p: The process to be woken up. ++ * ++ * Attempt to wake up the nominated process and move it to the set of runnable ++ * processes. ++ * ++ * Return: 1 if the process was woken up, 0 if it was already running. ++ * ++ * It may be assumed that this function implies a write memory barrier before ++ * changing the task state if and only if any tasks are woken up. ++ */ ++int wake_up_process(struct task_struct *p) ++{ ++ WARN_ON(task_is_stopped_or_traced(p)); ++ return try_to_wake_up(p, TASK_NORMAL, 0); ++} ++EXPORT_SYMBOL(wake_up_process); ++ ++int wake_up_state(struct task_struct *p, unsigned int state) ++{ ++ return try_to_wake_up(p, state, 0); ++} ++ ++/* ++ * This function clears the sched_dl_entity static params. ++ */ ++void __dl_clear_params(struct task_struct *p) ++{ ++ struct sched_dl_entity *dl_se = &p->dl; ++ ++ dl_se->dl_runtime = 0; ++ dl_se->dl_deadline = 0; ++ dl_se->dl_period = 0; ++ dl_se->flags = 0; ++ dl_se->dl_bw = 0; ++ ++ dl_se->dl_throttled = 0; ++ dl_se->dl_new = 1; ++ dl_se->dl_yielded = 0; ++} ++ ++/* ++ * Perform scheduler related setup for a newly forked process p. ++ * p is forked by current. ++ * ++ * __sched_fork() is basic setup used by init_idle() too: ++ */ ++static void __sched_fork(unsigned long clone_flags, struct task_struct *p) ++{ ++ p->on_rq = 0; ++ ++ p->se.on_rq = 0; ++ p->se.exec_start = 0; ++ p->se.sum_exec_runtime = 0; ++ p->se.prev_sum_exec_runtime = 0; ++ p->se.nr_migrations = 0; ++ p->se.vruntime = 0; ++#ifdef CONFIG_SMP ++ p->se.avg.decay_count = 0; ++#endif ++ INIT_LIST_HEAD(&p->se.group_node); ++ ++#ifdef CONFIG_SCHEDSTATS ++ memset(&p->se.statistics, 0, sizeof(p->se.statistics)); ++#endif ++ ++ RB_CLEAR_NODE(&p->dl.rb_node); ++ init_dl_task_timer(&p->dl); ++ __dl_clear_params(p); ++ ++ INIT_LIST_HEAD(&p->rt.run_list); ++ ++#ifdef CONFIG_PREEMPT_NOTIFIERS ++ INIT_HLIST_HEAD(&p->preempt_notifiers); ++#endif ++ ++#ifdef CONFIG_NUMA_BALANCING ++ if (p->mm && atomic_read(&p->mm->mm_users) == 1) { ++ p->mm->numa_next_scan = jiffies + msecs_to_jiffies(sysctl_numa_balancing_scan_delay); ++ p->mm->numa_scan_seq = 0; ++ } ++ ++ if (clone_flags & CLONE_VM) ++ p->numa_preferred_nid = current->numa_preferred_nid; ++ else ++ p->numa_preferred_nid = -1; ++ ++ p->node_stamp = 0ULL; ++ p->numa_scan_seq = p->mm ? p->mm->numa_scan_seq : 0; ++ p->numa_scan_period = sysctl_numa_balancing_scan_delay; ++ p->numa_work.next = &p->numa_work; ++ p->numa_faults = NULL; ++ p->last_task_numa_placement = 0; ++ p->last_sum_exec_runtime = 0; ++ ++ p->numa_group = NULL; ++#endif /* CONFIG_NUMA_BALANCING */ ++} ++ ++#ifdef CONFIG_NUMA_BALANCING ++#ifdef CONFIG_SCHED_DEBUG ++void set_numabalancing_state(bool enabled) ++{ ++ if (enabled) ++ sched_feat_set("NUMA"); ++ else ++ sched_feat_set("NO_NUMA"); ++} ++#else ++__read_mostly bool numabalancing_enabled; ++ ++void set_numabalancing_state(bool enabled) ++{ ++ numabalancing_enabled = enabled; ++} ++#endif /* CONFIG_SCHED_DEBUG */ ++ ++#ifdef CONFIG_PROC_SYSCTL ++int sysctl_numa_balancing(struct ctl_table *table, int write, ++ void __user *buffer, size_t *lenp, loff_t *ppos) ++{ ++ struct ctl_table t; ++ int err; ++ int state = numabalancing_enabled; ++ ++ if (write && !capable(CAP_SYS_ADMIN)) ++ return -EPERM; ++ ++ t = *table; ++ t.data = &state; ++ err = proc_dointvec_minmax(&t, write, buffer, lenp, ppos); ++ if (err < 0) ++ return err; ++ if (write) ++ set_numabalancing_state(state); ++ return err; ++} ++#endif ++#endif ++ ++/* ++ * fork()/clone()-time setup: ++ */ ++int sched_fork(unsigned long clone_flags, struct task_struct *p) ++{ ++ unsigned long flags; ++ int cpu = get_cpu(); ++ ++ __sched_fork(clone_flags, p); ++ /* ++ * We mark the process as running here. This guarantees that ++ * nobody will actually run it, and a signal or other external ++ * event cannot wake it up and insert it on the runqueue either. ++ */ ++ p->state = TASK_RUNNING; ++ ++ /* ++ * Make sure we do not leak PI boosting priority to the child. ++ */ ++ p->prio = current->normal_prio; ++ ++ /* ++ * Revert to default priority/policy on fork if requested. ++ */ ++ if (unlikely(p->sched_reset_on_fork)) { ++ if (task_has_dl_policy(p) || task_has_rt_policy(p)) { ++ p->policy = SCHED_NORMAL; ++ p->static_prio = NICE_TO_PRIO(0); ++ p->rt_priority = 0; ++ } else if (PRIO_TO_NICE(p->static_prio) < 0) ++ p->static_prio = NICE_TO_PRIO(0); ++ ++ p->prio = p->normal_prio = __normal_prio(p); ++ set_load_weight(p); ++ ++ /* ++ * We don't need the reset flag anymore after the fork. It has ++ * fulfilled its duty: ++ */ ++ p->sched_reset_on_fork = 0; ++ } ++ ++ if (dl_prio(p->prio)) { ++ put_cpu(); ++ return -EAGAIN; ++ } else if (rt_prio(p->prio)) { ++ p->sched_class = &rt_sched_class; ++ } else { ++ p->sched_class = &fair_sched_class; ++ } ++ ++ if (p->sched_class->task_fork) ++ p->sched_class->task_fork(p); ++ ++ /* ++ * The child is not yet in the pid-hash so no cgroup attach races, ++ * and the cgroup is pinned to this child due to cgroup_fork() ++ * is ran before sched_fork(). ++ * ++ * Silence PROVE_RCU. ++ */ ++ raw_spin_lock_irqsave(&p->pi_lock, flags); ++ set_task_cpu(p, cpu); ++ raw_spin_unlock_irqrestore(&p->pi_lock, flags); ++ ++#if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_TASK_DELAY_ACCT) ++ if (likely(sched_info_on())) ++ memset(&p->sched_info, 0, sizeof(p->sched_info)); ++#endif ++#if defined(CONFIG_SMP) ++ p->on_cpu = 0; ++#endif ++ init_task_preempt_count(p); ++#ifdef CONFIG_SMP ++ plist_node_init(&p->pushable_tasks, MAX_PRIO); ++ RB_CLEAR_NODE(&p->pushable_dl_tasks); ++#endif ++ ++ put_cpu(); ++ return 0; ++} ++ ++unsigned long to_ratio(u64 period, u64 runtime) ++{ ++ if (runtime == RUNTIME_INF) ++ return 1ULL << 20; ++ ++ /* ++ * Doing this here saves a lot of checks in all ++ * the calling paths, and returning zero seems ++ * safe for them anyway. ++ */ ++ if (period == 0) ++ return 0; ++ ++ return div64_u64(runtime << 20, period); ++} ++ ++#ifdef CONFIG_SMP ++inline struct dl_bw *dl_bw_of(int i) ++{ ++ rcu_lockdep_assert(rcu_read_lock_sched_held(), ++ "sched RCU must be held"); ++ return &cpu_rq(i)->rd->dl_bw; ++} ++ ++static inline int dl_bw_cpus(int i) ++{ ++ struct root_domain *rd = cpu_rq(i)->rd; ++ int cpus = 0; ++ ++ rcu_lockdep_assert(rcu_read_lock_sched_held(), ++ "sched RCU must be held"); ++ for_each_cpu_and(i, rd->span, cpu_active_mask) ++ cpus++; ++ ++ return cpus; ++} ++#else ++inline struct dl_bw *dl_bw_of(int i) ++{ ++ return &cpu_rq(i)->dl.dl_bw; ++} ++ ++static inline int dl_bw_cpus(int i) ++{ ++ return 1; ++} ++#endif ++ ++/* ++ * We must be sure that accepting a new task (or allowing changing the ++ * parameters of an existing one) is consistent with the bandwidth ++ * constraints. If yes, this function also accordingly updates the currently ++ * allocated bandwidth to reflect the new situation. ++ * ++ * This function is called while holding p's rq->lock. ++ * ++ * XXX we should delay bw change until the task's 0-lag point, see ++ * __setparam_dl(). ++ */ ++static int dl_overflow(struct task_struct *p, int policy, ++ const struct sched_attr *attr) ++{ ++ ++ struct dl_bw *dl_b = dl_bw_of(task_cpu(p)); ++ u64 period = attr->sched_period ?: attr->sched_deadline; ++ u64 runtime = attr->sched_runtime; ++ u64 new_bw = dl_policy(policy) ? to_ratio(period, runtime) : 0; ++ int cpus, err = -1; ++ ++ if (new_bw == p->dl.dl_bw) ++ return 0; ++ ++ /* ++ * Either if a task, enters, leave, or stays -deadline but changes ++ * its parameters, we may need to update accordingly the total ++ * allocated bandwidth of the container. ++ */ ++ raw_spin_lock(&dl_b->lock); ++ cpus = dl_bw_cpus(task_cpu(p)); ++ if (dl_policy(policy) && !task_has_dl_policy(p) && ++ !__dl_overflow(dl_b, cpus, 0, new_bw)) { ++ __dl_add(dl_b, new_bw); ++ err = 0; ++ } else if (dl_policy(policy) && task_has_dl_policy(p) && ++ !__dl_overflow(dl_b, cpus, p->dl.dl_bw, new_bw)) { ++ __dl_clear(dl_b, p->dl.dl_bw); ++ __dl_add(dl_b, new_bw); ++ err = 0; ++ } else if (!dl_policy(policy) && task_has_dl_policy(p)) { ++ __dl_clear(dl_b, p->dl.dl_bw); ++ err = 0; ++ } ++ raw_spin_unlock(&dl_b->lock); ++ ++ return err; ++} ++ ++extern void init_dl_bw(struct dl_bw *dl_b); ++ ++/* ++ * wake_up_new_task - wake up a newly created task for the first time. ++ * ++ * This function will do some initial scheduler statistics housekeeping ++ * that must be done for every newly created context, then puts the task ++ * on the runqueue and wakes it. ++ */ ++void wake_up_new_task(struct task_struct *p) ++{ ++ unsigned long flags; ++ struct rq *rq; ++ ++ raw_spin_lock_irqsave(&p->pi_lock, flags); ++#ifdef CONFIG_SMP ++ /* ++ * Fork balancing, do it here and not earlier because: ++ * - cpus_allowed can change in the fork path ++ * - any previously selected cpu might disappear through hotplug ++ */ ++ set_task_cpu(p, select_task_rq(p, task_cpu(p), SD_BALANCE_FORK, 0)); ++#endif ++ ++ /* Initialize new task's runnable average */ ++ init_task_runnable_average(p); ++ rq = __task_rq_lock(p); ++ activate_task(rq, p, 0); ++ p->on_rq = TASK_ON_RQ_QUEUED; ++ trace_sched_wakeup_new(p, true); ++ check_preempt_curr(rq, p, WF_FORK); ++#ifdef CONFIG_SMP ++ if (p->sched_class->task_woken) ++ p->sched_class->task_woken(rq, p); ++#endif ++ task_rq_unlock(rq, p, &flags); ++} ++ ++#ifdef CONFIG_PREEMPT_NOTIFIERS ++ ++/** ++ * preempt_notifier_register - tell me when current is being preempted & rescheduled ++ * @notifier: notifier struct to register ++ */ ++void preempt_notifier_register(struct preempt_notifier *notifier) ++{ ++ hlist_add_head(¬ifier->link, ¤t->preempt_notifiers); ++} ++EXPORT_SYMBOL_GPL(preempt_notifier_register); ++ ++/** ++ * preempt_notifier_unregister - no longer interested in preemption notifications ++ * @notifier: notifier struct to unregister ++ * ++ * This is safe to call from within a preemption notifier. ++ */ ++void preempt_notifier_unregister(struct preempt_notifier *notifier) ++{ ++ hlist_del(¬ifier->link); ++} ++EXPORT_SYMBOL_GPL(preempt_notifier_unregister); ++ ++static void fire_sched_in_preempt_notifiers(struct task_struct *curr) ++{ ++ struct preempt_notifier *notifier; ++ ++ hlist_for_each_entry(notifier, &curr->preempt_notifiers, link) ++ notifier->ops->sched_in(notifier, raw_smp_processor_id()); ++} ++ ++static void ++fire_sched_out_preempt_notifiers(struct task_struct *curr, ++ struct task_struct *next) ++{ ++ struct preempt_notifier *notifier; ++ ++ hlist_for_each_entry(notifier, &curr->preempt_notifiers, link) ++ notifier->ops->sched_out(notifier, next); ++} ++ ++#else /* !CONFIG_PREEMPT_NOTIFIERS */ ++ ++static void fire_sched_in_preempt_notifiers(struct task_struct *curr) ++{ ++} ++ ++static void ++fire_sched_out_preempt_notifiers(struct task_struct *curr, ++ struct task_struct *next) ++{ ++} ++ ++#endif /* CONFIG_PREEMPT_NOTIFIERS */ ++ ++/** ++ * prepare_task_switch - prepare to switch tasks ++ * @rq: the runqueue preparing to switch ++ * @prev: the current task that is being switched out ++ * @next: the task we are going to switch to. ++ * ++ * This is called with the rq lock held and interrupts off. It must ++ * be paired with a subsequent finish_task_switch after the context ++ * switch. ++ * ++ * prepare_task_switch sets up locking and calls architecture specific ++ * hooks. ++ */ ++static inline void ++prepare_task_switch(struct rq *rq, struct task_struct *prev, ++ struct task_struct *next) ++{ ++ trace_sched_switch(prev, next); ++ sched_info_switch(rq, prev, next); ++ perf_event_task_sched_out(prev, next); ++ fire_sched_out_preempt_notifiers(prev, next); ++ prepare_lock_switch(rq, next); ++ prepare_arch_switch(next); ++} ++ ++/** ++ * finish_task_switch - clean up after a task-switch ++ * @prev: the thread we just switched away from. ++ * ++ * finish_task_switch must be called after the context switch, paired ++ * with a prepare_task_switch call before the context switch. ++ * finish_task_switch will reconcile locking set up by prepare_task_switch, ++ * and do any other architecture-specific cleanup actions. ++ * ++ * Note that we may have delayed dropping an mm in context_switch(). If ++ * so, we finish that here outside of the runqueue lock. (Doing it ++ * with the lock held can cause deadlocks; see schedule() for ++ * details.) ++ * ++ * The context switch have flipped the stack from under us and restored the ++ * local variables which were saved when this task called schedule() in the ++ * past. prev == current is still correct but we need to recalculate this_rq ++ * because prev may have moved to another CPU. ++ */ ++static struct rq *finish_task_switch(struct task_struct *prev) ++ __releases(rq->lock) ++{ ++ struct rq *rq = this_rq(); ++ struct mm_struct *mm = rq->prev_mm; ++ long prev_state; ++ ++ rq->prev_mm = NULL; ++ ++ /* ++ * A task struct has one reference for the use as "current". ++ * If a task dies, then it sets TASK_DEAD in tsk->state and calls ++ * schedule one last time. The schedule call will never return, and ++ * the scheduled task must drop that reference. ++ * The test for TASK_DEAD must occur while the runqueue locks are ++ * still held, otherwise prev could be scheduled on another cpu, die ++ * there before we look at prev->state, and then the reference would ++ * be dropped twice. ++ * Manfred Spraul ++ */ ++ prev_state = prev->state; ++ vtime_task_switch(prev); ++ finish_arch_switch(prev); ++ perf_event_task_sched_in(prev, current); ++ finish_lock_switch(rq, prev); ++ finish_arch_post_lock_switch(); ++ ++ fire_sched_in_preempt_notifiers(current); ++ if (mm) ++ mmdrop(mm); ++ if (unlikely(prev_state == TASK_DEAD)) { ++ if (prev->sched_class->task_dead) ++ prev->sched_class->task_dead(prev); ++ ++ /* ++ * Remove function-return probe instances associated with this ++ * task and put them back on the free list. ++ */ ++ kprobe_flush_task(prev); ++ put_task_struct(prev); ++ } ++ ++ tick_nohz_task_switch(current); ++ return rq; ++} ++ ++#ifdef CONFIG_SMP ++ ++/* rq->lock is NOT held, but preemption is disabled */ ++static inline void post_schedule(struct rq *rq) ++{ ++ if (rq->post_schedule) { ++ unsigned long flags; ++ ++ raw_spin_lock_irqsave(&rq->lock, flags); ++ if (rq->curr->sched_class->post_schedule) ++ rq->curr->sched_class->post_schedule(rq); ++ raw_spin_unlock_irqrestore(&rq->lock, flags); ++ ++ rq->post_schedule = 0; ++ } ++} ++ ++#else ++ ++static inline void post_schedule(struct rq *rq) ++{ ++} ++ ++#endif ++ ++/** ++ * schedule_tail - first thing a freshly forked thread must call. ++ * @prev: the thread we just switched away from. ++ */ ++asmlinkage __visible void schedule_tail(struct task_struct *prev) ++ __releases(rq->lock) ++{ ++ struct rq *rq; ++ ++ /* finish_task_switch() drops rq->lock and enables preemtion */ ++ preempt_disable(); ++ rq = finish_task_switch(prev); ++ post_schedule(rq); ++ preempt_enable(); ++ ++ if (current->set_child_tid) ++ put_user(task_pid_vnr(current), current->set_child_tid); ++} ++ ++/* ++ * context_switch - switch to the new MM and the new thread's register state. ++ */ ++static inline struct rq * ++context_switch(struct rq *rq, struct task_struct *prev, ++ struct task_struct *next) ++{ ++ struct mm_struct *mm, *oldmm; ++ ++ prepare_task_switch(rq, prev, next); ++ ++ mm = next->mm; ++ oldmm = prev->active_mm; ++ /* ++ * For paravirt, this is coupled with an exit in switch_to to ++ * combine the page table reload and the switch backend into ++ * one hypercall. ++ */ ++ arch_start_context_switch(prev); ++ ++ if (!mm) { ++ next->active_mm = oldmm; ++ atomic_inc(&oldmm->mm_count); ++ enter_lazy_tlb(oldmm, next); ++ } else ++ switch_mm(oldmm, mm, next); ++ ++ if (!prev->mm) { ++ prev->active_mm = NULL; ++ rq->prev_mm = oldmm; ++ } ++ /* ++ * Since the runqueue lock will be released by the next ++ * task (which is an invalid locking op but in the case ++ * of the scheduler it's an obvious special-case), so we ++ * do an early lockdep release here: ++ */ ++ spin_release(&rq->lock.dep_map, 1, _THIS_IP_); ++ ++ context_tracking_task_switch(prev, next); ++ /* Here we just switch the register state and the stack. */ ++ switch_to(prev, next, prev); ++ barrier(); ++ ++ return finish_task_switch(prev); ++} ++ ++/* ++ * nr_running and nr_context_switches: ++ * ++ * externally visible scheduler statistics: current number of runnable ++ * threads, total number of context switches performed since bootup. ++ */ ++unsigned long nr_running(void) ++{ ++ unsigned long i, sum = 0; ++ ++ for_each_online_cpu(i) ++ sum += cpu_rq(i)->nr_running; ++ ++ return sum; ++} ++ ++/* ++ * Check if only the current task is running on the cpu. ++ */ ++bool single_task_running(void) ++{ ++ if (cpu_rq(smp_processor_id())->nr_running == 1) ++ return true; ++ else ++ return false; ++} ++EXPORT_SYMBOL(single_task_running); ++ ++unsigned long long nr_context_switches(void) ++{ ++ int i; ++ unsigned long long sum = 0; ++ ++ for_each_possible_cpu(i) ++ sum += cpu_rq(i)->nr_switches; ++ ++ return sum; ++} ++ ++unsigned long nr_iowait(void) ++{ ++ unsigned long i, sum = 0; ++ ++ for_each_possible_cpu(i) ++ sum += atomic_read(&cpu_rq(i)->nr_iowait); ++ ++ return sum; ++} ++ ++unsigned long nr_iowait_cpu(int cpu) ++{ ++ struct rq *this = cpu_rq(cpu); ++ return atomic_read(&this->nr_iowait); ++} ++ ++void get_iowait_load(unsigned long *nr_waiters, unsigned long *load) ++{ ++ struct rq *this = this_rq(); ++ *nr_waiters = atomic_read(&this->nr_iowait); ++ *load = this->cpu_load[0]; ++} ++ ++#ifdef CONFIG_SMP ++ ++/* ++ * sched_exec - execve() is a valuable balancing opportunity, because at ++ * this point the task has the smallest effective memory and cache footprint. ++ */ ++void sched_exec(void) ++{ ++ struct task_struct *p = current; ++ unsigned long flags; ++ int dest_cpu; ++ ++ raw_spin_lock_irqsave(&p->pi_lock, flags); ++ dest_cpu = p->sched_class->select_task_rq(p, task_cpu(p), SD_BALANCE_EXEC, 0); ++ if (dest_cpu == smp_processor_id()) ++ goto unlock; ++ ++ if (likely(cpu_active(dest_cpu))) { ++ struct migration_arg arg = { p, dest_cpu }; ++ ++ raw_spin_unlock_irqrestore(&p->pi_lock, flags); ++ stop_one_cpu(task_cpu(p), migration_cpu_stop, &arg); ++ return; ++ } ++unlock: ++ raw_spin_unlock_irqrestore(&p->pi_lock, flags); ++} ++ ++#endif ++ ++DEFINE_PER_CPU(struct kernel_stat, kstat); ++DEFINE_PER_CPU(struct kernel_cpustat, kernel_cpustat); ++ ++EXPORT_PER_CPU_SYMBOL(kstat); ++EXPORT_PER_CPU_SYMBOL(kernel_cpustat); ++ ++/* ++ * Return accounted runtime for the task. ++ * In case the task is currently running, return the runtime plus current's ++ * pending runtime that have not been accounted yet. ++ */ ++unsigned long long task_sched_runtime(struct task_struct *p) ++{ ++ unsigned long flags; ++ struct rq *rq; ++ u64 ns; ++ ++#if defined(CONFIG_64BIT) && defined(CONFIG_SMP) ++ /* ++ * 64-bit doesn't need locks to atomically read a 64bit value. ++ * So we have a optimization chance when the task's delta_exec is 0. ++ * Reading ->on_cpu is racy, but this is ok. ++ * ++ * If we race with it leaving cpu, we'll take a lock. So we're correct. ++ * If we race with it entering cpu, unaccounted time is 0. This is ++ * indistinguishable from the read occurring a few cycles earlier. ++ * If we see ->on_cpu without ->on_rq, the task is leaving, and has ++ * been accounted, so we're correct here as well. ++ */ ++ if (!p->on_cpu || !task_on_rq_queued(p)) ++ return p->se.sum_exec_runtime; ++#endif ++ ++ rq = task_rq_lock(p, &flags); ++ /* ++ * Must be ->curr _and_ ->on_rq. If dequeued, we would ++ * project cycles that may never be accounted to this ++ * thread, breaking clock_gettime(). ++ */ ++ if (task_current(rq, p) && task_on_rq_queued(p)) { ++ update_rq_clock(rq); ++ p->sched_class->update_curr(rq); ++ } ++ ns = p->se.sum_exec_runtime; ++ task_rq_unlock(rq, p, &flags); ++ ++ return ns; ++} ++ ++/* ++ * This function gets called by the timer code, with HZ frequency. ++ * We call it with interrupts disabled. ++ */ ++void scheduler_tick(void) ++{ ++ int cpu = smp_processor_id(); ++ struct rq *rq = cpu_rq(cpu); ++ struct task_struct *curr = rq->curr; ++ ++ sched_clock_tick(); ++ ++ raw_spin_lock(&rq->lock); ++ update_rq_clock(rq); ++ curr->sched_class->task_tick(rq, curr, 0); ++ update_cpu_load_active(rq); ++ raw_spin_unlock(&rq->lock); ++ ++ perf_event_task_tick(); ++ ++#ifdef CONFIG_SMP ++ rq->idle_balance = idle_cpu(cpu); ++ trigger_load_balance(rq); ++#endif ++ rq_last_tick_reset(rq); ++} ++ ++#ifdef CONFIG_NO_HZ_FULL ++/** ++ * scheduler_tick_max_deferment ++ * ++ * Keep at least one tick per second when a single ++ * active task is running because the scheduler doesn't ++ * yet completely support full dynticks environment. ++ * ++ * This makes sure that uptime, CFS vruntime, load ++ * balancing, etc... continue to move forward, even ++ * with a very low granularity. ++ * ++ * Return: Maximum deferment in nanoseconds. ++ */ ++u64 scheduler_tick_max_deferment(void) ++{ ++ struct rq *rq = this_rq(); ++ unsigned long next, now = ACCESS_ONCE(jiffies); ++ ++ next = rq->last_sched_tick + HZ; ++ ++ if (time_before_eq(next, now)) ++ return 0; ++ ++ return jiffies_to_nsecs(next - now); ++} ++#endif ++ ++notrace unsigned long get_parent_ip(unsigned long addr) ++{ ++ if (in_lock_functions(addr)) { ++ addr = CALLER_ADDR2; ++ if (in_lock_functions(addr)) ++ addr = CALLER_ADDR3; ++ } ++ return addr; ++} ++ ++#if defined(CONFIG_PREEMPT) && (defined(CONFIG_DEBUG_PREEMPT) || \ ++ defined(CONFIG_PREEMPT_TRACER)) ++ ++void preempt_count_add(int val) ++{ ++#ifdef CONFIG_DEBUG_PREEMPT ++ /* ++ * Underflow? ++ */ ++ if (DEBUG_LOCKS_WARN_ON((preempt_count() < 0))) ++ return; ++#endif ++ __preempt_count_add(val); ++#ifdef CONFIG_DEBUG_PREEMPT ++ /* ++ * Spinlock count overflowing soon? ++ */ ++ DEBUG_LOCKS_WARN_ON((preempt_count() & PREEMPT_MASK) >= ++ PREEMPT_MASK - 10); ++#endif ++ if (preempt_count() == val) { ++ unsigned long ip = get_parent_ip(CALLER_ADDR1); ++#ifdef CONFIG_DEBUG_PREEMPT ++ current->preempt_disable_ip = ip; ++#endif ++ trace_preempt_off(CALLER_ADDR0, ip); ++ } ++} ++EXPORT_SYMBOL(preempt_count_add); ++NOKPROBE_SYMBOL(preempt_count_add); ++ ++void preempt_count_sub(int val) ++{ ++#ifdef CONFIG_DEBUG_PREEMPT ++ /* ++ * Underflow? ++ */ ++ if (DEBUG_LOCKS_WARN_ON(val > preempt_count())) ++ return; ++ /* ++ * Is the spinlock portion underflowing? ++ */ ++ if (DEBUG_LOCKS_WARN_ON((val < PREEMPT_MASK) && ++ !(preempt_count() & PREEMPT_MASK))) ++ return; ++#endif ++ ++ if (preempt_count() == val) ++ trace_preempt_on(CALLER_ADDR0, get_parent_ip(CALLER_ADDR1)); ++ __preempt_count_sub(val); ++} ++EXPORT_SYMBOL(preempt_count_sub); ++NOKPROBE_SYMBOL(preempt_count_sub); ++ ++#endif ++ ++/* ++ * Print scheduling while atomic bug: ++ */ ++static noinline void __schedule_bug(struct task_struct *prev) ++{ ++ if (oops_in_progress) ++ return; ++ ++ printk(KERN_ERR "BUG: scheduling while atomic: %s/%d/0x%08x\n", ++ prev->comm, prev->pid, preempt_count()); ++ ++ debug_show_held_locks(prev); ++ print_modules(); ++ if (irqs_disabled()) ++ print_irqtrace_events(prev); ++#ifdef CONFIG_DEBUG_PREEMPT ++ if (in_atomic_preempt_off()) { ++ pr_err("Preemption disabled at:"); ++ print_ip_sym(current->preempt_disable_ip); ++ pr_cont("\n"); ++ } ++#endif ++ dump_stack(); ++ add_taint(TAINT_WARN, LOCKDEP_STILL_OK); ++} ++ ++/* ++ * Various schedule()-time debugging checks and statistics: ++ */ ++static inline void schedule_debug(struct task_struct *prev) ++{ ++#ifdef CONFIG_SCHED_STACK_END_CHECK ++ BUG_ON(unlikely(task_stack_end_corrupted(prev))); ++#endif ++ /* ++ * Test if we are atomic. Since do_exit() needs to call into ++ * schedule() atomically, we ignore that path. Otherwise whine ++ * if we are scheduling when we should not. ++ */ ++ if (unlikely(in_atomic_preempt_off() && prev->state != TASK_DEAD)) ++ __schedule_bug(prev); ++ rcu_sleep_check(); ++ ++ profile_hit(SCHED_PROFILING, __builtin_return_address(0)); ++ ++ schedstat_inc(this_rq(), sched_count); ++} ++ ++/* ++ * Pick up the highest-prio task: ++ */ ++static inline struct task_struct * ++pick_next_task(struct rq *rq, struct task_struct *prev) ++{ ++ const struct sched_class *class = &fair_sched_class; ++ struct task_struct *p; ++ ++ /* ++ * Optimization: we know that if all tasks are in ++ * the fair class we can call that function directly: ++ */ ++ if (likely(prev->sched_class == class && ++ rq->nr_running == rq->cfs.h_nr_running)) { ++ p = fair_sched_class.pick_next_task(rq, prev); ++ if (unlikely(p == RETRY_TASK)) ++ goto again; ++ ++ /* assumes fair_sched_class->next == idle_sched_class */ ++ if (unlikely(!p)) ++ p = idle_sched_class.pick_next_task(rq, prev); ++ ++ return p; ++ } ++ ++again: ++ for_each_class(class) { ++ p = class->pick_next_task(rq, prev); ++ if (p) { ++ if (unlikely(p == RETRY_TASK)) ++ goto again; ++ return p; ++ } ++ } ++ ++ BUG(); /* the idle class will always have a runnable task */ ++} ++ ++/* ++ * __schedule() is the main scheduler function. ++ * ++ * The main means of driving the scheduler and thus entering this function are: ++ * ++ * 1. Explicit blocking: mutex, semaphore, waitqueue, etc. ++ * ++ * 2. TIF_NEED_RESCHED flag is checked on interrupt and userspace return ++ * paths. For example, see arch/x86/entry_64.S. ++ * ++ * To drive preemption between tasks, the scheduler sets the flag in timer ++ * interrupt handler scheduler_tick(). ++ * ++ * 3. Wakeups don't really cause entry into schedule(). They add a ++ * task to the run-queue and that's it. ++ * ++ * Now, if the new task added to the run-queue preempts the current ++ * task, then the wakeup sets TIF_NEED_RESCHED and schedule() gets ++ * called on the nearest possible occasion: ++ * ++ * - If the kernel is preemptible (CONFIG_PREEMPT=y): ++ * ++ * - in syscall or exception context, at the next outmost ++ * preempt_enable(). (this might be as soon as the wake_up()'s ++ * spin_unlock()!) ++ * ++ * - in IRQ context, return from interrupt-handler to ++ * preemptible context ++ * ++ * - If the kernel is not preemptible (CONFIG_PREEMPT is not set) ++ * then at the next: ++ * ++ * - cond_resched() call ++ * - explicit schedule() call ++ * - return from syscall or exception to user-space ++ * - return from interrupt-handler to user-space ++ * ++ * WARNING: all callers must re-check need_resched() afterward and reschedule ++ * accordingly in case an event triggered the need for rescheduling (such as ++ * an interrupt waking up a task) while preemption was disabled in __schedule(). ++ */ ++static void __sched __schedule(void) ++{ ++ struct task_struct *prev, *next; ++ unsigned long *switch_count; ++ struct rq *rq; ++ int cpu; ++ ++ preempt_disable(); ++ cpu = smp_processor_id(); ++ rq = cpu_rq(cpu); ++ rcu_note_context_switch(); ++ prev = rq->curr; ++ ++ schedule_debug(prev); ++ ++ if (sched_feat(HRTICK)) ++ hrtick_clear(rq); ++ ++ /* ++ * Make sure that signal_pending_state()->signal_pending() below ++ * can't be reordered with __set_current_state(TASK_INTERRUPTIBLE) ++ * done by the caller to avoid the race with signal_wake_up(). ++ */ ++ smp_mb__before_spinlock(); ++ raw_spin_lock_irq(&rq->lock); ++ ++ rq->clock_skip_update <<= 1; /* promote REQ to ACT */ ++ ++ switch_count = &prev->nivcsw; ++ if (prev->state && !(preempt_count() & PREEMPT_ACTIVE)) { ++ if (unlikely(signal_pending_state(prev->state, prev))) { ++ prev->state = TASK_RUNNING; ++ } else { ++ deactivate_task(rq, prev, DEQUEUE_SLEEP); ++ prev->on_rq = 0; ++ ++ /* ++ * If a worker went to sleep, notify and ask workqueue ++ * whether it wants to wake up a task to maintain ++ * concurrency. ++ */ ++ if (prev->flags & PF_WQ_WORKER) { ++ struct task_struct *to_wakeup; ++ ++ to_wakeup = wq_worker_sleeping(prev, cpu); ++ if (to_wakeup) ++ try_to_wake_up_local(to_wakeup); ++ } ++ } ++ switch_count = &prev->nvcsw; ++ } ++ ++ if (task_on_rq_queued(prev)) ++ update_rq_clock(rq); ++ ++ next = pick_next_task(rq, prev); ++ clear_tsk_need_resched(prev); ++ clear_preempt_need_resched(); ++ rq->clock_skip_update = 0; ++ ++ if (likely(prev != next)) { ++ rq->nr_switches++; ++ rq->curr = next; ++ ++*switch_count; ++ ++ rq = context_switch(rq, prev, next); /* unlocks the rq */ ++ cpu = cpu_of(rq); ++ } else ++ raw_spin_unlock_irq(&rq->lock); ++ ++ post_schedule(rq); ++ ++ sched_preempt_enable_no_resched(); ++} ++ ++static inline void sched_submit_work(struct task_struct *tsk) ++{ ++ if (!tsk->state || tsk_is_pi_blocked(tsk)) ++ return; ++ /* ++ * If we are going to sleep and we have plugged IO queued, ++ * make sure to submit it to avoid deadlocks. ++ */ ++ if (blk_needs_flush_plug(tsk)) ++ blk_schedule_flush_plug(tsk); ++} ++ ++asmlinkage __visible void __sched schedule(void) ++{ ++ struct task_struct *tsk = current; ++ ++ sched_submit_work(tsk); ++ do { ++ __schedule(); ++ } while (need_resched()); ++} ++EXPORT_SYMBOL(schedule); ++ ++#ifdef CONFIG_CONTEXT_TRACKING ++asmlinkage __visible void __sched schedule_user(void) ++{ ++ /* ++ * If we come here after a random call to set_need_resched(), ++ * or we have been woken up remotely but the IPI has not yet arrived, ++ * we haven't yet exited the RCU idle mode. Do it here manually until ++ * we find a better solution. ++ * ++ * NB: There are buggy callers of this function. Ideally we ++ * should warn if prev_state != CONTEXT_USER, but that will trigger ++ * too frequently to make sense yet. ++ */ ++ enum ctx_state prev_state = exception_enter(); ++ schedule(); ++ exception_exit(prev_state); ++} ++#endif ++ ++/** ++ * schedule_preempt_disabled - called with preemption disabled ++ * ++ * Returns with preemption disabled. Note: preempt_count must be 1 ++ */ ++void __sched schedule_preempt_disabled(void) ++{ ++ sched_preempt_enable_no_resched(); ++ schedule(); ++ preempt_disable(); ++} ++ ++static void __sched notrace preempt_schedule_common(void) ++{ ++ do { ++ __preempt_count_add(PREEMPT_ACTIVE); ++ __schedule(); ++ __preempt_count_sub(PREEMPT_ACTIVE); ++ ++ /* ++ * Check again in case we missed a preemption opportunity ++ * between schedule and now. ++ */ ++ barrier(); ++ } while (need_resched()); ++} ++ ++#ifdef CONFIG_PREEMPT ++/* ++ * this is the entry point to schedule() from in-kernel preemption ++ * off of preempt_enable. Kernel preemptions off return from interrupt ++ * occur there and call schedule directly. ++ */ ++asmlinkage __visible void __sched notrace preempt_schedule(void) ++{ ++ /* ++ * If there is a non-zero preempt_count or interrupts are disabled, ++ * we do not want to preempt the current task. Just return.. ++ */ ++ if (likely(!preemptible())) ++ return; ++ ++ preempt_schedule_common(); ++} ++NOKPROBE_SYMBOL(preempt_schedule); ++EXPORT_SYMBOL(preempt_schedule); ++ ++#ifdef CONFIG_CONTEXT_TRACKING ++/** ++ * preempt_schedule_context - preempt_schedule called by tracing ++ * ++ * The tracing infrastructure uses preempt_enable_notrace to prevent ++ * recursion and tracing preempt enabling caused by the tracing ++ * infrastructure itself. But as tracing can happen in areas coming ++ * from userspace or just about to enter userspace, a preempt enable ++ * can occur before user_exit() is called. This will cause the scheduler ++ * to be called when the system is still in usermode. ++ * ++ * To prevent this, the preempt_enable_notrace will use this function ++ * instead of preempt_schedule() to exit user context if needed before ++ * calling the scheduler. ++ */ ++asmlinkage __visible void __sched notrace preempt_schedule_context(void) ++{ ++ enum ctx_state prev_ctx; ++ ++ if (likely(!preemptible())) ++ return; ++ ++ do { ++ __preempt_count_add(PREEMPT_ACTIVE); ++ /* ++ * Needs preempt disabled in case user_exit() is traced ++ * and the tracer calls preempt_enable_notrace() causing ++ * an infinite recursion. ++ */ ++ prev_ctx = exception_enter(); ++ __schedule(); ++ exception_exit(prev_ctx); ++ ++ __preempt_count_sub(PREEMPT_ACTIVE); ++ barrier(); ++ } while (need_resched()); ++} ++EXPORT_SYMBOL_GPL(preempt_schedule_context); ++#endif /* CONFIG_CONTEXT_TRACKING */ ++ ++#endif /* CONFIG_PREEMPT */ ++ ++/* ++ * this is the entry point to schedule() from kernel preemption ++ * off of irq context. ++ * Note, that this is called and return with irqs disabled. This will ++ * protect us against recursive calling from irq. ++ */ ++asmlinkage __visible void __sched preempt_schedule_irq(void) ++{ ++ enum ctx_state prev_state; ++ ++ /* Catch callers which need to be fixed */ ++ BUG_ON(preempt_count() || !irqs_disabled()); ++ ++ prev_state = exception_enter(); ++ ++ do { ++ __preempt_count_add(PREEMPT_ACTIVE); ++ local_irq_enable(); ++ __schedule(); ++ local_irq_disable(); ++ __preempt_count_sub(PREEMPT_ACTIVE); ++ ++ /* ++ * Check again in case we missed a preemption opportunity ++ * between schedule and now. ++ */ ++ barrier(); ++ } while (need_resched()); ++ ++ exception_exit(prev_state); ++} ++ ++int default_wake_function(wait_queue_t *curr, unsigned mode, int wake_flags, ++ void *key) ++{ ++ return try_to_wake_up(curr->private, mode, wake_flags); ++} ++EXPORT_SYMBOL(default_wake_function); ++ ++#ifdef CONFIG_RT_MUTEXES ++ ++/* ++ * rt_mutex_setprio - set the current priority of a task ++ * @p: task ++ * @prio: prio value (kernel-internal form) ++ * ++ * This function changes the 'effective' priority of a task. It does ++ * not touch ->normal_prio like __setscheduler(). ++ * ++ * Used by the rt_mutex code to implement priority inheritance ++ * logic. Call site only calls if the priority of the task changed. ++ */ ++void rt_mutex_setprio(struct task_struct *p, int prio) ++{ ++ int oldprio, queued, running, enqueue_flag = 0; ++ struct rq *rq; ++ const struct sched_class *prev_class; ++ ++ BUG_ON(prio > MAX_PRIO); ++ ++ rq = __task_rq_lock(p); ++ ++ /* ++ * Idle task boosting is a nono in general. There is one ++ * exception, when PREEMPT_RT and NOHZ is active: ++ * ++ * The idle task calls get_next_timer_interrupt() and holds ++ * the timer wheel base->lock on the CPU and another CPU wants ++ * to access the timer (probably to cancel it). We can safely ++ * ignore the boosting request, as the idle CPU runs this code ++ * with interrupts disabled and will complete the lock ++ * protected section without being interrupted. So there is no ++ * real need to boost. ++ */ ++ if (unlikely(p == rq->idle)) { ++ WARN_ON(p != rq->curr); ++ WARN_ON(p->pi_blocked_on); ++ goto out_unlock; ++ } ++ ++ trace_sched_pi_setprio(p, prio); ++ oldprio = p->prio; ++ prev_class = p->sched_class; ++ queued = task_on_rq_queued(p); ++ running = task_current(rq, p); ++ if (queued) ++ dequeue_task(rq, p, 0); ++ if (running) ++ put_prev_task(rq, p); ++ ++ /* ++ * Boosting condition are: ++ * 1. -rt task is running and holds mutex A ++ * --> -dl task blocks on mutex A ++ * ++ * 2. -dl task is running and holds mutex A ++ * --> -dl task blocks on mutex A and could preempt the ++ * running task ++ */ ++ if (dl_prio(prio)) { ++ struct task_struct *pi_task = rt_mutex_get_top_task(p); ++ if (!dl_prio(p->normal_prio) || ++ (pi_task && dl_entity_preempt(&pi_task->dl, &p->dl))) { ++ p->dl.dl_boosted = 1; ++ p->dl.dl_throttled = 0; ++ enqueue_flag = ENQUEUE_REPLENISH; ++ } else ++ p->dl.dl_boosted = 0; ++ p->sched_class = &dl_sched_class; ++ } else if (rt_prio(prio)) { ++ if (dl_prio(oldprio)) ++ p->dl.dl_boosted = 0; ++ if (oldprio < prio) ++ enqueue_flag = ENQUEUE_HEAD; ++ p->sched_class = &rt_sched_class; ++ } else { ++ if (dl_prio(oldprio)) ++ p->dl.dl_boosted = 0; ++ if (rt_prio(oldprio)) ++ p->rt.timeout = 0; ++ p->sched_class = &fair_sched_class; ++ } ++ ++ p->prio = prio; ++ ++ if (running) ++ p->sched_class->set_curr_task(rq); ++ if (queued) ++ enqueue_task(rq, p, enqueue_flag); ++ ++ check_class_changed(rq, p, prev_class, oldprio); ++out_unlock: ++ __task_rq_unlock(rq); ++} ++#endif ++ ++void set_user_nice(struct task_struct *p, long nice) ++{ ++ int old_prio, delta, queued; ++ unsigned long flags; ++ struct rq *rq; ++ ++ if (task_nice(p) == nice || nice < MIN_NICE || nice > MAX_NICE) ++ return; ++ /* ++ * We have to be careful, if called from sys_setpriority(), ++ * the task might be in the middle of scheduling on another CPU. ++ */ ++ rq = task_rq_lock(p, &flags); ++ /* ++ * The RT priorities are set via sched_setscheduler(), but we still ++ * allow the 'normal' nice value to be set - but as expected ++ * it wont have any effect on scheduling until the task is ++ * SCHED_DEADLINE, SCHED_FIFO or SCHED_RR: ++ */ ++ if (task_has_dl_policy(p) || task_has_rt_policy(p)) { ++ p->static_prio = NICE_TO_PRIO(nice); ++ goto out_unlock; ++ } ++ queued = task_on_rq_queued(p); ++ if (queued) ++ dequeue_task(rq, p, 0); ++ ++ p->static_prio = NICE_TO_PRIO(nice); ++ set_load_weight(p); ++ old_prio = p->prio; ++ p->prio = effective_prio(p); ++ delta = p->prio - old_prio; ++ ++ if (queued) { ++ enqueue_task(rq, p, 0); ++ /* ++ * If the task increased its priority or is running and ++ * lowered its priority, then reschedule its CPU: ++ */ ++ if (delta < 0 || (delta > 0 && task_running(rq, p))) ++ resched_curr(rq); ++ } ++out_unlock: ++ task_rq_unlock(rq, p, &flags); ++} ++EXPORT_SYMBOL(set_user_nice); ++ ++/* ++ * can_nice - check if a task can reduce its nice value ++ * @p: task ++ * @nice: nice value ++ */ ++int can_nice(const struct task_struct *p, const int nice) ++{ ++ /* convert nice value [19,-20] to rlimit style value [1,40] */ ++ int nice_rlim = nice_to_rlimit(nice); ++ ++ return (nice_rlim <= task_rlimit(p, RLIMIT_NICE) || ++ capable(CAP_SYS_NICE)); ++} ++ ++#ifdef __ARCH_WANT_SYS_NICE ++ ++/* ++ * sys_nice - change the priority of the current process. ++ * @increment: priority increment ++ * ++ * sys_setpriority is a more generic, but much slower function that ++ * does similar things. ++ */ ++SYSCALL_DEFINE1(nice, int, increment) ++{ ++ long nice, retval; ++ ++ /* ++ * Setpriority might change our priority at the same moment. ++ * We don't have to worry. Conceptually one call occurs first ++ * and we have a single winner. ++ */ ++ increment = clamp(increment, -NICE_WIDTH, NICE_WIDTH); ++ nice = task_nice(current) + increment; ++ ++ nice = clamp_val(nice, MIN_NICE, MAX_NICE); ++ if (increment < 0 && !can_nice(current, nice)) ++ return -EPERM; ++ ++ retval = security_task_setnice(current, nice); ++ if (retval) ++ return retval; ++ ++ set_user_nice(current, nice); ++ return 0; ++} ++ ++#endif ++ ++/** ++ * task_prio - return the priority value of a given task. ++ * @p: the task in question. ++ * ++ * Return: The priority value as seen by users in /proc. ++ * RT tasks are offset by -200. Normal tasks are centered ++ * around 0, value goes from -16 to +15. ++ */ ++int task_prio(const struct task_struct *p) ++{ ++ return p->prio - MAX_RT_PRIO; ++} ++ ++/** ++ * idle_cpu - is a given cpu idle currently? ++ * @cpu: the processor in question. ++ * ++ * Return: 1 if the CPU is currently idle. 0 otherwise. ++ */ ++int idle_cpu(int cpu) ++{ ++ struct rq *rq = cpu_rq(cpu); ++ ++ if (rq->curr != rq->idle) ++ return 0; ++ ++ if (rq->nr_running) ++ return 0; ++ ++#ifdef CONFIG_SMP ++ if (!llist_empty(&rq->wake_list)) ++ return 0; ++#endif ++ ++ return 1; ++} ++ ++/** ++ * idle_task - return the idle task for a given cpu. ++ * @cpu: the processor in question. ++ * ++ * Return: The idle task for the cpu @cpu. ++ */ ++struct task_struct *idle_task(int cpu) ++{ ++ return cpu_rq(cpu)->idle; ++} ++ ++/** ++ * find_process_by_pid - find a process with a matching PID value. ++ * @pid: the pid in question. ++ * ++ * The task of @pid, if found. %NULL otherwise. ++ */ ++static struct task_struct *find_process_by_pid(pid_t pid) ++{ ++ return pid ? find_task_by_vpid(pid) : current; ++} ++ ++/* ++ * This function initializes the sched_dl_entity of a newly becoming ++ * SCHED_DEADLINE task. ++ * ++ * Only the static values are considered here, the actual runtime and the ++ * absolute deadline will be properly calculated when the task is enqueued ++ * for the first time with its new policy. ++ */ ++static void ++__setparam_dl(struct task_struct *p, const struct sched_attr *attr) ++{ ++ struct sched_dl_entity *dl_se = &p->dl; ++ ++ dl_se->dl_runtime = attr->sched_runtime; ++ dl_se->dl_deadline = attr->sched_deadline; ++ dl_se->dl_period = attr->sched_period ?: dl_se->dl_deadline; ++ dl_se->flags = attr->sched_flags; ++ dl_se->dl_bw = to_ratio(dl_se->dl_period, dl_se->dl_runtime); ++ ++ /* ++ * Changing the parameters of a task is 'tricky' and we're not doing ++ * the correct thing -- also see task_dead_dl() and switched_from_dl(). ++ * ++ * What we SHOULD do is delay the bandwidth release until the 0-lag ++ * point. This would include retaining the task_struct until that time ++ * and change dl_overflow() to not immediately decrement the current ++ * amount. ++ * ++ * Instead we retain the current runtime/deadline and let the new ++ * parameters take effect after the current reservation period lapses. ++ * This is safe (albeit pessimistic) because the 0-lag point is always ++ * before the current scheduling deadline. ++ * ++ * We can still have temporary overloads because we do not delay the ++ * change in bandwidth until that time; so admission control is ++ * not on the safe side. It does however guarantee tasks will never ++ * consume more than promised. ++ */ ++} ++ ++/* ++ * sched_setparam() passes in -1 for its policy, to let the functions ++ * it calls know not to change it. ++ */ ++#define SETPARAM_POLICY -1 ++ ++static void __setscheduler_params(struct task_struct *p, ++ const struct sched_attr *attr) ++{ ++ int policy = attr->sched_policy; ++ ++ if (policy == SETPARAM_POLICY) ++ policy = p->policy; ++ ++ p->policy = policy; ++ ++ if (dl_policy(policy)) ++ __setparam_dl(p, attr); ++ else if (fair_policy(policy)) ++ p->static_prio = NICE_TO_PRIO(attr->sched_nice); ++ ++ /* ++ * __sched_setscheduler() ensures attr->sched_priority == 0 when ++ * !rt_policy. Always setting this ensures that things like ++ * getparam()/getattr() don't report silly values for !rt tasks. ++ */ ++ p->rt_priority = attr->sched_priority; ++ p->normal_prio = normal_prio(p); ++ set_load_weight(p); ++} ++ ++/* Actually do priority change: must hold pi & rq lock. */ ++static void __setscheduler(struct rq *rq, struct task_struct *p, ++ const struct sched_attr *attr, bool keep_boost) ++{ ++ __setscheduler_params(p, attr); ++ ++ /* ++ * Keep a potential priority boosting if called from ++ * sched_setscheduler(). ++ */ ++ if (keep_boost) ++ p->prio = rt_mutex_get_effective_prio(p, normal_prio(p)); ++ else ++ p->prio = normal_prio(p); ++ ++ if (dl_prio(p->prio)) ++ p->sched_class = &dl_sched_class; ++ else if (rt_prio(p->prio)) ++ p->sched_class = &rt_sched_class; ++ else ++ p->sched_class = &fair_sched_class; ++} ++ ++static void ++__getparam_dl(struct task_struct *p, struct sched_attr *attr) ++{ ++ struct sched_dl_entity *dl_se = &p->dl; ++ ++ attr->sched_priority = p->rt_priority; ++ attr->sched_runtime = dl_se->dl_runtime; ++ attr->sched_deadline = dl_se->dl_deadline; ++ attr->sched_period = dl_se->dl_period; ++ attr->sched_flags = dl_se->flags; ++} ++ ++/* ++ * This function validates the new parameters of a -deadline task. ++ * We ask for the deadline not being zero, and greater or equal ++ * than the runtime, as well as the period of being zero or ++ * greater than deadline. Furthermore, we have to be sure that ++ * user parameters are above the internal resolution of 1us (we ++ * check sched_runtime only since it is always the smaller one) and ++ * below 2^63 ns (we have to check both sched_deadline and ++ * sched_period, as the latter can be zero). ++ */ ++static bool ++__checkparam_dl(const struct sched_attr *attr) ++{ ++ /* deadline != 0 */ ++ if (attr->sched_deadline == 0) ++ return false; ++ ++ /* ++ * Since we truncate DL_SCALE bits, make sure we're at least ++ * that big. ++ */ ++ if (attr->sched_runtime < (1ULL << DL_SCALE)) ++ return false; ++ ++ /* ++ * Since we use the MSB for wrap-around and sign issues, make ++ * sure it's not set (mind that period can be equal to zero). ++ */ ++ if (attr->sched_deadline & (1ULL << 63) || ++ attr->sched_period & (1ULL << 63)) ++ return false; ++ ++ /* runtime <= deadline <= period (if period != 0) */ ++ if ((attr->sched_period != 0 && ++ attr->sched_period < attr->sched_deadline) || ++ attr->sched_deadline < attr->sched_runtime) ++ return false; ++ ++ return true; ++} ++ ++/* ++ * check the target process has a UID that matches the current process's ++ */ ++static bool check_same_owner(struct task_struct *p) ++{ ++ const struct cred *cred = current_cred(), *pcred; ++ bool match; ++ ++ rcu_read_lock(); ++ pcred = __task_cred(p); ++ match = (uid_eq(cred->euid, pcred->euid) || ++ uid_eq(cred->euid, pcred->uid)); ++ rcu_read_unlock(); ++ return match; ++} ++ ++static bool dl_param_changed(struct task_struct *p, ++ const struct sched_attr *attr) ++{ ++ struct sched_dl_entity *dl_se = &p->dl; ++ ++ if (dl_se->dl_runtime != attr->sched_runtime || ++ dl_se->dl_deadline != attr->sched_deadline || ++ dl_se->dl_period != attr->sched_period || ++ dl_se->flags != attr->sched_flags) ++ return true; ++ ++ return false; ++} ++ ++static int __sched_setscheduler(struct task_struct *p, ++ const struct sched_attr *attr, ++ bool user) ++{ ++ int newprio = dl_policy(attr->sched_policy) ? MAX_DL_PRIO - 1 : ++ MAX_RT_PRIO - 1 - attr->sched_priority; ++ int retval, oldprio, oldpolicy = -1, queued, running; ++ int new_effective_prio, policy = attr->sched_policy; ++ unsigned long flags; ++ const struct sched_class *prev_class; ++ struct rq *rq; ++ int reset_on_fork; ++ ++ /* may grab non-irq protected spin_locks */ ++ BUG_ON(in_interrupt()); ++recheck: ++ /* double check policy once rq lock held */ ++ if (policy < 0) { ++ reset_on_fork = p->sched_reset_on_fork; ++ policy = oldpolicy = p->policy; ++ } else { ++ reset_on_fork = !!(attr->sched_flags & SCHED_FLAG_RESET_ON_FORK); ++ ++ if (policy != SCHED_DEADLINE && ++ policy != SCHED_FIFO && policy != SCHED_RR && ++ policy != SCHED_NORMAL && policy != SCHED_BATCH && ++ policy != SCHED_IDLE) ++ return -EINVAL; ++ } ++ ++ if (attr->sched_flags & ~(SCHED_FLAG_RESET_ON_FORK)) ++ return -EINVAL; ++ ++ /* ++ * Valid priorities for SCHED_FIFO and SCHED_RR are ++ * 1..MAX_USER_RT_PRIO-1, valid priority for SCHED_NORMAL, ++ * SCHED_BATCH and SCHED_IDLE is 0. ++ */ ++ if ((p->mm && attr->sched_priority > MAX_USER_RT_PRIO-1) || ++ (!p->mm && attr->sched_priority > MAX_RT_PRIO-1)) ++ return -EINVAL; ++ if ((dl_policy(policy) && !__checkparam_dl(attr)) || ++ (rt_policy(policy) != (attr->sched_priority != 0))) ++ return -EINVAL; ++ ++ /* ++ * Allow unprivileged RT tasks to decrease priority: ++ */ ++ if (user && !capable(CAP_SYS_NICE)) { ++ if (fair_policy(policy)) { ++ if (attr->sched_nice < task_nice(p) && ++ !can_nice(p, attr->sched_nice)) ++ return -EPERM; ++ } ++ ++ if (rt_policy(policy)) { ++ unsigned long rlim_rtprio = ++ task_rlimit(p, RLIMIT_RTPRIO); ++ ++ /* can't set/change the rt policy */ ++ if (policy != p->policy && !rlim_rtprio) ++ return -EPERM; ++ ++ /* can't increase priority */ ++ if (attr->sched_priority > p->rt_priority && ++ attr->sched_priority > rlim_rtprio) ++ return -EPERM; ++ } ++ ++ /* ++ * Can't set/change SCHED_DEADLINE policy at all for now ++ * (safest behavior); in the future we would like to allow ++ * unprivileged DL tasks to increase their relative deadline ++ * or reduce their runtime (both ways reducing utilization) ++ */ ++ if (dl_policy(policy)) ++ return -EPERM; ++ ++ /* ++ * Treat SCHED_IDLE as nice 20. Only allow a switch to ++ * SCHED_NORMAL if the RLIMIT_NICE would normally permit it. ++ */ ++ if (p->policy == SCHED_IDLE && policy != SCHED_IDLE) { ++ if (!can_nice(p, task_nice(p))) ++ return -EPERM; ++ } ++ ++ /* can't change other user's priorities */ ++ if (!check_same_owner(p)) ++ return -EPERM; ++ ++ /* Normal users shall not reset the sched_reset_on_fork flag */ ++ if (p->sched_reset_on_fork && !reset_on_fork) ++ return -EPERM; ++ } ++ ++ if (user) { ++ retval = security_task_setscheduler(p); ++ if (retval) ++ return retval; ++ } ++ ++ /* ++ * make sure no PI-waiters arrive (or leave) while we are ++ * changing the priority of the task: ++ * ++ * To be able to change p->policy safely, the appropriate ++ * runqueue lock must be held. ++ */ ++ rq = task_rq_lock(p, &flags); ++ ++ /* ++ * Changing the policy of the stop threads its a very bad idea ++ */ ++ if (p == rq->stop) { ++ task_rq_unlock(rq, p, &flags); ++ return -EINVAL; ++ } ++ ++ /* ++ * If not changing anything there's no need to proceed further, ++ * but store a possible modification of reset_on_fork. ++ */ ++ if (unlikely(policy == p->policy)) { ++ if (fair_policy(policy) && attr->sched_nice != task_nice(p)) ++ goto change; ++ if (rt_policy(policy) && attr->sched_priority != p->rt_priority) ++ goto change; ++ if (dl_policy(policy) && dl_param_changed(p, attr)) ++ goto change; ++ ++ p->sched_reset_on_fork = reset_on_fork; ++ task_rq_unlock(rq, p, &flags); ++ return 0; ++ } ++change: ++ ++ if (user) { ++#ifdef CONFIG_RT_GROUP_SCHED ++ /* ++ * Do not allow realtime tasks into groups that have no runtime ++ * assigned. ++ */ ++ if (rt_bandwidth_enabled() && rt_policy(policy) && ++ task_group(p)->rt_bandwidth.rt_runtime == 0 && ++ !task_group_is_autogroup(task_group(p))) { ++ task_rq_unlock(rq, p, &flags); ++ return -EPERM; ++ } ++#endif ++#ifdef CONFIG_SMP ++ if (dl_bandwidth_enabled() && dl_policy(policy)) { ++ cpumask_t *span = rq->rd->span; ++ ++ /* ++ * Don't allow tasks with an affinity mask smaller than ++ * the entire root_domain to become SCHED_DEADLINE. We ++ * will also fail if there's no bandwidth available. ++ */ ++ if (!cpumask_subset(span, &p->cpus_allowed) || ++ rq->rd->dl_bw.bw == 0) { ++ task_rq_unlock(rq, p, &flags); ++ return -EPERM; ++ } ++ } ++#endif ++ } ++ ++ /* recheck policy now with rq lock held */ ++ if (unlikely(oldpolicy != -1 && oldpolicy != p->policy)) { ++ policy = oldpolicy = -1; ++ task_rq_unlock(rq, p, &flags); ++ goto recheck; ++ } ++ ++ /* ++ * If setscheduling to SCHED_DEADLINE (or changing the parameters ++ * of a SCHED_DEADLINE task) we need to check if enough bandwidth ++ * is available. ++ */ ++ if ((dl_policy(policy) || dl_task(p)) && dl_overflow(p, policy, attr)) { ++ task_rq_unlock(rq, p, &flags); ++ return -EBUSY; ++ } ++ ++ p->sched_reset_on_fork = reset_on_fork; ++ oldprio = p->prio; ++ ++ /* ++ * Take priority boosted tasks into account. If the new ++ * effective priority is unchanged, we just store the new ++ * normal parameters and do not touch the scheduler class and ++ * the runqueue. This will be done when the task deboost ++ * itself. ++ */ ++ new_effective_prio = rt_mutex_get_effective_prio(p, newprio); ++ if (new_effective_prio == oldprio) { ++ __setscheduler_params(p, attr); ++ task_rq_unlock(rq, p, &flags); ++ return 0; ++ } ++ ++ queued = task_on_rq_queued(p); ++ running = task_current(rq, p); ++ if (queued) ++ dequeue_task(rq, p, 0); ++ if (running) ++ put_prev_task(rq, p); ++ ++ prev_class = p->sched_class; ++ __setscheduler(rq, p, attr, true); ++ ++ if (running) ++ p->sched_class->set_curr_task(rq); ++ if (queued) { ++ /* ++ * We enqueue to tail when the priority of a task is ++ * increased (user space view). ++ */ ++ enqueue_task(rq, p, oldprio <= p->prio ? ENQUEUE_HEAD : 0); ++ } ++ ++ check_class_changed(rq, p, prev_class, oldprio); ++ task_rq_unlock(rq, p, &flags); ++ ++ rt_mutex_adjust_pi(p); ++ ++ return 0; ++} ++ ++static int _sched_setscheduler(struct task_struct *p, int policy, ++ const struct sched_param *param, bool check) ++{ ++ struct sched_attr attr = { ++ .sched_policy = policy, ++ .sched_priority = param->sched_priority, ++ .sched_nice = PRIO_TO_NICE(p->static_prio), ++ }; ++ ++ /* Fixup the legacy SCHED_RESET_ON_FORK hack. */ ++ if ((policy != SETPARAM_POLICY) && (policy & SCHED_RESET_ON_FORK)) { ++ attr.sched_flags |= SCHED_FLAG_RESET_ON_FORK; ++ policy &= ~SCHED_RESET_ON_FORK; ++ attr.sched_policy = policy; ++ } ++ ++ return __sched_setscheduler(p, &attr, check); ++} ++/** ++ * sched_setscheduler - change the scheduling policy and/or RT priority of a thread. ++ * @p: the task in question. ++ * @policy: new policy. ++ * @param: structure containing the new RT priority. ++ * ++ * Return: 0 on success. An error code otherwise. ++ * ++ * NOTE that the task may be already dead. ++ */ ++int sched_setscheduler(struct task_struct *p, int policy, ++ const struct sched_param *param) ++{ ++ return _sched_setscheduler(p, policy, param, true); ++} ++EXPORT_SYMBOL_GPL(sched_setscheduler); ++ ++int sched_setattr(struct task_struct *p, const struct sched_attr *attr) ++{ ++ return __sched_setscheduler(p, attr, true); ++} ++EXPORT_SYMBOL_GPL(sched_setattr); ++ ++/** ++ * sched_setscheduler_nocheck - change the scheduling policy and/or RT priority of a thread from kernelspace. ++ * @p: the task in question. ++ * @policy: new policy. ++ * @param: structure containing the new RT priority. ++ * ++ * Just like sched_setscheduler, only don't bother checking if the ++ * current context has permission. For example, this is needed in ++ * stop_machine(): we create temporary high priority worker threads, ++ * but our caller might not have that capability. ++ * ++ * Return: 0 on success. An error code otherwise. ++ */ ++int sched_setscheduler_nocheck(struct task_struct *p, int policy, ++ const struct sched_param *param) ++{ ++ return _sched_setscheduler(p, policy, param, false); ++} ++ ++static int ++do_sched_setscheduler(pid_t pid, int policy, struct sched_param __user *param) ++{ ++ struct sched_param lparam; ++ struct task_struct *p; ++ int retval; ++ ++ if (!param || pid < 0) ++ return -EINVAL; ++ if (copy_from_user(&lparam, param, sizeof(struct sched_param))) ++ return -EFAULT; ++ ++ rcu_read_lock(); ++ retval = -ESRCH; ++ p = find_process_by_pid(pid); ++ if (p != NULL) ++ retval = sched_setscheduler(p, policy, &lparam); ++ rcu_read_unlock(); ++ ++ return retval; ++} ++ ++/* ++ * Mimics kernel/events/core.c perf_copy_attr(). ++ */ ++static int sched_copy_attr(struct sched_attr __user *uattr, ++ struct sched_attr *attr) ++{ ++ u32 size; ++ int ret; ++ ++ if (!access_ok(VERIFY_WRITE, uattr, SCHED_ATTR_SIZE_VER0)) ++ return -EFAULT; ++ ++ /* ++ * zero the full structure, so that a short copy will be nice. ++ */ ++ memset(attr, 0, sizeof(*attr)); ++ ++ ret = get_user(size, &uattr->size); ++ if (ret) ++ return ret; ++ ++ if (size > PAGE_SIZE) /* silly large */ ++ goto err_size; ++ ++ if (!size) /* abi compat */ ++ size = SCHED_ATTR_SIZE_VER0; ++ ++ if (size < SCHED_ATTR_SIZE_VER0) ++ goto err_size; ++ ++ /* ++ * If we're handed a bigger struct than we know of, ++ * ensure all the unknown bits are 0 - i.e. new ++ * user-space does not rely on any kernel feature ++ * extensions we dont know about yet. ++ */ ++ if (size > sizeof(*attr)) { ++ unsigned char __user *addr; ++ unsigned char __user *end; ++ unsigned char val; ++ ++ addr = (void __user *)uattr + sizeof(*attr); ++ end = (void __user *)uattr + size; ++ ++ for (; addr < end; addr++) { ++ ret = get_user(val, addr); ++ if (ret) ++ return ret; ++ if (val) ++ goto err_size; ++ } ++ size = sizeof(*attr); ++ } ++ ++ ret = copy_from_user(attr, uattr, size); ++ if (ret) ++ return -EFAULT; ++ ++ /* ++ * XXX: do we want to be lenient like existing syscalls; or do we want ++ * to be strict and return an error on out-of-bounds values? ++ */ ++ attr->sched_nice = clamp(attr->sched_nice, MIN_NICE, MAX_NICE); ++ ++ return 0; ++ ++err_size: ++ put_user(sizeof(*attr), &uattr->size); ++ return -E2BIG; ++} ++ ++/** ++ * sys_sched_setscheduler - set/change the scheduler policy and RT priority ++ * @pid: the pid in question. ++ * @policy: new policy. ++ * @param: structure containing the new RT priority. ++ * ++ * Return: 0 on success. An error code otherwise. ++ */ ++SYSCALL_DEFINE3(sched_setscheduler, pid_t, pid, int, policy, ++ struct sched_param __user *, param) ++{ ++ /* negative values for policy are not valid */ ++ if (policy < 0) ++ return -EINVAL; ++ ++ return do_sched_setscheduler(pid, policy, param); ++} ++ ++/** ++ * sys_sched_setparam - set/change the RT priority of a thread ++ * @pid: the pid in question. ++ * @param: structure containing the new RT priority. ++ * ++ * Return: 0 on success. An error code otherwise. ++ */ ++SYSCALL_DEFINE2(sched_setparam, pid_t, pid, struct sched_param __user *, param) ++{ ++ return do_sched_setscheduler(pid, SETPARAM_POLICY, param); ++} ++ ++/** ++ * sys_sched_setattr - same as above, but with extended sched_attr ++ * @pid: the pid in question. ++ * @uattr: structure containing the extended parameters. ++ * @flags: for future extension. ++ */ ++SYSCALL_DEFINE3(sched_setattr, pid_t, pid, struct sched_attr __user *, uattr, ++ unsigned int, flags) ++{ ++ struct sched_attr attr; ++ struct task_struct *p; ++ int retval; ++ ++ if (!uattr || pid < 0 || flags) ++ return -EINVAL; ++ ++ retval = sched_copy_attr(uattr, &attr); ++ if (retval) ++ return retval; ++ ++ if ((int)attr.sched_policy < 0) ++ return -EINVAL; ++ ++ rcu_read_lock(); ++ retval = -ESRCH; ++ p = find_process_by_pid(pid); ++ if (p != NULL) ++ retval = sched_setattr(p, &attr); ++ rcu_read_unlock(); ++ ++ return retval; ++} ++ ++/** ++ * sys_sched_getscheduler - get the policy (scheduling class) of a thread ++ * @pid: the pid in question. ++ * ++ * Return: On success, the policy of the thread. Otherwise, a negative error ++ * code. ++ */ ++SYSCALL_DEFINE1(sched_getscheduler, pid_t, pid) ++{ ++ struct task_struct *p; ++ int retval; ++ ++ if (pid < 0) ++ return -EINVAL; ++ ++ retval = -ESRCH; ++ rcu_read_lock(); ++ p = find_process_by_pid(pid); ++ if (p) { ++ retval = security_task_getscheduler(p); ++ if (!retval) ++ retval = p->policy ++ | (p->sched_reset_on_fork ? SCHED_RESET_ON_FORK : 0); ++ } ++ rcu_read_unlock(); ++ return retval; ++} ++ ++/** ++ * sys_sched_getparam - get the RT priority of a thread ++ * @pid: the pid in question. ++ * @param: structure containing the RT priority. ++ * ++ * Return: On success, 0 and the RT priority is in @param. Otherwise, an error ++ * code. ++ */ ++SYSCALL_DEFINE2(sched_getparam, pid_t, pid, struct sched_param __user *, param) ++{ ++ struct sched_param lp = { .sched_priority = 0 }; ++ struct task_struct *p; ++ int retval; ++ ++ if (!param || pid < 0) ++ return -EINVAL; ++ ++ rcu_read_lock(); ++ p = find_process_by_pid(pid); ++ retval = -ESRCH; ++ if (!p) ++ goto out_unlock; ++ ++ retval = security_task_getscheduler(p); ++ if (retval) ++ goto out_unlock; ++ ++ if (task_has_rt_policy(p)) ++ lp.sched_priority = p->rt_priority; ++ rcu_read_unlock(); ++ ++ /* ++ * This one might sleep, we cannot do it with a spinlock held ... ++ */ ++ retval = copy_to_user(param, &lp, sizeof(*param)) ? -EFAULT : 0; ++ ++ return retval; ++ ++out_unlock: ++ rcu_read_unlock(); ++ return retval; ++} ++ ++static int sched_read_attr(struct sched_attr __user *uattr, ++ struct sched_attr *attr, ++ unsigned int usize) ++{ ++ int ret; ++ ++ if (!access_ok(VERIFY_WRITE, uattr, usize)) ++ return -EFAULT; ++ ++ /* ++ * If we're handed a smaller struct than we know of, ++ * ensure all the unknown bits are 0 - i.e. old ++ * user-space does not get uncomplete information. ++ */ ++ if (usize < sizeof(*attr)) { ++ unsigned char *addr; ++ unsigned char *end; ++ ++ addr = (void *)attr + usize; ++ end = (void *)attr + sizeof(*attr); ++ ++ for (; addr < end; addr++) { ++ if (*addr) ++ return -EFBIG; ++ } ++ ++ attr->size = usize; ++ } ++ ++ ret = copy_to_user(uattr, attr, attr->size); ++ if (ret) ++ return -EFAULT; ++ ++ return 0; ++} ++ ++/** ++ * sys_sched_getattr - similar to sched_getparam, but with sched_attr ++ * @pid: the pid in question. ++ * @uattr: structure containing the extended parameters. ++ * @size: sizeof(attr) for fwd/bwd comp. ++ * @flags: for future extension. ++ */ ++SYSCALL_DEFINE4(sched_getattr, pid_t, pid, struct sched_attr __user *, uattr, ++ unsigned int, size, unsigned int, flags) ++{ ++ struct sched_attr attr = { ++ .size = sizeof(struct sched_attr), ++ }; ++ struct task_struct *p; ++ int retval; ++ ++ if (!uattr || pid < 0 || size > PAGE_SIZE || ++ size < SCHED_ATTR_SIZE_VER0 || flags) ++ return -EINVAL; ++ ++ rcu_read_lock(); ++ p = find_process_by_pid(pid); ++ retval = -ESRCH; ++ if (!p) ++ goto out_unlock; ++ ++ retval = security_task_getscheduler(p); ++ if (retval) ++ goto out_unlock; ++ ++ attr.sched_policy = p->policy; ++ if (p->sched_reset_on_fork) ++ attr.sched_flags |= SCHED_FLAG_RESET_ON_FORK; ++ if (task_has_dl_policy(p)) ++ __getparam_dl(p, &attr); ++ else if (task_has_rt_policy(p)) ++ attr.sched_priority = p->rt_priority; ++ else ++ attr.sched_nice = task_nice(p); ++ ++ rcu_read_unlock(); ++ ++ retval = sched_read_attr(uattr, &attr, size); ++ return retval; ++ ++out_unlock: ++ rcu_read_unlock(); ++ return retval; ++} ++ ++long sched_setaffinity(pid_t pid, const struct cpumask *in_mask) ++{ ++ cpumask_var_t cpus_allowed, new_mask; ++ struct task_struct *p; ++ int retval; ++ ++ rcu_read_lock(); ++ ++ p = find_process_by_pid(pid); ++ if (!p) { ++ rcu_read_unlock(); ++ return -ESRCH; ++ } ++ ++ /* Prevent p going away */ ++ get_task_struct(p); ++ rcu_read_unlock(); ++ ++ if (p->flags & PF_NO_SETAFFINITY) { ++ retval = -EINVAL; ++ goto out_put_task; ++ } ++ if (!alloc_cpumask_var(&cpus_allowed, GFP_KERNEL)) { ++ retval = -ENOMEM; ++ goto out_put_task; ++ } ++ if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) { ++ retval = -ENOMEM; ++ goto out_free_cpus_allowed; ++ } ++ retval = -EPERM; ++ if (!check_same_owner(p)) { ++ rcu_read_lock(); ++ if (!ns_capable(__task_cred(p)->user_ns, CAP_SYS_NICE)) { ++ rcu_read_unlock(); ++ goto out_free_new_mask; ++ } ++ rcu_read_unlock(); ++ } ++ ++ retval = security_task_setscheduler(p); ++ if (retval) ++ goto out_free_new_mask; ++ ++ ++ cpuset_cpus_allowed(p, cpus_allowed); ++ cpumask_and(new_mask, in_mask, cpus_allowed); ++ ++ /* ++ * Since bandwidth control happens on root_domain basis, ++ * if admission test is enabled, we only admit -deadline ++ * tasks allowed to run on all the CPUs in the task's ++ * root_domain. ++ */ ++#ifdef CONFIG_SMP ++ if (task_has_dl_policy(p) && dl_bandwidth_enabled()) { ++ rcu_read_lock(); ++ if (!cpumask_subset(task_rq(p)->rd->span, new_mask)) { ++ retval = -EBUSY; ++ rcu_read_unlock(); ++ goto out_free_new_mask; ++ } ++ rcu_read_unlock(); ++ } ++#endif ++again: ++ retval = set_cpus_allowed_ptr(p, new_mask); ++ ++ if (!retval) { ++ cpuset_cpus_allowed(p, cpus_allowed); ++ if (!cpumask_subset(new_mask, cpus_allowed)) { ++ /* ++ * We must have raced with a concurrent cpuset ++ * update. Just reset the cpus_allowed to the ++ * cpuset's cpus_allowed ++ */ ++ cpumask_copy(new_mask, cpus_allowed); ++ goto again; ++ } ++ } ++out_free_new_mask: ++ free_cpumask_var(new_mask); ++out_free_cpus_allowed: ++ free_cpumask_var(cpus_allowed); ++out_put_task: ++ put_task_struct(p); ++ return retval; ++} ++ ++static int get_user_cpu_mask(unsigned long __user *user_mask_ptr, unsigned len, ++ struct cpumask *new_mask) ++{ ++ if (len < cpumask_size()) ++ cpumask_clear(new_mask); ++ else if (len > cpumask_size()) ++ len = cpumask_size(); ++ ++ return copy_from_user(new_mask, user_mask_ptr, len) ? -EFAULT : 0; ++} ++ ++/** ++ * sys_sched_setaffinity - set the cpu affinity of a process ++ * @pid: pid of the process ++ * @len: length in bytes of the bitmask pointed to by user_mask_ptr ++ * @user_mask_ptr: user-space pointer to the new cpu mask ++ * ++ * Return: 0 on success. An error code otherwise. ++ */ ++SYSCALL_DEFINE3(sched_setaffinity, pid_t, pid, unsigned int, len, ++ unsigned long __user *, user_mask_ptr) ++{ ++ cpumask_var_t new_mask; ++ int retval; ++ ++ if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) ++ return -ENOMEM; ++ ++ retval = get_user_cpu_mask(user_mask_ptr, len, new_mask); ++ if (retval == 0) ++ retval = sched_setaffinity(pid, new_mask); ++ free_cpumask_var(new_mask); ++ return retval; ++} ++ ++long sched_getaffinity(pid_t pid, struct cpumask *mask) ++{ ++ struct task_struct *p; ++ unsigned long flags; ++ int retval; ++ ++ rcu_read_lock(); ++ ++ retval = -ESRCH; ++ p = find_process_by_pid(pid); ++ if (!p) ++ goto out_unlock; ++ ++ retval = security_task_getscheduler(p); ++ if (retval) ++ goto out_unlock; ++ ++ raw_spin_lock_irqsave(&p->pi_lock, flags); ++ cpumask_and(mask, &p->cpus_allowed, cpu_active_mask); ++ raw_spin_unlock_irqrestore(&p->pi_lock, flags); ++ ++out_unlock: ++ rcu_read_unlock(); ++ ++ return retval; ++} ++ ++/** ++ * sys_sched_getaffinity - get the cpu affinity of a process ++ * @pid: pid of the process ++ * @len: length in bytes of the bitmask pointed to by user_mask_ptr ++ * @user_mask_ptr: user-space pointer to hold the current cpu mask ++ * ++ * Return: 0 on success. An error code otherwise. ++ */ ++SYSCALL_DEFINE3(sched_getaffinity, pid_t, pid, unsigned int, len, ++ unsigned long __user *, user_mask_ptr) ++{ ++ int ret; ++ cpumask_var_t mask; ++ ++ if ((len * BITS_PER_BYTE) < nr_cpu_ids) ++ return -EINVAL; ++ if (len & (sizeof(unsigned long)-1)) ++ return -EINVAL; ++ ++ if (!alloc_cpumask_var(&mask, GFP_KERNEL)) ++ return -ENOMEM; ++ ++ ret = sched_getaffinity(pid, mask); ++ if (ret == 0) { ++ size_t retlen = min_t(size_t, len, cpumask_size()); ++ ++ if (copy_to_user(user_mask_ptr, mask, retlen)) ++ ret = -EFAULT; ++ else ++ ret = retlen; ++ } ++ free_cpumask_var(mask); ++ ++ return ret; ++} ++ ++/** ++ * sys_sched_yield - yield the current processor to other threads. ++ * ++ * This function yields the current CPU to other tasks. If there are no ++ * other threads running on this CPU then this function will return. ++ * ++ * Return: 0. ++ */ ++SYSCALL_DEFINE0(sched_yield) ++{ ++ struct rq *rq = this_rq_lock(); ++ ++ schedstat_inc(rq, yld_count); ++ current->sched_class->yield_task(rq); ++ ++ /* ++ * Since we are going to call schedule() anyway, there's ++ * no need to preempt or enable interrupts: ++ */ ++ __release(rq->lock); ++ spin_release(&rq->lock.dep_map, 1, _THIS_IP_); ++ do_raw_spin_unlock(&rq->lock); ++ sched_preempt_enable_no_resched(); ++ ++ schedule(); ++ ++ return 0; ++} ++ ++int __sched _cond_resched(void) ++{ ++ if (should_resched()) { ++ preempt_schedule_common(); ++ return 1; ++ } ++ return 0; ++} ++EXPORT_SYMBOL(_cond_resched); ++ ++/* ++ * __cond_resched_lock() - if a reschedule is pending, drop the given lock, ++ * call schedule, and on return reacquire the lock. ++ * ++ * This works OK both with and without CONFIG_PREEMPT. We do strange low-level ++ * operations here to prevent schedule() from being called twice (once via ++ * spin_unlock(), once by hand). ++ */ ++int __cond_resched_lock(spinlock_t *lock) ++{ ++ int resched = should_resched(); ++ int ret = 0; ++ ++ lockdep_assert_held(lock); ++ ++ if (spin_needbreak(lock) || resched) { ++ spin_unlock(lock); ++ if (resched) ++ preempt_schedule_common(); ++ else ++ cpu_relax(); ++ ret = 1; ++ spin_lock(lock); ++ } ++ return ret; ++} ++EXPORT_SYMBOL(__cond_resched_lock); ++ ++int __sched __cond_resched_softirq(void) ++{ ++ BUG_ON(!in_softirq()); ++ ++ if (should_resched()) { ++ local_bh_enable(); ++ preempt_schedule_common(); ++ local_bh_disable(); ++ return 1; ++ } ++ return 0; ++} ++EXPORT_SYMBOL(__cond_resched_softirq); ++ ++/** ++ * yield - yield the current processor to other threads. ++ * ++ * Do not ever use this function, there's a 99% chance you're doing it wrong. ++ * ++ * The scheduler is at all times free to pick the calling task as the most ++ * eligible task to run, if removing the yield() call from your code breaks ++ * it, its already broken. ++ * ++ * Typical broken usage is: ++ * ++ * while (!event) ++ * yield(); ++ * ++ * where one assumes that yield() will let 'the other' process run that will ++ * make event true. If the current task is a SCHED_FIFO task that will never ++ * happen. Never use yield() as a progress guarantee!! ++ * ++ * If you want to use yield() to wait for something, use wait_event(). ++ * If you want to use yield() to be 'nice' for others, use cond_resched(). ++ * If you still want to use yield(), do not! ++ */ ++void __sched yield(void) ++{ ++ set_current_state(TASK_RUNNING); ++ sys_sched_yield(); ++} ++EXPORT_SYMBOL(yield); ++ ++/** ++ * yield_to - yield the current processor to another thread in ++ * your thread group, or accelerate that thread toward the ++ * processor it's on. ++ * @p: target task ++ * @preempt: whether task preemption is allowed or not ++ * ++ * It's the caller's job to ensure that the target task struct ++ * can't go away on us before we can do any checks. ++ * ++ * Return: ++ * true (>0) if we indeed boosted the target task. ++ * false (0) if we failed to boost the target. ++ * -ESRCH if there's no task to yield to. ++ */ ++int __sched yield_to(struct task_struct *p, bool preempt) ++{ ++ struct task_struct *curr = current; ++ struct rq *rq, *p_rq; ++ unsigned long flags; ++ int yielded = 0; ++ ++ local_irq_save(flags); ++ rq = this_rq(); ++ ++again: ++ p_rq = task_rq(p); ++ /* ++ * If we're the only runnable task on the rq and target rq also ++ * has only one task, there's absolutely no point in yielding. ++ */ ++ if (rq->nr_running == 1 && p_rq->nr_running == 1) { ++ yielded = -ESRCH; ++ goto out_irq; ++ } ++ ++ double_rq_lock(rq, p_rq); ++ if (task_rq(p) != p_rq) { ++ double_rq_unlock(rq, p_rq); ++ goto again; ++ } ++ ++ if (!curr->sched_class->yield_to_task) ++ goto out_unlock; ++ ++ if (curr->sched_class != p->sched_class) ++ goto out_unlock; ++ ++ if (task_running(p_rq, p) || p->state) ++ goto out_unlock; ++ ++ yielded = curr->sched_class->yield_to_task(rq, p, preempt); ++ if (yielded) { ++ schedstat_inc(rq, yld_count); ++ /* ++ * Make p's CPU reschedule; pick_next_entity takes care of ++ * fairness. ++ */ ++ if (preempt && rq != p_rq) ++ resched_curr(p_rq); ++ } ++ ++out_unlock: ++ double_rq_unlock(rq, p_rq); ++out_irq: ++ local_irq_restore(flags); ++ ++ if (yielded > 0) ++ schedule(); ++ ++ return yielded; ++} ++EXPORT_SYMBOL_GPL(yield_to); ++ ++/* ++ * This task is about to go to sleep on IO. Increment rq->nr_iowait so ++ * that process accounting knows that this is a task in IO wait state. ++ */ ++long __sched io_schedule_timeout(long timeout) ++{ ++ int old_iowait = current->in_iowait; ++ struct rq *rq; ++ long ret; ++ ++ current->in_iowait = 1; ++ blk_schedule_flush_plug(current); ++ ++ delayacct_blkio_start(); ++ rq = raw_rq(); ++ atomic_inc(&rq->nr_iowait); ++ ret = schedule_timeout(timeout); ++ current->in_iowait = old_iowait; ++ atomic_dec(&rq->nr_iowait); ++ delayacct_blkio_end(); ++ ++ return ret; ++} ++EXPORT_SYMBOL(io_schedule_timeout); ++ ++/** ++ * sys_sched_get_priority_max - return maximum RT priority. ++ * @policy: scheduling class. ++ * ++ * Return: On success, this syscall returns the maximum ++ * rt_priority that can be used by a given scheduling class. ++ * On failure, a negative error code is returned. ++ */ ++SYSCALL_DEFINE1(sched_get_priority_max, int, policy) ++{ ++ int ret = -EINVAL; ++ ++ switch (policy) { ++ case SCHED_FIFO: ++ case SCHED_RR: ++ ret = MAX_USER_RT_PRIO-1; ++ break; ++ case SCHED_DEADLINE: ++ case SCHED_NORMAL: ++ case SCHED_BATCH: ++ case SCHED_IDLE: ++ ret = 0; ++ break; ++ } ++ return ret; ++} ++ ++/** ++ * sys_sched_get_priority_min - return minimum RT priority. ++ * @policy: scheduling class. ++ * ++ * Return: On success, this syscall returns the minimum ++ * rt_priority that can be used by a given scheduling class. ++ * On failure, a negative error code is returned. ++ */ ++SYSCALL_DEFINE1(sched_get_priority_min, int, policy) ++{ ++ int ret = -EINVAL; ++ ++ switch (policy) { ++ case SCHED_FIFO: ++ case SCHED_RR: ++ ret = 1; ++ break; ++ case SCHED_DEADLINE: ++ case SCHED_NORMAL: ++ case SCHED_BATCH: ++ case SCHED_IDLE: ++ ret = 0; ++ } ++ return ret; ++} ++ ++/** ++ * sys_sched_rr_get_interval - return the default timeslice of a process. ++ * @pid: pid of the process. ++ * @interval: userspace pointer to the timeslice value. ++ * ++ * this syscall writes the default timeslice value of a given process ++ * into the user-space timespec buffer. A value of '0' means infinity. ++ * ++ * Return: On success, 0 and the timeslice is in @interval. Otherwise, ++ * an error code. ++ */ ++SYSCALL_DEFINE2(sched_rr_get_interval, pid_t, pid, ++ struct timespec __user *, interval) ++{ ++ struct task_struct *p; ++ unsigned int time_slice; ++ unsigned long flags; ++ struct rq *rq; ++ int retval; ++ struct timespec t; ++ ++ if (pid < 0) ++ return -EINVAL; ++ ++ retval = -ESRCH; ++ rcu_read_lock(); ++ p = find_process_by_pid(pid); ++ if (!p) ++ goto out_unlock; ++ ++ retval = security_task_getscheduler(p); ++ if (retval) ++ goto out_unlock; ++ ++ rq = task_rq_lock(p, &flags); ++ time_slice = 0; ++ if (p->sched_class->get_rr_interval) ++ time_slice = p->sched_class->get_rr_interval(rq, p); ++ task_rq_unlock(rq, p, &flags); ++ ++ rcu_read_unlock(); ++ jiffies_to_timespec(time_slice, &t); ++ retval = copy_to_user(interval, &t, sizeof(t)) ? -EFAULT : 0; ++ return retval; ++ ++out_unlock: ++ rcu_read_unlock(); ++ return retval; ++} ++ ++static const char stat_nam[] = TASK_STATE_TO_CHAR_STR; ++ ++void sched_show_task(struct task_struct *p) ++{ ++ unsigned long free = 0; ++ int ppid; ++ unsigned long state = p->state; ++ ++ if (state) ++ state = __ffs(state) + 1; ++ printk(KERN_INFO "%-15.15s %c", p->comm, ++ state < sizeof(stat_nam) - 1 ? stat_nam[state] : '?'); ++#if BITS_PER_LONG == 32 ++ if (state == TASK_RUNNING) ++ printk(KERN_CONT " running "); ++ else ++ printk(KERN_CONT " %08lx ", thread_saved_pc(p)); ++#else ++ if (state == TASK_RUNNING) ++ printk(KERN_CONT " running task "); ++ else ++ printk(KERN_CONT " %016lx ", thread_saved_pc(p)); ++#endif ++#ifdef CONFIG_DEBUG_STACK_USAGE ++ free = stack_not_used(p); ++#endif ++ ppid = 0; ++ rcu_read_lock(); ++ if (pid_alive(p)) ++ ppid = task_pid_nr(rcu_dereference(p->real_parent)); ++ rcu_read_unlock(); ++ printk(KERN_CONT "%5lu %5d %6d 0x%08lx\n", free, ++ task_pid_nr(p), ppid, ++ (unsigned long)task_thread_info(p)->flags); ++ ++ print_worker_info(KERN_INFO, p); ++ show_stack(p, NULL); ++} ++ ++void show_state_filter(unsigned long state_filter) ++{ ++ struct task_struct *g, *p; ++ ++#if BITS_PER_LONG == 32 ++ printk(KERN_INFO ++ " task PC stack pid father\n"); ++#else ++ printk(KERN_INFO ++ " task PC stack pid father\n"); ++#endif ++ rcu_read_lock(); ++ for_each_process_thread(g, p) { ++ /* ++ * reset the NMI-timeout, listing all files on a slow ++ * console might take a lot of time: ++ */ ++ touch_nmi_watchdog(); ++ if (!state_filter || (p->state & state_filter)) ++ sched_show_task(p); ++ } ++ ++ touch_all_softlockup_watchdogs(); ++ ++#ifdef CONFIG_SCHED_DEBUG ++ sysrq_sched_debug_show(); ++#endif ++ rcu_read_unlock(); ++ /* ++ * Only show locks if all tasks are dumped: ++ */ ++ if (!state_filter) ++ debug_show_all_locks(); ++} ++ ++void init_idle_bootup_task(struct task_struct *idle) ++{ ++ idle->sched_class = &idle_sched_class; ++} ++ ++/** ++ * init_idle - set up an idle thread for a given CPU ++ * @idle: task in question ++ * @cpu: cpu the idle task belongs to ++ * ++ * NOTE: this function does not set the idle thread's NEED_RESCHED ++ * flag, to make booting more robust. ++ */ ++void init_idle(struct task_struct *idle, int cpu) ++{ ++ struct rq *rq = cpu_rq(cpu); ++ unsigned long flags; ++ ++ raw_spin_lock_irqsave(&rq->lock, flags); ++ ++ __sched_fork(0, idle); ++ idle->state = TASK_RUNNING; ++ idle->se.exec_start = sched_clock(); ++ ++ do_set_cpus_allowed(idle, cpumask_of(cpu)); ++ /* ++ * We're having a chicken and egg problem, even though we are ++ * holding rq->lock, the cpu isn't yet set to this cpu so the ++ * lockdep check in task_group() will fail. ++ * ++ * Similar case to sched_fork(). / Alternatively we could ++ * use task_rq_lock() here and obtain the other rq->lock. ++ * ++ * Silence PROVE_RCU ++ */ ++ rcu_read_lock(); ++ __set_task_cpu(idle, cpu); ++ rcu_read_unlock(); ++ ++ rq->curr = rq->idle = idle; ++ idle->on_rq = TASK_ON_RQ_QUEUED; ++#if defined(CONFIG_SMP) ++ idle->on_cpu = 1; ++#endif ++ raw_spin_unlock_irqrestore(&rq->lock, flags); ++ ++ /* Set the preempt count _outside_ the spinlocks! */ ++ init_idle_preempt_count(idle, cpu); ++ ++ /* ++ * The idle tasks have their own, simple scheduling class: ++ */ ++ idle->sched_class = &idle_sched_class; ++ ftrace_graph_init_idle_task(idle, cpu); ++ vtime_init_idle(idle, cpu); ++#if defined(CONFIG_SMP) ++ sprintf(idle->comm, "%s/%d", INIT_TASK_COMM, cpu); ++#endif ++} ++ ++int cpuset_cpumask_can_shrink(const struct cpumask *cur, ++ const struct cpumask *trial) ++{ ++ int ret = 1, trial_cpus; ++ struct dl_bw *cur_dl_b; ++ unsigned long flags; ++ ++ if (!cpumask_weight(cur)) ++ return ret; ++ ++ rcu_read_lock_sched(); ++ cur_dl_b = dl_bw_of(cpumask_any(cur)); ++ trial_cpus = cpumask_weight(trial); ++ ++ raw_spin_lock_irqsave(&cur_dl_b->lock, flags); ++ if (cur_dl_b->bw != -1 && ++ cur_dl_b->bw * trial_cpus < cur_dl_b->total_bw) ++ ret = 0; ++ raw_spin_unlock_irqrestore(&cur_dl_b->lock, flags); ++ rcu_read_unlock_sched(); ++ ++ return ret; ++} ++ ++int task_can_attach(struct task_struct *p, ++ const struct cpumask *cs_cpus_allowed) ++{ ++ int ret = 0; ++ ++ /* ++ * Kthreads which disallow setaffinity shouldn't be moved ++ * to a new cpuset; we don't want to change their cpu ++ * affinity and isolating such threads by their set of ++ * allowed nodes is unnecessary. Thus, cpusets are not ++ * applicable for such threads. This prevents checking for ++ * success of set_cpus_allowed_ptr() on all attached tasks ++ * before cpus_allowed may be changed. ++ */ ++ if (p->flags & PF_NO_SETAFFINITY) { ++ ret = -EINVAL; ++ goto out; ++ } ++ ++#ifdef CONFIG_SMP ++ if (dl_task(p) && !cpumask_intersects(task_rq(p)->rd->span, ++ cs_cpus_allowed)) { ++ unsigned int dest_cpu = cpumask_any_and(cpu_active_mask, ++ cs_cpus_allowed); ++ struct dl_bw *dl_b; ++ bool overflow; ++ int cpus; ++ unsigned long flags; ++ ++ rcu_read_lock_sched(); ++ dl_b = dl_bw_of(dest_cpu); ++ raw_spin_lock_irqsave(&dl_b->lock, flags); ++ cpus = dl_bw_cpus(dest_cpu); ++ overflow = __dl_overflow(dl_b, cpus, 0, p->dl.dl_bw); ++ if (overflow) ++ ret = -EBUSY; ++ else { ++ /* ++ * We reserve space for this task in the destination ++ * root_domain, as we can't fail after this point. ++ * We will free resources in the source root_domain ++ * later on (see set_cpus_allowed_dl()). ++ */ ++ __dl_add(dl_b, p->dl.dl_bw); ++ } ++ raw_spin_unlock_irqrestore(&dl_b->lock, flags); ++ rcu_read_unlock_sched(); ++ ++ } ++#endif ++out: ++ return ret; ++} ++ ++#ifdef CONFIG_SMP ++/* ++ * move_queued_task - move a queued task to new rq. ++ * ++ * Returns (locked) new rq. Old rq's lock is released. ++ */ ++static struct rq *move_queued_task(struct task_struct *p, int new_cpu) ++{ ++ struct rq *rq = task_rq(p); ++ ++ lockdep_assert_held(&rq->lock); ++ ++ dequeue_task(rq, p, 0); ++ p->on_rq = TASK_ON_RQ_MIGRATING; ++ set_task_cpu(p, new_cpu); ++ raw_spin_unlock(&rq->lock); ++ ++ rq = cpu_rq(new_cpu); ++ ++ raw_spin_lock(&rq->lock); ++ BUG_ON(task_cpu(p) != new_cpu); ++ p->on_rq = TASK_ON_RQ_QUEUED; ++ enqueue_task(rq, p, 0); ++ check_preempt_curr(rq, p, 0); ++ ++ return rq; ++} ++ ++void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask) ++{ ++ if (p->sched_class->set_cpus_allowed) ++ p->sched_class->set_cpus_allowed(p, new_mask); ++ ++ cpumask_copy(&p->cpus_allowed, new_mask); ++ p->nr_cpus_allowed = cpumask_weight(new_mask); ++} ++ ++/* ++ * This is how migration works: ++ * ++ * 1) we invoke migration_cpu_stop() on the target CPU using ++ * stop_one_cpu(). ++ * 2) stopper starts to run (implicitly forcing the migrated thread ++ * off the CPU) ++ * 3) it checks whether the migrated task is still in the wrong runqueue. ++ * 4) if it's in the wrong runqueue then the migration thread removes ++ * it and puts it into the right queue. ++ * 5) stopper completes and stop_one_cpu() returns and the migration ++ * is done. ++ */ ++ ++/* ++ * Change a given task's CPU affinity. Migrate the thread to a ++ * proper CPU and schedule it away if the CPU it's executing on ++ * is removed from the allowed bitmask. ++ * ++ * NOTE: the caller must have a valid reference to the task, the ++ * task must not exit() & deallocate itself prematurely. The ++ * call is not atomic; no spinlocks may be held. ++ */ ++int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask) ++{ ++ unsigned long flags; ++ struct rq *rq; ++ unsigned int dest_cpu; ++ int ret = 0; ++ ++ rq = task_rq_lock(p, &flags); ++ ++ if (cpumask_equal(&p->cpus_allowed, new_mask)) ++ goto out; ++ ++ if (!cpumask_intersects(new_mask, cpu_active_mask)) { ++ ret = -EINVAL; ++ goto out; ++ } ++ ++ do_set_cpus_allowed(p, new_mask); ++ ++ /* Can the task run on the task's current CPU? If so, we're done */ ++ if (cpumask_test_cpu(task_cpu(p), new_mask)) ++ goto out; ++ ++ dest_cpu = cpumask_any_and(cpu_active_mask, new_mask); ++ if (task_running(rq, p) || p->state == TASK_WAKING) { ++ struct migration_arg arg = { p, dest_cpu }; ++ /* Need help from migration thread: drop lock and wait. */ ++ task_rq_unlock(rq, p, &flags); ++ stop_one_cpu(cpu_of(rq), migration_cpu_stop, &arg); ++ tlb_migrate_finish(p->mm); ++ return 0; ++ } else if (task_on_rq_queued(p)) ++ rq = move_queued_task(p, dest_cpu); ++out: ++ task_rq_unlock(rq, p, &flags); ++ ++ return ret; ++} ++EXPORT_SYMBOL_GPL(set_cpus_allowed_ptr); ++ ++/* ++ * Move (not current) task off this cpu, onto dest cpu. We're doing ++ * this because either it can't run here any more (set_cpus_allowed() ++ * away from this CPU, or CPU going down), or because we're ++ * attempting to rebalance this task on exec (sched_exec). ++ * ++ * So we race with normal scheduler movements, but that's OK, as long ++ * as the task is no longer on this CPU. ++ * ++ * Returns non-zero if task was successfully migrated. ++ */ ++static int __migrate_task(struct task_struct *p, int src_cpu, int dest_cpu) ++{ ++ struct rq *rq; ++ int ret = 0; ++ ++ if (unlikely(!cpu_active(dest_cpu))) ++ return ret; ++ ++ rq = cpu_rq(src_cpu); ++ ++ raw_spin_lock(&p->pi_lock); ++ raw_spin_lock(&rq->lock); ++ /* Already moved. */ ++ if (task_cpu(p) != src_cpu) ++ goto done; ++ ++ /* Affinity changed (again). */ ++ if (!cpumask_test_cpu(dest_cpu, tsk_cpus_allowed(p))) ++ goto fail; ++ ++ /* ++ * If we're not on a rq, the next wake-up will ensure we're ++ * placed properly. ++ */ ++ if (task_on_rq_queued(p)) ++ rq = move_queued_task(p, dest_cpu); ++done: ++ ret = 1; ++fail: ++ raw_spin_unlock(&rq->lock); ++ raw_spin_unlock(&p->pi_lock); ++ return ret; ++} ++ ++#ifdef CONFIG_NUMA_BALANCING ++/* Migrate current task p to target_cpu */ ++int migrate_task_to(struct task_struct *p, int target_cpu) ++{ ++ struct migration_arg arg = { p, target_cpu }; ++ int curr_cpu = task_cpu(p); ++ ++ if (curr_cpu == target_cpu) ++ return 0; ++ ++ if (!cpumask_test_cpu(target_cpu, tsk_cpus_allowed(p))) ++ return -EINVAL; ++ ++ /* TODO: This is not properly updating schedstats */ ++ ++ trace_sched_move_numa(p, curr_cpu, target_cpu); ++ return stop_one_cpu(curr_cpu, migration_cpu_stop, &arg); ++} ++ ++/* ++ * Requeue a task on a given node and accurately track the number of NUMA ++ * tasks on the runqueues ++ */ ++void sched_setnuma(struct task_struct *p, int nid) ++{ ++ struct rq *rq; ++ unsigned long flags; ++ bool queued, running; ++ ++ rq = task_rq_lock(p, &flags); ++ queued = task_on_rq_queued(p); ++ running = task_current(rq, p); ++ ++ if (queued) ++ dequeue_task(rq, p, 0); ++ if (running) ++ put_prev_task(rq, p); ++ ++ p->numa_preferred_nid = nid; ++ ++ if (running) ++ p->sched_class->set_curr_task(rq); ++ if (queued) ++ enqueue_task(rq, p, 0); ++ task_rq_unlock(rq, p, &flags); ++} ++#endif ++ ++/* ++ * migration_cpu_stop - this will be executed by a highprio stopper thread ++ * and performs thread migration by bumping thread off CPU then ++ * 'pushing' onto another runqueue. ++ */ ++static int migration_cpu_stop(void *data) ++{ ++ struct migration_arg *arg = data; ++ ++ /* ++ * The original target cpu might have gone down and we might ++ * be on another cpu but it doesn't matter. ++ */ ++ local_irq_disable(); ++ /* ++ * We need to explicitly wake pending tasks before running ++ * __migrate_task() such that we will not miss enforcing cpus_allowed ++ * during wakeups, see set_cpus_allowed_ptr()'s TASK_WAKING test. ++ */ ++ sched_ttwu_pending(); ++ __migrate_task(arg->task, raw_smp_processor_id(), arg->dest_cpu); ++ local_irq_enable(); ++ return 0; ++} ++ ++#ifdef CONFIG_HOTPLUG_CPU ++ ++/* ++ * Ensures that the idle task is using init_mm right before its cpu goes ++ * offline. ++ */ ++void idle_task_exit(void) ++{ ++ struct mm_struct *mm = current->active_mm; ++ ++ BUG_ON(cpu_online(smp_processor_id())); ++ ++ if (mm != &init_mm) { ++ switch_mm(mm, &init_mm, current); ++ finish_arch_post_lock_switch(); ++ } ++ mmdrop(mm); ++} ++ ++/* ++ * Since this CPU is going 'away' for a while, fold any nr_active delta ++ * we might have. Assumes we're called after migrate_tasks() so that the ++ * nr_active count is stable. ++ * ++ * Also see the comment "Global load-average calculations". ++ */ ++static void calc_load_migrate(struct rq *rq) ++{ ++ long delta = calc_load_fold_active(rq); ++ if (delta) ++ atomic_long_add(delta, &calc_load_tasks); ++} ++ ++static void put_prev_task_fake(struct rq *rq, struct task_struct *prev) ++{ ++} ++ ++static const struct sched_class fake_sched_class = { ++ .put_prev_task = put_prev_task_fake, ++}; ++ ++static struct task_struct fake_task = { ++ /* ++ * Avoid pull_{rt,dl}_task() ++ */ ++ .prio = MAX_PRIO + 1, ++ .sched_class = &fake_sched_class, ++}; ++ ++/* ++ * Migrate all tasks from the rq, sleeping tasks will be migrated by ++ * try_to_wake_up()->select_task_rq(). ++ * ++ * Called with rq->lock held even though we'er in stop_machine() and ++ * there's no concurrency possible, we hold the required locks anyway ++ * because of lock validation efforts. ++ */ ++static void migrate_tasks(unsigned int dead_cpu) ++{ ++ struct rq *rq = cpu_rq(dead_cpu); ++ struct task_struct *next, *stop = rq->stop; ++ int dest_cpu; ++ ++ /* ++ * Fudge the rq selection such that the below task selection loop ++ * doesn't get stuck on the currently eligible stop task. ++ * ++ * We're currently inside stop_machine() and the rq is either stuck ++ * in the stop_machine_cpu_stop() loop, or we're executing this code, ++ * either way we should never end up calling schedule() until we're ++ * done here. ++ */ ++ rq->stop = NULL; ++ ++ /* ++ * put_prev_task() and pick_next_task() sched ++ * class method both need to have an up-to-date ++ * value of rq->clock[_task] ++ */ ++ update_rq_clock(rq); ++ ++ for ( ; ; ) { ++ /* ++ * There's this thread running, bail when that's the only ++ * remaining thread. ++ */ ++ if (rq->nr_running == 1) ++ break; ++ ++ next = pick_next_task(rq, &fake_task); ++ BUG_ON(!next); ++ next->sched_class->put_prev_task(rq, next); ++ ++ /* Find suitable destination for @next, with force if needed. */ ++ dest_cpu = select_fallback_rq(dead_cpu, next); ++ raw_spin_unlock(&rq->lock); ++ ++ __migrate_task(next, dead_cpu, dest_cpu); ++ ++ raw_spin_lock(&rq->lock); ++ } ++ ++ rq->stop = stop; ++} ++ ++#endif /* CONFIG_HOTPLUG_CPU */ ++ ++#if defined(CONFIG_SCHED_DEBUG) && defined(CONFIG_SYSCTL) ++ ++static struct ctl_table sd_ctl_dir[] = { ++ { ++ .procname = "sched_domain", ++ .mode = 0555, ++ }, ++ {} ++}; ++ ++static struct ctl_table sd_ctl_root[] = { ++ { ++ .procname = "kernel", ++ .mode = 0555, ++ .child = sd_ctl_dir, ++ }, ++ {} ++}; ++ ++static struct ctl_table *sd_alloc_ctl_entry(int n) ++{ ++ struct ctl_table *entry = ++ kcalloc(n, sizeof(struct ctl_table), GFP_KERNEL); ++ ++ return entry; ++} ++ ++static void sd_free_ctl_entry(struct ctl_table **tablep) ++{ ++ struct ctl_table *entry; ++ ++ /* ++ * In the intermediate directories, both the child directory and ++ * procname are dynamically allocated and could fail but the mode ++ * will always be set. In the lowest directory the names are ++ * static strings and all have proc handlers. ++ */ ++ for (entry = *tablep; entry->mode; entry++) { ++ if (entry->child) ++ sd_free_ctl_entry(&entry->child); ++ if (entry->proc_handler == NULL) ++ kfree(entry->procname); ++ } ++ ++ kfree(*tablep); ++ *tablep = NULL; ++} ++ ++static int min_load_idx = 0; ++static int max_load_idx = CPU_LOAD_IDX_MAX-1; ++ ++static void ++set_table_entry(struct ctl_table *entry, ++ const char *procname, void *data, int maxlen, ++ umode_t mode, proc_handler *proc_handler, ++ bool load_idx) ++{ ++ entry->procname = procname; ++ entry->data = data; ++ entry->maxlen = maxlen; ++ entry->mode = mode; ++ entry->proc_handler = proc_handler; ++ ++ if (load_idx) { ++ entry->extra1 = &min_load_idx; ++ entry->extra2 = &max_load_idx; ++ } ++} ++ ++static struct ctl_table * ++sd_alloc_ctl_domain_table(struct sched_domain *sd) ++{ ++ struct ctl_table *table = sd_alloc_ctl_entry(14); ++ ++ if (table == NULL) ++ return NULL; ++ ++ set_table_entry(&table[0], "min_interval", &sd->min_interval, ++ sizeof(long), 0644, proc_doulongvec_minmax, false); ++ set_table_entry(&table[1], "max_interval", &sd->max_interval, ++ sizeof(long), 0644, proc_doulongvec_minmax, false); ++ set_table_entry(&table[2], "busy_idx", &sd->busy_idx, ++ sizeof(int), 0644, proc_dointvec_minmax, true); ++ set_table_entry(&table[3], "idle_idx", &sd->idle_idx, ++ sizeof(int), 0644, proc_dointvec_minmax, true); ++ set_table_entry(&table[4], "newidle_idx", &sd->newidle_idx, ++ sizeof(int), 0644, proc_dointvec_minmax, true); ++ set_table_entry(&table[5], "wake_idx", &sd->wake_idx, ++ sizeof(int), 0644, proc_dointvec_minmax, true); ++ set_table_entry(&table[6], "forkexec_idx", &sd->forkexec_idx, ++ sizeof(int), 0644, proc_dointvec_minmax, true); ++ set_table_entry(&table[7], "busy_factor", &sd->busy_factor, ++ sizeof(int), 0644, proc_dointvec_minmax, false); ++ set_table_entry(&table[8], "imbalance_pct", &sd->imbalance_pct, ++ sizeof(int), 0644, proc_dointvec_minmax, false); ++ set_table_entry(&table[9], "cache_nice_tries", ++ &sd->cache_nice_tries, ++ sizeof(int), 0644, proc_dointvec_minmax, false); ++ set_table_entry(&table[10], "flags", &sd->flags, ++ sizeof(int), 0644, proc_dointvec_minmax, false); ++ set_table_entry(&table[11], "max_newidle_lb_cost", ++ &sd->max_newidle_lb_cost, ++ sizeof(long), 0644, proc_doulongvec_minmax, false); ++ set_table_entry(&table[12], "name", sd->name, ++ CORENAME_MAX_SIZE, 0444, proc_dostring, false); ++ /* &table[13] is terminator */ ++ ++ return table; ++} ++ ++static struct ctl_table *sd_alloc_ctl_cpu_table(int cpu) ++{ ++ struct ctl_table *entry, *table; ++ struct sched_domain *sd; ++ int domain_num = 0, i; ++ char buf[32]; ++ ++ for_each_domain(cpu, sd) ++ domain_num++; ++ entry = table = sd_alloc_ctl_entry(domain_num + 1); ++ if (table == NULL) ++ return NULL; ++ ++ i = 0; ++ for_each_domain(cpu, sd) { ++ snprintf(buf, 32, "domain%d", i); ++ entry->procname = kstrdup(buf, GFP_KERNEL); ++ entry->mode = 0555; ++ entry->child = sd_alloc_ctl_domain_table(sd); ++ entry++; ++ i++; ++ } ++ return table; ++} ++ ++static struct ctl_table_header *sd_sysctl_header; ++static void register_sched_domain_sysctl(void) ++{ ++ int i, cpu_num = num_possible_cpus(); ++ struct ctl_table *entry = sd_alloc_ctl_entry(cpu_num + 1); ++ char buf[32]; ++ ++ WARN_ON(sd_ctl_dir[0].child); ++ sd_ctl_dir[0].child = entry; ++ ++ if (entry == NULL) ++ return; ++ ++ for_each_possible_cpu(i) { ++ snprintf(buf, 32, "cpu%d", i); ++ entry->procname = kstrdup(buf, GFP_KERNEL); ++ entry->mode = 0555; ++ entry->child = sd_alloc_ctl_cpu_table(i); ++ entry++; ++ } ++ ++ WARN_ON(sd_sysctl_header); ++ sd_sysctl_header = register_sysctl_table(sd_ctl_root); ++} ++ ++/* may be called multiple times per register */ ++static void unregister_sched_domain_sysctl(void) ++{ ++ if (sd_sysctl_header) ++ unregister_sysctl_table(sd_sysctl_header); ++ sd_sysctl_header = NULL; ++ if (sd_ctl_dir[0].child) ++ sd_free_ctl_entry(&sd_ctl_dir[0].child); ++} ++#else ++static void register_sched_domain_sysctl(void) ++{ ++} ++static void unregister_sched_domain_sysctl(void) ++{ ++} ++#endif ++ ++static void set_rq_online(struct rq *rq) ++{ ++ if (!rq->online) { ++ const struct sched_class *class; ++ ++ cpumask_set_cpu(rq->cpu, rq->rd->online); ++ rq->online = 1; ++ ++ for_each_class(class) { ++ if (class->rq_online) ++ class->rq_online(rq); ++ } ++ } ++} ++ ++static void set_rq_offline(struct rq *rq) ++{ ++ if (rq->online) { ++ const struct sched_class *class; ++ ++ for_each_class(class) { ++ if (class->rq_offline) ++ class->rq_offline(rq); ++ } ++ ++ cpumask_clear_cpu(rq->cpu, rq->rd->online); ++ rq->online = 0; ++ } ++} ++ ++/* ++ * migration_call - callback that gets triggered when a CPU is added. ++ * Here we can start up the necessary migration thread for the new CPU. ++ */ ++static int ++migration_call(struct notifier_block *nfb, unsigned long action, void *hcpu) ++{ ++ int cpu = (long)hcpu; ++ unsigned long flags; ++ struct rq *rq = cpu_rq(cpu); ++ ++ switch (action & ~CPU_TASKS_FROZEN) { ++ ++ case CPU_UP_PREPARE: ++ rq->calc_load_update = calc_load_update; ++ break; ++ ++ case CPU_ONLINE: ++ /* Update our root-domain */ ++ raw_spin_lock_irqsave(&rq->lock, flags); ++ if (rq->rd) { ++ BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span)); ++ ++ set_rq_online(rq); ++ } ++ raw_spin_unlock_irqrestore(&rq->lock, flags); ++ break; ++ ++#ifdef CONFIG_HOTPLUG_CPU ++ case CPU_DYING: ++ sched_ttwu_pending(); ++ /* Update our root-domain */ ++ raw_spin_lock_irqsave(&rq->lock, flags); ++ if (rq->rd) { ++ BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span)); ++ set_rq_offline(rq); ++ } ++ migrate_tasks(cpu); ++ BUG_ON(rq->nr_running != 1); /* the migration thread */ ++ raw_spin_unlock_irqrestore(&rq->lock, flags); ++ break; ++ ++ case CPU_DEAD: ++ calc_load_migrate(rq); ++ break; ++#endif ++ } ++ ++ update_max_interval(); ++ ++ return NOTIFY_OK; ++} ++ ++/* ++ * Register at high priority so that task migration (migrate_all_tasks) ++ * happens before everything else. This has to be lower priority than ++ * the notifier in the perf_event subsystem, though. ++ */ ++static struct notifier_block migration_notifier = { ++ .notifier_call = migration_call, ++ .priority = CPU_PRI_MIGRATION, ++}; ++ ++static void __cpuinit set_cpu_rq_start_time(void) ++{ ++ int cpu = smp_processor_id(); ++ struct rq *rq = cpu_rq(cpu); ++ rq->age_stamp = sched_clock_cpu(cpu); ++} ++ ++static int sched_cpu_active(struct notifier_block *nfb, ++ unsigned long action, void *hcpu) ++{ ++ switch (action & ~CPU_TASKS_FROZEN) { ++ case CPU_STARTING: ++ set_cpu_rq_start_time(); ++ return NOTIFY_OK; ++ case CPU_ONLINE: ++ /* ++ * At this point a starting CPU has marked itself as online via ++ * set_cpu_online(). But it might not yet have marked itself ++ * as active, which is essential from here on. ++ * ++ * Thus, fall-through and help the starting CPU along. ++ */ ++ case CPU_DOWN_FAILED: ++ set_cpu_active((long)hcpu, true); ++ return NOTIFY_OK; ++ default: ++ return NOTIFY_DONE; ++ } ++} ++ ++static int sched_cpu_inactive(struct notifier_block *nfb, ++ unsigned long action, void *hcpu) ++{ ++ switch (action & ~CPU_TASKS_FROZEN) { ++ case CPU_DOWN_PREPARE: ++ set_cpu_active((long)hcpu, false); ++ return NOTIFY_OK; ++ default: ++ return NOTIFY_DONE; ++ } ++} ++ ++static int __init migration_init(void) ++{ ++ void *cpu = (void *)(long)smp_processor_id(); ++ int err; ++ ++ /* Initialize migration for the boot CPU */ ++ err = migration_call(&migration_notifier, CPU_UP_PREPARE, cpu); ++ BUG_ON(err == NOTIFY_BAD); ++ migration_call(&migration_notifier, CPU_ONLINE, cpu); ++ register_cpu_notifier(&migration_notifier); ++ ++ /* Register cpu active notifiers */ ++ cpu_notifier(sched_cpu_active, CPU_PRI_SCHED_ACTIVE); ++ cpu_notifier(sched_cpu_inactive, CPU_PRI_SCHED_INACTIVE); ++ ++ return 0; ++} ++early_initcall(migration_init); ++#endif ++ ++#ifdef CONFIG_SMP ++ ++static cpumask_var_t sched_domains_tmpmask; /* sched_domains_mutex */ ++ ++#ifdef CONFIG_SCHED_DEBUG ++ ++static __read_mostly int sched_debug_enabled; ++ ++static int __init sched_debug_setup(char *str) ++{ ++ sched_debug_enabled = 1; ++ ++ return 0; ++} ++early_param("sched_debug", sched_debug_setup); ++ ++static inline bool sched_debug(void) ++{ ++ return sched_debug_enabled; ++} ++ ++static int sched_domain_debug_one(struct sched_domain *sd, int cpu, int level, ++ struct cpumask *groupmask) ++{ ++ struct sched_group *group = sd->groups; ++ ++ cpumask_clear(groupmask); ++ ++ printk(KERN_DEBUG "%*s domain %d: ", level, "", level); ++ ++ if (!(sd->flags & SD_LOAD_BALANCE)) { ++ printk("does not load-balance\n"); ++ if (sd->parent) ++ printk(KERN_ERR "ERROR: !SD_LOAD_BALANCE domain" ++ " has parent"); ++ return -1; ++ } ++ ++ printk(KERN_CONT "span %*pbl level %s\n", ++ cpumask_pr_args(sched_domain_span(sd)), sd->name); ++ ++ if (!cpumask_test_cpu(cpu, sched_domain_span(sd))) { ++ printk(KERN_ERR "ERROR: domain->span does not contain " ++ "CPU%d\n", cpu); ++ } ++ if (!cpumask_test_cpu(cpu, sched_group_cpus(group))) { ++ printk(KERN_ERR "ERROR: domain->groups does not contain" ++ " CPU%d\n", cpu); ++ } ++ ++ printk(KERN_DEBUG "%*s groups:", level + 1, ""); ++ do { ++ if (!group) { ++ printk("\n"); ++ printk(KERN_ERR "ERROR: group is NULL\n"); ++ break; ++ } ++ ++ if (!cpumask_weight(sched_group_cpus(group))) { ++ printk(KERN_CONT "\n"); ++ printk(KERN_ERR "ERROR: empty group\n"); ++ break; ++ } ++ ++ if (!(sd->flags & SD_OVERLAP) && ++ cpumask_intersects(groupmask, sched_group_cpus(group))) { ++ printk(KERN_CONT "\n"); ++ printk(KERN_ERR "ERROR: repeated CPUs\n"); ++ break; ++ } ++ ++ cpumask_or(groupmask, groupmask, sched_group_cpus(group)); ++ ++ printk(KERN_CONT " %*pbl", ++ cpumask_pr_args(sched_group_cpus(group))); ++ if (group->sgc->capacity != SCHED_CAPACITY_SCALE) { ++ printk(KERN_CONT " (cpu_capacity = %d)", ++ group->sgc->capacity); ++ } ++ ++ group = group->next; ++ } while (group != sd->groups); ++ printk(KERN_CONT "\n"); ++ ++ if (!cpumask_equal(sched_domain_span(sd), groupmask)) ++ printk(KERN_ERR "ERROR: groups don't span domain->span\n"); ++ ++ if (sd->parent && ++ !cpumask_subset(groupmask, sched_domain_span(sd->parent))) ++ printk(KERN_ERR "ERROR: parent span is not a superset " ++ "of domain->span\n"); ++ return 0; ++} ++ ++static void sched_domain_debug(struct sched_domain *sd, int cpu) ++{ ++ int level = 0; ++ ++ if (!sched_debug_enabled) ++ return; ++ ++ if (!sd) { ++ printk(KERN_DEBUG "CPU%d attaching NULL sched-domain.\n", cpu); ++ return; ++ } ++ ++ printk(KERN_DEBUG "CPU%d attaching sched-domain:\n", cpu); ++ ++ for (;;) { ++ if (sched_domain_debug_one(sd, cpu, level, sched_domains_tmpmask)) ++ break; ++ level++; ++ sd = sd->parent; ++ if (!sd) ++ break; ++ } ++} ++#else /* !CONFIG_SCHED_DEBUG */ ++# define sched_domain_debug(sd, cpu) do { } while (0) ++static inline bool sched_debug(void) ++{ ++ return false; ++} ++#endif /* CONFIG_SCHED_DEBUG */ ++ ++static int sd_degenerate(struct sched_domain *sd) ++{ ++ if (cpumask_weight(sched_domain_span(sd)) == 1) ++ return 1; ++ ++ /* Following flags need at least 2 groups */ ++ if (sd->flags & (SD_LOAD_BALANCE | ++ SD_BALANCE_NEWIDLE | ++ SD_BALANCE_FORK | ++ SD_BALANCE_EXEC | ++ SD_SHARE_CPUCAPACITY | ++ SD_SHARE_PKG_RESOURCES | ++ SD_SHARE_POWERDOMAIN)) { ++ if (sd->groups != sd->groups->next) ++ return 0; ++ } ++ ++ /* Following flags don't use groups */ ++ if (sd->flags & (SD_WAKE_AFFINE)) ++ return 0; ++ ++ return 1; ++} ++ ++static int ++sd_parent_degenerate(struct sched_domain *sd, struct sched_domain *parent) ++{ ++ unsigned long cflags = sd->flags, pflags = parent->flags; ++ ++ if (sd_degenerate(parent)) ++ return 1; ++ ++ if (!cpumask_equal(sched_domain_span(sd), sched_domain_span(parent))) ++ return 0; ++ ++ /* Flags needing groups don't count if only 1 group in parent */ ++ if (parent->groups == parent->groups->next) { ++ pflags &= ~(SD_LOAD_BALANCE | ++ SD_BALANCE_NEWIDLE | ++ SD_BALANCE_FORK | ++ SD_BALANCE_EXEC | ++ SD_SHARE_CPUCAPACITY | ++ SD_SHARE_PKG_RESOURCES | ++ SD_PREFER_SIBLING | ++ SD_SHARE_POWERDOMAIN); ++ if (nr_node_ids == 1) ++ pflags &= ~SD_SERIALIZE; ++ } ++ if (~cflags & pflags) ++ return 0; ++ ++ return 1; ++} ++ ++static void free_rootdomain(struct rcu_head *rcu) ++{ ++ struct root_domain *rd = container_of(rcu, struct root_domain, rcu); ++ ++ cpupri_cleanup(&rd->cpupri); ++ cpudl_cleanup(&rd->cpudl); ++ free_cpumask_var(rd->dlo_mask); ++ free_cpumask_var(rd->rto_mask); ++ free_cpumask_var(rd->online); ++ free_cpumask_var(rd->span); ++ kfree(rd); ++} ++ ++static void rq_attach_root(struct rq *rq, struct root_domain *rd) ++{ ++ struct root_domain *old_rd = NULL; ++ unsigned long flags; ++ ++ raw_spin_lock_irqsave(&rq->lock, flags); ++ ++ if (rq->rd) { ++ old_rd = rq->rd; ++ ++ if (cpumask_test_cpu(rq->cpu, old_rd->online)) ++ set_rq_offline(rq); ++ ++ cpumask_clear_cpu(rq->cpu, old_rd->span); ++ ++ /* ++ * If we dont want to free the old_rd yet then ++ * set old_rd to NULL to skip the freeing later ++ * in this function: ++ */ ++ if (!atomic_dec_and_test(&old_rd->refcount)) ++ old_rd = NULL; ++ } ++ ++ atomic_inc(&rd->refcount); ++ rq->rd = rd; ++ ++ cpumask_set_cpu(rq->cpu, rd->span); ++ if (cpumask_test_cpu(rq->cpu, cpu_active_mask)) ++ set_rq_online(rq); ++ ++ raw_spin_unlock_irqrestore(&rq->lock, flags); ++ ++ if (old_rd) ++ call_rcu_sched(&old_rd->rcu, free_rootdomain); ++} ++ ++static int init_rootdomain(struct root_domain *rd) ++{ ++ memset(rd, 0, sizeof(*rd)); ++ ++ if (!alloc_cpumask_var(&rd->span, GFP_KERNEL)) ++ goto out; ++ if (!alloc_cpumask_var(&rd->online, GFP_KERNEL)) ++ goto free_span; ++ if (!alloc_cpumask_var(&rd->dlo_mask, GFP_KERNEL)) ++ goto free_online; ++ if (!alloc_cpumask_var(&rd->rto_mask, GFP_KERNEL)) ++ goto free_dlo_mask; ++ ++ init_dl_bw(&rd->dl_bw); ++ if (cpudl_init(&rd->cpudl) != 0) ++ goto free_dlo_mask; ++ ++ if (cpupri_init(&rd->cpupri) != 0) ++ goto free_rto_mask; ++ return 0; ++ ++free_rto_mask: ++ free_cpumask_var(rd->rto_mask); ++free_dlo_mask: ++ free_cpumask_var(rd->dlo_mask); ++free_online: ++ free_cpumask_var(rd->online); ++free_span: ++ free_cpumask_var(rd->span); ++out: ++ return -ENOMEM; ++} ++ ++/* ++ * By default the system creates a single root-domain with all cpus as ++ * members (mimicking the global state we have today). ++ */ ++struct root_domain def_root_domain; ++ ++static void init_defrootdomain(void) ++{ ++ init_rootdomain(&def_root_domain); ++ ++ atomic_set(&def_root_domain.refcount, 1); ++} ++ ++static struct root_domain *alloc_rootdomain(void) ++{ ++ struct root_domain *rd; ++ ++ rd = kmalloc(sizeof(*rd), GFP_KERNEL); ++ if (!rd) ++ return NULL; ++ ++ if (init_rootdomain(rd) != 0) { ++ kfree(rd); ++ return NULL; ++ } ++ ++ return rd; ++} ++ ++static void free_sched_groups(struct sched_group *sg, int free_sgc) ++{ ++ struct sched_group *tmp, *first; ++ ++ if (!sg) ++ return; ++ ++ first = sg; ++ do { ++ tmp = sg->next; ++ ++ if (free_sgc && atomic_dec_and_test(&sg->sgc->ref)) ++ kfree(sg->sgc); ++ ++ kfree(sg); ++ sg = tmp; ++ } while (sg != first); ++} ++ ++static void free_sched_domain(struct rcu_head *rcu) ++{ ++ struct sched_domain *sd = container_of(rcu, struct sched_domain, rcu); ++ ++ /* ++ * If its an overlapping domain it has private groups, iterate and ++ * nuke them all. ++ */ ++ if (sd->flags & SD_OVERLAP) { ++ free_sched_groups(sd->groups, 1); ++ } else if (atomic_dec_and_test(&sd->groups->ref)) { ++ kfree(sd->groups->sgc); ++ kfree(sd->groups); ++ } ++ kfree(sd); ++} ++ ++static void destroy_sched_domain(struct sched_domain *sd, int cpu) ++{ ++ call_rcu(&sd->rcu, free_sched_domain); ++} ++ ++static void destroy_sched_domains(struct sched_domain *sd, int cpu) ++{ ++ for (; sd; sd = sd->parent) ++ destroy_sched_domain(sd, cpu); ++} ++ ++/* ++ * Keep a special pointer to the highest sched_domain that has ++ * SD_SHARE_PKG_RESOURCE set (Last Level Cache Domain) for this ++ * allows us to avoid some pointer chasing select_idle_sibling(). ++ * ++ * Also keep a unique ID per domain (we use the first cpu number in ++ * the cpumask of the domain), this allows us to quickly tell if ++ * two cpus are in the same cache domain, see cpus_share_cache(). ++ */ ++DEFINE_PER_CPU(struct sched_domain *, sd_llc); ++DEFINE_PER_CPU(int, sd_llc_size); ++DEFINE_PER_CPU(int, sd_llc_id); ++DEFINE_PER_CPU(struct sched_domain *, sd_numa); ++DEFINE_PER_CPU(struct sched_domain *, sd_busy); ++DEFINE_PER_CPU(struct sched_domain *, sd_asym); ++ ++static void update_top_cache_domain(int cpu) ++{ ++ struct sched_domain *sd; ++ struct sched_domain *busy_sd = NULL; ++ int id = cpu; ++ int size = 1; ++ ++ sd = highest_flag_domain(cpu, SD_SHARE_PKG_RESOURCES); ++ if (sd) { ++ id = cpumask_first(sched_domain_span(sd)); ++ size = cpumask_weight(sched_domain_span(sd)); ++ busy_sd = sd->parent; /* sd_busy */ ++ } ++ rcu_assign_pointer(per_cpu(sd_busy, cpu), busy_sd); ++ ++ rcu_assign_pointer(per_cpu(sd_llc, cpu), sd); ++ per_cpu(sd_llc_size, cpu) = size; ++ per_cpu(sd_llc_id, cpu) = id; ++ ++ sd = lowest_flag_domain(cpu, SD_NUMA); ++ rcu_assign_pointer(per_cpu(sd_numa, cpu), sd); ++ ++ sd = highest_flag_domain(cpu, SD_ASYM_PACKING); ++ rcu_assign_pointer(per_cpu(sd_asym, cpu), sd); ++} ++ ++/* ++ * Attach the domain 'sd' to 'cpu' as its base domain. Callers must ++ * hold the hotplug lock. ++ */ ++static void ++cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu) ++{ ++ struct rq *rq = cpu_rq(cpu); ++ struct sched_domain *tmp; ++ ++ /* Remove the sched domains which do not contribute to scheduling. */ ++ for (tmp = sd; tmp; ) { ++ struct sched_domain *parent = tmp->parent; ++ if (!parent) ++ break; ++ ++ if (sd_parent_degenerate(tmp, parent)) { ++ tmp->parent = parent->parent; ++ if (parent->parent) ++ parent->parent->child = tmp; ++ /* ++ * Transfer SD_PREFER_SIBLING down in case of a ++ * degenerate parent; the spans match for this ++ * so the property transfers. ++ */ ++ if (parent->flags & SD_PREFER_SIBLING) ++ tmp->flags |= SD_PREFER_SIBLING; ++ destroy_sched_domain(parent, cpu); ++ } else ++ tmp = tmp->parent; ++ } ++ ++ if (sd && sd_degenerate(sd)) { ++ tmp = sd; ++ sd = sd->parent; ++ destroy_sched_domain(tmp, cpu); ++ if (sd) ++ sd->child = NULL; ++ } ++ ++ sched_domain_debug(sd, cpu); ++ ++ rq_attach_root(rq, rd); ++ tmp = rq->sd; ++ rcu_assign_pointer(rq->sd, sd); ++ destroy_sched_domains(tmp, cpu); ++ ++ update_top_cache_domain(cpu); ++} ++ ++/* Setup the mask of cpus configured for isolated domains */ ++static int __init isolated_cpu_setup(char *str) ++{ ++ alloc_bootmem_cpumask_var(&cpu_isolated_map); ++ cpulist_parse(str, cpu_isolated_map); ++ return 1; ++} ++ ++__setup("isolcpus=", isolated_cpu_setup); ++ ++struct s_data { ++ struct sched_domain ** __percpu sd; ++ struct root_domain *rd; ++}; ++ ++enum s_alloc { ++ sa_rootdomain, ++ sa_sd, ++ sa_sd_storage, ++ sa_none, ++}; ++ ++/* ++ * Build an iteration mask that can exclude certain CPUs from the upwards ++ * domain traversal. ++ * ++ * Asymmetric node setups can result in situations where the domain tree is of ++ * unequal depth, make sure to skip domains that already cover the entire ++ * range. ++ * ++ * In that case build_sched_domains() will have terminated the iteration early ++ * and our sibling sd spans will be empty. Domains should always include the ++ * cpu they're built on, so check that. ++ * ++ */ ++static void build_group_mask(struct sched_domain *sd, struct sched_group *sg) ++{ ++ const struct cpumask *span = sched_domain_span(sd); ++ struct sd_data *sdd = sd->private; ++ struct sched_domain *sibling; ++ int i; ++ ++ for_each_cpu(i, span) { ++ sibling = *per_cpu_ptr(sdd->sd, i); ++ if (!cpumask_test_cpu(i, sched_domain_span(sibling))) ++ continue; ++ ++ cpumask_set_cpu(i, sched_group_mask(sg)); ++ } ++} ++ ++/* ++ * Return the canonical balance cpu for this group, this is the first cpu ++ * of this group that's also in the iteration mask. ++ */ ++int group_balance_cpu(struct sched_group *sg) ++{ ++ return cpumask_first_and(sched_group_cpus(sg), sched_group_mask(sg)); ++} ++ ++static int ++build_overlap_sched_groups(struct sched_domain *sd, int cpu) ++{ ++ struct sched_group *first = NULL, *last = NULL, *groups = NULL, *sg; ++ const struct cpumask *span = sched_domain_span(sd); ++ struct cpumask *covered = sched_domains_tmpmask; ++ struct sd_data *sdd = sd->private; ++ struct sched_domain *sibling; ++ int i; ++ ++ cpumask_clear(covered); ++ ++ for_each_cpu(i, span) { ++ struct cpumask *sg_span; ++ ++ if (cpumask_test_cpu(i, covered)) ++ continue; ++ ++ sibling = *per_cpu_ptr(sdd->sd, i); ++ ++ /* See the comment near build_group_mask(). */ ++ if (!cpumask_test_cpu(i, sched_domain_span(sibling))) ++ continue; ++ ++ sg = kzalloc_node(sizeof(struct sched_group) + cpumask_size(), ++ GFP_KERNEL, cpu_to_node(cpu)); ++ ++ if (!sg) ++ goto fail; ++ ++ sg_span = sched_group_cpus(sg); ++ if (sibling->child) ++ cpumask_copy(sg_span, sched_domain_span(sibling->child)); ++ else ++ cpumask_set_cpu(i, sg_span); ++ ++ cpumask_or(covered, covered, sg_span); ++ ++ sg->sgc = *per_cpu_ptr(sdd->sgc, i); ++ if (atomic_inc_return(&sg->sgc->ref) == 1) ++ build_group_mask(sd, sg); ++ ++ /* ++ * Initialize sgc->capacity such that even if we mess up the ++ * domains and no possible iteration will get us here, we won't ++ * die on a /0 trap. ++ */ ++ sg->sgc->capacity = SCHED_CAPACITY_SCALE * cpumask_weight(sg_span); ++ ++ /* ++ * Make sure the first group of this domain contains the ++ * canonical balance cpu. Otherwise the sched_domain iteration ++ * breaks. See update_sg_lb_stats(). ++ */ ++ if ((!groups && cpumask_test_cpu(cpu, sg_span)) || ++ group_balance_cpu(sg) == cpu) ++ groups = sg; ++ ++ if (!first) ++ first = sg; ++ if (last) ++ last->next = sg; ++ last = sg; ++ last->next = first; ++ } ++ sd->groups = groups; ++ ++ return 0; ++ ++fail: ++ free_sched_groups(first, 0); ++ ++ return -ENOMEM; ++} ++ ++static int get_group(int cpu, struct sd_data *sdd, struct sched_group **sg) ++{ ++ struct sched_domain *sd = *per_cpu_ptr(sdd->sd, cpu); ++ struct sched_domain *child = sd->child; ++ ++ if (child) ++ cpu = cpumask_first(sched_domain_span(child)); ++ ++ if (sg) { ++ *sg = *per_cpu_ptr(sdd->sg, cpu); ++ (*sg)->sgc = *per_cpu_ptr(sdd->sgc, cpu); ++ atomic_set(&(*sg)->sgc->ref, 1); /* for claim_allocations */ ++ } ++ ++ return cpu; ++} ++ ++/* ++ * build_sched_groups will build a circular linked list of the groups ++ * covered by the given span, and will set each group's ->cpumask correctly, ++ * and ->cpu_capacity to 0. ++ * ++ * Assumes the sched_domain tree is fully constructed ++ */ ++static int ++build_sched_groups(struct sched_domain *sd, int cpu) ++{ ++ struct sched_group *first = NULL, *last = NULL; ++ struct sd_data *sdd = sd->private; ++ const struct cpumask *span = sched_domain_span(sd); ++ struct cpumask *covered; ++ int i; ++ ++ get_group(cpu, sdd, &sd->groups); ++ atomic_inc(&sd->groups->ref); ++ ++ if (cpu != cpumask_first(span)) ++ return 0; ++ ++ lockdep_assert_held(&sched_domains_mutex); ++ covered = sched_domains_tmpmask; ++ ++ cpumask_clear(covered); ++ ++ for_each_cpu(i, span) { ++ struct sched_group *sg; ++ int group, j; ++ ++ if (cpumask_test_cpu(i, covered)) ++ continue; ++ ++ group = get_group(i, sdd, &sg); ++ cpumask_setall(sched_group_mask(sg)); ++ ++ for_each_cpu(j, span) { ++ if (get_group(j, sdd, NULL) != group) ++ continue; ++ ++ cpumask_set_cpu(j, covered); ++ cpumask_set_cpu(j, sched_group_cpus(sg)); ++ } ++ ++ if (!first) ++ first = sg; ++ if (last) ++ last->next = sg; ++ last = sg; ++ } ++ last->next = first; ++ ++ return 0; ++} ++ ++/* ++ * Initialize sched groups cpu_capacity. ++ * ++ * cpu_capacity indicates the capacity of sched group, which is used while ++ * distributing the load between different sched groups in a sched domain. ++ * Typically cpu_capacity for all the groups in a sched domain will be same ++ * unless there are asymmetries in the topology. If there are asymmetries, ++ * group having more cpu_capacity will pickup more load compared to the ++ * group having less cpu_capacity. ++ */ ++static void init_sched_groups_capacity(int cpu, struct sched_domain *sd) ++{ ++ struct sched_group *sg = sd->groups; ++ ++ WARN_ON(!sg); ++ ++ do { ++ sg->group_weight = cpumask_weight(sched_group_cpus(sg)); ++ sg = sg->next; ++ } while (sg != sd->groups); ++ ++ if (cpu != group_balance_cpu(sg)) ++ return; ++ ++ update_group_capacity(sd, cpu); ++ atomic_set(&sg->sgc->nr_busy_cpus, sg->group_weight); ++} ++ ++/* ++ * Initializers for schedule domains ++ * Non-inlined to reduce accumulated stack pressure in build_sched_domains() ++ */ ++ ++static int default_relax_domain_level = -1; ++int sched_domain_level_max; ++ ++static int __init setup_relax_domain_level(char *str) ++{ ++ if (kstrtoint(str, 0, &default_relax_domain_level)) ++ pr_warn("Unable to set relax_domain_level\n"); ++ ++ return 1; ++} ++__setup("relax_domain_level=", setup_relax_domain_level); ++ ++static void set_domain_attribute(struct sched_domain *sd, ++ struct sched_domain_attr *attr) ++{ ++ int request; ++ ++ if (!attr || attr->relax_domain_level < 0) { ++ if (default_relax_domain_level < 0) ++ return; ++ else ++ request = default_relax_domain_level; ++ } else ++ request = attr->relax_domain_level; ++ if (request < sd->level) { ++ /* turn off idle balance on this domain */ ++ sd->flags &= ~(SD_BALANCE_WAKE|SD_BALANCE_NEWIDLE); ++ } else { ++ /* turn on idle balance on this domain */ ++ sd->flags |= (SD_BALANCE_WAKE|SD_BALANCE_NEWIDLE); ++ } ++} ++ ++static void __sdt_free(const struct cpumask *cpu_map); ++static int __sdt_alloc(const struct cpumask *cpu_map); ++ ++static void __free_domain_allocs(struct s_data *d, enum s_alloc what, ++ const struct cpumask *cpu_map) ++{ ++ switch (what) { ++ case sa_rootdomain: ++ if (!atomic_read(&d->rd->refcount)) ++ free_rootdomain(&d->rd->rcu); /* fall through */ ++ case sa_sd: ++ free_percpu(d->sd); /* fall through */ ++ case sa_sd_storage: ++ __sdt_free(cpu_map); /* fall through */ ++ case sa_none: ++ break; ++ } ++} ++ ++static enum s_alloc __visit_domain_allocation_hell(struct s_data *d, ++ const struct cpumask *cpu_map) ++{ ++ memset(d, 0, sizeof(*d)); ++ ++ if (__sdt_alloc(cpu_map)) ++ return sa_sd_storage; ++ d->sd = alloc_percpu(struct sched_domain *); ++ if (!d->sd) ++ return sa_sd_storage; ++ d->rd = alloc_rootdomain(); ++ if (!d->rd) ++ return sa_sd; ++ return sa_rootdomain; ++} ++ ++/* ++ * NULL the sd_data elements we've used to build the sched_domain and ++ * sched_group structure so that the subsequent __free_domain_allocs() ++ * will not free the data we're using. ++ */ ++static void claim_allocations(int cpu, struct sched_domain *sd) ++{ ++ struct sd_data *sdd = sd->private; ++ ++ WARN_ON_ONCE(*per_cpu_ptr(sdd->sd, cpu) != sd); ++ *per_cpu_ptr(sdd->sd, cpu) = NULL; ++ ++ if (atomic_read(&(*per_cpu_ptr(sdd->sg, cpu))->ref)) ++ *per_cpu_ptr(sdd->sg, cpu) = NULL; ++ ++ if (atomic_read(&(*per_cpu_ptr(sdd->sgc, cpu))->ref)) ++ *per_cpu_ptr(sdd->sgc, cpu) = NULL; ++} ++ ++#ifdef CONFIG_NUMA ++static int sched_domains_numa_levels; ++enum numa_topology_type sched_numa_topology_type; ++static int *sched_domains_numa_distance; ++int sched_max_numa_distance; ++static struct cpumask ***sched_domains_numa_masks; ++static int sched_domains_curr_level; ++#endif ++ ++/* ++ * SD_flags allowed in topology descriptions. ++ * ++ * SD_SHARE_CPUCAPACITY - describes SMT topologies ++ * SD_SHARE_PKG_RESOURCES - describes shared caches ++ * SD_NUMA - describes NUMA topologies ++ * SD_SHARE_POWERDOMAIN - describes shared power domain ++ * ++ * Odd one out: ++ * SD_ASYM_PACKING - describes SMT quirks ++ */ ++#define TOPOLOGY_SD_FLAGS \ ++ (SD_SHARE_CPUCAPACITY | \ ++ SD_SHARE_PKG_RESOURCES | \ ++ SD_NUMA | \ ++ SD_ASYM_PACKING | \ ++ SD_SHARE_POWERDOMAIN) ++ ++static struct sched_domain * ++sd_init(struct sched_domain_topology_level *tl, int cpu) ++{ ++ struct sched_domain *sd = *per_cpu_ptr(tl->data.sd, cpu); ++ int sd_weight, sd_flags = 0; ++ ++#ifdef CONFIG_NUMA ++ /* ++ * Ugly hack to pass state to sd_numa_mask()... ++ */ ++ sched_domains_curr_level = tl->numa_level; ++#endif ++ ++ sd_weight = cpumask_weight(tl->mask(cpu)); ++ ++ if (tl->sd_flags) ++ sd_flags = (*tl->sd_flags)(); ++ if (WARN_ONCE(sd_flags & ~TOPOLOGY_SD_FLAGS, ++ "wrong sd_flags in topology description\n")) ++ sd_flags &= ~TOPOLOGY_SD_FLAGS; ++ ++ *sd = (struct sched_domain){ ++ .min_interval = sd_weight, ++ .max_interval = 2*sd_weight, ++ .busy_factor = 32, ++ .imbalance_pct = 125, ++ ++ .cache_nice_tries = 0, ++ .busy_idx = 0, ++ .idle_idx = 0, ++ .newidle_idx = 0, ++ .wake_idx = 0, ++ .forkexec_idx = 0, ++ ++ .flags = 1*SD_LOAD_BALANCE ++ | 1*SD_BALANCE_NEWIDLE ++ | 1*SD_BALANCE_EXEC ++ | 1*SD_BALANCE_FORK ++ | 0*SD_BALANCE_WAKE ++ | 1*SD_WAKE_AFFINE ++ | 0*SD_SHARE_CPUCAPACITY ++ | 0*SD_SHARE_PKG_RESOURCES ++ | 0*SD_SERIALIZE ++ | 0*SD_PREFER_SIBLING ++ | 0*SD_NUMA ++ | sd_flags ++ , ++ ++ .last_balance = jiffies, ++ .balance_interval = sd_weight, ++ .smt_gain = 0, ++ .max_newidle_lb_cost = 0, ++ .next_decay_max_lb_cost = jiffies, ++#ifdef CONFIG_SCHED_DEBUG ++ .name = tl->name, ++#endif ++ }; ++ ++ /* ++ * Convert topological properties into behaviour. ++ */ ++ ++ if (sd->flags & SD_SHARE_CPUCAPACITY) { ++ sd->flags |= SD_PREFER_SIBLING; ++ sd->imbalance_pct = 110; ++ sd->smt_gain = 1178; /* ~15% */ ++ ++ } else if (sd->flags & SD_SHARE_PKG_RESOURCES) { ++ sd->imbalance_pct = 117; ++ sd->cache_nice_tries = 1; ++ sd->busy_idx = 2; ++ ++#ifdef CONFIG_NUMA ++ } else if (sd->flags & SD_NUMA) { ++ sd->cache_nice_tries = 2; ++ sd->busy_idx = 3; ++ sd->idle_idx = 2; ++ ++ sd->flags |= SD_SERIALIZE; ++ if (sched_domains_numa_distance[tl->numa_level] > RECLAIM_DISTANCE) { ++ sd->flags &= ~(SD_BALANCE_EXEC | ++ SD_BALANCE_FORK | ++ SD_WAKE_AFFINE); ++ } ++ ++#endif ++ } else { ++ sd->flags |= SD_PREFER_SIBLING; ++ sd->cache_nice_tries = 1; ++ sd->busy_idx = 2; ++ sd->idle_idx = 1; ++ } ++ ++ sd->private = &tl->data; ++ ++ return sd; ++} ++ ++/* ++ * Topology list, bottom-up. ++ */ ++static struct sched_domain_topology_level default_topology[] = { ++#ifdef CONFIG_SCHED_SMT ++ { cpu_smt_mask, cpu_smt_flags, SD_INIT_NAME(SMT) }, ++#endif ++#ifdef CONFIG_SCHED_MC ++ { cpu_coregroup_mask, cpu_core_flags, SD_INIT_NAME(MC) }, ++#endif ++ { cpu_cpu_mask, SD_INIT_NAME(DIE) }, ++ { NULL, }, ++}; ++ ++struct sched_domain_topology_level *sched_domain_topology = default_topology; ++ ++#define for_each_sd_topology(tl) \ ++ for (tl = sched_domain_topology; tl->mask; tl++) ++ ++void set_sched_topology(struct sched_domain_topology_level *tl) ++{ ++ sched_domain_topology = tl; ++} ++ ++#ifdef CONFIG_NUMA ++ ++static const struct cpumask *sd_numa_mask(int cpu) ++{ ++ return sched_domains_numa_masks[sched_domains_curr_level][cpu_to_node(cpu)]; ++} ++ ++static void sched_numa_warn(const char *str) ++{ ++ static int done = false; ++ int i,j; ++ ++ if (done) ++ return; ++ ++ done = true; ++ ++ printk(KERN_WARNING "ERROR: %s\n\n", str); ++ ++ for (i = 0; i < nr_node_ids; i++) { ++ printk(KERN_WARNING " "); ++ for (j = 0; j < nr_node_ids; j++) ++ printk(KERN_CONT "%02d ", node_distance(i,j)); ++ printk(KERN_CONT "\n"); ++ } ++ printk(KERN_WARNING "\n"); ++} ++ ++bool find_numa_distance(int distance) ++{ ++ int i; ++ ++ if (distance == node_distance(0, 0)) ++ return true; ++ ++ for (i = 0; i < sched_domains_numa_levels; i++) { ++ if (sched_domains_numa_distance[i] == distance) ++ return true; ++ } ++ ++ return false; ++} ++ ++/* ++ * A system can have three types of NUMA topology: ++ * NUMA_DIRECT: all nodes are directly connected, or not a NUMA system ++ * NUMA_GLUELESS_MESH: some nodes reachable through intermediary nodes ++ * NUMA_BACKPLANE: nodes can reach other nodes through a backplane ++ * ++ * The difference between a glueless mesh topology and a backplane ++ * topology lies in whether communication between not directly ++ * connected nodes goes through intermediary nodes (where programs ++ * could run), or through backplane controllers. This affects ++ * placement of programs. ++ * ++ * The type of topology can be discerned with the following tests: ++ * - If the maximum distance between any nodes is 1 hop, the system ++ * is directly connected. ++ * - If for two nodes A and B, located N > 1 hops away from each other, ++ * there is an intermediary node C, which is < N hops away from both ++ * nodes A and B, the system is a glueless mesh. ++ */ ++static void init_numa_topology_type(void) ++{ ++ int a, b, c, n; ++ ++ n = sched_max_numa_distance; ++ ++ if (n <= 1) ++ sched_numa_topology_type = NUMA_DIRECT; ++ ++ for_each_online_node(a) { ++ for_each_online_node(b) { ++ /* Find two nodes furthest removed from each other. */ ++ if (node_distance(a, b) < n) ++ continue; ++ ++ /* Is there an intermediary node between a and b? */ ++ for_each_online_node(c) { ++ if (node_distance(a, c) < n && ++ node_distance(b, c) < n) { ++ sched_numa_topology_type = ++ NUMA_GLUELESS_MESH; ++ return; ++ } ++ } ++ ++ sched_numa_topology_type = NUMA_BACKPLANE; ++ return; ++ } ++ } ++} ++ ++static void sched_init_numa(void) ++{ ++ int next_distance, curr_distance = node_distance(0, 0); ++ struct sched_domain_topology_level *tl; ++ int level = 0; ++ int i, j, k; ++ ++ sched_domains_numa_distance = kzalloc(sizeof(int) * nr_node_ids, GFP_KERNEL); ++ if (!sched_domains_numa_distance) ++ return; ++ ++ /* ++ * O(nr_nodes^2) deduplicating selection sort -- in order to find the ++ * unique distances in the node_distance() table. ++ * ++ * Assumes node_distance(0,j) includes all distances in ++ * node_distance(i,j) in order to avoid cubic time. ++ */ ++ next_distance = curr_distance; ++ for (i = 0; i < nr_node_ids; i++) { ++ for (j = 0; j < nr_node_ids; j++) { ++ for (k = 0; k < nr_node_ids; k++) { ++ int distance = node_distance(i, k); ++ ++ if (distance > curr_distance && ++ (distance < next_distance || ++ next_distance == curr_distance)) ++ next_distance = distance; ++ ++ /* ++ * While not a strong assumption it would be nice to know ++ * about cases where if node A is connected to B, B is not ++ * equally connected to A. ++ */ ++ if (sched_debug() && node_distance(k, i) != distance) ++ sched_numa_warn("Node-distance not symmetric"); ++ ++ if (sched_debug() && i && !find_numa_distance(distance)) ++ sched_numa_warn("Node-0 not representative"); ++ } ++ if (next_distance != curr_distance) { ++ sched_domains_numa_distance[level++] = next_distance; ++ sched_domains_numa_levels = level; ++ curr_distance = next_distance; ++ } else break; ++ } ++ ++ /* ++ * In case of sched_debug() we verify the above assumption. ++ */ ++ if (!sched_debug()) ++ break; ++ } ++ ++ if (!level) ++ return; ++ ++ /* ++ * 'level' contains the number of unique distances, excluding the ++ * identity distance node_distance(i,i). ++ * ++ * The sched_domains_numa_distance[] array includes the actual distance ++ * numbers. ++ */ ++ ++ /* ++ * Here, we should temporarily reset sched_domains_numa_levels to 0. ++ * If it fails to allocate memory for array sched_domains_numa_masks[][], ++ * the array will contain less then 'level' members. This could be ++ * dangerous when we use it to iterate array sched_domains_numa_masks[][] ++ * in other functions. ++ * ++ * We reset it to 'level' at the end of this function. ++ */ ++ sched_domains_numa_levels = 0; ++ ++ sched_domains_numa_masks = kzalloc(sizeof(void *) * level, GFP_KERNEL); ++ if (!sched_domains_numa_masks) ++ return; ++ ++ /* ++ * Now for each level, construct a mask per node which contains all ++ * cpus of nodes that are that many hops away from us. ++ */ ++ for (i = 0; i < level; i++) { ++ sched_domains_numa_masks[i] = ++ kzalloc(nr_node_ids * sizeof(void *), GFP_KERNEL); ++ if (!sched_domains_numa_masks[i]) ++ return; ++ ++ for (j = 0; j < nr_node_ids; j++) { ++ struct cpumask *mask = kzalloc(cpumask_size(), GFP_KERNEL); ++ if (!mask) ++ return; ++ ++ sched_domains_numa_masks[i][j] = mask; ++ ++ for (k = 0; k < nr_node_ids; k++) { ++ if (node_distance(j, k) > sched_domains_numa_distance[i]) ++ continue; ++ ++ cpumask_or(mask, mask, cpumask_of_node(k)); ++ } ++ } ++ } ++ ++ /* Compute default topology size */ ++ for (i = 0; sched_domain_topology[i].mask; i++); ++ ++ tl = kzalloc((i + level + 1) * ++ sizeof(struct sched_domain_topology_level), GFP_KERNEL); ++ if (!tl) ++ return; ++ ++ /* ++ * Copy the default topology bits.. ++ */ ++ for (i = 0; sched_domain_topology[i].mask; i++) ++ tl[i] = sched_domain_topology[i]; ++ ++ /* ++ * .. and append 'j' levels of NUMA goodness. ++ */ ++ for (j = 0; j < level; i++, j++) { ++ tl[i] = (struct sched_domain_topology_level){ ++ .mask = sd_numa_mask, ++ .sd_flags = cpu_numa_flags, ++ .flags = SDTL_OVERLAP, ++ .numa_level = j, ++ SD_INIT_NAME(NUMA) ++ }; ++ } ++ ++ sched_domain_topology = tl; ++ ++ sched_domains_numa_levels = level; ++ sched_max_numa_distance = sched_domains_numa_distance[level - 1]; ++ ++ init_numa_topology_type(); ++} ++ ++static void sched_domains_numa_masks_set(int cpu) ++{ ++ int i, j; ++ int node = cpu_to_node(cpu); ++ ++ for (i = 0; i < sched_domains_numa_levels; i++) { ++ for (j = 0; j < nr_node_ids; j++) { ++ if (node_distance(j, node) <= sched_domains_numa_distance[i]) ++ cpumask_set_cpu(cpu, sched_domains_numa_masks[i][j]); ++ } ++ } ++} ++ ++static void sched_domains_numa_masks_clear(int cpu) ++{ ++ int i, j; ++ for (i = 0; i < sched_domains_numa_levels; i++) { ++ for (j = 0; j < nr_node_ids; j++) ++ cpumask_clear_cpu(cpu, sched_domains_numa_masks[i][j]); ++ } ++} ++ ++/* ++ * Update sched_domains_numa_masks[level][node] array when new cpus ++ * are onlined. ++ */ ++static int sched_domains_numa_masks_update(struct notifier_block *nfb, ++ unsigned long action, ++ void *hcpu) ++{ ++ int cpu = (long)hcpu; ++ ++ switch (action & ~CPU_TASKS_FROZEN) { ++ case CPU_ONLINE: ++ sched_domains_numa_masks_set(cpu); ++ break; ++ ++ case CPU_DEAD: ++ sched_domains_numa_masks_clear(cpu); ++ break; ++ ++ default: ++ return NOTIFY_DONE; ++ } ++ ++ return NOTIFY_OK; ++} ++#else ++static inline void sched_init_numa(void) ++{ ++} ++ ++static int sched_domains_numa_masks_update(struct notifier_block *nfb, ++ unsigned long action, ++ void *hcpu) ++{ ++ return 0; ++} ++#endif /* CONFIG_NUMA */ ++ ++static int __sdt_alloc(const struct cpumask *cpu_map) ++{ ++ struct sched_domain_topology_level *tl; ++ int j; ++ ++ for_each_sd_topology(tl) { ++ struct sd_data *sdd = &tl->data; ++ ++ sdd->sd = alloc_percpu(struct sched_domain *); ++ if (!sdd->sd) ++ return -ENOMEM; ++ ++ sdd->sg = alloc_percpu(struct sched_group *); ++ if (!sdd->sg) ++ return -ENOMEM; ++ ++ sdd->sgc = alloc_percpu(struct sched_group_capacity *); ++ if (!sdd->sgc) ++ return -ENOMEM; ++ ++ for_each_cpu(j, cpu_map) { ++ struct sched_domain *sd; ++ struct sched_group *sg; ++ struct sched_group_capacity *sgc; ++ ++ sd = kzalloc_node(sizeof(struct sched_domain) + cpumask_size(), ++ GFP_KERNEL, cpu_to_node(j)); ++ if (!sd) ++ return -ENOMEM; ++ ++ *per_cpu_ptr(sdd->sd, j) = sd; ++ ++ sg = kzalloc_node(sizeof(struct sched_group) + cpumask_size(), ++ GFP_KERNEL, cpu_to_node(j)); ++ if (!sg) ++ return -ENOMEM; ++ ++ sg->next = sg; ++ ++ *per_cpu_ptr(sdd->sg, j) = sg; ++ ++ sgc = kzalloc_node(sizeof(struct sched_group_capacity) + cpumask_size(), ++ GFP_KERNEL, cpu_to_node(j)); ++ if (!sgc) ++ return -ENOMEM; ++ ++ *per_cpu_ptr(sdd->sgc, j) = sgc; ++ } ++ } ++ ++ return 0; ++} ++ ++static void __sdt_free(const struct cpumask *cpu_map) ++{ ++ struct sched_domain_topology_level *tl; ++ int j; ++ ++ for_each_sd_topology(tl) { ++ struct sd_data *sdd = &tl->data; ++ ++ for_each_cpu(j, cpu_map) { ++ struct sched_domain *sd; ++ ++ if (sdd->sd) { ++ sd = *per_cpu_ptr(sdd->sd, j); ++ if (sd && (sd->flags & SD_OVERLAP)) ++ free_sched_groups(sd->groups, 0); ++ kfree(*per_cpu_ptr(sdd->sd, j)); ++ } ++ ++ if (sdd->sg) ++ kfree(*per_cpu_ptr(sdd->sg, j)); ++ if (sdd->sgc) ++ kfree(*per_cpu_ptr(sdd->sgc, j)); ++ } ++ free_percpu(sdd->sd); ++ sdd->sd = NULL; ++ free_percpu(sdd->sg); ++ sdd->sg = NULL; ++ free_percpu(sdd->sgc); ++ sdd->sgc = NULL; ++ } ++} ++ ++struct sched_domain *build_sched_domain(struct sched_domain_topology_level *tl, ++ const struct cpumask *cpu_map, struct sched_domain_attr *attr, ++ struct sched_domain *child, int cpu) ++{ ++ struct sched_domain *sd = sd_init(tl, cpu); ++ if (!sd) ++ return child; ++ ++ cpumask_and(sched_domain_span(sd), cpu_map, tl->mask(cpu)); ++ if (child) { ++ sd->level = child->level + 1; ++ sched_domain_level_max = max(sched_domain_level_max, sd->level); ++ child->parent = sd; ++ sd->child = child; ++ ++ if (!cpumask_subset(sched_domain_span(child), ++ sched_domain_span(sd))) { ++ pr_err("BUG: arch topology borken\n"); ++#ifdef CONFIG_SCHED_DEBUG ++ pr_err(" the %s domain not a subset of the %s domain\n", ++ child->name, sd->name); ++#endif ++ /* Fixup, ensure @sd has at least @child cpus. */ ++ cpumask_or(sched_domain_span(sd), ++ sched_domain_span(sd), ++ sched_domain_span(child)); ++ } ++ ++ } ++ set_domain_attribute(sd, attr); ++ ++ return sd; ++} ++ ++/* ++ * Build sched domains for a given set of cpus and attach the sched domains ++ * to the individual cpus ++ */ ++static int build_sched_domains(const struct cpumask *cpu_map, ++ struct sched_domain_attr *attr) ++{ ++ enum s_alloc alloc_state; ++ struct sched_domain *sd; ++ struct s_data d; ++ int i, ret = -ENOMEM; ++ ++ alloc_state = __visit_domain_allocation_hell(&d, cpu_map); ++ if (alloc_state != sa_rootdomain) ++ goto error; ++ ++ /* Set up domains for cpus specified by the cpu_map. */ ++ for_each_cpu(i, cpu_map) { ++ struct sched_domain_topology_level *tl; ++ ++ sd = NULL; ++ for_each_sd_topology(tl) { ++ sd = build_sched_domain(tl, cpu_map, attr, sd, i); ++ if (tl == sched_domain_topology) ++ *per_cpu_ptr(d.sd, i) = sd; ++ if (tl->flags & SDTL_OVERLAP || sched_feat(FORCE_SD_OVERLAP)) ++ sd->flags |= SD_OVERLAP; ++ if (cpumask_equal(cpu_map, sched_domain_span(sd))) ++ break; ++ } ++ } ++ ++ /* Build the groups for the domains */ ++ for_each_cpu(i, cpu_map) { ++ for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) { ++ sd->span_weight = cpumask_weight(sched_domain_span(sd)); ++ if (sd->flags & SD_OVERLAP) { ++ if (build_overlap_sched_groups(sd, i)) ++ goto error; ++ } else { ++ if (build_sched_groups(sd, i)) ++ goto error; ++ } ++ } ++ } ++ ++ /* Calculate CPU capacity for physical packages and nodes */ ++ for (i = nr_cpumask_bits-1; i >= 0; i--) { ++ if (!cpumask_test_cpu(i, cpu_map)) ++ continue; ++ ++ for (sd = *per_cpu_ptr(d.sd, i); sd; sd = sd->parent) { ++ claim_allocations(i, sd); ++ init_sched_groups_capacity(i, sd); ++ } ++ } ++ ++ /* Attach the domains */ ++ rcu_read_lock(); ++ for_each_cpu(i, cpu_map) { ++ sd = *per_cpu_ptr(d.sd, i); ++ cpu_attach_domain(sd, d.rd, i); ++ } ++ rcu_read_unlock(); ++ ++ ret = 0; ++error: ++ __free_domain_allocs(&d, alloc_state, cpu_map); ++ return ret; ++} ++ ++static cpumask_var_t *doms_cur; /* current sched domains */ ++static int ndoms_cur; /* number of sched domains in 'doms_cur' */ ++static struct sched_domain_attr *dattr_cur; ++ /* attribues of custom domains in 'doms_cur' */ ++ ++/* ++ * Special case: If a kmalloc of a doms_cur partition (array of ++ * cpumask) fails, then fallback to a single sched domain, ++ * as determined by the single cpumask fallback_doms. ++ */ ++static cpumask_var_t fallback_doms; ++ ++/* ++ * arch_update_cpu_topology lets virtualized architectures update the ++ * cpu core maps. It is supposed to return 1 if the topology changed ++ * or 0 if it stayed the same. ++ */ ++int __weak arch_update_cpu_topology(void) ++{ ++ return 0; ++} ++ ++cpumask_var_t *alloc_sched_domains(unsigned int ndoms) ++{ ++ int i; ++ cpumask_var_t *doms; ++ ++ doms = kmalloc(sizeof(*doms) * ndoms, GFP_KERNEL); ++ if (!doms) ++ return NULL; ++ for (i = 0; i < ndoms; i++) { ++ if (!alloc_cpumask_var(&doms[i], GFP_KERNEL)) { ++ free_sched_domains(doms, i); ++ return NULL; ++ } ++ } ++ return doms; ++} ++ ++void free_sched_domains(cpumask_var_t doms[], unsigned int ndoms) ++{ ++ unsigned int i; ++ for (i = 0; i < ndoms; i++) ++ free_cpumask_var(doms[i]); ++ kfree(doms); ++} ++ ++/* ++ * Set up scheduler domains and groups. Callers must hold the hotplug lock. ++ * For now this just excludes isolated cpus, but could be used to ++ * exclude other special cases in the future. ++ */ ++static int init_sched_domains(const struct cpumask *cpu_map) ++{ ++ int err; ++ ++ arch_update_cpu_topology(); ++ ndoms_cur = 1; ++ doms_cur = alloc_sched_domains(ndoms_cur); ++ if (!doms_cur) ++ doms_cur = &fallback_doms; ++ cpumask_andnot(doms_cur[0], cpu_map, cpu_isolated_map); ++ err = build_sched_domains(doms_cur[0], NULL); ++ register_sched_domain_sysctl(); ++ ++ return err; ++} ++ ++/* ++ * Detach sched domains from a group of cpus specified in cpu_map ++ * These cpus will now be attached to the NULL domain ++ */ ++static void detach_destroy_domains(const struct cpumask *cpu_map) ++{ ++ int i; ++ ++ rcu_read_lock(); ++ for_each_cpu(i, cpu_map) ++ cpu_attach_domain(NULL, &def_root_domain, i); ++ rcu_read_unlock(); ++} ++ ++/* handle null as "default" */ ++static int dattrs_equal(struct sched_domain_attr *cur, int idx_cur, ++ struct sched_domain_attr *new, int idx_new) ++{ ++ struct sched_domain_attr tmp; ++ ++ /* fast path */ ++ if (!new && !cur) ++ return 1; ++ ++ tmp = SD_ATTR_INIT; ++ return !memcmp(cur ? (cur + idx_cur) : &tmp, ++ new ? (new + idx_new) : &tmp, ++ sizeof(struct sched_domain_attr)); ++} ++ ++/* ++ * Partition sched domains as specified by the 'ndoms_new' ++ * cpumasks in the array doms_new[] of cpumasks. This compares ++ * doms_new[] to the current sched domain partitioning, doms_cur[]. ++ * It destroys each deleted domain and builds each new domain. ++ * ++ * 'doms_new' is an array of cpumask_var_t's of length 'ndoms_new'. ++ * The masks don't intersect (don't overlap.) We should setup one ++ * sched domain for each mask. CPUs not in any of the cpumasks will ++ * not be load balanced. If the same cpumask appears both in the ++ * current 'doms_cur' domains and in the new 'doms_new', we can leave ++ * it as it is. ++ * ++ * The passed in 'doms_new' should be allocated using ++ * alloc_sched_domains. This routine takes ownership of it and will ++ * free_sched_domains it when done with it. If the caller failed the ++ * alloc call, then it can pass in doms_new == NULL && ndoms_new == 1, ++ * and partition_sched_domains() will fallback to the single partition ++ * 'fallback_doms', it also forces the domains to be rebuilt. ++ * ++ * If doms_new == NULL it will be replaced with cpu_online_mask. ++ * ndoms_new == 0 is a special case for destroying existing domains, ++ * and it will not create the default domain. ++ * ++ * Call with hotplug lock held ++ */ ++void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[], ++ struct sched_domain_attr *dattr_new) ++{ ++ int i, j, n; ++ int new_topology; ++ ++ mutex_lock(&sched_domains_mutex); ++ ++ /* always unregister in case we don't destroy any domains */ ++ unregister_sched_domain_sysctl(); ++ ++ /* Let architecture update cpu core mappings. */ ++ new_topology = arch_update_cpu_topology(); ++ ++ n = doms_new ? ndoms_new : 0; ++ ++ /* Destroy deleted domains */ ++ for (i = 0; i < ndoms_cur; i++) { ++ for (j = 0; j < n && !new_topology; j++) { ++ if (cpumask_equal(doms_cur[i], doms_new[j]) ++ && dattrs_equal(dattr_cur, i, dattr_new, j)) ++ goto match1; ++ } ++ /* no match - a current sched domain not in new doms_new[] */ ++ detach_destroy_domains(doms_cur[i]); ++match1: ++ ; ++ } ++ ++ n = ndoms_cur; ++ if (doms_new == NULL) { ++ n = 0; ++ doms_new = &fallback_doms; ++ cpumask_andnot(doms_new[0], cpu_active_mask, cpu_isolated_map); ++ WARN_ON_ONCE(dattr_new); ++ } ++ ++ /* Build new domains */ ++ for (i = 0; i < ndoms_new; i++) { ++ for (j = 0; j < n && !new_topology; j++) { ++ if (cpumask_equal(doms_new[i], doms_cur[j]) ++ && dattrs_equal(dattr_new, i, dattr_cur, j)) ++ goto match2; ++ } ++ /* no match - add a new doms_new */ ++ build_sched_domains(doms_new[i], dattr_new ? dattr_new + i : NULL); ++match2: ++ ; ++ } ++ ++ /* Remember the new sched domains */ ++ if (doms_cur != &fallback_doms) ++ free_sched_domains(doms_cur, ndoms_cur); ++ kfree(dattr_cur); /* kfree(NULL) is safe */ ++ doms_cur = doms_new; ++ dattr_cur = dattr_new; ++ ndoms_cur = ndoms_new; ++ ++ register_sched_domain_sysctl(); ++ ++ mutex_unlock(&sched_domains_mutex); ++} ++ ++static int num_cpus_frozen; /* used to mark begin/end of suspend/resume */ ++ ++/* ++ * Update cpusets according to cpu_active mask. If cpusets are ++ * disabled, cpuset_update_active_cpus() becomes a simple wrapper ++ * around partition_sched_domains(). ++ * ++ * If we come here as part of a suspend/resume, don't touch cpusets because we ++ * want to restore it back to its original state upon resume anyway. ++ */ ++static int cpuset_cpu_active(struct notifier_block *nfb, unsigned long action, ++ void *hcpu) ++{ ++ switch (action) { ++ case CPU_ONLINE_FROZEN: ++ case CPU_DOWN_FAILED_FROZEN: ++ ++ /* ++ * num_cpus_frozen tracks how many CPUs are involved in suspend ++ * resume sequence. As long as this is not the last online ++ * operation in the resume sequence, just build a single sched ++ * domain, ignoring cpusets. ++ */ ++ num_cpus_frozen--; ++ if (likely(num_cpus_frozen)) { ++ partition_sched_domains(1, NULL, NULL); ++ break; ++ } ++ ++ /* ++ * This is the last CPU online operation. So fall through and ++ * restore the original sched domains by considering the ++ * cpuset configurations. ++ */ ++ ++ case CPU_ONLINE: ++ cpuset_update_active_cpus(true); ++ break; ++ default: ++ return NOTIFY_DONE; ++ } ++ return NOTIFY_OK; ++} ++ ++static int cpuset_cpu_inactive(struct notifier_block *nfb, unsigned long action, ++ void *hcpu) ++{ ++ unsigned long flags; ++ long cpu = (long)hcpu; ++ struct dl_bw *dl_b; ++ bool overflow; ++ int cpus; ++ ++ switch (action) { ++ case CPU_DOWN_PREPARE: ++ rcu_read_lock_sched(); ++ dl_b = dl_bw_of(cpu); ++ ++ raw_spin_lock_irqsave(&dl_b->lock, flags); ++ cpus = dl_bw_cpus(cpu); ++ overflow = __dl_overflow(dl_b, cpus, 0, 0); ++ raw_spin_unlock_irqrestore(&dl_b->lock, flags); ++ ++ rcu_read_unlock_sched(); ++ ++ if (overflow) ++ return notifier_from_errno(-EBUSY); ++ cpuset_update_active_cpus(false); ++ break; ++ case CPU_DOWN_PREPARE_FROZEN: ++ num_cpus_frozen++; ++ partition_sched_domains(1, NULL, NULL); ++ break; ++ default: ++ return NOTIFY_DONE; ++ } ++ return NOTIFY_OK; ++} ++ ++void __init sched_init_smp(void) ++{ ++ cpumask_var_t non_isolated_cpus; ++ ++ alloc_cpumask_var(&non_isolated_cpus, GFP_KERNEL); ++ alloc_cpumask_var(&fallback_doms, GFP_KERNEL); ++ ++ sched_init_numa(); ++ ++ /* ++ * There's no userspace yet to cause hotplug operations; hence all the ++ * cpu masks are stable and all blatant races in the below code cannot ++ * happen. ++ */ ++ mutex_lock(&sched_domains_mutex); ++ init_sched_domains(cpu_active_mask); ++ cpumask_andnot(non_isolated_cpus, cpu_possible_mask, cpu_isolated_map); ++ if (cpumask_empty(non_isolated_cpus)) ++ cpumask_set_cpu(smp_processor_id(), non_isolated_cpus); ++ mutex_unlock(&sched_domains_mutex); ++ ++ hotcpu_notifier(sched_domains_numa_masks_update, CPU_PRI_SCHED_ACTIVE); ++ hotcpu_notifier(cpuset_cpu_active, CPU_PRI_CPUSET_ACTIVE); ++ hotcpu_notifier(cpuset_cpu_inactive, CPU_PRI_CPUSET_INACTIVE); ++ ++ init_hrtick(); ++ ++ /* Move init over to a non-isolated CPU */ ++ if (set_cpus_allowed_ptr(current, non_isolated_cpus) < 0) ++ BUG(); ++ sched_init_granularity(); ++ free_cpumask_var(non_isolated_cpus); ++ ++ init_sched_rt_class(); ++ init_sched_dl_class(); ++} ++#else ++void __init sched_init_smp(void) ++{ ++ sched_init_granularity(); ++} ++#endif /* CONFIG_SMP */ ++ ++const_debug unsigned int sysctl_timer_migration = 1; ++ ++int in_sched_functions(unsigned long addr) ++{ ++ return in_lock_functions(addr) || ++ (addr >= (unsigned long)__sched_text_start ++ && addr < (unsigned long)__sched_text_end); ++} ++ ++#ifdef CONFIG_CGROUP_SCHED ++/* ++ * Default task group. ++ * Every task in system belongs to this group at bootup. ++ */ ++struct task_group root_task_group; ++LIST_HEAD(task_groups); ++#endif ++ ++DECLARE_PER_CPU(cpumask_var_t, load_balance_mask); ++ ++void __init sched_init(void) ++{ ++ int i, j; ++ unsigned long alloc_size = 0, ptr; ++ ++#ifdef CONFIG_FAIR_GROUP_SCHED ++ alloc_size += 2 * nr_cpu_ids * sizeof(void **); ++#endif ++#ifdef CONFIG_RT_GROUP_SCHED ++ alloc_size += 2 * nr_cpu_ids * sizeof(void **); ++#endif ++ if (alloc_size) { ++ ptr = (unsigned long)kzalloc(alloc_size, GFP_NOWAIT); ++ ++#ifdef CONFIG_FAIR_GROUP_SCHED ++ root_task_group.se = (struct sched_entity **)ptr; ++ ptr += nr_cpu_ids * sizeof(void **); ++ ++ root_task_group.cfs_rq = (struct cfs_rq **)ptr; ++ ptr += nr_cpu_ids * sizeof(void **); ++ ++#endif /* CONFIG_FAIR_GROUP_SCHED */ ++#ifdef CONFIG_RT_GROUP_SCHED ++ root_task_group.rt_se = (struct sched_rt_entity **)ptr; ++ ptr += nr_cpu_ids * sizeof(void **); ++ ++ root_task_group.rt_rq = (struct rt_rq **)ptr; ++ ptr += nr_cpu_ids * sizeof(void **); ++ ++#endif /* CONFIG_RT_GROUP_SCHED */ ++ } ++#ifdef CONFIG_CPUMASK_OFFSTACK ++ for_each_possible_cpu(i) { ++ per_cpu(load_balance_mask, i) = (cpumask_var_t)kzalloc_node( ++ cpumask_size(), GFP_KERNEL, cpu_to_node(i)); ++ } ++#endif /* CONFIG_CPUMASK_OFFSTACK */ ++ ++ init_rt_bandwidth(&def_rt_bandwidth, ++ global_rt_period(), global_rt_runtime()); ++ init_dl_bandwidth(&def_dl_bandwidth, ++ global_rt_period(), global_rt_runtime()); ++ ++#ifdef CONFIG_SMP ++ init_defrootdomain(); ++#endif ++ ++#ifdef CONFIG_RT_GROUP_SCHED ++ init_rt_bandwidth(&root_task_group.rt_bandwidth, ++ global_rt_period(), global_rt_runtime()); ++#endif /* CONFIG_RT_GROUP_SCHED */ ++ ++#ifdef CONFIG_CGROUP_SCHED ++ list_add(&root_task_group.list, &task_groups); ++ INIT_LIST_HEAD(&root_task_group.children); ++ INIT_LIST_HEAD(&root_task_group.siblings); ++ autogroup_init(&init_task); ++ ++#endif /* CONFIG_CGROUP_SCHED */ ++ ++ for_each_possible_cpu(i) { ++ struct rq *rq; ++ ++ rq = cpu_rq(i); ++ raw_spin_lock_init(&rq->lock); ++ rq->nr_running = 0; ++ rq->calc_load_active = 0; ++ rq->calc_load_update = jiffies + LOAD_FREQ; ++ init_cfs_rq(&rq->cfs); ++ init_rt_rq(&rq->rt); ++ init_dl_rq(&rq->dl); ++#ifdef CONFIG_FAIR_GROUP_SCHED ++ root_task_group.shares = ROOT_TASK_GROUP_LOAD; ++ INIT_LIST_HEAD(&rq->leaf_cfs_rq_list); ++ /* ++ * How much cpu bandwidth does root_task_group get? ++ * ++ * In case of task-groups formed thr' the cgroup filesystem, it ++ * gets 100% of the cpu resources in the system. This overall ++ * system cpu resource is divided among the tasks of ++ * root_task_group and its child task-groups in a fair manner, ++ * based on each entity's (task or task-group's) weight ++ * (se->load.weight). ++ * ++ * In other words, if root_task_group has 10 tasks of weight ++ * 1024) and two child groups A0 and A1 (of weight 1024 each), ++ * then A0's share of the cpu resource is: ++ * ++ * A0's bandwidth = 1024 / (10*1024 + 1024 + 1024) = 8.33% ++ * ++ * We achieve this by letting root_task_group's tasks sit ++ * directly in rq->cfs (i.e root_task_group->se[] = NULL). ++ */ ++ init_cfs_bandwidth(&root_task_group.cfs_bandwidth); ++ init_tg_cfs_entry(&root_task_group, &rq->cfs, NULL, i, NULL); ++#endif /* CONFIG_FAIR_GROUP_SCHED */ ++ ++ rq->rt.rt_runtime = def_rt_bandwidth.rt_runtime; ++#ifdef CONFIG_RT_GROUP_SCHED ++ init_tg_rt_entry(&root_task_group, &rq->rt, NULL, i, NULL); ++#endif ++ ++ for (j = 0; j < CPU_LOAD_IDX_MAX; j++) ++ rq->cpu_load[j] = 0; ++ ++ rq->last_load_update_tick = jiffies; ++ ++#ifdef CONFIG_SMP ++ rq->sd = NULL; ++ rq->rd = NULL; ++ rq->cpu_capacity = rq->cpu_capacity_orig = SCHED_CAPACITY_SCALE; ++ rq->post_schedule = 0; ++ rq->active_balance = 0; ++ rq->next_balance = jiffies; ++ rq->push_cpu = 0; ++ rq->cpu = i; ++ rq->online = 0; ++ rq->idle_stamp = 0; ++ rq->avg_idle = 2*sysctl_sched_migration_cost; ++ rq->max_idle_balance_cost = sysctl_sched_migration_cost; ++ ++ INIT_LIST_HEAD(&rq->cfs_tasks); ++ ++ rq_attach_root(rq, &def_root_domain); ++#ifdef CONFIG_NO_HZ_COMMON ++ rq->nohz_flags = 0; ++#endif ++#ifdef CONFIG_NO_HZ_FULL ++ rq->last_sched_tick = 0; ++#endif ++#endif ++ init_rq_hrtick(rq); ++ atomic_set(&rq->nr_iowait, 0); ++ } ++ ++ set_load_weight(&init_task); ++ ++#ifdef CONFIG_PREEMPT_NOTIFIERS ++ INIT_HLIST_HEAD(&init_task.preempt_notifiers); ++#endif ++ ++ /* ++ * The boot idle thread does lazy MMU switching as well: ++ */ ++ atomic_inc(&init_mm.mm_count); ++ enter_lazy_tlb(&init_mm, current); ++ ++ /* ++ * During early bootup we pretend to be a normal task: ++ */ ++ current->sched_class = &fair_sched_class; ++ ++ /* ++ * Make us the idle thread. Technically, schedule() should not be ++ * called from this thread, however somewhere below it might be, ++ * but because we are the idle thread, we just pick up running again ++ * when this runqueue becomes "idle". ++ */ ++ init_idle(current, smp_processor_id()); ++ ++ calc_load_update = jiffies + LOAD_FREQ; ++ ++#ifdef CONFIG_SMP ++ zalloc_cpumask_var(&sched_domains_tmpmask, GFP_NOWAIT); ++ /* May be allocated at isolcpus cmdline parse time */ ++ if (cpu_isolated_map == NULL) ++ zalloc_cpumask_var(&cpu_isolated_map, GFP_NOWAIT); ++ idle_thread_set_boot_cpu(); ++ set_cpu_rq_start_time(); ++#endif ++ init_sched_fair_class(); ++ ++ scheduler_running = 1; ++} ++ ++#ifdef CONFIG_DEBUG_ATOMIC_SLEEP ++static inline int preempt_count_equals(int preempt_offset) ++{ ++ int nested = (preempt_count() & ~PREEMPT_ACTIVE) + rcu_preempt_depth(); ++ ++ return (nested == preempt_offset); ++} ++ ++void __might_sleep(const char *file, int line, int preempt_offset) ++{ ++ /* ++ * Blocking primitives will set (and therefore destroy) current->state, ++ * since we will exit with TASK_RUNNING make sure we enter with it, ++ * otherwise we will destroy state. ++ */ ++ WARN_ONCE(current->state != TASK_RUNNING && current->task_state_change, ++ "do not call blocking ops when !TASK_RUNNING; " ++ "state=%lx set at [<%p>] %pS\n", ++ current->state, ++ (void *)current->task_state_change, ++ (void *)current->task_state_change); ++ ++ ___might_sleep(file, line, preempt_offset); ++} ++EXPORT_SYMBOL(__might_sleep); ++ ++void ___might_sleep(const char *file, int line, int preempt_offset) ++{ ++ static unsigned long prev_jiffy; /* ratelimiting */ ++ ++ rcu_sleep_check(); /* WARN_ON_ONCE() by default, no rate limit reqd. */ ++ if ((preempt_count_equals(preempt_offset) && !irqs_disabled() && ++ !is_idle_task(current)) || ++ system_state != SYSTEM_RUNNING || oops_in_progress) ++ return; ++ if (time_before(jiffies, prev_jiffy + HZ) && prev_jiffy) ++ return; ++ prev_jiffy = jiffies; ++ ++ printk(KERN_ERR ++ "BUG: sleeping function called from invalid context at %s:%d\n", ++ file, line); ++ printk(KERN_ERR ++ "in_atomic(): %d, irqs_disabled(): %d, pid: %d, name: %s\n", ++ in_atomic(), irqs_disabled(), ++ current->pid, current->comm); ++ ++ if (task_stack_end_corrupted(current)) ++ printk(KERN_EMERG "Thread overran stack, or stack corrupted\n"); ++ ++ debug_show_held_locks(current); ++ if (irqs_disabled()) ++ print_irqtrace_events(current); ++#ifdef CONFIG_DEBUG_PREEMPT ++ if (!preempt_count_equals(preempt_offset)) { ++ pr_err("Preemption disabled at:"); ++ print_ip_sym(current->preempt_disable_ip); ++ pr_cont("\n"); ++ } ++#endif ++ dump_stack(); ++} ++EXPORT_SYMBOL(___might_sleep); ++#endif ++ ++#ifdef CONFIG_MAGIC_SYSRQ ++static void normalize_task(struct rq *rq, struct task_struct *p) ++{ ++ const struct sched_class *prev_class = p->sched_class; ++ struct sched_attr attr = { ++ .sched_policy = SCHED_NORMAL, ++ }; ++ int old_prio = p->prio; ++ int queued; ++ ++ queued = task_on_rq_queued(p); ++ if (queued) ++ dequeue_task(rq, p, 0); ++ __setscheduler(rq, p, &attr, false); ++ if (queued) { ++ enqueue_task(rq, p, 0); ++ resched_curr(rq); ++ } ++ ++ check_class_changed(rq, p, prev_class, old_prio); ++} ++ ++void normalize_rt_tasks(void) ++{ ++ struct task_struct *g, *p; ++ unsigned long flags; ++ struct rq *rq; ++ ++ read_lock(&tasklist_lock); ++ for_each_process_thread(g, p) { ++ /* ++ * Only normalize user tasks: ++ */ ++ if (p->flags & PF_KTHREAD) ++ continue; ++ ++ p->se.exec_start = 0; ++#ifdef CONFIG_SCHEDSTATS ++ p->se.statistics.wait_start = 0; ++ p->se.statistics.sleep_start = 0; ++ p->se.statistics.block_start = 0; ++#endif ++ ++ if (!dl_task(p) && !rt_task(p)) { ++ /* ++ * Renice negative nice level userspace ++ * tasks back to 0: ++ */ ++ if (task_nice(p) < 0) ++ set_user_nice(p, 0); ++ continue; ++ } ++ ++ rq = task_rq_lock(p, &flags); ++ normalize_task(rq, p); ++ task_rq_unlock(rq, p, &flags); ++ } ++ read_unlock(&tasklist_lock); ++} ++ ++#endif /* CONFIG_MAGIC_SYSRQ */ ++ ++#if defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB) ++/* ++ * These functions are only useful for the IA64 MCA handling, or kdb. ++ * ++ * They can only be called when the whole system has been ++ * stopped - every CPU needs to be quiescent, and no scheduling ++ * activity can take place. Using them for anything else would ++ * be a serious bug, and as a result, they aren't even visible ++ * under any other configuration. ++ */ ++ ++/** ++ * curr_task - return the current task for a given cpu. ++ * @cpu: the processor in question. ++ * ++ * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED! ++ * ++ * Return: The current task for @cpu. ++ */ ++struct task_struct *curr_task(int cpu) ++{ ++ return cpu_curr(cpu); ++} ++ ++#endif /* defined(CONFIG_IA64) || defined(CONFIG_KGDB_KDB) */ ++ ++#ifdef CONFIG_IA64 ++/** ++ * set_curr_task - set the current task for a given cpu. ++ * @cpu: the processor in question. ++ * @p: the task pointer to set. ++ * ++ * Description: This function must only be used when non-maskable interrupts ++ * are serviced on a separate stack. It allows the architecture to switch the ++ * notion of the current task on a cpu in a non-blocking manner. This function ++ * must be called with all CPU's synchronized, and interrupts disabled, the ++ * and caller must save the original value of the current task (see ++ * curr_task() above) and restore that value before reenabling interrupts and ++ * re-starting the system. ++ * ++ * ONLY VALID WHEN THE WHOLE SYSTEM IS STOPPED! ++ */ ++void set_curr_task(int cpu, struct task_struct *p) ++{ ++ cpu_curr(cpu) = p; ++} ++ ++#endif ++ ++#ifdef CONFIG_CGROUP_SCHED ++/* task_group_lock serializes the addition/removal of task groups */ ++static DEFINE_SPINLOCK(task_group_lock); ++ ++static void free_sched_group(struct task_group *tg) ++{ ++ free_fair_sched_group(tg); ++ free_rt_sched_group(tg); ++ autogroup_free(tg); ++ kfree(tg); ++} ++ ++/* allocate runqueue etc for a new task group */ ++struct task_group *sched_create_group(struct task_group *parent) ++{ ++ struct task_group *tg; ++ ++ tg = kzalloc(sizeof(*tg), GFP_KERNEL); ++ if (!tg) ++ return ERR_PTR(-ENOMEM); ++ ++ if (!alloc_fair_sched_group(tg, parent)) ++ goto err; ++ ++ if (!alloc_rt_sched_group(tg, parent)) ++ goto err; ++ ++ return tg; ++ ++err: ++ free_sched_group(tg); ++ return ERR_PTR(-ENOMEM); ++} ++ ++void sched_online_group(struct task_group *tg, struct task_group *parent) ++{ ++ unsigned long flags; ++ ++ spin_lock_irqsave(&task_group_lock, flags); ++ list_add_rcu(&tg->list, &task_groups); ++ ++ WARN_ON(!parent); /* root should already exist */ ++ ++ tg->parent = parent; ++ INIT_LIST_HEAD(&tg->children); ++ list_add_rcu(&tg->siblings, &parent->children); ++ spin_unlock_irqrestore(&task_group_lock, flags); ++} ++ ++/* rcu callback to free various structures associated with a task group */ ++static void free_sched_group_rcu(struct rcu_head *rhp) ++{ ++ /* now it should be safe to free those cfs_rqs */ ++ free_sched_group(container_of(rhp, struct task_group, rcu)); ++} ++ ++/* Destroy runqueue etc associated with a task group */ ++void sched_destroy_group(struct task_group *tg) ++{ ++ /* wait for possible concurrent references to cfs_rqs complete */ ++ call_rcu(&tg->rcu, free_sched_group_rcu); ++} ++ ++void sched_offline_group(struct task_group *tg) ++{ ++ unsigned long flags; ++ int i; ++ ++ /* end participation in shares distribution */ ++ for_each_possible_cpu(i) ++ unregister_fair_sched_group(tg, i); ++ ++ spin_lock_irqsave(&task_group_lock, flags); ++ list_del_rcu(&tg->list); ++ list_del_rcu(&tg->siblings); ++ spin_unlock_irqrestore(&task_group_lock, flags); ++} ++ ++/* change task's runqueue when it moves between groups. ++ * The caller of this function should have put the task in its new group ++ * by now. This function just updates tsk->se.cfs_rq and tsk->se.parent to ++ * reflect its new group. ++ */ ++void sched_move_task(struct task_struct *tsk) ++{ ++ struct task_group *tg; ++ int queued, running; ++ unsigned long flags; ++ struct rq *rq; ++ ++ rq = task_rq_lock(tsk, &flags); ++ ++ running = task_current(rq, tsk); ++ queued = task_on_rq_queued(tsk); ++ ++ if (queued) ++ dequeue_task(rq, tsk, 0); ++ if (unlikely(running)) ++ put_prev_task(rq, tsk); ++ ++ /* ++ * All callers are synchronized by task_rq_lock(); we do not use RCU ++ * which is pointless here. Thus, we pass "true" to task_css_check() ++ * to prevent lockdep warnings. ++ */ ++ tg = container_of(task_css_check(tsk, cpu_cgrp_id, true), ++ struct task_group, css); ++ tg = autogroup_task_group(tsk, tg); ++ tsk->sched_task_group = tg; ++ ++#ifdef CONFIG_FAIR_GROUP_SCHED ++ if (tsk->sched_class->task_move_group) ++ tsk->sched_class->task_move_group(tsk, queued); ++ else ++#endif ++ set_task_rq(tsk, task_cpu(tsk)); ++ ++ if (unlikely(running)) ++ tsk->sched_class->set_curr_task(rq); ++ if (queued) ++ enqueue_task(rq, tsk, 0); ++ ++ task_rq_unlock(rq, tsk, &flags); ++} ++#endif /* CONFIG_CGROUP_SCHED */ ++ ++#ifdef CONFIG_RT_GROUP_SCHED ++/* ++ * Ensure that the real time constraints are schedulable. ++ */ ++static DEFINE_MUTEX(rt_constraints_mutex); ++ ++/* Must be called with tasklist_lock held */ ++static inline int tg_has_rt_tasks(struct task_group *tg) ++{ ++ struct task_struct *g, *p; ++ ++ /* ++ * Autogroups do not have RT tasks; see autogroup_create(). ++ */ ++ if (task_group_is_autogroup(tg)) ++ return 0; ++ ++ for_each_process_thread(g, p) { ++ if (rt_task(p) && task_group(p) == tg) ++ return 1; ++ } ++ ++ return 0; ++} ++ ++struct rt_schedulable_data { ++ struct task_group *tg; ++ u64 rt_period; ++ u64 rt_runtime; ++}; ++ ++static int tg_rt_schedulable(struct task_group *tg, void *data) ++{ ++ struct rt_schedulable_data *d = data; ++ struct task_group *child; ++ unsigned long total, sum = 0; ++ u64 period, runtime; ++ ++ period = ktime_to_ns(tg->rt_bandwidth.rt_period); ++ runtime = tg->rt_bandwidth.rt_runtime; ++ ++ if (tg == d->tg) { ++ period = d->rt_period; ++ runtime = d->rt_runtime; ++ } ++ ++ /* ++ * Cannot have more runtime than the period. ++ */ ++ if (runtime > period && runtime != RUNTIME_INF) ++ return -EINVAL; ++ ++ /* ++ * Ensure we don't starve existing RT tasks. ++ */ ++ if (rt_bandwidth_enabled() && !runtime && tg_has_rt_tasks(tg)) ++ return -EBUSY; ++ ++ total = to_ratio(period, runtime); ++ ++ /* ++ * Nobody can have more than the global setting allows. ++ */ ++ if (total > to_ratio(global_rt_period(), global_rt_runtime())) ++ return -EINVAL; ++ ++ /* ++ * The sum of our children's runtime should not exceed our own. ++ */ ++ list_for_each_entry_rcu(child, &tg->children, siblings) { ++ period = ktime_to_ns(child->rt_bandwidth.rt_period); ++ runtime = child->rt_bandwidth.rt_runtime; ++ ++ if (child == d->tg) { ++ period = d->rt_period; ++ runtime = d->rt_runtime; ++ } ++ ++ sum += to_ratio(period, runtime); ++ } ++ ++ if (sum > total) ++ return -EINVAL; ++ ++ return 0; ++} ++ ++static int __rt_schedulable(struct task_group *tg, u64 period, u64 runtime) ++{ ++ int ret; ++ ++ struct rt_schedulable_data data = { ++ .tg = tg, ++ .rt_period = period, ++ .rt_runtime = runtime, ++ }; ++ ++ rcu_read_lock(); ++ ret = walk_tg_tree(tg_rt_schedulable, tg_nop, &data); ++ rcu_read_unlock(); ++ ++ return ret; ++} ++ ++static int tg_set_rt_bandwidth(struct task_group *tg, ++ u64 rt_period, u64 rt_runtime) ++{ ++ int i, err = 0; ++ ++ /* ++ * Disallowing the root group RT runtime is BAD, it would disallow the ++ * kernel creating (and or operating) RT threads. ++ */ ++ if (tg == &root_task_group && rt_runtime == 0) ++ return -EINVAL; ++ ++ /* No period doesn't make any sense. */ ++ if (rt_period == 0) ++ return -EINVAL; ++ ++ mutex_lock(&rt_constraints_mutex); ++ read_lock(&tasklist_lock); ++ err = __rt_schedulable(tg, rt_period, rt_runtime); ++ if (err) ++ goto unlock; ++ ++ raw_spin_lock_irq(&tg->rt_bandwidth.rt_runtime_lock); ++ tg->rt_bandwidth.rt_period = ns_to_ktime(rt_period); ++ tg->rt_bandwidth.rt_runtime = rt_runtime; ++ ++ for_each_possible_cpu(i) { ++ struct rt_rq *rt_rq = tg->rt_rq[i]; ++ ++ raw_spin_lock(&rt_rq->rt_runtime_lock); ++ rt_rq->rt_runtime = rt_runtime; ++ raw_spin_unlock(&rt_rq->rt_runtime_lock); ++ } ++ raw_spin_unlock_irq(&tg->rt_bandwidth.rt_runtime_lock); ++unlock: ++ read_unlock(&tasklist_lock); ++ mutex_unlock(&rt_constraints_mutex); ++ ++ return err; ++} ++ ++static int sched_group_set_rt_runtime(struct task_group *tg, long rt_runtime_us) ++{ ++ u64 rt_runtime, rt_period; ++ ++ rt_period = ktime_to_ns(tg->rt_bandwidth.rt_period); ++ rt_runtime = (u64)rt_runtime_us * NSEC_PER_USEC; ++ if (rt_runtime_us < 0) ++ rt_runtime = RUNTIME_INF; ++ ++ return tg_set_rt_bandwidth(tg, rt_period, rt_runtime); ++} ++ ++static long sched_group_rt_runtime(struct task_group *tg) ++{ ++ u64 rt_runtime_us; ++ ++ if (tg->rt_bandwidth.rt_runtime == RUNTIME_INF) ++ return -1; ++ ++ rt_runtime_us = tg->rt_bandwidth.rt_runtime; ++ do_div(rt_runtime_us, NSEC_PER_USEC); ++ return rt_runtime_us; ++} ++ ++static int sched_group_set_rt_period(struct task_group *tg, long rt_period_us) ++{ ++ u64 rt_runtime, rt_period; ++ ++ rt_period = (u64)rt_period_us * NSEC_PER_USEC; ++ rt_runtime = tg->rt_bandwidth.rt_runtime; ++ ++ return tg_set_rt_bandwidth(tg, rt_period, rt_runtime); ++} ++ ++static long sched_group_rt_period(struct task_group *tg) ++{ ++ u64 rt_period_us; ++ ++ rt_period_us = ktime_to_ns(tg->rt_bandwidth.rt_period); ++ do_div(rt_period_us, NSEC_PER_USEC); ++ return rt_period_us; ++} ++#endif /* CONFIG_RT_GROUP_SCHED */ ++ ++#ifdef CONFIG_RT_GROUP_SCHED ++static int sched_rt_global_constraints(void) ++{ ++ int ret = 0; ++ ++ mutex_lock(&rt_constraints_mutex); ++ read_lock(&tasklist_lock); ++ ret = __rt_schedulable(NULL, 0, 0); ++ read_unlock(&tasklist_lock); ++ mutex_unlock(&rt_constraints_mutex); ++ ++ return ret; ++} ++ ++static int sched_rt_can_attach(struct task_group *tg, struct task_struct *tsk) ++{ ++ /* Don't accept realtime tasks when there is no way for them to run */ ++ if (rt_task(tsk) && tg->rt_bandwidth.rt_runtime == 0) ++ return 0; ++ ++ return 1; ++} ++ ++#else /* !CONFIG_RT_GROUP_SCHED */ ++static int sched_rt_global_constraints(void) ++{ ++ unsigned long flags; ++ int i, ret = 0; ++ ++ raw_spin_lock_irqsave(&def_rt_bandwidth.rt_runtime_lock, flags); ++ for_each_possible_cpu(i) { ++ struct rt_rq *rt_rq = &cpu_rq(i)->rt; ++ ++ raw_spin_lock(&rt_rq->rt_runtime_lock); ++ rt_rq->rt_runtime = global_rt_runtime(); ++ raw_spin_unlock(&rt_rq->rt_runtime_lock); ++ } ++ raw_spin_unlock_irqrestore(&def_rt_bandwidth.rt_runtime_lock, flags); ++ ++ return ret; ++} ++#endif /* CONFIG_RT_GROUP_SCHED */ ++ ++static int sched_dl_global_validate(void) ++{ ++ u64 runtime = global_rt_runtime(); ++ u64 period = global_rt_period(); ++ u64 new_bw = to_ratio(period, runtime); ++ struct dl_bw *dl_b; ++ int cpu, ret = 0; ++ unsigned long flags; ++ ++ /* ++ * Here we want to check the bandwidth not being set to some ++ * value smaller than the currently allocated bandwidth in ++ * any of the root_domains. ++ * ++ * FIXME: Cycling on all the CPUs is overdoing, but simpler than ++ * cycling on root_domains... Discussion on different/better ++ * solutions is welcome! ++ */ ++ for_each_possible_cpu(cpu) { ++ rcu_read_lock_sched(); ++ dl_b = dl_bw_of(cpu); ++ ++ raw_spin_lock_irqsave(&dl_b->lock, flags); ++ if (new_bw < dl_b->total_bw) ++ ret = -EBUSY; ++ raw_spin_unlock_irqrestore(&dl_b->lock, flags); ++ ++ rcu_read_unlock_sched(); ++ ++ if (ret) ++ break; ++ } ++ ++ return ret; ++} ++ ++static void sched_dl_do_global(void) ++{ ++ u64 new_bw = -1; ++ struct dl_bw *dl_b; ++ int cpu; ++ unsigned long flags; ++ ++ def_dl_bandwidth.dl_period = global_rt_period(); ++ def_dl_bandwidth.dl_runtime = global_rt_runtime(); ++ ++ if (global_rt_runtime() != RUNTIME_INF) ++ new_bw = to_ratio(global_rt_period(), global_rt_runtime()); ++ ++ /* ++ * FIXME: As above... ++ */ ++ for_each_possible_cpu(cpu) { ++ rcu_read_lock_sched(); ++ dl_b = dl_bw_of(cpu); ++ ++ raw_spin_lock_irqsave(&dl_b->lock, flags); ++ dl_b->bw = new_bw; ++ raw_spin_unlock_irqrestore(&dl_b->lock, flags); ++ ++ rcu_read_unlock_sched(); ++ } ++} ++ ++static int sched_rt_global_validate(void) ++{ ++ if (sysctl_sched_rt_period <= 0) ++ return -EINVAL; ++ ++ if ((sysctl_sched_rt_runtime != RUNTIME_INF) && ++ (sysctl_sched_rt_runtime > sysctl_sched_rt_period)) ++ return -EINVAL; ++ ++ return 0; ++} ++ ++static void sched_rt_do_global(void) ++{ ++ def_rt_bandwidth.rt_runtime = global_rt_runtime(); ++ def_rt_bandwidth.rt_period = ns_to_ktime(global_rt_period()); ++} ++ ++int sched_rt_handler(struct ctl_table *table, int write, ++ void __user *buffer, size_t *lenp, ++ loff_t *ppos) ++{ ++ int old_period, old_runtime; ++ static DEFINE_MUTEX(mutex); ++ int ret; ++ ++ mutex_lock(&mutex); ++ old_period = sysctl_sched_rt_period; ++ old_runtime = sysctl_sched_rt_runtime; ++ ++ ret = proc_dointvec(table, write, buffer, lenp, ppos); ++ ++ if (!ret && write) { ++ ret = sched_rt_global_validate(); ++ if (ret) ++ goto undo; ++ ++ ret = sched_dl_global_validate(); ++ if (ret) ++ goto undo; ++ ++ ret = sched_rt_global_constraints(); ++ if (ret) ++ goto undo; ++ ++ sched_rt_do_global(); ++ sched_dl_do_global(); ++ } ++ if (0) { ++undo: ++ sysctl_sched_rt_period = old_period; ++ sysctl_sched_rt_runtime = old_runtime; ++ } ++ mutex_unlock(&mutex); ++ ++ return ret; ++} ++ ++int sched_rr_handler(struct ctl_table *table, int write, ++ void __user *buffer, size_t *lenp, ++ loff_t *ppos) ++{ ++ int ret; ++ static DEFINE_MUTEX(mutex); ++ ++ mutex_lock(&mutex); ++ ret = proc_dointvec(table, write, buffer, lenp, ppos); ++ /* make sure that internally we keep jiffies */ ++ /* also, writing zero resets timeslice to default */ ++ if (!ret && write) { ++ sched_rr_timeslice = sched_rr_timeslice <= 0 ? ++ RR_TIMESLICE : msecs_to_jiffies(sched_rr_timeslice); ++ } ++ mutex_unlock(&mutex); ++ return ret; ++} ++ ++#ifdef CONFIG_CGROUP_SCHED ++ ++static inline struct task_group *css_tg(struct cgroup_subsys_state *css) ++{ ++ return css ? container_of(css, struct task_group, css) : NULL; ++} ++ ++static struct cgroup_subsys_state * ++cpu_cgroup_css_alloc(struct cgroup_subsys_state *parent_css) ++{ ++ struct task_group *parent = css_tg(parent_css); ++ struct task_group *tg; ++ ++ if (!parent) { ++ /* This is early initialization for the top cgroup */ ++ return &root_task_group.css; ++ } ++ ++ tg = sched_create_group(parent); ++ if (IS_ERR(tg)) ++ return ERR_PTR(-ENOMEM); ++ ++ return &tg->css; ++} ++ ++static int cpu_cgroup_css_online(struct cgroup_subsys_state *css) ++{ ++ struct task_group *tg = css_tg(css); ++ struct task_group *parent = css_tg(css->parent); ++ ++ if (parent) ++ sched_online_group(tg, parent); ++ return 0; ++} ++ ++static void cpu_cgroup_css_free(struct cgroup_subsys_state *css) ++{ ++ struct task_group *tg = css_tg(css); ++ ++ sched_destroy_group(tg); ++} ++ ++static void cpu_cgroup_css_offline(struct cgroup_subsys_state *css) ++{ ++ struct task_group *tg = css_tg(css); ++ ++ sched_offline_group(tg); ++} ++ ++static void cpu_cgroup_fork(struct task_struct *task) ++{ ++ sched_move_task(task); ++} ++ ++static int cpu_cgroup_can_attach(struct cgroup_subsys_state *css, ++ struct cgroup_taskset *tset) ++{ ++ struct task_struct *task; ++ ++ cgroup_taskset_for_each(task, tset) { ++#ifdef CONFIG_RT_GROUP_SCHED ++ if (!sched_rt_can_attach(css_tg(css), task)) ++ return -EINVAL; ++#else ++ /* We don't support RT-tasks being in separate groups */ ++ if (task->sched_class != &fair_sched_class) ++ return -EINVAL; ++#endif ++ } ++ return 0; ++} ++ ++static void cpu_cgroup_attach(struct cgroup_subsys_state *css, ++ struct cgroup_taskset *tset) ++{ ++ struct task_struct *task; ++ ++ cgroup_taskset_for_each(task, tset) ++ sched_move_task(task); ++} ++ ++static void cpu_cgroup_exit(struct cgroup_subsys_state *css, ++ struct cgroup_subsys_state *old_css, ++ struct task_struct *task) ++{ ++ /* ++ * cgroup_exit() is called in the copy_process() failure path. ++ * Ignore this case since the task hasn't ran yet, this avoids ++ * trying to poke a half freed task state from generic code. ++ */ ++ if (!(task->flags & PF_EXITING)) ++ return; ++ ++ sched_move_task(task); ++} ++ ++#ifdef CONFIG_FAIR_GROUP_SCHED ++static int cpu_shares_write_u64(struct cgroup_subsys_state *css, ++ struct cftype *cftype, u64 shareval) ++{ ++ return sched_group_set_shares(css_tg(css), scale_load(shareval)); ++} ++ ++static u64 cpu_shares_read_u64(struct cgroup_subsys_state *css, ++ struct cftype *cft) ++{ ++ struct task_group *tg = css_tg(css); ++ ++ return (u64) scale_load_down(tg->shares); ++} ++ ++#ifdef CONFIG_CFS_BANDWIDTH ++static DEFINE_MUTEX(cfs_constraints_mutex); ++ ++const u64 max_cfs_quota_period = 1 * NSEC_PER_SEC; /* 1s */ ++const u64 min_cfs_quota_period = 1 * NSEC_PER_MSEC; /* 1ms */ ++ ++static int __cfs_schedulable(struct task_group *tg, u64 period, u64 runtime); ++ ++static int tg_set_cfs_bandwidth(struct task_group *tg, u64 period, u64 quota) ++{ ++ int i, ret = 0, runtime_enabled, runtime_was_enabled; ++ struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; ++ ++ if (tg == &root_task_group) ++ return -EINVAL; ++ ++ /* ++ * Ensure we have at some amount of bandwidth every period. This is ++ * to prevent reaching a state of large arrears when throttled via ++ * entity_tick() resulting in prolonged exit starvation. ++ */ ++ if (quota < min_cfs_quota_period || period < min_cfs_quota_period) ++ return -EINVAL; ++ ++ /* ++ * Likewise, bound things on the otherside by preventing insane quota ++ * periods. This also allows us to normalize in computing quota ++ * feasibility. ++ */ ++ if (period > max_cfs_quota_period) ++ return -EINVAL; ++ ++ /* ++ * Prevent race between setting of cfs_rq->runtime_enabled and ++ * unthrottle_offline_cfs_rqs(). ++ */ ++ get_online_cpus(); ++ mutex_lock(&cfs_constraints_mutex); ++ ret = __cfs_schedulable(tg, period, quota); ++ if (ret) ++ goto out_unlock; ++ ++ runtime_enabled = quota != RUNTIME_INF; ++ runtime_was_enabled = cfs_b->quota != RUNTIME_INF; ++ /* ++ * If we need to toggle cfs_bandwidth_used, off->on must occur ++ * before making related changes, and on->off must occur afterwards ++ */ ++ if (runtime_enabled && !runtime_was_enabled) ++ cfs_bandwidth_usage_inc(); ++ raw_spin_lock_irq(&cfs_b->lock); ++ cfs_b->period = ns_to_ktime(period); ++ cfs_b->quota = quota; ++ ++ __refill_cfs_bandwidth_runtime(cfs_b); ++ /* restart the period timer (if active) to handle new period expiry */ ++ if (runtime_enabled && cfs_b->timer_active) { ++ /* force a reprogram */ ++ __start_cfs_bandwidth(cfs_b, true); ++ } ++ raw_spin_unlock_irq(&cfs_b->lock); ++ ++ for_each_online_cpu(i) { ++ struct cfs_rq *cfs_rq = tg->cfs_rq[i]; ++ struct rq *rq = cfs_rq->rq; ++ ++ raw_spin_lock_irq(&rq->lock); ++ cfs_rq->runtime_enabled = runtime_enabled; ++ cfs_rq->runtime_remaining = 0; ++ ++ if (cfs_rq->throttled) ++ unthrottle_cfs_rq(cfs_rq); ++ raw_spin_unlock_irq(&rq->lock); ++ } ++ if (runtime_was_enabled && !runtime_enabled) ++ cfs_bandwidth_usage_dec(); ++out_unlock: ++ mutex_unlock(&cfs_constraints_mutex); ++ put_online_cpus(); ++ ++ return ret; ++} ++ ++int tg_set_cfs_quota(struct task_group *tg, long cfs_quota_us) ++{ ++ u64 quota, period; ++ ++ period = ktime_to_ns(tg->cfs_bandwidth.period); ++ if (cfs_quota_us < 0) ++ quota = RUNTIME_INF; ++ else ++ quota = (u64)cfs_quota_us * NSEC_PER_USEC; ++ ++ return tg_set_cfs_bandwidth(tg, period, quota); ++} ++ ++long tg_get_cfs_quota(struct task_group *tg) ++{ ++ u64 quota_us; ++ ++ if (tg->cfs_bandwidth.quota == RUNTIME_INF) ++ return -1; ++ ++ quota_us = tg->cfs_bandwidth.quota; ++ do_div(quota_us, NSEC_PER_USEC); ++ ++ return quota_us; ++} ++ ++int tg_set_cfs_period(struct task_group *tg, long cfs_period_us) ++{ ++ u64 quota, period; ++ ++ period = (u64)cfs_period_us * NSEC_PER_USEC; ++ quota = tg->cfs_bandwidth.quota; ++ ++ return tg_set_cfs_bandwidth(tg, period, quota); ++} ++ ++long tg_get_cfs_period(struct task_group *tg) ++{ ++ u64 cfs_period_us; ++ ++ cfs_period_us = ktime_to_ns(tg->cfs_bandwidth.period); ++ do_div(cfs_period_us, NSEC_PER_USEC); ++ ++ return cfs_period_us; ++} ++ ++static s64 cpu_cfs_quota_read_s64(struct cgroup_subsys_state *css, ++ struct cftype *cft) ++{ ++ return tg_get_cfs_quota(css_tg(css)); ++} ++ ++static int cpu_cfs_quota_write_s64(struct cgroup_subsys_state *css, ++ struct cftype *cftype, s64 cfs_quota_us) ++{ ++ return tg_set_cfs_quota(css_tg(css), cfs_quota_us); ++} ++ ++static u64 cpu_cfs_period_read_u64(struct cgroup_subsys_state *css, ++ struct cftype *cft) ++{ ++ return tg_get_cfs_period(css_tg(css)); ++} ++ ++static int cpu_cfs_period_write_u64(struct cgroup_subsys_state *css, ++ struct cftype *cftype, u64 cfs_period_us) ++{ ++ return tg_set_cfs_period(css_tg(css), cfs_period_us); ++} ++ ++struct cfs_schedulable_data { ++ struct task_group *tg; ++ u64 period, quota; ++}; ++ ++/* ++ * normalize group quota/period to be quota/max_period ++ * note: units are usecs ++ */ ++static u64 normalize_cfs_quota(struct task_group *tg, ++ struct cfs_schedulable_data *d) ++{ ++ u64 quota, period; ++ ++ if (tg == d->tg) { ++ period = d->period; ++ quota = d->quota; ++ } else { ++ period = tg_get_cfs_period(tg); ++ quota = tg_get_cfs_quota(tg); ++ } ++ ++ /* note: these should typically be equivalent */ ++ if (quota == RUNTIME_INF || quota == -1) ++ return RUNTIME_INF; ++ ++ return to_ratio(period, quota); ++} ++ ++static int tg_cfs_schedulable_down(struct task_group *tg, void *data) ++{ ++ struct cfs_schedulable_data *d = data; ++ struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; ++ s64 quota = 0, parent_quota = -1; ++ ++ if (!tg->parent) { ++ quota = RUNTIME_INF; ++ } else { ++ struct cfs_bandwidth *parent_b = &tg->parent->cfs_bandwidth; ++ ++ quota = normalize_cfs_quota(tg, d); ++ parent_quota = parent_b->hierarchical_quota; ++ ++ /* ++ * ensure max(child_quota) <= parent_quota, inherit when no ++ * limit is set ++ */ ++ if (quota == RUNTIME_INF) ++ quota = parent_quota; ++ else if (parent_quota != RUNTIME_INF && quota > parent_quota) ++ return -EINVAL; ++ } ++ cfs_b->hierarchical_quota = quota; ++ ++ return 0; ++} ++ ++static int __cfs_schedulable(struct task_group *tg, u64 period, u64 quota) ++{ ++ int ret; ++ struct cfs_schedulable_data data = { ++ .tg = tg, ++ .period = period, ++ .quota = quota, ++ }; ++ ++ if (quota != RUNTIME_INF) { ++ do_div(data.period, NSEC_PER_USEC); ++ do_div(data.quota, NSEC_PER_USEC); ++ } ++ ++ rcu_read_lock(); ++ ret = walk_tg_tree(tg_cfs_schedulable_down, tg_nop, &data); ++ rcu_read_unlock(); ++ ++ return ret; ++} ++ ++static int cpu_stats_show(struct seq_file *sf, void *v) ++{ ++ struct task_group *tg = css_tg(seq_css(sf)); ++ struct cfs_bandwidth *cfs_b = &tg->cfs_bandwidth; ++ ++ seq_printf(sf, "nr_periods %d\n", cfs_b->nr_periods); ++ seq_printf(sf, "nr_throttled %d\n", cfs_b->nr_throttled); ++ seq_printf(sf, "throttled_time %llu\n", cfs_b->throttled_time); ++ ++ return 0; ++} ++#endif /* CONFIG_CFS_BANDWIDTH */ ++#endif /* CONFIG_FAIR_GROUP_SCHED */ ++ ++#ifdef CONFIG_RT_GROUP_SCHED ++static int cpu_rt_runtime_write(struct cgroup_subsys_state *css, ++ struct cftype *cft, s64 val) ++{ ++ return sched_group_set_rt_runtime(css_tg(css), val); ++} ++ ++static s64 cpu_rt_runtime_read(struct cgroup_subsys_state *css, ++ struct cftype *cft) ++{ ++ return sched_group_rt_runtime(css_tg(css)); ++} ++ ++static int cpu_rt_period_write_uint(struct cgroup_subsys_state *css, ++ struct cftype *cftype, u64 rt_period_us) ++{ ++ return sched_group_set_rt_period(css_tg(css), rt_period_us); ++} ++ ++static u64 cpu_rt_period_read_uint(struct cgroup_subsys_state *css, ++ struct cftype *cft) ++{ ++ return sched_group_rt_period(css_tg(css)); ++} ++#endif /* CONFIG_RT_GROUP_SCHED */ ++ ++static struct cftype cpu_files[] = { ++#ifdef CONFIG_FAIR_GROUP_SCHED ++ { ++ .name = "shares", ++ .read_u64 = cpu_shares_read_u64, ++ .write_u64 = cpu_shares_write_u64, ++ }, ++#endif ++#ifdef CONFIG_CFS_BANDWIDTH ++ { ++ .name = "cfs_quota_us", ++ .read_s64 = cpu_cfs_quota_read_s64, ++ .write_s64 = cpu_cfs_quota_write_s64, ++ }, ++ { ++ .name = "cfs_period_us", ++ .read_u64 = cpu_cfs_period_read_u64, ++ .write_u64 = cpu_cfs_period_write_u64, ++ }, ++ { ++ .name = "stat", ++ .seq_show = cpu_stats_show, ++ }, ++#endif ++#ifdef CONFIG_RT_GROUP_SCHED ++ { ++ .name = "rt_runtime_us", ++ .read_s64 = cpu_rt_runtime_read, ++ .write_s64 = cpu_rt_runtime_write, ++ }, ++ { ++ .name = "rt_period_us", ++ .read_u64 = cpu_rt_period_read_uint, ++ .write_u64 = cpu_rt_period_write_uint, ++ }, ++#endif ++ { } /* terminate */ ++}; ++ ++struct cgroup_subsys cpu_cgrp_subsys = { ++ .css_alloc = cpu_cgroup_css_alloc, ++ .css_free = cpu_cgroup_css_free, ++ .css_online = cpu_cgroup_css_online, ++ .css_offline = cpu_cgroup_css_offline, ++ .fork = cpu_cgroup_fork, ++ .can_attach = cpu_cgroup_can_attach, ++ .attach = cpu_cgroup_attach, ++ .exit = cpu_cgroup_exit, ++ .legacy_cftypes = cpu_files, ++ .early_init = 1, ++}; ++ ++#endif /* CONFIG_CGROUP_SCHED */ ++ ++void dump_cpu_task(int cpu) ++{ ++ pr_info("Task dump for CPU %d:\n", cpu); ++ sched_show_task(cpu_curr(cpu)); ++} +diff -Nur linux-4.1.10.orig/kernel/sched/cputime.c linux-4.1.10/kernel/sched/cputime.c +--- linux-4.1.10.orig/kernel/sched/cputime.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/sched/cputime.c 2015-10-07 18:00:08.000000000 +0200 +@@ -675,37 +675,45 @@ + + void vtime_account_system(struct task_struct *tsk) + { +- write_seqlock(&tsk->vtime_seqlock); ++ raw_spin_lock(&tsk->vtime_lock); ++ write_seqcount_begin(&tsk->vtime_seq); + __vtime_account_system(tsk); +- write_sequnlock(&tsk->vtime_seqlock); ++ write_seqcount_end(&tsk->vtime_seq); ++ raw_spin_unlock(&tsk->vtime_lock); + } + + void vtime_gen_account_irq_exit(struct task_struct *tsk) + { +- write_seqlock(&tsk->vtime_seqlock); ++ raw_spin_lock(&tsk->vtime_lock); ++ write_seqcount_begin(&tsk->vtime_seq); + __vtime_account_system(tsk); + if (context_tracking_in_user()) + tsk->vtime_snap_whence = VTIME_USER; +- write_sequnlock(&tsk->vtime_seqlock); ++ write_seqcount_end(&tsk->vtime_seq); ++ raw_spin_unlock(&tsk->vtime_lock); + } + + void vtime_account_user(struct task_struct *tsk) + { + cputime_t delta_cpu; + +- write_seqlock(&tsk->vtime_seqlock); ++ raw_spin_lock(&tsk->vtime_lock); ++ write_seqcount_begin(&tsk->vtime_seq); + delta_cpu = get_vtime_delta(tsk); + tsk->vtime_snap_whence = VTIME_SYS; + account_user_time(tsk, delta_cpu, cputime_to_scaled(delta_cpu)); +- write_sequnlock(&tsk->vtime_seqlock); ++ write_seqcount_end(&tsk->vtime_seq); ++ raw_spin_unlock(&tsk->vtime_lock); + } + + void vtime_user_enter(struct task_struct *tsk) + { +- write_seqlock(&tsk->vtime_seqlock); ++ raw_spin_lock(&tsk->vtime_lock); ++ write_seqcount_begin(&tsk->vtime_seq); + __vtime_account_system(tsk); + tsk->vtime_snap_whence = VTIME_USER; +- write_sequnlock(&tsk->vtime_seqlock); ++ write_seqcount_end(&tsk->vtime_seq); ++ raw_spin_unlock(&tsk->vtime_lock); + } + + void vtime_guest_enter(struct task_struct *tsk) +@@ -717,19 +725,23 @@ + * synchronization against the reader (task_gtime()) + * that can thus safely catch up with a tickless delta. + */ +- write_seqlock(&tsk->vtime_seqlock); ++ raw_spin_lock(&tsk->vtime_lock); ++ write_seqcount_begin(&tsk->vtime_seq); + __vtime_account_system(tsk); + current->flags |= PF_VCPU; +- write_sequnlock(&tsk->vtime_seqlock); ++ write_seqcount_end(&tsk->vtime_seq); ++ raw_spin_unlock(&tsk->vtime_lock); + } + EXPORT_SYMBOL_GPL(vtime_guest_enter); + + void vtime_guest_exit(struct task_struct *tsk) + { +- write_seqlock(&tsk->vtime_seqlock); ++ raw_spin_lock(&tsk->vtime_lock); ++ write_seqcount_begin(&tsk->vtime_seq); + __vtime_account_system(tsk); + current->flags &= ~PF_VCPU; +- write_sequnlock(&tsk->vtime_seqlock); ++ write_seqcount_end(&tsk->vtime_seq); ++ raw_spin_unlock(&tsk->vtime_lock); + } + EXPORT_SYMBOL_GPL(vtime_guest_exit); + +@@ -742,24 +754,30 @@ + + void arch_vtime_task_switch(struct task_struct *prev) + { +- write_seqlock(&prev->vtime_seqlock); ++ raw_spin_lock(&prev->vtime_lock); ++ write_seqcount_begin(&prev->vtime_seq); + prev->vtime_snap_whence = VTIME_SLEEPING; +- write_sequnlock(&prev->vtime_seqlock); ++ write_seqcount_end(&prev->vtime_seq); ++ raw_spin_unlock(&prev->vtime_lock); + +- write_seqlock(¤t->vtime_seqlock); ++ raw_spin_lock(¤t->vtime_lock); ++ write_seqcount_begin(¤t->vtime_seq); + current->vtime_snap_whence = VTIME_SYS; + current->vtime_snap = sched_clock_cpu(smp_processor_id()); +- write_sequnlock(¤t->vtime_seqlock); ++ write_seqcount_end(¤t->vtime_seq); ++ raw_spin_unlock(¤t->vtime_lock); + } + + void vtime_init_idle(struct task_struct *t, int cpu) + { + unsigned long flags; + +- write_seqlock_irqsave(&t->vtime_seqlock, flags); ++ raw_spin_lock_irqsave(&t->vtime_lock, flags); ++ write_seqcount_begin(&t->vtime_seq); + t->vtime_snap_whence = VTIME_SYS; + t->vtime_snap = sched_clock_cpu(cpu); +- write_sequnlock_irqrestore(&t->vtime_seqlock, flags); ++ write_seqcount_end(&t->vtime_seq); ++ raw_spin_unlock_irqrestore(&t->vtime_lock, flags); + } + + cputime_t task_gtime(struct task_struct *t) +@@ -768,13 +786,13 @@ + cputime_t gtime; + + do { +- seq = read_seqbegin(&t->vtime_seqlock); ++ seq = read_seqcount_begin(&t->vtime_seq); + + gtime = t->gtime; + if (t->flags & PF_VCPU) + gtime += vtime_delta(t); + +- } while (read_seqretry(&t->vtime_seqlock, seq)); ++ } while (read_seqcount_retry(&t->vtime_seq, seq)); + + return gtime; + } +@@ -797,7 +815,7 @@ + *udelta = 0; + *sdelta = 0; + +- seq = read_seqbegin(&t->vtime_seqlock); ++ seq = read_seqcount_begin(&t->vtime_seq); + + if (u_dst) + *u_dst = *u_src; +@@ -821,7 +839,7 @@ + if (t->vtime_snap_whence == VTIME_SYS) + *sdelta = delta; + } +- } while (read_seqretry(&t->vtime_seqlock, seq)); ++ } while (read_seqcount_retry(&t->vtime_seq, seq)); + } + + +diff -Nur linux-4.1.10.orig/kernel/sched/deadline.c linux-4.1.10/kernel/sched/deadline.c +--- linux-4.1.10.orig/kernel/sched/deadline.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/sched/deadline.c 2015-10-07 18:00:08.000000000 +0200 +@@ -637,6 +637,7 @@ + + hrtimer_init(timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + timer->function = dl_task_timer; ++ timer->irqsafe = 1; + } + + static +diff -Nur linux-4.1.10.orig/kernel/sched/debug.c linux-4.1.10/kernel/sched/debug.c +--- linux-4.1.10.orig/kernel/sched/debug.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/sched/debug.c 2015-10-07 18:00:08.000000000 +0200 +@@ -260,6 +260,9 @@ + P(rt_throttled); + PN(rt_time); + PN(rt_runtime); ++#ifdef CONFIG_SMP ++ P(rt_nr_migratory); ++#endif + + #undef PN + #undef P +@@ -648,6 +651,10 @@ + #endif + P(policy); + P(prio); ++#ifdef CONFIG_PREEMPT_RT_FULL ++ P(migrate_disable); ++#endif ++ P(nr_cpus_allowed); + #undef PN + #undef __PN + #undef P +diff -Nur linux-4.1.10.orig/kernel/sched/fair.c linux-4.1.10/kernel/sched/fair.c +--- linux-4.1.10.orig/kernel/sched/fair.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/sched/fair.c 2015-10-07 18:00:08.000000000 +0200 +@@ -3201,7 +3201,7 @@ + ideal_runtime = sched_slice(cfs_rq, curr); + delta_exec = curr->sum_exec_runtime - curr->prev_sum_exec_runtime; + if (delta_exec > ideal_runtime) { +- resched_curr(rq_of(cfs_rq)); ++ resched_curr_lazy(rq_of(cfs_rq)); + /* + * The current task ran long enough, ensure it doesn't get + * re-elected due to buddy favours. +@@ -3225,7 +3225,7 @@ + return; + + if (delta > ideal_runtime) +- resched_curr(rq_of(cfs_rq)); ++ resched_curr_lazy(rq_of(cfs_rq)); + } + + static void +@@ -3366,7 +3366,7 @@ + * validating it and just reschedule. + */ + if (queued) { +- resched_curr(rq_of(cfs_rq)); ++ resched_curr_lazy(rq_of(cfs_rq)); + return; + } + /* +@@ -3557,7 +3557,7 @@ + * hierarchy can be throttled + */ + if (!assign_cfs_rq_runtime(cfs_rq) && likely(cfs_rq->curr)) +- resched_curr(rq_of(cfs_rq)); ++ resched_curr_lazy(rq_of(cfs_rq)); + } + + static __always_inline +@@ -4180,7 +4180,7 @@ + + if (delta < 0) { + if (rq->curr == p) +- resched_curr(rq); ++ resched_curr_lazy(rq); + return; + } + hrtick_start(rq, delta); +@@ -5076,7 +5076,7 @@ + return; + + preempt: +- resched_curr(rq); ++ resched_curr_lazy(rq); + /* + * Only set the backward buddy when the current task is still + * on the rq. This can happen when a wakeup gets interleaved +@@ -7866,7 +7866,7 @@ + * 'current' within the tree based on its new key value. + */ + swap(curr->vruntime, se->vruntime); +- resched_curr(rq); ++ resched_curr_lazy(rq); + } + + se->vruntime -= cfs_rq->min_vruntime; +@@ -7891,7 +7891,7 @@ + */ + if (rq->curr == p) { + if (p->prio > oldprio) +- resched_curr(rq); ++ resched_curr_lazy(rq); + } else + check_preempt_curr(rq, p, 0); + } +diff -Nur linux-4.1.10.orig/kernel/sched/features.h linux-4.1.10/kernel/sched/features.h +--- linux-4.1.10.orig/kernel/sched/features.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/sched/features.h 2015-10-07 18:00:08.000000000 +0200 +@@ -50,11 +50,19 @@ + */ + SCHED_FEAT(NONTASK_CAPACITY, true) + ++#ifdef CONFIG_PREEMPT_RT_FULL ++SCHED_FEAT(TTWU_QUEUE, false) ++# ifdef CONFIG_PREEMPT_LAZY ++SCHED_FEAT(PREEMPT_LAZY, true) ++# endif ++#else ++ + /* + * Queue remote wakeups on the target CPU and process them + * using the scheduler IPI. Reduces rq->lock contention/bounces. + */ + SCHED_FEAT(TTWU_QUEUE, true) ++#endif + + #ifdef HAVE_RT_PUSH_IPI + /* +diff -Nur linux-4.1.10.orig/kernel/sched/Makefile linux-4.1.10/kernel/sched/Makefile +--- linux-4.1.10.orig/kernel/sched/Makefile 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/sched/Makefile 2015-10-07 18:00:08.000000000 +0200 +@@ -13,7 +13,7 @@ + + obj-y += core.o proc.o clock.o cputime.o + obj-y += idle_task.o fair.o rt.o deadline.o stop_task.o +-obj-y += wait.o completion.o idle.o ++obj-y += wait.o wait-simple.o work-simple.o completion.o idle.o + obj-$(CONFIG_SMP) += cpupri.o cpudeadline.o + obj-$(CONFIG_SCHED_AUTOGROUP) += auto_group.o + obj-$(CONFIG_SCHEDSTATS) += stats.o +diff -Nur linux-4.1.10.orig/kernel/sched/rt.c linux-4.1.10/kernel/sched/rt.c +--- linux-4.1.10.orig/kernel/sched/rt.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/sched/rt.c 2015-10-07 18:00:08.000000000 +0200 +@@ -44,6 +44,7 @@ + + hrtimer_init(&rt_b->rt_period_timer, + CLOCK_MONOTONIC, HRTIMER_MODE_REL); ++ rt_b->rt_period_timer.irqsafe = 1; + rt_b->rt_period_timer.function = sched_rt_period_timer; + } + +@@ -89,6 +90,7 @@ + rt_rq->push_cpu = nr_cpu_ids; + raw_spin_lock_init(&rt_rq->push_lock); + init_irq_work(&rt_rq->push_work, push_irq_work_func); ++ rt_rq->push_work.flags |= IRQ_WORK_HARD_IRQ; + #endif + #endif /* CONFIG_SMP */ + /* We start is dequeued state, because no RT tasks are queued */ +diff -Nur linux-4.1.10.orig/kernel/sched/sched.h linux-4.1.10/kernel/sched/sched.h +--- linux-4.1.10.orig/kernel/sched/sched.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/sched/sched.h 2015-10-07 18:00:08.000000000 +0200 +@@ -1092,6 +1092,7 @@ + #define WF_SYNC 0x01 /* waker goes to sleep after wakeup */ + #define WF_FORK 0x02 /* child wakeup after fork */ + #define WF_MIGRATED 0x4 /* internal use, task got migrated */ ++#define WF_LOCK_SLEEPER 0x08 /* wakeup spinlock "sleeper" */ + + /* + * To aid in avoiding the subversion of "niceness" due to uneven distribution +@@ -1289,6 +1290,15 @@ + extern void resched_curr(struct rq *rq); + extern void resched_cpu(int cpu); + ++#ifdef CONFIG_PREEMPT_LAZY ++extern void resched_curr_lazy(struct rq *rq); ++#else ++static inline void resched_curr_lazy(struct rq *rq) ++{ ++ resched_curr(rq); ++} ++#endif ++ + extern struct rt_bandwidth def_rt_bandwidth; + extern void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 runtime); + +diff -Nur linux-4.1.10.orig/kernel/sched/wait-simple.c linux-4.1.10/kernel/sched/wait-simple.c +--- linux-4.1.10.orig/kernel/sched/wait-simple.c 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/kernel/sched/wait-simple.c 2015-10-07 18:00:08.000000000 +0200 +@@ -0,0 +1,115 @@ ++/* ++ * Simple waitqueues without fancy flags and callbacks ++ * ++ * (C) 2011 Thomas Gleixner ++ * ++ * Based on kernel/wait.c ++ * ++ * For licencing details see kernel-base/COPYING ++ */ ++#include ++#include ++#include ++#include ++ ++/* Adds w to head->list. Must be called with head->lock locked. */ ++static inline void __swait_enqueue(struct swait_head *head, struct swaiter *w) ++{ ++ list_add(&w->node, &head->list); ++ /* We can't let the condition leak before the setting of head */ ++ smp_mb(); ++} ++ ++/* Removes w from head->list. Must be called with head->lock locked. */ ++static inline void __swait_dequeue(struct swaiter *w) ++{ ++ list_del_init(&w->node); ++} ++ ++void __init_swait_head(struct swait_head *head, struct lock_class_key *key) ++{ ++ raw_spin_lock_init(&head->lock); ++ lockdep_set_class(&head->lock, key); ++ INIT_LIST_HEAD(&head->list); ++} ++EXPORT_SYMBOL(__init_swait_head); ++ ++void swait_prepare_locked(struct swait_head *head, struct swaiter *w) ++{ ++ w->task = current; ++ if (list_empty(&w->node)) ++ __swait_enqueue(head, w); ++} ++ ++void swait_prepare(struct swait_head *head, struct swaiter *w, int state) ++{ ++ unsigned long flags; ++ ++ raw_spin_lock_irqsave(&head->lock, flags); ++ swait_prepare_locked(head, w); ++ __set_current_state(state); ++ raw_spin_unlock_irqrestore(&head->lock, flags); ++} ++EXPORT_SYMBOL(swait_prepare); ++ ++void swait_finish_locked(struct swait_head *head, struct swaiter *w) ++{ ++ __set_current_state(TASK_RUNNING); ++ if (w->task) ++ __swait_dequeue(w); ++} ++ ++void swait_finish(struct swait_head *head, struct swaiter *w) ++{ ++ unsigned long flags; ++ ++ __set_current_state(TASK_RUNNING); ++ if (w->task) { ++ raw_spin_lock_irqsave(&head->lock, flags); ++ __swait_dequeue(w); ++ raw_spin_unlock_irqrestore(&head->lock, flags); ++ } ++} ++EXPORT_SYMBOL(swait_finish); ++ ++unsigned int ++__swait_wake_locked(struct swait_head *head, unsigned int state, unsigned int num) ++{ ++ struct swaiter *curr, *next; ++ int woken = 0; ++ ++ list_for_each_entry_safe(curr, next, &head->list, node) { ++ if (wake_up_state(curr->task, state)) { ++ __swait_dequeue(curr); ++ /* ++ * The waiting task can free the waiter as ++ * soon as curr->task = NULL is written, ++ * without taking any locks. A memory barrier ++ * is required here to prevent the following ++ * store to curr->task from getting ahead of ++ * the dequeue operation. ++ */ ++ smp_wmb(); ++ curr->task = NULL; ++ if (++woken == num) ++ break; ++ } ++ } ++ return woken; ++} ++ ++unsigned int ++__swait_wake(struct swait_head *head, unsigned int state, unsigned int num) ++{ ++ unsigned long flags; ++ int woken; ++ ++ if (!swaitqueue_active(head)) ++ return 0; ++ ++ raw_spin_lock_irqsave(&head->lock, flags); ++ woken = __swait_wake_locked(head, state, num); ++ raw_spin_unlock_irqrestore(&head->lock, flags); ++ return woken; ++} ++EXPORT_SYMBOL(__swait_wake); +diff -Nur linux-4.1.10.orig/kernel/sched/work-simple.c linux-4.1.10/kernel/sched/work-simple.c +--- linux-4.1.10.orig/kernel/sched/work-simple.c 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/kernel/sched/work-simple.c 2015-10-07 18:00:08.000000000 +0200 +@@ -0,0 +1,172 @@ ++/* ++ * Copyright (C) 2014 BMW Car IT GmbH, Daniel Wagner daniel.wagner@bmw-carit.de ++ * ++ * Provides a framework for enqueuing callbacks from irq context ++ * PREEMPT_RT_FULL safe. The callbacks are executed in kthread context. ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++ ++#define SWORK_EVENT_PENDING (1 << 0) ++ ++static DEFINE_MUTEX(worker_mutex); ++static struct sworker *glob_worker; ++ ++struct sworker { ++ struct list_head events; ++ struct swait_head wq; ++ ++ raw_spinlock_t lock; ++ ++ struct task_struct *task; ++ int refs; ++}; ++ ++static bool swork_readable(struct sworker *worker) ++{ ++ bool r; ++ ++ if (kthread_should_stop()) ++ return true; ++ ++ raw_spin_lock_irq(&worker->lock); ++ r = !list_empty(&worker->events); ++ raw_spin_unlock_irq(&worker->lock); ++ ++ return r; ++} ++ ++static int swork_kthread(void *arg) ++{ ++ struct sworker *worker = arg; ++ ++ for (;;) { ++ swait_event_interruptible(worker->wq, ++ swork_readable(worker)); ++ if (kthread_should_stop()) ++ break; ++ ++ raw_spin_lock_irq(&worker->lock); ++ while (!list_empty(&worker->events)) { ++ struct swork_event *sev; ++ ++ sev = list_first_entry(&worker->events, ++ struct swork_event, item); ++ list_del(&sev->item); ++ raw_spin_unlock_irq(&worker->lock); ++ ++ WARN_ON_ONCE(!test_and_clear_bit(SWORK_EVENT_PENDING, ++ &sev->flags)); ++ sev->func(sev); ++ raw_spin_lock_irq(&worker->lock); ++ } ++ raw_spin_unlock_irq(&worker->lock); ++ } ++ return 0; ++} ++ ++static struct sworker *swork_create(void) ++{ ++ struct sworker *worker; ++ ++ worker = kzalloc(sizeof(*worker), GFP_KERNEL); ++ if (!worker) ++ return ERR_PTR(-ENOMEM); ++ ++ INIT_LIST_HEAD(&worker->events); ++ raw_spin_lock_init(&worker->lock); ++ init_swait_head(&worker->wq); ++ ++ worker->task = kthread_run(swork_kthread, worker, "kswork"); ++ if (IS_ERR(worker->task)) { ++ kfree(worker); ++ return ERR_PTR(-ENOMEM); ++ } ++ ++ return worker; ++} ++ ++static void swork_destroy(struct sworker *worker) ++{ ++ kthread_stop(worker->task); ++ ++ WARN_ON(!list_empty(&worker->events)); ++ kfree(worker); ++} ++ ++/** ++ * swork_queue - queue swork ++ * ++ * Returns %false if @work was already on a queue, %true otherwise. ++ * ++ * The work is queued and processed on a random CPU ++ */ ++bool swork_queue(struct swork_event *sev) ++{ ++ unsigned long flags; ++ ++ if (test_and_set_bit(SWORK_EVENT_PENDING, &sev->flags)) ++ return false; ++ ++ raw_spin_lock_irqsave(&glob_worker->lock, flags); ++ list_add_tail(&sev->item, &glob_worker->events); ++ raw_spin_unlock_irqrestore(&glob_worker->lock, flags); ++ ++ swait_wake(&glob_worker->wq); ++ return true; ++} ++EXPORT_SYMBOL_GPL(swork_queue); ++ ++/** ++ * swork_get - get an instance of the sworker ++ * ++ * Returns an negative error code if the initialization if the worker did not ++ * work, %0 otherwise. ++ * ++ */ ++int swork_get(void) ++{ ++ struct sworker *worker; ++ ++ mutex_lock(&worker_mutex); ++ if (!glob_worker) { ++ worker = swork_create(); ++ if (IS_ERR(worker)) { ++ mutex_unlock(&worker_mutex); ++ return -ENOMEM; ++ } ++ ++ glob_worker = worker; ++ } ++ ++ glob_worker->refs++; ++ mutex_unlock(&worker_mutex); ++ ++ return 0; ++} ++EXPORT_SYMBOL_GPL(swork_get); ++ ++/** ++ * swork_put - puts an instance of the sworker ++ * ++ * Will destroy the sworker thread. This function must not be called until all ++ * queued events have been completed. ++ */ ++void swork_put(void) ++{ ++ mutex_lock(&worker_mutex); ++ ++ glob_worker->refs--; ++ if (glob_worker->refs > 0) ++ goto out; ++ ++ swork_destroy(glob_worker); ++ glob_worker = NULL; ++out: ++ mutex_unlock(&worker_mutex); ++} ++EXPORT_SYMBOL_GPL(swork_put); +diff -Nur linux-4.1.10.orig/kernel/signal.c linux-4.1.10/kernel/signal.c +--- linux-4.1.10.orig/kernel/signal.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/signal.c 2015-10-07 18:00:08.000000000 +0200 +@@ -14,6 +14,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -352,13 +353,45 @@ + return false; + } + ++#ifdef __HAVE_ARCH_CMPXCHG ++static inline struct sigqueue *get_task_cache(struct task_struct *t) ++{ ++ struct sigqueue *q = t->sigqueue_cache; ++ ++ if (cmpxchg(&t->sigqueue_cache, q, NULL) != q) ++ return NULL; ++ return q; ++} ++ ++static inline int put_task_cache(struct task_struct *t, struct sigqueue *q) ++{ ++ if (cmpxchg(&t->sigqueue_cache, NULL, q) == NULL) ++ return 0; ++ return 1; ++} ++ ++#else ++ ++static inline struct sigqueue *get_task_cache(struct task_struct *t) ++{ ++ return NULL; ++} ++ ++static inline int put_task_cache(struct task_struct *t, struct sigqueue *q) ++{ ++ return 1; ++} ++ ++#endif ++ + /* + * allocate a new signal queue record + * - this may be called without locks if and only if t == current, otherwise an + * appropriate lock must be held to stop the target task from exiting + */ + static struct sigqueue * +-__sigqueue_alloc(int sig, struct task_struct *t, gfp_t flags, int override_rlimit) ++__sigqueue_do_alloc(int sig, struct task_struct *t, gfp_t flags, ++ int override_rlimit, int fromslab) + { + struct sigqueue *q = NULL; + struct user_struct *user; +@@ -375,7 +408,10 @@ + if (override_rlimit || + atomic_read(&user->sigpending) <= + task_rlimit(t, RLIMIT_SIGPENDING)) { +- q = kmem_cache_alloc(sigqueue_cachep, flags); ++ if (!fromslab) ++ q = get_task_cache(t); ++ if (!q) ++ q = kmem_cache_alloc(sigqueue_cachep, flags); + } else { + print_dropped_signal(sig); + } +@@ -392,6 +428,13 @@ + return q; + } + ++static struct sigqueue * ++__sigqueue_alloc(int sig, struct task_struct *t, gfp_t flags, ++ int override_rlimit) ++{ ++ return __sigqueue_do_alloc(sig, t, flags, override_rlimit, 0); ++} ++ + static void __sigqueue_free(struct sigqueue *q) + { + if (q->flags & SIGQUEUE_PREALLOC) +@@ -401,6 +444,21 @@ + kmem_cache_free(sigqueue_cachep, q); + } + ++static void sigqueue_free_current(struct sigqueue *q) ++{ ++ struct user_struct *up; ++ ++ if (q->flags & SIGQUEUE_PREALLOC) ++ return; ++ ++ up = q->user; ++ if (rt_prio(current->normal_prio) && !put_task_cache(current, q)) { ++ atomic_dec(&up->sigpending); ++ free_uid(up); ++ } else ++ __sigqueue_free(q); ++} ++ + void flush_sigqueue(struct sigpending *queue) + { + struct sigqueue *q; +@@ -414,6 +472,21 @@ + } + + /* ++ * Called from __exit_signal. Flush tsk->pending and ++ * tsk->sigqueue_cache ++ */ ++void flush_task_sigqueue(struct task_struct *tsk) ++{ ++ struct sigqueue *q; ++ ++ flush_sigqueue(&tsk->pending); ++ ++ q = get_task_cache(tsk); ++ if (q) ++ kmem_cache_free(sigqueue_cachep, q); ++} ++ ++/* + * Flush all pending signals for a task. + */ + void __flush_signals(struct task_struct *t) +@@ -565,7 +638,7 @@ + still_pending: + list_del_init(&first->list); + copy_siginfo(info, &first->info); +- __sigqueue_free(first); ++ sigqueue_free_current(first); + } else { + /* + * Ok, it wasn't in the queue. This must be +@@ -611,6 +684,8 @@ + { + int signr; + ++ WARN_ON_ONCE(tsk != current); ++ + /* We only dequeue private signals from ourselves, we don't let + * signalfd steal them + */ +@@ -1207,8 +1282,8 @@ + * We don't want to have recursive SIGSEGV's etc, for example, + * that is why we also clear SIGNAL_UNKILLABLE. + */ +-int +-force_sig_info(int sig, struct siginfo *info, struct task_struct *t) ++static int ++do_force_sig_info(int sig, struct siginfo *info, struct task_struct *t) + { + unsigned long int flags; + int ret, blocked, ignored; +@@ -1233,6 +1308,39 @@ + return ret; + } + ++int force_sig_info(int sig, struct siginfo *info, struct task_struct *t) ++{ ++/* ++ * On some archs, PREEMPT_RT has to delay sending a signal from a trap ++ * since it can not enable preemption, and the signal code's spin_locks ++ * turn into mutexes. Instead, it must set TIF_NOTIFY_RESUME which will ++ * send the signal on exit of the trap. ++ */ ++#ifdef ARCH_RT_DELAYS_SIGNAL_SEND ++ if (in_atomic()) { ++ if (WARN_ON_ONCE(t != current)) ++ return 0; ++ if (WARN_ON_ONCE(t->forced_info.si_signo)) ++ return 0; ++ ++ if (is_si_special(info)) { ++ WARN_ON_ONCE(info != SEND_SIG_PRIV); ++ t->forced_info.si_signo = sig; ++ t->forced_info.si_errno = 0; ++ t->forced_info.si_code = SI_KERNEL; ++ t->forced_info.si_pid = 0; ++ t->forced_info.si_uid = 0; ++ } else { ++ t->forced_info = *info; ++ } ++ ++ set_tsk_thread_flag(t, TIF_NOTIFY_RESUME); ++ return 0; ++ } ++#endif ++ return do_force_sig_info(sig, info, t); ++} ++ + /* + * Nuke all other threads in the group. + */ +@@ -1267,12 +1375,12 @@ + * Disable interrupts early to avoid deadlocks. + * See rcu_read_unlock() comment header for details. + */ +- local_irq_save(*flags); ++ local_irq_save_nort(*flags); + rcu_read_lock(); + sighand = rcu_dereference(tsk->sighand); + if (unlikely(sighand == NULL)) { + rcu_read_unlock(); +- local_irq_restore(*flags); ++ local_irq_restore_nort(*flags); + break; + } + /* +@@ -1293,7 +1401,7 @@ + } + spin_unlock(&sighand->siglock); + rcu_read_unlock(); +- local_irq_restore(*flags); ++ local_irq_restore_nort(*flags); + } + + return sighand; +@@ -1536,7 +1644,8 @@ + */ + struct sigqueue *sigqueue_alloc(void) + { +- struct sigqueue *q = __sigqueue_alloc(-1, current, GFP_KERNEL, 0); ++ /* Preallocated sigqueue objects always from the slabcache ! */ ++ struct sigqueue *q = __sigqueue_do_alloc(-1, current, GFP_KERNEL, 0, 1); + + if (q) + q->flags |= SIGQUEUE_PREALLOC; +@@ -1897,15 +2006,7 @@ + if (gstop_done && ptrace_reparented(current)) + do_notify_parent_cldstop(current, false, why); + +- /* +- * Don't want to allow preemption here, because +- * sys_ptrace() needs this task to be inactive. +- * +- * XXX: implement read_unlock_no_resched(). +- */ +- preempt_disable(); + read_unlock(&tasklist_lock); +- preempt_enable_no_resched(); + freezable_schedule(); + } else { + /* +diff -Nur linux-4.1.10.orig/kernel/softirq.c linux-4.1.10/kernel/softirq.c +--- linux-4.1.10.orig/kernel/softirq.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/softirq.c 2015-10-07 18:00:08.000000000 +0200 +@@ -21,10 +21,12 @@ + #include + #include + #include ++#include + #include + #include + #include + #include ++#include + #include + + #define CREATE_TRACE_POINTS +@@ -62,6 +64,98 @@ + "TASKLET", "SCHED", "HRTIMER", "RCU" + }; + ++#ifdef CONFIG_NO_HZ_COMMON ++# ifdef CONFIG_PREEMPT_RT_FULL ++ ++struct softirq_runner { ++ struct task_struct *runner[NR_SOFTIRQS]; ++}; ++ ++static DEFINE_PER_CPU(struct softirq_runner, softirq_runners); ++ ++static inline void softirq_set_runner(unsigned int sirq) ++{ ++ struct softirq_runner *sr = this_cpu_ptr(&softirq_runners); ++ ++ sr->runner[sirq] = current; ++} ++ ++static inline void softirq_clr_runner(unsigned int sirq) ++{ ++ struct softirq_runner *sr = this_cpu_ptr(&softirq_runners); ++ ++ sr->runner[sirq] = NULL; ++} ++ ++/* ++ * On preempt-rt a softirq running context might be blocked on a ++ * lock. There might be no other runnable task on this CPU because the ++ * lock owner runs on some other CPU. So we have to go into idle with ++ * the pending bit set. Therefor we need to check this otherwise we ++ * warn about false positives which confuses users and defeats the ++ * whole purpose of this test. ++ * ++ * This code is called with interrupts disabled. ++ */ ++void softirq_check_pending_idle(void) ++{ ++ static int rate_limit; ++ struct softirq_runner *sr = this_cpu_ptr(&softirq_runners); ++ u32 warnpending; ++ int i; ++ ++ if (rate_limit >= 10) ++ return; ++ ++ warnpending = local_softirq_pending() & SOFTIRQ_STOP_IDLE_MASK; ++ for (i = 0; i < NR_SOFTIRQS; i++) { ++ struct task_struct *tsk = sr->runner[i]; ++ ++ /* ++ * The wakeup code in rtmutex.c wakes up the task ++ * _before_ it sets pi_blocked_on to NULL under ++ * tsk->pi_lock. So we need to check for both: state ++ * and pi_blocked_on. ++ */ ++ if (tsk) { ++ raw_spin_lock(&tsk->pi_lock); ++ if (tsk->pi_blocked_on || tsk->state == TASK_RUNNING) { ++ /* Clear all bits pending in that task */ ++ warnpending &= ~(tsk->softirqs_raised); ++ warnpending &= ~(1 << i); ++ } ++ raw_spin_unlock(&tsk->pi_lock); ++ } ++ } ++ ++ if (warnpending) { ++ printk(KERN_ERR "NOHZ: local_softirq_pending %02x\n", ++ warnpending); ++ rate_limit++; ++ } ++} ++# else ++/* ++ * On !PREEMPT_RT we just printk rate limited: ++ */ ++void softirq_check_pending_idle(void) ++{ ++ static int rate_limit; ++ ++ if (rate_limit < 10 && ++ (local_softirq_pending() & SOFTIRQ_STOP_IDLE_MASK)) { ++ printk(KERN_ERR "NOHZ: local_softirq_pending %02x\n", ++ local_softirq_pending()); ++ rate_limit++; ++ } ++} ++# endif ++ ++#else /* !CONFIG_NO_HZ_COMMON */ ++static inline void softirq_set_runner(unsigned int sirq) { } ++static inline void softirq_clr_runner(unsigned int sirq) { } ++#endif ++ + /* + * we cannot loop indefinitely here to avoid userspace starvation, + * but we also don't want to introduce a worst case 1/HZ latency +@@ -77,6 +171,68 @@ + wake_up_process(tsk); + } + ++static void handle_softirq(unsigned int vec_nr) ++{ ++ struct softirq_action *h = softirq_vec + vec_nr; ++ int prev_count; ++ ++ prev_count = preempt_count(); ++ ++ kstat_incr_softirqs_this_cpu(vec_nr); ++ ++ trace_softirq_entry(vec_nr); ++ h->action(h); ++ trace_softirq_exit(vec_nr); ++ if (unlikely(prev_count != preempt_count())) { ++ pr_err("huh, entered softirq %u %s %p with preempt_count %08x, exited with %08x?\n", ++ vec_nr, softirq_to_name[vec_nr], h->action, ++ prev_count, preempt_count()); ++ preempt_count_set(prev_count); ++ } ++} ++ ++#ifndef CONFIG_PREEMPT_RT_FULL ++static inline int ksoftirqd_softirq_pending(void) ++{ ++ return local_softirq_pending(); ++} ++ ++static void handle_pending_softirqs(u32 pending) ++{ ++ struct softirq_action *h = softirq_vec; ++ int softirq_bit; ++ ++ local_irq_enable(); ++ ++ h = softirq_vec; ++ ++ while ((softirq_bit = ffs(pending))) { ++ unsigned int vec_nr; ++ ++ h += softirq_bit - 1; ++ vec_nr = h - softirq_vec; ++ handle_softirq(vec_nr); ++ ++ h++; ++ pending >>= softirq_bit; ++ } ++ ++ rcu_bh_qs(); ++ local_irq_disable(); ++} ++ ++static void run_ksoftirqd(unsigned int cpu) ++{ ++ local_irq_disable(); ++ if (ksoftirqd_softirq_pending()) { ++ __do_softirq(); ++ local_irq_enable(); ++ cond_resched_rcu_qs(); ++ return; ++ } ++ local_irq_enable(); ++} ++ + /* + * preempt_count and SOFTIRQ_OFFSET usage: + * - preempt_count is changed by SOFTIRQ_OFFSET on entering or leaving +@@ -232,10 +388,8 @@ + unsigned long end = jiffies + MAX_SOFTIRQ_TIME; + unsigned long old_flags = current->flags; + int max_restart = MAX_SOFTIRQ_RESTART; +- struct softirq_action *h; + bool in_hardirq; + __u32 pending; +- int softirq_bit; + + /* + * Mask out PF_MEMALLOC s current task context is borrowed for the +@@ -254,36 +408,7 @@ + /* Reset the pending bitmask before enabling irqs */ + set_softirq_pending(0); + +- local_irq_enable(); +- +- h = softirq_vec; +- +- while ((softirq_bit = ffs(pending))) { +- unsigned int vec_nr; +- int prev_count; +- +- h += softirq_bit - 1; +- +- vec_nr = h - softirq_vec; +- prev_count = preempt_count(); +- +- kstat_incr_softirqs_this_cpu(vec_nr); +- +- trace_softirq_entry(vec_nr); +- h->action(h); +- trace_softirq_exit(vec_nr); +- if (unlikely(prev_count != preempt_count())) { +- pr_err("huh, entered softirq %u %s %p with preempt_count %08x, exited with %08x?\n", +- vec_nr, softirq_to_name[vec_nr], h->action, +- prev_count, preempt_count()); +- preempt_count_set(prev_count); +- } +- h++; +- pending >>= softirq_bit; +- } +- +- rcu_bh_qs(); +- local_irq_disable(); ++ handle_pending_softirqs(pending); + + pending = local_softirq_pending(); + if (pending) { +@@ -320,6 +445,285 @@ + } + + /* ++ * This function must run with irqs disabled! ++ */ ++void raise_softirq_irqoff(unsigned int nr) ++{ ++ __raise_softirq_irqoff(nr); ++ ++ /* ++ * If we're in an interrupt or softirq, we're done ++ * (this also catches softirq-disabled code). We will ++ * actually run the softirq once we return from ++ * the irq or softirq. ++ * ++ * Otherwise we wake up ksoftirqd to make sure we ++ * schedule the softirq soon. ++ */ ++ if (!in_interrupt()) ++ wakeup_softirqd(); ++} ++ ++void __raise_softirq_irqoff(unsigned int nr) ++{ ++ trace_softirq_raise(nr); ++ or_softirq_pending(1UL << nr); ++} ++ ++static inline void local_bh_disable_nort(void) { local_bh_disable(); } ++static inline void _local_bh_enable_nort(void) { _local_bh_enable(); } ++static void ksoftirqd_set_sched_params(unsigned int cpu) { } ++static void ksoftirqd_clr_sched_params(unsigned int cpu, bool online) { } ++ ++#else /* !PREEMPT_RT_FULL */ ++ ++/* ++ * On RT we serialize softirq execution with a cpu local lock per softirq ++ */ ++static DEFINE_PER_CPU(struct local_irq_lock [NR_SOFTIRQS], local_softirq_locks); ++ ++void __init softirq_early_init(void) ++{ ++ int i; ++ ++ for (i = 0; i < NR_SOFTIRQS; i++) ++ local_irq_lock_init(local_softirq_locks[i]); ++} ++ ++static void lock_softirq(int which) ++{ ++ local_lock(local_softirq_locks[which]); ++} ++ ++static void unlock_softirq(int which) ++{ ++ local_unlock(local_softirq_locks[which]); ++} ++ ++static void do_single_softirq(int which) ++{ ++ unsigned long old_flags = current->flags; ++ ++ current->flags &= ~PF_MEMALLOC; ++ vtime_account_irq_enter(current); ++ current->flags |= PF_IN_SOFTIRQ; ++ lockdep_softirq_enter(); ++ local_irq_enable(); ++ handle_softirq(which); ++ local_irq_disable(); ++ lockdep_softirq_exit(); ++ current->flags &= ~PF_IN_SOFTIRQ; ++ vtime_account_irq_enter(current); ++ tsk_restore_flags(current, old_flags, PF_MEMALLOC); ++} ++ ++/* ++ * Called with interrupts disabled. Process softirqs which were raised ++ * in current context (or on behalf of ksoftirqd). ++ */ ++static void do_current_softirqs(void) ++{ ++ while (current->softirqs_raised) { ++ int i = __ffs(current->softirqs_raised); ++ unsigned int pending, mask = (1U << i); ++ ++ current->softirqs_raised &= ~mask; ++ local_irq_enable(); ++ ++ /* ++ * If the lock is contended, we boost the owner to ++ * process the softirq or leave the critical section ++ * now. ++ */ ++ lock_softirq(i); ++ local_irq_disable(); ++ softirq_set_runner(i); ++ /* ++ * Check with the local_softirq_pending() bits, ++ * whether we need to process this still or if someone ++ * else took care of it. ++ */ ++ pending = local_softirq_pending(); ++ if (pending & mask) { ++ set_softirq_pending(pending & ~mask); ++ do_single_softirq(i); ++ } ++ softirq_clr_runner(i); ++ unlock_softirq(i); ++ WARN_ON(current->softirq_nestcnt != 1); ++ } ++} ++ ++static void __local_bh_disable(void) ++{ ++ if (++current->softirq_nestcnt == 1) ++ migrate_disable(); ++} ++ ++void local_bh_disable(void) ++{ ++ __local_bh_disable(); ++} ++EXPORT_SYMBOL(local_bh_disable); ++ ++void __local_bh_disable_ip(unsigned long ip, unsigned int cnt) ++{ ++ __local_bh_disable(); ++ if (cnt & PREEMPT_CHECK_OFFSET) ++ preempt_disable(); ++} ++ ++static void __local_bh_enable(void) ++{ ++ if (WARN_ON(current->softirq_nestcnt == 0)) ++ return; ++ ++ local_irq_disable(); ++ if (current->softirq_nestcnt == 1 && current->softirqs_raised) ++ do_current_softirqs(); ++ local_irq_enable(); ++ ++ if (--current->softirq_nestcnt == 0) ++ migrate_enable(); ++} ++ ++void local_bh_enable(void) ++{ ++ __local_bh_enable(); ++} ++EXPORT_SYMBOL(local_bh_enable); ++ ++extern void __local_bh_enable_ip(unsigned long ip, unsigned int cnt) ++{ ++ __local_bh_enable(); ++ if (cnt & PREEMPT_CHECK_OFFSET) ++ preempt_enable(); ++} ++ ++void local_bh_enable_ip(unsigned long ip) ++{ ++ local_bh_enable(); ++} ++EXPORT_SYMBOL(local_bh_enable_ip); ++ ++void _local_bh_enable(void) ++{ ++ if (WARN_ON(current->softirq_nestcnt == 0)) ++ return; ++ if (--current->softirq_nestcnt == 0) ++ migrate_enable(); ++} ++EXPORT_SYMBOL(_local_bh_enable); ++ ++int in_serving_softirq(void) ++{ ++ return current->flags & PF_IN_SOFTIRQ; ++} ++EXPORT_SYMBOL(in_serving_softirq); ++ ++/* Called with preemption disabled */ ++static void run_ksoftirqd(unsigned int cpu) ++{ ++ local_irq_disable(); ++ current->softirq_nestcnt++; ++ ++ do_current_softirqs(); ++ current->softirq_nestcnt--; ++ rcu_note_context_switch(); ++ local_irq_enable(); ++} ++ ++/* ++ * Called from netif_rx_ni(). Preemption enabled, but migration ++ * disabled. So the cpu can't go away under us. ++ */ ++void thread_do_softirq(void) ++{ ++ if (!in_serving_softirq() && current->softirqs_raised) { ++ current->softirq_nestcnt++; ++ do_current_softirqs(); ++ current->softirq_nestcnt--; ++ } ++} ++ ++static void do_raise_softirq_irqoff(unsigned int nr) ++{ ++ trace_softirq_raise(nr); ++ or_softirq_pending(1UL << nr); ++ ++ /* ++ * If we are not in a hard interrupt and inside a bh disabled ++ * region, we simply raise the flag on current. local_bh_enable() ++ * will make sure that the softirq is executed. Otherwise we ++ * delegate it to ksoftirqd. ++ */ ++ if (!in_irq() && current->softirq_nestcnt) ++ current->softirqs_raised |= (1U << nr); ++ else if (__this_cpu_read(ksoftirqd)) ++ __this_cpu_read(ksoftirqd)->softirqs_raised |= (1U << nr); ++} ++ ++void __raise_softirq_irqoff(unsigned int nr) ++{ ++ do_raise_softirq_irqoff(nr); ++ if (!in_irq() && !current->softirq_nestcnt) ++ wakeup_softirqd(); ++} ++ ++/* ++ * This function must run with irqs disabled! ++ */ ++void raise_softirq_irqoff(unsigned int nr) ++{ ++ do_raise_softirq_irqoff(nr); ++ ++ /* ++ * If we're in an hard interrupt we let irq return code deal ++ * with the wakeup of ksoftirqd. ++ */ ++ if (in_irq()) ++ return; ++ /* ++ * If we are in thread context but outside of a bh disabled ++ * region, we need to wake ksoftirqd as well. ++ * ++ * CHECKME: Some of the places which do that could be wrapped ++ * into local_bh_disable/enable pairs. Though it's unclear ++ * whether this is worth the effort. To find those places just ++ * raise a WARN() if the condition is met. ++ */ ++ if (!current->softirq_nestcnt) ++ wakeup_softirqd(); ++} ++ ++static inline int ksoftirqd_softirq_pending(void) ++{ ++ return current->softirqs_raised; ++} ++ ++static inline void local_bh_disable_nort(void) { } ++static inline void _local_bh_enable_nort(void) { } ++ ++static inline void ksoftirqd_set_sched_params(unsigned int cpu) ++{ ++ struct sched_param param = { .sched_priority = 1 }; ++ ++ sched_setscheduler(current, SCHED_FIFO, ¶m); ++ /* Take over all pending softirqs when starting */ ++ local_irq_disable(); ++ current->softirqs_raised = local_softirq_pending(); ++ local_irq_enable(); ++} ++ ++static inline void ksoftirqd_clr_sched_params(unsigned int cpu, bool online) ++{ ++ struct sched_param param = { .sched_priority = 0 }; ++ ++ sched_setscheduler(current, SCHED_NORMAL, ¶m); ++} ++ ++#endif /* PREEMPT_RT_FULL */ ++/* + * Enter an interrupt context. + */ + void irq_enter(void) +@@ -330,9 +734,9 @@ + * Prevent raise_softirq from needlessly waking up ksoftirqd + * here, as softirq will be serviced on return from interrupt. + */ +- local_bh_disable(); ++ local_bh_disable_nort(); + tick_irq_enter(); +- _local_bh_enable(); ++ _local_bh_enable_nort(); + } + + __irq_enter(); +@@ -340,6 +744,7 @@ + + static inline void invoke_softirq(void) + { ++#ifndef CONFIG_PREEMPT_RT_FULL + if (!force_irqthreads) { + #ifdef CONFIG_HAVE_IRQ_EXIT_ON_IRQ_STACK + /* +@@ -359,6 +764,15 @@ + } else { + wakeup_softirqd(); + } ++#else /* PREEMPT_RT_FULL */ ++ unsigned long flags; ++ ++ local_irq_save(flags); ++ if (__this_cpu_read(ksoftirqd) && ++ __this_cpu_read(ksoftirqd)->softirqs_raised) ++ wakeup_softirqd(); ++ local_irq_restore(flags); ++#endif + } + + static inline void tick_irq_exit(void) +@@ -395,26 +809,6 @@ + trace_hardirq_exit(); /* must be last! */ + } + +-/* +- * This function must run with irqs disabled! +- */ +-inline void raise_softirq_irqoff(unsigned int nr) +-{ +- __raise_softirq_irqoff(nr); +- +- /* +- * If we're in an interrupt or softirq, we're done +- * (this also catches softirq-disabled code). We will +- * actually run the softirq once we return from +- * the irq or softirq. +- * +- * Otherwise we wake up ksoftirqd to make sure we +- * schedule the softirq soon. +- */ +- if (!in_interrupt()) +- wakeup_softirqd(); +-} +- + void raise_softirq(unsigned int nr) + { + unsigned long flags; +@@ -424,12 +818,6 @@ + local_irq_restore(flags); + } + +-void __raise_softirq_irqoff(unsigned int nr) +-{ +- trace_softirq_raise(nr); +- or_softirq_pending(1UL << nr); +-} +- + void open_softirq(int nr, void (*action)(struct softirq_action *)) + { + softirq_vec[nr].action = action; +@@ -446,15 +834,45 @@ + static DEFINE_PER_CPU(struct tasklet_head, tasklet_vec); + static DEFINE_PER_CPU(struct tasklet_head, tasklet_hi_vec); + ++static void inline ++__tasklet_common_schedule(struct tasklet_struct *t, struct tasklet_head *head, unsigned int nr) ++{ ++ if (tasklet_trylock(t)) { ++again: ++ /* We may have been preempted before tasklet_trylock ++ * and __tasklet_action may have already run. ++ * So double check the sched bit while the takslet ++ * is locked before adding it to the list. ++ */ ++ if (test_bit(TASKLET_STATE_SCHED, &t->state)) { ++ t->next = NULL; ++ *head->tail = t; ++ head->tail = &(t->next); ++ raise_softirq_irqoff(nr); ++ tasklet_unlock(t); ++ } else { ++ /* This is subtle. If we hit the corner case above ++ * It is possible that we get preempted right here, ++ * and another task has successfully called ++ * tasklet_schedule(), then this function, and ++ * failed on the trylock. Thus we must be sure ++ * before releasing the tasklet lock, that the ++ * SCHED_BIT is clear. Otherwise the tasklet ++ * may get its SCHED_BIT set, but not added to the ++ * list ++ */ ++ if (!tasklet_tryunlock(t)) ++ goto again; ++ } ++ } ++} ++ + void __tasklet_schedule(struct tasklet_struct *t) + { + unsigned long flags; + + local_irq_save(flags); +- t->next = NULL; +- *__this_cpu_read(tasklet_vec.tail) = t; +- __this_cpu_write(tasklet_vec.tail, &(t->next)); +- raise_softirq_irqoff(TASKLET_SOFTIRQ); ++ __tasklet_common_schedule(t, this_cpu_ptr(&tasklet_vec), TASKLET_SOFTIRQ); + local_irq_restore(flags); + } + EXPORT_SYMBOL(__tasklet_schedule); +@@ -464,10 +882,7 @@ + unsigned long flags; + + local_irq_save(flags); +- t->next = NULL; +- *__this_cpu_read(tasklet_hi_vec.tail) = t; +- __this_cpu_write(tasklet_hi_vec.tail, &(t->next)); +- raise_softirq_irqoff(HI_SOFTIRQ); ++ __tasklet_common_schedule(t, this_cpu_ptr(&tasklet_hi_vec), HI_SOFTIRQ); + local_irq_restore(flags); + } + EXPORT_SYMBOL(__tasklet_hi_schedule); +@@ -476,82 +891,122 @@ + { + BUG_ON(!irqs_disabled()); + +- t->next = __this_cpu_read(tasklet_hi_vec.head); +- __this_cpu_write(tasklet_hi_vec.head, t); +- __raise_softirq_irqoff(HI_SOFTIRQ); ++ __tasklet_hi_schedule(t); + } + EXPORT_SYMBOL(__tasklet_hi_schedule_first); + +-static void tasklet_action(struct softirq_action *a) ++void tasklet_enable(struct tasklet_struct *t) + { +- struct tasklet_struct *list; ++ if (!atomic_dec_and_test(&t->count)) ++ return; ++ if (test_and_clear_bit(TASKLET_STATE_PENDING, &t->state)) ++ tasklet_schedule(t); ++} ++EXPORT_SYMBOL(tasklet_enable); + +- local_irq_disable(); +- list = __this_cpu_read(tasklet_vec.head); +- __this_cpu_write(tasklet_vec.head, NULL); +- __this_cpu_write(tasklet_vec.tail, this_cpu_ptr(&tasklet_vec.head)); +- local_irq_enable(); ++static void __tasklet_action(struct softirq_action *a, ++ struct tasklet_struct *list) ++{ ++ int loops = 1000000; + + while (list) { + struct tasklet_struct *t = list; + + list = list->next; + +- if (tasklet_trylock(t)) { +- if (!atomic_read(&t->count)) { +- if (!test_and_clear_bit(TASKLET_STATE_SCHED, +- &t->state)) +- BUG(); +- t->func(t->data); +- tasklet_unlock(t); +- continue; +- } +- tasklet_unlock(t); ++ /* ++ * Should always succeed - after a tasklist got on the ++ * list (after getting the SCHED bit set from 0 to 1), ++ * nothing but the tasklet softirq it got queued to can ++ * lock it: ++ */ ++ if (!tasklet_trylock(t)) { ++ WARN_ON(1); ++ continue; + } + +- local_irq_disable(); + t->next = NULL; +- *__this_cpu_read(tasklet_vec.tail) = t; +- __this_cpu_write(tasklet_vec.tail, &(t->next)); +- __raise_softirq_irqoff(TASKLET_SOFTIRQ); +- local_irq_enable(); ++ ++ /* ++ * If we cannot handle the tasklet because it's disabled, ++ * mark it as pending. tasklet_enable() will later ++ * re-schedule the tasklet. ++ */ ++ if (unlikely(atomic_read(&t->count))) { ++out_disabled: ++ /* implicit unlock: */ ++ wmb(); ++ t->state = TASKLET_STATEF_PENDING; ++ continue; ++ } ++ ++ /* ++ * After this point on the tasklet might be rescheduled ++ * on another CPU, but it can only be added to another ++ * CPU's tasklet list if we unlock the tasklet (which we ++ * dont do yet). ++ */ ++ if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state)) ++ WARN_ON(1); ++ ++again: ++ t->func(t->data); ++ ++ /* ++ * Try to unlock the tasklet. We must use cmpxchg, because ++ * another CPU might have scheduled or disabled the tasklet. ++ * We only allow the STATE_RUN -> 0 transition here. ++ */ ++ while (!tasklet_tryunlock(t)) { ++ /* ++ * If it got disabled meanwhile, bail out: ++ */ ++ if (atomic_read(&t->count)) ++ goto out_disabled; ++ /* ++ * If it got scheduled meanwhile, re-execute ++ * the tasklet function: ++ */ ++ if (test_and_clear_bit(TASKLET_STATE_SCHED, &t->state)) ++ goto again; ++ if (!--loops) { ++ printk("hm, tasklet state: %08lx\n", t->state); ++ WARN_ON(1); ++ tasklet_unlock(t); ++ break; ++ } ++ } + } + } + ++static void tasklet_action(struct softirq_action *a) ++{ ++ struct tasklet_struct *list; ++ ++ local_irq_disable(); ++ ++ list = __this_cpu_read(tasklet_vec.head); ++ __this_cpu_write(tasklet_vec.head, NULL); ++ __this_cpu_write(tasklet_vec.tail, this_cpu_ptr(&tasklet_vec.head)); ++ ++ local_irq_enable(); ++ ++ __tasklet_action(a, list); ++} ++ + static void tasklet_hi_action(struct softirq_action *a) + { + struct tasklet_struct *list; + + local_irq_disable(); ++ + list = __this_cpu_read(tasklet_hi_vec.head); + __this_cpu_write(tasklet_hi_vec.head, NULL); + __this_cpu_write(tasklet_hi_vec.tail, this_cpu_ptr(&tasklet_hi_vec.head)); +- local_irq_enable(); +- +- while (list) { +- struct tasklet_struct *t = list; +- +- list = list->next; + +- if (tasklet_trylock(t)) { +- if (!atomic_read(&t->count)) { +- if (!test_and_clear_bit(TASKLET_STATE_SCHED, +- &t->state)) +- BUG(); +- t->func(t->data); +- tasklet_unlock(t); +- continue; +- } +- tasklet_unlock(t); +- } ++ local_irq_enable(); + +- local_irq_disable(); +- t->next = NULL; +- *__this_cpu_read(tasklet_hi_vec.tail) = t; +- __this_cpu_write(tasklet_hi_vec.tail, &(t->next)); +- __raise_softirq_irqoff(HI_SOFTIRQ); +- local_irq_enable(); +- } ++ __tasklet_action(a, list); + } + + void tasklet_init(struct tasklet_struct *t, +@@ -572,7 +1027,7 @@ + + while (test_and_set_bit(TASKLET_STATE_SCHED, &t->state)) { + do { +- yield(); ++ msleep(1); + } while (test_bit(TASKLET_STATE_SCHED, &t->state)); + } + tasklet_unlock_wait(t); +@@ -646,25 +1101,26 @@ + open_softirq(HI_SOFTIRQ, tasklet_hi_action); + } + +-static int ksoftirqd_should_run(unsigned int cpu) +-{ +- return local_softirq_pending(); +-} +- +-static void run_ksoftirqd(unsigned int cpu) ++#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT_FULL) ++void tasklet_unlock_wait(struct tasklet_struct *t) + { +- local_irq_disable(); +- if (local_softirq_pending()) { ++ while (test_bit(TASKLET_STATE_RUN, &(t)->state)) { + /* +- * We can safely run softirq on inline stack, as we are not deep +- * in the task stack here. ++ * Hack for now to avoid this busy-loop: + */ +- __do_softirq(); +- local_irq_enable(); +- cond_resched_rcu_qs(); +- return; ++#ifdef CONFIG_PREEMPT_RT_FULL ++ msleep(1); ++#else ++ barrier(); ++#endif + } +- local_irq_enable(); ++} ++EXPORT_SYMBOL(tasklet_unlock_wait); ++#endif ++ ++static int ksoftirqd_should_run(unsigned int cpu) ++{ ++ return ksoftirqd_softirq_pending(); + } + + #ifdef CONFIG_HOTPLUG_CPU +@@ -746,6 +1202,8 @@ + + static struct smp_hotplug_thread softirq_threads = { + .store = &ksoftirqd, ++ .setup = ksoftirqd_set_sched_params, ++ .cleanup = ksoftirqd_clr_sched_params, + .thread_should_run = ksoftirqd_should_run, + .thread_fn = run_ksoftirqd, + .thread_comm = "ksoftirqd/%u", +diff -Nur linux-4.1.10.orig/kernel/stop_machine.c linux-4.1.10/kernel/stop_machine.c +--- linux-4.1.10.orig/kernel/stop_machine.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/stop_machine.c 2015-10-07 18:00:08.000000000 +0200 +@@ -30,12 +30,12 @@ + atomic_t nr_todo; /* nr left to execute */ + bool executed; /* actually executed? */ + int ret; /* collected return value */ +- struct completion completion; /* fired if nr_todo reaches 0 */ ++ struct task_struct *waiter; /* woken when nr_todo reaches 0 */ + }; + + /* the actual stopper, one per every possible cpu, enabled on online cpus */ + struct cpu_stopper { +- spinlock_t lock; ++ raw_spinlock_t lock; + bool enabled; /* is this stopper enabled? */ + struct list_head works; /* list of pending works */ + }; +@@ -56,7 +56,7 @@ + { + memset(done, 0, sizeof(*done)); + atomic_set(&done->nr_todo, nr_todo); +- init_completion(&done->completion); ++ done->waiter = current; + } + + /* signal completion unless @done is NULL */ +@@ -65,8 +65,10 @@ + if (done) { + if (executed) + done->executed = true; +- if (atomic_dec_and_test(&done->nr_todo)) +- complete(&done->completion); ++ if (atomic_dec_and_test(&done->nr_todo)) { ++ wake_up_process(done->waiter); ++ done->waiter = NULL; ++ } + } + } + +@@ -78,7 +80,7 @@ + + unsigned long flags; + +- spin_lock_irqsave(&stopper->lock, flags); ++ raw_spin_lock_irqsave(&stopper->lock, flags); + + if (stopper->enabled) { + list_add_tail(&work->list, &stopper->works); +@@ -86,7 +88,23 @@ + } else + cpu_stop_signal_done(work->done, false); + +- spin_unlock_irqrestore(&stopper->lock, flags); ++ raw_spin_unlock_irqrestore(&stopper->lock, flags); ++} ++ ++static void wait_for_stop_done(struct cpu_stop_done *done) ++{ ++ set_current_state(TASK_UNINTERRUPTIBLE); ++ while (atomic_read(&done->nr_todo)) { ++ schedule(); ++ set_current_state(TASK_UNINTERRUPTIBLE); ++ } ++ /* ++ * We need to wait until cpu_stop_signal_done() has cleared ++ * done->waiter. ++ */ ++ while (done->waiter) ++ cpu_relax(); ++ set_current_state(TASK_RUNNING); + } + + /** +@@ -120,7 +138,7 @@ + + cpu_stop_init_done(&done, 1); + cpu_stop_queue_work(cpu, &work); +- wait_for_completion(&done.completion); ++ wait_for_stop_done(&done); + return done.executed ? done.ret : -ENOENT; + } + +@@ -248,7 +266,7 @@ + struct irq_cpu_stop_queue_work_info call_args; + struct multi_stop_data msdata; + +- preempt_disable(); ++ preempt_disable_nort(); + msdata = (struct multi_stop_data){ + .fn = fn, + .data = arg, +@@ -281,7 +299,7 @@ + * This relies on the stopper workqueues to be FIFO. + */ + if (!cpu_active(cpu1) || !cpu_active(cpu2)) { +- preempt_enable(); ++ preempt_enable_nort(); + return -ENOENT; + } + +@@ -295,9 +313,9 @@ + &irq_cpu_stop_queue_work, + &call_args, 1); + lg_local_unlock(&stop_cpus_lock); +- preempt_enable(); ++ preempt_enable_nort(); + +- wait_for_completion(&done.completion); ++ wait_for_stop_done(&done); + + return done.executed ? done.ret : -ENOENT; + } +@@ -329,7 +347,7 @@ + + static void queue_stop_cpus_work(const struct cpumask *cpumask, + cpu_stop_fn_t fn, void *arg, +- struct cpu_stop_done *done) ++ struct cpu_stop_done *done, bool inactive) + { + struct cpu_stop_work *work; + unsigned int cpu; +@@ -343,11 +361,13 @@ + } + + /* +- * Disable preemption while queueing to avoid getting +- * preempted by a stopper which might wait for other stoppers +- * to enter @fn which can lead to deadlock. ++ * Make sure that all work is queued on all cpus before ++ * any of the cpus can execute it. + */ +- lg_global_lock(&stop_cpus_lock); ++ if (!inactive) ++ lg_global_lock(&stop_cpus_lock); ++ else ++ lg_global_trylock_relax(&stop_cpus_lock); + for_each_cpu(cpu, cpumask) + cpu_stop_queue_work(cpu, &per_cpu(stop_cpus_work, cpu)); + lg_global_unlock(&stop_cpus_lock); +@@ -359,8 +379,8 @@ + struct cpu_stop_done done; + + cpu_stop_init_done(&done, cpumask_weight(cpumask)); +- queue_stop_cpus_work(cpumask, fn, arg, &done); +- wait_for_completion(&done.completion); ++ queue_stop_cpus_work(cpumask, fn, arg, &done, false); ++ wait_for_stop_done(&done); + return done.executed ? done.ret : -ENOENT; + } + +@@ -439,9 +459,9 @@ + unsigned long flags; + int run; + +- spin_lock_irqsave(&stopper->lock, flags); ++ raw_spin_lock_irqsave(&stopper->lock, flags); + run = !list_empty(&stopper->works); +- spin_unlock_irqrestore(&stopper->lock, flags); ++ raw_spin_unlock_irqrestore(&stopper->lock, flags); + return run; + } + +@@ -453,13 +473,13 @@ + + repeat: + work = NULL; +- spin_lock_irq(&stopper->lock); ++ raw_spin_lock_irq(&stopper->lock); + if (!list_empty(&stopper->works)) { + work = list_first_entry(&stopper->works, + struct cpu_stop_work, list); + list_del_init(&work->list); + } +- spin_unlock_irq(&stopper->lock); ++ raw_spin_unlock_irq(&stopper->lock); + + if (work) { + cpu_stop_fn_t fn = work->fn; +@@ -467,6 +487,16 @@ + struct cpu_stop_done *done = work->done; + char ksym_buf[KSYM_NAME_LEN] __maybe_unused; + ++ /* ++ * Wait until the stopper finished scheduling on all ++ * cpus ++ */ ++ lg_global_lock(&stop_cpus_lock); ++ /* ++ * Let other cpu threads continue as well ++ */ ++ lg_global_unlock(&stop_cpus_lock); ++ + /* cpu stop callbacks are not allowed to sleep */ + preempt_disable(); + +@@ -481,7 +511,13 @@ + kallsyms_lookup((unsigned long)fn, NULL, NULL, NULL, + ksym_buf), arg); + ++ /* ++ * Make sure that the wakeup and setting done->waiter ++ * to NULL is atomic. ++ */ ++ local_irq_disable(); + cpu_stop_signal_done(done, true); ++ local_irq_enable(); + goto repeat; + } + } +@@ -500,20 +536,20 @@ + unsigned long flags; + + /* drain remaining works */ +- spin_lock_irqsave(&stopper->lock, flags); ++ raw_spin_lock_irqsave(&stopper->lock, flags); + list_for_each_entry(work, &stopper->works, list) + cpu_stop_signal_done(work->done, false); + stopper->enabled = false; +- spin_unlock_irqrestore(&stopper->lock, flags); ++ raw_spin_unlock_irqrestore(&stopper->lock, flags); + } + + static void cpu_stop_unpark(unsigned int cpu) + { + struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu); + +- spin_lock_irq(&stopper->lock); ++ raw_spin_lock_irq(&stopper->lock); + stopper->enabled = true; +- spin_unlock_irq(&stopper->lock); ++ raw_spin_unlock_irq(&stopper->lock); + } + + static struct smp_hotplug_thread cpu_stop_threads = { +@@ -535,10 +571,12 @@ + for_each_possible_cpu(cpu) { + struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu); + +- spin_lock_init(&stopper->lock); ++ raw_spin_lock_init(&stopper->lock); + INIT_LIST_HEAD(&stopper->works); + } + ++ lg_lock_init(&stop_cpus_lock, "stop_cpus_lock"); ++ + BUG_ON(smpboot_register_percpu_thread(&cpu_stop_threads)); + stop_machine_initialized = true; + return 0; +@@ -634,11 +672,11 @@ + set_state(&msdata, MULTI_STOP_PREPARE); + cpu_stop_init_done(&done, num_active_cpus()); + queue_stop_cpus_work(cpu_active_mask, multi_cpu_stop, &msdata, +- &done); ++ &done, true); + ret = multi_cpu_stop(&msdata); + + /* Busy wait for completion. */ +- while (!completion_done(&done.completion)) ++ while (atomic_read(&done.nr_todo)) + cpu_relax(); + + mutex_unlock(&stop_cpus_mutex); +diff -Nur linux-4.1.10.orig/kernel/time/hrtimer.c linux-4.1.10/kernel/time/hrtimer.c +--- linux-4.1.10.orig/kernel/time/hrtimer.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/time/hrtimer.c 2015-10-07 18:00:08.000000000 +0200 +@@ -48,11 +48,13 @@ + #include + #include + #include ++#include + #include + + #include + + #include ++#include + + #include "tick-internal.h" + +@@ -576,8 +578,7 @@ + * When the callback is running, we do not reprogram the clock event + * device. The timer callback is either running on a different CPU or + * the callback is executed in the hrtimer_interrupt context. The +- * reprogramming is handled either by the softirq, which called the +- * callback or at the end of the hrtimer_interrupt. ++ * reprogramming is handled at the end of the hrtimer_interrupt. + */ + if (hrtimer_callback_running(timer)) + return 0; +@@ -621,6 +622,9 @@ + return res; + } + ++static void __run_hrtimer(struct hrtimer *timer, ktime_t *now); ++static int hrtimer_rt_defer(struct hrtimer *timer); ++ + /* + * Initialize the high resolution related parts of cpu_base + */ +@@ -630,6 +634,21 @@ + base->hres_active = 0; + } + ++static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer, ++ struct hrtimer_clock_base *base, ++ int wakeup) ++{ ++ if (!hrtimer_reprogram(timer, base)) ++ return 0; ++ if (!wakeup) ++ return -ETIME; ++#ifdef CONFIG_PREEMPT_RT_BASE ++ if (!hrtimer_rt_defer(timer)) ++ return -ETIME; ++#endif ++ return 1; ++} ++ + static inline ktime_t hrtimer_update_base(struct hrtimer_cpu_base *base) + { + ktime_t *offs_real = &base->clock_base[HRTIMER_BASE_REALTIME].offset; +@@ -695,6 +714,44 @@ + + static DECLARE_WORK(hrtimer_work, clock_was_set_work); + ++#ifdef CONFIG_PREEMPT_RT_FULL ++/* ++ * RT can not call schedule_work from real interrupt context. ++ * Need to make a thread to do the real work. ++ */ ++static struct task_struct *clock_set_delay_thread; ++static bool do_clock_set_delay; ++ ++static int run_clock_set_delay(void *ignore) ++{ ++ while (!kthread_should_stop()) { ++ set_current_state(TASK_INTERRUPTIBLE); ++ if (do_clock_set_delay) { ++ do_clock_set_delay = false; ++ schedule_work(&hrtimer_work); ++ } ++ schedule(); ++ } ++ __set_current_state(TASK_RUNNING); ++ return 0; ++} ++ ++void clock_was_set_delayed(void) ++{ ++ do_clock_set_delay = true; ++ /* Make visible before waking up process */ ++ smp_wmb(); ++ wake_up_process(clock_set_delay_thread); ++} ++ ++static __init int create_clock_set_delay_thread(void) ++{ ++ clock_set_delay_thread = kthread_run(run_clock_set_delay, NULL, "kclksetdelayd"); ++ BUG_ON(!clock_set_delay_thread); ++ return 0; ++} ++early_initcall(create_clock_set_delay_thread); ++#else /* PREEMPT_RT_FULL */ + /* + * Called from timekeeping and resume code to reprogramm the hrtimer + * interrupt device on all cpus. +@@ -703,6 +760,7 @@ + { + schedule_work(&hrtimer_work); + } ++#endif + + #else + +@@ -711,6 +769,13 @@ + static inline int hrtimer_switch_to_hres(void) { return 0; } + static inline void + hrtimer_force_reprogram(struct hrtimer_cpu_base *base, int skip_equal) { } ++static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer, ++ struct hrtimer_clock_base *base, ++ int wakeup) ++{ ++ return 0; ++} ++ + static inline int hrtimer_reprogram(struct hrtimer *timer, + struct hrtimer_clock_base *base) + { +@@ -718,7 +783,6 @@ + } + static inline void hrtimer_init_hres(struct hrtimer_cpu_base *base) { } + static inline void retrigger_next_event(void *arg) { } +- + #endif /* CONFIG_HIGH_RES_TIMERS */ + + /* +@@ -836,6 +900,32 @@ + } + EXPORT_SYMBOL_GPL(hrtimer_forward); + ++#ifdef CONFIG_PREEMPT_RT_BASE ++# define wake_up_timer_waiters(b) wake_up(&(b)->wait) ++ ++/** ++ * hrtimer_wait_for_timer - Wait for a running timer ++ * ++ * @timer: timer to wait for ++ * ++ * The function waits in case the timers callback function is ++ * currently executed on the waitqueue of the timer base. The ++ * waitqueue is woken up after the timer callback function has ++ * finished execution. ++ */ ++void hrtimer_wait_for_timer(const struct hrtimer *timer) ++{ ++ struct hrtimer_clock_base *base = timer->base; ++ ++ if (base && base->cpu_base && !timer->irqsafe) ++ wait_event(base->cpu_base->wait, ++ !(timer->state & HRTIMER_STATE_CALLBACK)); ++} ++ ++#else ++# define wake_up_timer_waiters(b) do { } while (0) ++#endif ++ + /* + * enqueue_hrtimer - internal function to (re)start a timer + * +@@ -879,6 +969,11 @@ + if (!(timer->state & HRTIMER_STATE_ENQUEUED)) + goto out; + ++ if (unlikely(!list_empty(&timer->cb_entry))) { ++ list_del_init(&timer->cb_entry); ++ goto out; ++ } ++ + next_timer = timerqueue_getnext(&base->active); + timerqueue_del(&base->active, &timer->node); + if (&timer->node == next_timer) { +@@ -966,7 +1061,16 @@ + new_base = switch_hrtimer_base(timer, base, mode & HRTIMER_MODE_PINNED); + + timer_stats_hrtimer_set_start_info(timer); ++#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST ++ { ++ ktime_t now = new_base->get_time(); + ++ if (ktime_to_ns(tim) < ktime_to_ns(now)) ++ timer->praecox = now; ++ else ++ timer->praecox = ktime_set(0, 0); ++ } ++#endif + leftmost = enqueue_hrtimer(timer, new_base); + + if (!leftmost) { +@@ -980,15 +1084,26 @@ + * on dynticks target. + */ + wake_up_nohz_cpu(new_base->cpu_base->cpu); +- } else if (new_base->cpu_base == this_cpu_ptr(&hrtimer_bases) && +- hrtimer_reprogram(timer, new_base)) { ++ } else if (new_base->cpu_base == this_cpu_ptr(&hrtimer_bases)) { ++ ++ ret = hrtimer_enqueue_reprogram(timer, new_base, wakeup); ++ if (ret < 0) { ++ /* ++ * In case we failed to reprogram the timer (mostly ++ * because out current timer is already elapsed), ++ * remove it again and report a failure. This avoids ++ * stale base->first entries. ++ */ ++ debug_deactivate(timer); ++ __remove_hrtimer(timer, new_base, ++ timer->state & HRTIMER_STATE_CALLBACK, 0); ++ } else if (ret > 0) { + /* + * Only allow reprogramming if the new base is on this CPU. + * (it might still be on another CPU if the timer was pending) + * + * XXX send_remote_softirq() ? + */ +- if (wakeup) { + /* + * We need to drop cpu_base->lock to avoid a + * lock ordering issue vs. rq->lock. +@@ -996,9 +1111,7 @@ + raw_spin_unlock(&new_base->cpu_base->lock); + raise_softirq_irqoff(HRTIMER_SOFTIRQ); + local_irq_restore(flags); +- return ret; +- } else { +- __raise_softirq_irqoff(HRTIMER_SOFTIRQ); ++ return 0; + } + } + +@@ -1089,7 +1202,7 @@ + + if (ret >= 0) + return ret; +- cpu_relax(); ++ hrtimer_wait_for_timer(timer); + } + } + EXPORT_SYMBOL_GPL(hrtimer_cancel); +@@ -1153,6 +1266,7 @@ + + base = hrtimer_clockid_to_base(clock_id); + timer->base = &cpu_base->clock_base[base]; ++ INIT_LIST_HEAD(&timer->cb_entry); + timerqueue_init(&timer->node); + + #ifdef CONFIG_TIMER_STATS +@@ -1236,6 +1350,126 @@ + timer->state &= ~HRTIMER_STATE_CALLBACK; + } + ++static enum hrtimer_restart hrtimer_wakeup(struct hrtimer *timer); ++ ++#ifdef CONFIG_PREEMPT_RT_BASE ++static void hrtimer_rt_reprogram(int restart, struct hrtimer *timer, ++ struct hrtimer_clock_base *base) ++{ ++ /* ++ * Note, we clear the callback flag before we requeue the ++ * timer otherwise we trigger the callback_running() check ++ * in hrtimer_reprogram(). ++ */ ++ timer->state &= ~HRTIMER_STATE_CALLBACK; ++ ++ if (restart != HRTIMER_NORESTART) { ++ BUG_ON(hrtimer_active(timer)); ++ /* ++ * Enqueue the timer, if it's the leftmost timer then ++ * we need to reprogram it. ++ */ ++ if (!enqueue_hrtimer(timer, base)) ++ return; ++ ++#ifndef CONFIG_HIGH_RES_TIMERS ++ } ++#else ++ if (base->cpu_base->hres_active && ++ hrtimer_reprogram(timer, base)) ++ goto requeue; ++ ++ } else if (hrtimer_active(timer)) { ++ /* ++ * If the timer was rearmed on another CPU, reprogram ++ * the event device. ++ */ ++ if (&timer->node == base->active.next && ++ base->cpu_base->hres_active && ++ hrtimer_reprogram(timer, base)) ++ goto requeue; ++ } ++ return; ++ ++requeue: ++ /* ++ * Timer is expired. Thus move it from tree to pending list ++ * again. ++ */ ++ __remove_hrtimer(timer, base, timer->state, 0); ++ list_add_tail(&timer->cb_entry, &base->expired); ++#endif ++} ++ ++/* ++ * The changes in mainline which removed the callback modes from ++ * hrtimer are not yet working with -rt. The non wakeup_process() ++ * based callbacks which involve sleeping locks need to be treated ++ * seperately. ++ */ ++static void hrtimer_rt_run_pending(void) ++{ ++ enum hrtimer_restart (*fn)(struct hrtimer *); ++ struct hrtimer_cpu_base *cpu_base; ++ struct hrtimer_clock_base *base; ++ struct hrtimer *timer; ++ int index, restart; ++ ++ local_irq_disable(); ++ cpu_base = &per_cpu(hrtimer_bases, smp_processor_id()); ++ ++ raw_spin_lock(&cpu_base->lock); ++ ++ for (index = 0; index < HRTIMER_MAX_CLOCK_BASES; index++) { ++ base = &cpu_base->clock_base[index]; ++ ++ while (!list_empty(&base->expired)) { ++ timer = list_first_entry(&base->expired, ++ struct hrtimer, cb_entry); ++ ++ /* ++ * Same as the above __run_hrtimer function ++ * just we run with interrupts enabled. ++ */ ++ debug_hrtimer_deactivate(timer); ++ __remove_hrtimer(timer, base, HRTIMER_STATE_CALLBACK, 0); ++ timer_stats_account_hrtimer(timer); ++ fn = timer->function; ++ ++ raw_spin_unlock_irq(&cpu_base->lock); ++ restart = fn(timer); ++ raw_spin_lock_irq(&cpu_base->lock); ++ ++ hrtimer_rt_reprogram(restart, timer, base); ++ } ++ } ++ ++ raw_spin_unlock_irq(&cpu_base->lock); ++ ++ wake_up_timer_waiters(cpu_base); ++} ++ ++static int hrtimer_rt_defer(struct hrtimer *timer) ++{ ++ if (timer->irqsafe) ++ return 0; ++ ++ __remove_hrtimer(timer, timer->base, timer->state, 0); ++ list_add_tail(&timer->cb_entry, &timer->base->expired); ++ return 1; ++} ++ ++#else ++ ++static inline void hrtimer_rt_run_pending(void) ++{ ++ hrtimer_peek_ahead_timers(); ++} ++ ++static inline int hrtimer_rt_defer(struct hrtimer *timer) { return 0; } ++ ++#endif ++ + #ifdef CONFIG_HIGH_RES_TIMERS + + /* +@@ -1246,7 +1480,7 @@ + { + struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases); + ktime_t expires_next, now, entry_time, delta; +- int i, retries = 0; ++ int i, retries = 0, raise = 0; + + BUG_ON(!cpu_base->hres_active); + cpu_base->nr_events++; +@@ -1281,6 +1515,15 @@ + + timer = container_of(node, struct hrtimer, node); + ++ trace_hrtimer_interrupt(raw_smp_processor_id(), ++ ktime_to_ns(ktime_sub(ktime_to_ns(timer->praecox) ? ++ timer->praecox : hrtimer_get_expires(timer), ++ basenow)), ++ current, ++ timer->function == hrtimer_wakeup ? ++ container_of(timer, struct hrtimer_sleeper, ++ timer)->task : NULL); ++ + /* + * The immediate goal for using the softexpires is + * minimizing wakeups, not running timers at the +@@ -1296,7 +1539,10 @@ + if (basenow.tv64 < hrtimer_get_softexpires_tv64(timer)) + break; + +- __run_hrtimer(timer, &basenow); ++ if (!hrtimer_rt_defer(timer)) ++ __run_hrtimer(timer, &basenow); ++ else ++ raise = 1; + } + } + /* Reevaluate the clock bases for the next expiry */ +@@ -1313,7 +1559,7 @@ + if (expires_next.tv64 == KTIME_MAX || + !tick_program_event(expires_next, 0)) { + cpu_base->hang_detected = 0; +- return; ++ goto out; + } + + /* +@@ -1357,6 +1603,9 @@ + tick_program_event(expires_next, 1); + printk_once(KERN_WARNING "hrtimer: interrupt took %llu ns\n", + ktime_to_ns(delta)); ++out: ++ if (raise) ++ raise_softirq_irqoff(HRTIMER_SOFTIRQ); + } + + /* +@@ -1392,18 +1641,18 @@ + __hrtimer_peek_ahead_timers(); + local_irq_restore(flags); + } +- +-static void run_hrtimer_softirq(struct softirq_action *h) +-{ +- hrtimer_peek_ahead_timers(); +-} +- + #else /* CONFIG_HIGH_RES_TIMERS */ + + static inline void __hrtimer_peek_ahead_timers(void) { } + + #endif /* !CONFIG_HIGH_RES_TIMERS */ + ++ ++static void run_hrtimer_softirq(struct softirq_action *h) ++{ ++ hrtimer_rt_run_pending(); ++} ++ + /* + * Called from timer softirq every jiffy, expire hrtimers: + * +@@ -1436,7 +1685,7 @@ + struct timerqueue_node *node; + struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases); + struct hrtimer_clock_base *base; +- int index, gettime = 1; ++ int index, gettime = 1, raise = 0; + + if (hrtimer_hres_active()) + return; +@@ -1461,10 +1710,16 @@ + hrtimer_get_expires_tv64(timer)) + break; + +- __run_hrtimer(timer, &base->softirq_time); ++ if (!hrtimer_rt_defer(timer)) ++ __run_hrtimer(timer, &base->softirq_time); ++ else ++ raise = 1; + } + raw_spin_unlock(&cpu_base->lock); + } ++ ++ if (raise) ++ raise_softirq_irqoff(HRTIMER_SOFTIRQ); + } + + /* +@@ -1486,16 +1741,18 @@ + void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, struct task_struct *task) + { + sl->timer.function = hrtimer_wakeup; ++ sl->timer.irqsafe = 1; + sl->task = task; + } + EXPORT_SYMBOL_GPL(hrtimer_init_sleeper); + +-static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mode) ++static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mode, ++ unsigned long state) + { + hrtimer_init_sleeper(t, current); + + do { +- set_current_state(TASK_INTERRUPTIBLE); ++ set_current_state(state); + hrtimer_start_expires(&t->timer, mode); + if (!hrtimer_active(&t->timer)) + t->task = NULL; +@@ -1539,7 +1796,8 @@ + HRTIMER_MODE_ABS); + hrtimer_set_expires_tv64(&t.timer, restart->nanosleep.expires); + +- if (do_nanosleep(&t, HRTIMER_MODE_ABS)) ++ /* cpu_chill() does not care about restart state. */ ++ if (do_nanosleep(&t, HRTIMER_MODE_ABS, TASK_INTERRUPTIBLE)) + goto out; + + rmtp = restart->nanosleep.rmtp; +@@ -1556,8 +1814,10 @@ + return ret; + } + +-long hrtimer_nanosleep(struct timespec *rqtp, struct timespec __user *rmtp, +- const enum hrtimer_mode mode, const clockid_t clockid) ++static long ++__hrtimer_nanosleep(struct timespec *rqtp, struct timespec __user *rmtp, ++ const enum hrtimer_mode mode, const clockid_t clockid, ++ unsigned long state) + { + struct restart_block *restart; + struct hrtimer_sleeper t; +@@ -1570,7 +1830,7 @@ + + hrtimer_init_on_stack(&t.timer, clockid, mode); + hrtimer_set_expires_range_ns(&t.timer, timespec_to_ktime(*rqtp), slack); +- if (do_nanosleep(&t, mode)) ++ if (do_nanosleep(&t, mode, state)) + goto out; + + /* Absolute timers do not update the rmtp value and restart: */ +@@ -1597,6 +1857,12 @@ + return ret; + } + ++long hrtimer_nanosleep(struct timespec *rqtp, struct timespec __user *rmtp, ++ const enum hrtimer_mode mode, const clockid_t clockid) ++{ ++ return __hrtimer_nanosleep(rqtp, rmtp, mode, clockid, TASK_INTERRUPTIBLE); ++} ++ + SYSCALL_DEFINE2(nanosleep, struct timespec __user *, rqtp, + struct timespec __user *, rmtp) + { +@@ -1611,6 +1877,26 @@ + return hrtimer_nanosleep(&tu, rmtp, HRTIMER_MODE_REL, CLOCK_MONOTONIC); + } + ++#ifdef CONFIG_PREEMPT_RT_FULL ++/* ++ * Sleep for 1 ms in hope whoever holds what we want will let it go. ++ */ ++void cpu_chill(void) ++{ ++ struct timespec tu = { ++ .tv_nsec = NSEC_PER_MSEC, ++ }; ++ unsigned int freeze_flag = current->flags & PF_NOFREEZE; ++ ++ current->flags |= PF_NOFREEZE; ++ __hrtimer_nanosleep(&tu, NULL, HRTIMER_MODE_REL, CLOCK_MONOTONIC, ++ TASK_UNINTERRUPTIBLE); ++ if (!freeze_flag) ++ current->flags &= ~PF_NOFREEZE; ++} ++EXPORT_SYMBOL(cpu_chill); ++#endif ++ + /* + * Functions related to boot-time initialization: + */ +@@ -1622,10 +1908,14 @@ + for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) { + cpu_base->clock_base[i].cpu_base = cpu_base; + timerqueue_init_head(&cpu_base->clock_base[i].active); ++ INIT_LIST_HEAD(&cpu_base->clock_base[i].expired); + } + + cpu_base->cpu = cpu; + hrtimer_init_hres(cpu_base); ++#ifdef CONFIG_PREEMPT_RT_BASE ++ init_waitqueue_head(&cpu_base->wait); ++#endif + } + + #ifdef CONFIG_HOTPLUG_CPU +@@ -1731,9 +2021,7 @@ + hrtimer_cpu_notify(&hrtimers_nb, (unsigned long)CPU_UP_PREPARE, + (void *)(long)smp_processor_id()); + register_cpu_notifier(&hrtimers_nb); +-#ifdef CONFIG_HIGH_RES_TIMERS + open_softirq(HRTIMER_SOFTIRQ, run_hrtimer_softirq); +-#endif + } + + /** +diff -Nur linux-4.1.10.orig/kernel/time/itimer.c linux-4.1.10/kernel/time/itimer.c +--- linux-4.1.10.orig/kernel/time/itimer.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/time/itimer.c 2015-10-07 18:00:08.000000000 +0200 +@@ -213,6 +213,7 @@ + /* We are sharing ->siglock with it_real_fn() */ + if (hrtimer_try_to_cancel(timer) < 0) { + spin_unlock_irq(&tsk->sighand->siglock); ++ hrtimer_wait_for_timer(&tsk->signal->real_timer); + goto again; + } + expires = timeval_to_ktime(value->it_value); +diff -Nur linux-4.1.10.orig/kernel/time/jiffies.c linux-4.1.10/kernel/time/jiffies.c +--- linux-4.1.10.orig/kernel/time/jiffies.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/time/jiffies.c 2015-10-07 18:00:08.000000000 +0200 +@@ -74,7 +74,8 @@ + .max_cycles = 10, + }; + +-__cacheline_aligned_in_smp DEFINE_SEQLOCK(jiffies_lock); ++__cacheline_aligned_in_smp DEFINE_RAW_SPINLOCK(jiffies_lock); ++__cacheline_aligned_in_smp seqcount_t jiffies_seq; + + #if (BITS_PER_LONG < 64) + u64 get_jiffies_64(void) +@@ -83,9 +84,9 @@ + u64 ret; + + do { +- seq = read_seqbegin(&jiffies_lock); ++ seq = read_seqcount_begin(&jiffies_seq); + ret = jiffies_64; +- } while (read_seqretry(&jiffies_lock, seq)); ++ } while (read_seqcount_retry(&jiffies_seq, seq)); + return ret; + } + EXPORT_SYMBOL(get_jiffies_64); +diff -Nur linux-4.1.10.orig/kernel/time/ntp.c linux-4.1.10/kernel/time/ntp.c +--- linux-4.1.10.orig/kernel/time/ntp.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/time/ntp.c 2015-10-07 18:00:08.000000000 +0200 +@@ -10,6 +10,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -529,10 +530,52 @@ + &sync_cmos_work, timespec_to_jiffies(&next)); + } + ++#ifdef CONFIG_PREEMPT_RT_FULL ++/* ++ * RT can not call schedule_delayed_work from real interrupt context. ++ * Need to make a thread to do the real work. ++ */ ++static struct task_struct *cmos_delay_thread; ++static bool do_cmos_delay; ++ ++static int run_cmos_delay(void *ignore) ++{ ++ while (!kthread_should_stop()) { ++ set_current_state(TASK_INTERRUPTIBLE); ++ if (do_cmos_delay) { ++ do_cmos_delay = false; ++ queue_delayed_work(system_power_efficient_wq, ++ &sync_cmos_work, 0); ++ } ++ schedule(); ++ } ++ __set_current_state(TASK_RUNNING); ++ return 0; ++} ++ ++void ntp_notify_cmos_timer(void) ++{ ++ do_cmos_delay = true; ++ /* Make visible before waking up process */ ++ smp_wmb(); ++ wake_up_process(cmos_delay_thread); ++} ++ ++static __init int create_cmos_delay_thread(void) ++{ ++ cmos_delay_thread = kthread_run(run_cmos_delay, NULL, "kcmosdelayd"); ++ BUG_ON(!cmos_delay_thread); ++ return 0; ++} ++early_initcall(create_cmos_delay_thread); ++ ++#else ++ + void ntp_notify_cmos_timer(void) + { + queue_delayed_work(system_power_efficient_wq, &sync_cmos_work, 0); + } ++#endif /* CONFIG_PREEMPT_RT_FULL */ + + #else + void ntp_notify_cmos_timer(void) { } +diff -Nur linux-4.1.10.orig/kernel/time/posix-cpu-timers.c linux-4.1.10/kernel/time/posix-cpu-timers.c +--- linux-4.1.10.orig/kernel/time/posix-cpu-timers.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/time/posix-cpu-timers.c 2015-10-07 18:00:08.000000000 +0200 +@@ -3,6 +3,7 @@ + */ + + #include ++#include + #include + #include + #include +@@ -626,7 +627,7 @@ + /* + * Disarm any old timer after extracting its expiry time. + */ +- WARN_ON_ONCE(!irqs_disabled()); ++ WARN_ON_ONCE_NONRT(!irqs_disabled()); + + ret = 0; + old_incr = timer->it.cpu.incr; +@@ -1047,7 +1048,7 @@ + /* + * Now re-arm for the new expiry time. + */ +- WARN_ON_ONCE(!irqs_disabled()); ++ WARN_ON_ONCE_NONRT(!irqs_disabled()); + arm_timer(timer); + unlock_task_sighand(p, &flags); + +@@ -1113,10 +1114,11 @@ + sig = tsk->signal; + if (sig->cputimer.running) { + struct task_cputime group_sample; ++ unsigned long flags; + +- raw_spin_lock(&sig->cputimer.lock); ++ raw_spin_lock_irqsave(&sig->cputimer.lock, flags); + group_sample = sig->cputimer.cputime; +- raw_spin_unlock(&sig->cputimer.lock); ++ raw_spin_unlock_irqrestore(&sig->cputimer.lock, flags); + + if (task_cputime_expired(&group_sample, &sig->cputime_expires)) + return 1; +@@ -1130,13 +1132,13 @@ + * already updated our counts. We need to check if any timers fire now. + * Interrupts are disabled. + */ +-void run_posix_cpu_timers(struct task_struct *tsk) ++static void __run_posix_cpu_timers(struct task_struct *tsk) + { + LIST_HEAD(firing); + struct k_itimer *timer, *next; + unsigned long flags; + +- WARN_ON_ONCE(!irqs_disabled()); ++ WARN_ON_ONCE_NONRT(!irqs_disabled()); + + /* + * The fast path checks that there are no expired thread or thread +@@ -1194,6 +1196,190 @@ + } + } + ++#ifdef CONFIG_PREEMPT_RT_BASE ++#include ++#include ++DEFINE_PER_CPU(struct task_struct *, posix_timer_task); ++DEFINE_PER_CPU(struct task_struct *, posix_timer_tasklist); ++ ++static int posix_cpu_timers_thread(void *data) ++{ ++ int cpu = (long)data; ++ ++ BUG_ON(per_cpu(posix_timer_task,cpu) != current); ++ ++ while (!kthread_should_stop()) { ++ struct task_struct *tsk = NULL; ++ struct task_struct *next = NULL; ++ ++ if (cpu_is_offline(cpu)) ++ goto wait_to_die; ++ ++ /* grab task list */ ++ raw_local_irq_disable(); ++ tsk = per_cpu(posix_timer_tasklist, cpu); ++ per_cpu(posix_timer_tasklist, cpu) = NULL; ++ raw_local_irq_enable(); ++ ++ /* its possible the list is empty, just return */ ++ if (!tsk) { ++ set_current_state(TASK_INTERRUPTIBLE); ++ schedule(); ++ __set_current_state(TASK_RUNNING); ++ continue; ++ } ++ ++ /* Process task list */ ++ while (1) { ++ /* save next */ ++ next = tsk->posix_timer_list; ++ ++ /* run the task timers, clear its ptr and ++ * unreference it ++ */ ++ __run_posix_cpu_timers(tsk); ++ tsk->posix_timer_list = NULL; ++ put_task_struct(tsk); ++ ++ /* check if this is the last on the list */ ++ if (next == tsk) ++ break; ++ tsk = next; ++ } ++ } ++ return 0; ++ ++wait_to_die: ++ /* Wait for kthread_stop */ ++ set_current_state(TASK_INTERRUPTIBLE); ++ while (!kthread_should_stop()) { ++ schedule(); ++ set_current_state(TASK_INTERRUPTIBLE); ++ } ++ __set_current_state(TASK_RUNNING); ++ return 0; ++} ++ ++static inline int __fastpath_timer_check(struct task_struct *tsk) ++{ ++ /* tsk == current, ensure it is safe to use ->signal/sighand */ ++ if (unlikely(tsk->exit_state)) ++ return 0; ++ ++ if (!task_cputime_zero(&tsk->cputime_expires)) ++ return 1; ++ ++ if (!task_cputime_zero(&tsk->signal->cputime_expires)) ++ return 1; ++ ++ return 0; ++} ++ ++void run_posix_cpu_timers(struct task_struct *tsk) ++{ ++ unsigned long cpu = smp_processor_id(); ++ struct task_struct *tasklist; ++ ++ BUG_ON(!irqs_disabled()); ++ if(!per_cpu(posix_timer_task, cpu)) ++ return; ++ /* get per-cpu references */ ++ tasklist = per_cpu(posix_timer_tasklist, cpu); ++ ++ /* check to see if we're already queued */ ++ if (!tsk->posix_timer_list && __fastpath_timer_check(tsk)) { ++ get_task_struct(tsk); ++ if (tasklist) { ++ tsk->posix_timer_list = tasklist; ++ } else { ++ /* ++ * The list is terminated by a self-pointing ++ * task_struct ++ */ ++ tsk->posix_timer_list = tsk; ++ } ++ per_cpu(posix_timer_tasklist, cpu) = tsk; ++ ++ wake_up_process(per_cpu(posix_timer_task, cpu)); ++ } ++} ++ ++/* ++ * posix_cpu_thread_call - callback that gets triggered when a CPU is added. ++ * Here we can start up the necessary migration thread for the new CPU. ++ */ ++static int posix_cpu_thread_call(struct notifier_block *nfb, ++ unsigned long action, void *hcpu) ++{ ++ int cpu = (long)hcpu; ++ struct task_struct *p; ++ struct sched_param param; ++ ++ switch (action) { ++ case CPU_UP_PREPARE: ++ p = kthread_create(posix_cpu_timers_thread, hcpu, ++ "posixcputmr/%d",cpu); ++ if (IS_ERR(p)) ++ return NOTIFY_BAD; ++ p->flags |= PF_NOFREEZE; ++ kthread_bind(p, cpu); ++ /* Must be high prio to avoid getting starved */ ++ param.sched_priority = MAX_RT_PRIO-1; ++ sched_setscheduler(p, SCHED_FIFO, ¶m); ++ per_cpu(posix_timer_task,cpu) = p; ++ break; ++ case CPU_ONLINE: ++ /* Strictly unneccessary, as first user will wake it. */ ++ wake_up_process(per_cpu(posix_timer_task,cpu)); ++ break; ++#ifdef CONFIG_HOTPLUG_CPU ++ case CPU_UP_CANCELED: ++ /* Unbind it from offline cpu so it can run. Fall thru. */ ++ kthread_bind(per_cpu(posix_timer_task, cpu), ++ cpumask_any(cpu_online_mask)); ++ kthread_stop(per_cpu(posix_timer_task,cpu)); ++ per_cpu(posix_timer_task,cpu) = NULL; ++ break; ++ case CPU_DEAD: ++ kthread_stop(per_cpu(posix_timer_task,cpu)); ++ per_cpu(posix_timer_task,cpu) = NULL; ++ break; ++#endif ++ } ++ return NOTIFY_OK; ++} ++ ++/* Register at highest priority so that task migration (migrate_all_tasks) ++ * happens before everything else. ++ */ ++static struct notifier_block posix_cpu_thread_notifier = { ++ .notifier_call = posix_cpu_thread_call, ++ .priority = 10 ++}; ++ ++static int __init posix_cpu_thread_init(void) ++{ ++ void *hcpu = (void *)(long)smp_processor_id(); ++ /* Start one for boot CPU. */ ++ unsigned long cpu; ++ ++ /* init the per-cpu posix_timer_tasklets */ ++ for_each_possible_cpu(cpu) ++ per_cpu(posix_timer_tasklist, cpu) = NULL; ++ ++ posix_cpu_thread_call(&posix_cpu_thread_notifier, CPU_UP_PREPARE, hcpu); ++ posix_cpu_thread_call(&posix_cpu_thread_notifier, CPU_ONLINE, hcpu); ++ register_cpu_notifier(&posix_cpu_thread_notifier); ++ return 0; ++} ++early_initcall(posix_cpu_thread_init); ++#else /* CONFIG_PREEMPT_RT_BASE */ ++void run_posix_cpu_timers(struct task_struct *tsk) ++{ ++ __run_posix_cpu_timers(tsk); ++} ++#endif /* CONFIG_PREEMPT_RT_BASE */ ++ + /* + * Set one of the process-wide special case CPU timers or RLIMIT_CPU. + * The tsk->sighand->siglock must be held by the caller. +diff -Nur linux-4.1.10.orig/kernel/time/posix-timers.c linux-4.1.10/kernel/time/posix-timers.c +--- linux-4.1.10.orig/kernel/time/posix-timers.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/time/posix-timers.c 2015-10-07 18:00:08.000000000 +0200 +@@ -499,6 +499,7 @@ + static struct pid *good_sigevent(sigevent_t * event) + { + struct task_struct *rtn = current->group_leader; ++ int sig = event->sigev_signo; + + if ((event->sigev_notify & SIGEV_THREAD_ID ) && + (!(rtn = find_task_by_vpid(event->sigev_notify_thread_id)) || +@@ -507,7 +508,8 @@ + return NULL; + + if (((event->sigev_notify & ~SIGEV_THREAD_ID) != SIGEV_NONE) && +- ((event->sigev_signo <= 0) || (event->sigev_signo > SIGRTMAX))) ++ (sig <= 0 || sig > SIGRTMAX || sig_kernel_only(sig) || ++ sig_kernel_coredump(sig))) + return NULL; + + return task_pid(rtn); +@@ -819,6 +821,20 @@ + return overrun; + } + ++/* ++ * Protected by RCU! ++ */ ++static void timer_wait_for_callback(struct k_clock *kc, struct k_itimer *timr) ++{ ++#ifdef CONFIG_PREEMPT_RT_FULL ++ if (kc->timer_set == common_timer_set) ++ hrtimer_wait_for_timer(&timr->it.real.timer); ++ else ++ /* FIXME: Whacky hack for posix-cpu-timers */ ++ schedule_timeout(1); ++#endif ++} ++ + /* Set a POSIX.1b interval timer. */ + /* timr->it_lock is taken. */ + static int +@@ -896,6 +912,7 @@ + if (!timr) + return -EINVAL; + ++ rcu_read_lock(); + kc = clockid_to_kclock(timr->it_clock); + if (WARN_ON_ONCE(!kc || !kc->timer_set)) + error = -EINVAL; +@@ -904,9 +921,12 @@ + + unlock_timer(timr, flag); + if (error == TIMER_RETRY) { ++ timer_wait_for_callback(kc, timr); + rtn = NULL; // We already got the old time... ++ rcu_read_unlock(); + goto retry; + } ++ rcu_read_unlock(); + + if (old_setting && !error && + copy_to_user(old_setting, &old_spec, sizeof (old_spec))) +@@ -944,10 +964,15 @@ + if (!timer) + return -EINVAL; + ++ rcu_read_lock(); + if (timer_delete_hook(timer) == TIMER_RETRY) { + unlock_timer(timer, flags); ++ timer_wait_for_callback(clockid_to_kclock(timer->it_clock), ++ timer); ++ rcu_read_unlock(); + goto retry_delete; + } ++ rcu_read_unlock(); + + spin_lock(¤t->sighand->siglock); + list_del(&timer->list); +@@ -973,8 +998,18 @@ + retry_delete: + spin_lock_irqsave(&timer->it_lock, flags); + ++ /* On RT we can race with a deletion */ ++ if (!timer->it_signal) { ++ unlock_timer(timer, flags); ++ return; ++ } ++ + if (timer_delete_hook(timer) == TIMER_RETRY) { ++ rcu_read_lock(); + unlock_timer(timer, flags); ++ timer_wait_for_callback(clockid_to_kclock(timer->it_clock), ++ timer); ++ rcu_read_unlock(); + goto retry_delete; + } + list_del(&timer->list); +diff -Nur linux-4.1.10.orig/kernel/time/tick-common.c linux-4.1.10/kernel/time/tick-common.c +--- linux-4.1.10.orig/kernel/time/tick-common.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/time/tick-common.c 2015-10-07 18:00:08.000000000 +0200 +@@ -78,13 +78,15 @@ + static void tick_periodic(int cpu) + { + if (tick_do_timer_cpu == cpu) { +- write_seqlock(&jiffies_lock); ++ raw_spin_lock(&jiffies_lock); ++ write_seqcount_begin(&jiffies_seq); + + /* Keep track of the next tick event */ + tick_next_period = ktime_add(tick_next_period, tick_period); + + do_timer(1); +- write_sequnlock(&jiffies_lock); ++ write_seqcount_end(&jiffies_seq); ++ raw_spin_unlock(&jiffies_lock); + update_wall_time(); + } + +@@ -146,9 +148,9 @@ + ktime_t next; + + do { +- seq = read_seqbegin(&jiffies_lock); ++ seq = read_seqcount_begin(&jiffies_seq); + next = tick_next_period; +- } while (read_seqretry(&jiffies_lock, seq)); ++ } while (read_seqcount_retry(&jiffies_seq, seq)); + + clockevents_set_state(dev, CLOCK_EVT_STATE_ONESHOT); + +diff -Nur linux-4.1.10.orig/kernel/time/tick-sched.c linux-4.1.10/kernel/time/tick-sched.c +--- linux-4.1.10.orig/kernel/time/tick-sched.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/time/tick-sched.c 2015-10-07 18:00:08.000000000 +0200 +@@ -62,7 +62,8 @@ + return; + + /* Reevalute with jiffies_lock held */ +- write_seqlock(&jiffies_lock); ++ raw_spin_lock(&jiffies_lock); ++ write_seqcount_begin(&jiffies_seq); + + delta = ktime_sub(now, last_jiffies_update); + if (delta.tv64 >= tick_period.tv64) { +@@ -85,10 +86,12 @@ + /* Keep the tick_next_period variable up to date */ + tick_next_period = ktime_add(last_jiffies_update, tick_period); + } else { +- write_sequnlock(&jiffies_lock); ++ write_seqcount_end(&jiffies_seq); ++ raw_spin_unlock(&jiffies_lock); + return; + } +- write_sequnlock(&jiffies_lock); ++ write_seqcount_end(&jiffies_seq); ++ raw_spin_unlock(&jiffies_lock); + update_wall_time(); + } + +@@ -99,12 +102,14 @@ + { + ktime_t period; + +- write_seqlock(&jiffies_lock); ++ raw_spin_lock(&jiffies_lock); ++ write_seqcount_begin(&jiffies_seq); + /* Did we start the jiffies update yet ? */ + if (last_jiffies_update.tv64 == 0) + last_jiffies_update = tick_next_period; + period = last_jiffies_update; +- write_sequnlock(&jiffies_lock); ++ write_seqcount_end(&jiffies_seq); ++ raw_spin_unlock(&jiffies_lock); + return period; + } + +@@ -176,6 +181,11 @@ + return false; + } + ++ if (!arch_irq_work_has_interrupt()) { ++ trace_tick_stop(0, "missing irq work interrupt\n"); ++ return false; ++ } ++ + /* sched_clock_tick() needs us? */ + #ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK + /* +@@ -222,6 +232,7 @@ + + static DEFINE_PER_CPU(struct irq_work, nohz_full_kick_work) = { + .func = nohz_full_kick_work_func, ++ .flags = IRQ_WORK_HARD_IRQ, + }; + + /* +@@ -578,10 +589,10 @@ + + /* Read jiffies and the time when jiffies were updated last */ + do { +- seq = read_seqbegin(&jiffies_lock); ++ seq = read_seqcount_begin(&jiffies_seq); + last_update = last_jiffies_update; + last_jiffies = jiffies; +- } while (read_seqretry(&jiffies_lock, seq)); ++ } while (read_seqcount_retry(&jiffies_seq, seq)); + + if (rcu_needs_cpu(&rcu_delta_jiffies) || + arch_needs_cpu() || irq_work_needs_cpu()) { +@@ -759,14 +770,7 @@ + return false; + + if (unlikely(local_softirq_pending() && cpu_online(cpu))) { +- static int ratelimit; +- +- if (ratelimit < 10 && +- (local_softirq_pending() & SOFTIRQ_STOP_IDLE_MASK)) { +- pr_warn("NOHZ: local_softirq_pending %02x\n", +- (unsigned int) local_softirq_pending()); +- ratelimit++; +- } ++ softirq_check_pending_idle(); + return false; + } + +@@ -1154,6 +1158,7 @@ + * Emulate tick processing via per-CPU hrtimers: + */ + hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS); ++ ts->sched_timer.irqsafe = 1; + ts->sched_timer.function = tick_sched_timer; + + /* Get the next period (per cpu) */ +diff -Nur linux-4.1.10.orig/kernel/time/timekeeping.c linux-4.1.10/kernel/time/timekeeping.c +--- linux-4.1.10.orig/kernel/time/timekeeping.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/time/timekeeping.c 2015-10-07 18:00:08.000000000 +0200 +@@ -2065,8 +2065,10 @@ + */ + void xtime_update(unsigned long ticks) + { +- write_seqlock(&jiffies_lock); ++ raw_spin_lock(&jiffies_lock); ++ write_seqcount_begin(&jiffies_seq); + do_timer(ticks); +- write_sequnlock(&jiffies_lock); ++ write_seqcount_end(&jiffies_seq); ++ raw_spin_unlock(&jiffies_lock); + update_wall_time(); + } +diff -Nur linux-4.1.10.orig/kernel/time/timekeeping.h linux-4.1.10/kernel/time/timekeeping.h +--- linux-4.1.10.orig/kernel/time/timekeeping.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/time/timekeeping.h 2015-10-07 18:00:08.000000000 +0200 +@@ -22,7 +22,8 @@ + extern void do_timer(unsigned long ticks); + extern void update_wall_time(void); + +-extern seqlock_t jiffies_lock; ++extern raw_spinlock_t jiffies_lock; ++extern seqcount_t jiffies_seq; + + #define CS_NAME_LEN 32 + +diff -Nur linux-4.1.10.orig/kernel/time/timer.c linux-4.1.10/kernel/time/timer.c +--- linux-4.1.10.orig/kernel/time/timer.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/time/timer.c 2015-10-07 18:00:08.000000000 +0200 +@@ -78,6 +78,9 @@ + struct tvec_base { + spinlock_t lock; + struct timer_list *running_timer; ++#ifdef CONFIG_PREEMPT_RT_FULL ++ wait_queue_head_t wait_for_running_timer; ++#endif + unsigned long timer_jiffies; + unsigned long next_timer; + unsigned long active_timers; +@@ -768,6 +771,36 @@ + } + } + ++#ifndef CONFIG_PREEMPT_RT_FULL ++static inline struct tvec_base *switch_timer_base(struct timer_list *timer, ++ struct tvec_base *old, ++ struct tvec_base *new) ++{ ++ /* See the comment in lock_timer_base() */ ++ timer_set_base(timer, NULL); ++ spin_unlock(&old->lock); ++ spin_lock(&new->lock); ++ timer_set_base(timer, new); ++ return new; ++} ++#else ++static inline struct tvec_base *switch_timer_base(struct timer_list *timer, ++ struct tvec_base *old, ++ struct tvec_base *new) ++{ ++ /* ++ * We cannot do the above because we might be preempted and ++ * then the preempter would see NULL and loop forever. ++ */ ++ if (spin_trylock(&new->lock)) { ++ timer_set_base(timer, new); ++ spin_unlock(&old->lock); ++ return new; ++ } ++ return old; ++} ++#endif ++ + static inline int + __mod_timer(struct timer_list *timer, unsigned long expires, + bool pending_only, int pinned) +@@ -798,14 +831,8 @@ + * handler yet has not finished. This also guarantees that + * the timer is serialized wrt itself. + */ +- if (likely(base->running_timer != timer)) { +- /* See the comment in lock_timer_base() */ +- timer_set_base(timer, NULL); +- spin_unlock(&base->lock); +- base = new_base; +- spin_lock(&base->lock); +- timer_set_base(timer, base); +- } ++ if (likely(base->running_timer != timer)) ++ base = switch_timer_base(timer, base, new_base); + } + + timer->expires = expires; +@@ -979,6 +1006,29 @@ + } + EXPORT_SYMBOL_GPL(add_timer_on); + ++#ifdef CONFIG_PREEMPT_RT_FULL ++/* ++ * Wait for a running timer ++ */ ++static void wait_for_running_timer(struct timer_list *timer) ++{ ++ struct tvec_base *base = timer->base; ++ ++ if (base->running_timer == timer) ++ wait_event(base->wait_for_running_timer, ++ base->running_timer != timer); ++} ++ ++# define wakeup_timer_waiters(b) wake_up(&(b)->wait_for_running_timer) ++#else ++static inline void wait_for_running_timer(struct timer_list *timer) ++{ ++ cpu_relax(); ++} ++ ++# define wakeup_timer_waiters(b) do { } while (0) ++#endif ++ + /** + * del_timer - deactive a timer. + * @timer: the timer to be deactivated +@@ -1036,7 +1086,7 @@ + } + EXPORT_SYMBOL(try_to_del_timer_sync); + +-#ifdef CONFIG_SMP ++#if defined(CONFIG_SMP) || defined(CONFIG_PREEMPT_RT_FULL) + static DEFINE_PER_CPU(struct tvec_base, __tvec_bases); + + /** +@@ -1098,7 +1148,7 @@ + int ret = try_to_del_timer_sync(timer); + if (ret >= 0) + return ret; +- cpu_relax(); ++ wait_for_running_timer(timer); + } + } + EXPORT_SYMBOL(del_timer_sync); +@@ -1219,15 +1269,17 @@ + if (irqsafe) { + spin_unlock(&base->lock); + call_timer_fn(timer, fn, data); ++ base->running_timer = NULL; + spin_lock(&base->lock); + } else { + spin_unlock_irq(&base->lock); + call_timer_fn(timer, fn, data); ++ base->running_timer = NULL; + spin_lock_irq(&base->lock); + } + } + } +- base->running_timer = NULL; ++ wakeup_timer_waiters(base); + spin_unlock_irq(&base->lock); + } + +@@ -1367,6 +1419,14 @@ + if (cpu_is_offline(smp_processor_id())) + return expires; + ++#ifdef CONFIG_PREEMPT_RT_FULL ++ /* ++ * On PREEMPT_RT we cannot sleep here. As a result we can't take ++ * the base lock to check when the next timer is pending and so ++ * we assume the next jiffy. ++ */ ++ return now + 1; ++#endif + spin_lock(&base->lock); + if (base->active_timers) { + if (time_before_eq(base->next_timer, base->timer_jiffies)) +@@ -1392,13 +1452,13 @@ + + /* Note: this timer irq context must be accounted for as well. */ + account_process_tick(p, user_tick); ++ scheduler_tick(); + run_local_timers(); + rcu_check_callbacks(user_tick); +-#ifdef CONFIG_IRQ_WORK ++#if defined(CONFIG_IRQ_WORK) && !defined(CONFIG_PREEMPT_RT_FULL) + if (in_irq()) + irq_work_tick(); + #endif +- scheduler_tick(); + run_posix_cpu_timers(p); + } + +@@ -1411,6 +1471,10 @@ + + hrtimer_run_pending(); + ++#if defined(CONFIG_IRQ_WORK) && defined(CONFIG_PREEMPT_RT_FULL) ++ irq_work_tick(); ++#endif ++ + if (time_after_eq(jiffies, base->timer_jiffies)) + __run_timers(base); + } +@@ -1566,7 +1630,7 @@ + + BUG_ON(cpu_online(cpu)); + old_base = per_cpu(tvec_bases, cpu); +- new_base = get_cpu_var(tvec_bases); ++ new_base = get_local_var(tvec_bases); + /* + * The caller is globally serialized and nobody else + * takes two locks at once, deadlock is not possible. +@@ -1590,7 +1654,7 @@ + + spin_unlock(&old_base->lock); + spin_unlock_irq(&new_base->lock); +- put_cpu_var(tvec_bases); ++ put_local_var(tvec_bases); + } + + static int timer_cpu_notify(struct notifier_block *self, +@@ -1625,6 +1689,9 @@ + base->cpu = cpu; + per_cpu(tvec_bases, cpu) = base; + spin_lock_init(&base->lock); ++#ifdef CONFIG_PREEMPT_RT_FULL ++ init_waitqueue_head(&base->wait_for_running_timer); ++#endif + + for (j = 0; j < TVN_SIZE; j++) { + INIT_LIST_HEAD(base->tv5.vec + j); +diff -Nur linux-4.1.10.orig/kernel/trace/Kconfig linux-4.1.10/kernel/trace/Kconfig +--- linux-4.1.10.orig/kernel/trace/Kconfig 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/trace/Kconfig 2015-10-07 18:00:08.000000000 +0200 +@@ -187,6 +187,24 @@ + enabled. This option and the preempt-off timing option can be + used together or separately.) + ++config INTERRUPT_OFF_HIST ++ bool "Interrupts-off Latency Histogram" ++ depends on IRQSOFF_TRACER ++ help ++ This option generates continuously updated histograms (one per cpu) ++ of the duration of time periods with interrupts disabled. The ++ histograms are disabled by default. To enable them, write a non-zero ++ number to ++ ++ /sys/kernel/debug/tracing/latency_hist/enable/preemptirqsoff ++ ++ If PREEMPT_OFF_HIST is also selected, additional histograms (one ++ per cpu) are generated that accumulate the duration of time periods ++ when both interrupts and preemption are disabled. The histogram data ++ will be located in the debug file system at ++ ++ /sys/kernel/debug/tracing/latency_hist/irqsoff ++ + config PREEMPT_TRACER + bool "Preemption-off Latency Tracer" + default n +@@ -211,6 +229,24 @@ + enabled. This option and the irqs-off timing option can be + used together or separately.) + ++config PREEMPT_OFF_HIST ++ bool "Preemption-off Latency Histogram" ++ depends on PREEMPT_TRACER ++ help ++ This option generates continuously updated histograms (one per cpu) ++ of the duration of time periods with preemption disabled. The ++ histograms are disabled by default. To enable them, write a non-zero ++ number to ++ ++ /sys/kernel/debug/tracing/latency_hist/enable/preemptirqsoff ++ ++ If INTERRUPT_OFF_HIST is also selected, additional histograms (one ++ per cpu) are generated that accumulate the duration of time periods ++ when both interrupts and preemption are disabled. The histogram data ++ will be located in the debug file system at ++ ++ /sys/kernel/debug/tracing/latency_hist/preemptoff ++ + config SCHED_TRACER + bool "Scheduling Latency Tracer" + select GENERIC_TRACER +@@ -221,6 +257,74 @@ + This tracer tracks the latency of the highest priority task + to be scheduled in, starting from the point it has woken up. + ++config WAKEUP_LATENCY_HIST ++ bool "Scheduling Latency Histogram" ++ depends on SCHED_TRACER ++ help ++ This option generates continuously updated histograms (one per cpu) ++ of the scheduling latency of the highest priority task. ++ The histograms are disabled by default. To enable them, write a ++ non-zero number to ++ ++ /sys/kernel/debug/tracing/latency_hist/enable/wakeup ++ ++ Two different algorithms are used, one to determine the latency of ++ processes that exclusively use the highest priority of the system and ++ another one to determine the latency of processes that share the ++ highest system priority with other processes. The former is used to ++ improve hardware and system software, the latter to optimize the ++ priority design of a given system. The histogram data will be ++ located in the debug file system at ++ ++ /sys/kernel/debug/tracing/latency_hist/wakeup ++ ++ and ++ ++ /sys/kernel/debug/tracing/latency_hist/wakeup/sharedprio ++ ++ If both Scheduling Latency Histogram and Missed Timer Offsets ++ Histogram are selected, additional histogram data will be collected ++ that contain, in addition to the wakeup latency, the timer latency, in ++ case the wakeup was triggered by an expired timer. These histograms ++ are available in the ++ ++ /sys/kernel/debug/tracing/latency_hist/timerandwakeup ++ ++ directory. They reflect the apparent interrupt and scheduling latency ++ and are best suitable to determine the worst-case latency of a given ++ system. To enable these histograms, write a non-zero number to ++ ++ /sys/kernel/debug/tracing/latency_hist/enable/timerandwakeup ++ ++config MISSED_TIMER_OFFSETS_HIST ++ depends on HIGH_RES_TIMERS ++ select GENERIC_TRACER ++ bool "Missed Timer Offsets Histogram" ++ help ++ Generate a histogram of missed timer offsets in microseconds. The ++ histograms are disabled by default. To enable them, write a non-zero ++ number to ++ ++ /sys/kernel/debug/tracing/latency_hist/enable/missed_timer_offsets ++ ++ The histogram data will be located in the debug file system at ++ ++ /sys/kernel/debug/tracing/latency_hist/missed_timer_offsets ++ ++ If both Scheduling Latency Histogram and Missed Timer Offsets ++ Histogram are selected, additional histogram data will be collected ++ that contain, in addition to the wakeup latency, the timer latency, in ++ case the wakeup was triggered by an expired timer. These histograms ++ are available in the ++ ++ /sys/kernel/debug/tracing/latency_hist/timerandwakeup ++ ++ directory. They reflect the apparent interrupt and scheduling latency ++ and are best suitable to determine the worst-case latency of a given ++ system. To enable these histograms, write a non-zero number to ++ ++ /sys/kernel/debug/tracing/latency_hist/enable/timerandwakeup ++ + config ENABLE_DEFAULT_TRACERS + bool "Trace process context switches and events" + depends on !GENERIC_TRACER +diff -Nur linux-4.1.10.orig/kernel/trace/latency_hist.c linux-4.1.10/kernel/trace/latency_hist.c +--- linux-4.1.10.orig/kernel/trace/latency_hist.c 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/kernel/trace/latency_hist.c 2015-10-07 18:00:08.000000000 +0200 +@@ -0,0 +1,1178 @@ ++/* ++ * kernel/trace/latency_hist.c ++ * ++ * Add support for histograms of preemption-off latency and ++ * interrupt-off latency and wakeup latency, it depends on ++ * Real-Time Preemption Support. ++ * ++ * Copyright (C) 2005 MontaVista Software, Inc. ++ * Yi Yang ++ * ++ * Converted to work with the new latency tracer. ++ * Copyright (C) 2008 Red Hat, Inc. ++ * Steven Rostedt ++ * ++ */ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "trace.h" ++#include ++ ++#define NSECS_PER_USECS 1000L ++ ++#define CREATE_TRACE_POINTS ++#include ++ ++enum { ++ IRQSOFF_LATENCY = 0, ++ PREEMPTOFF_LATENCY, ++ PREEMPTIRQSOFF_LATENCY, ++ WAKEUP_LATENCY, ++ WAKEUP_LATENCY_SHAREDPRIO, ++ MISSED_TIMER_OFFSETS, ++ TIMERANDWAKEUP_LATENCY, ++ MAX_LATENCY_TYPE, ++}; ++ ++#define MAX_ENTRY_NUM 10240 ++ ++struct hist_data { ++ atomic_t hist_mode; /* 0 log, 1 don't log */ ++ long offset; /* set it to MAX_ENTRY_NUM/2 for a bipolar scale */ ++ long min_lat; ++ long max_lat; ++ unsigned long long below_hist_bound_samples; ++ unsigned long long above_hist_bound_samples; ++ long long accumulate_lat; ++ unsigned long long total_samples; ++ unsigned long long hist_array[MAX_ENTRY_NUM]; ++}; ++ ++struct enable_data { ++ int latency_type; ++ int enabled; ++}; ++ ++static char *latency_hist_dir_root = "latency_hist"; ++ ++#ifdef CONFIG_INTERRUPT_OFF_HIST ++static DEFINE_PER_CPU(struct hist_data, irqsoff_hist); ++static char *irqsoff_hist_dir = "irqsoff"; ++static DEFINE_PER_CPU(cycles_t, hist_irqsoff_start); ++static DEFINE_PER_CPU(int, hist_irqsoff_counting); ++#endif ++ ++#ifdef CONFIG_PREEMPT_OFF_HIST ++static DEFINE_PER_CPU(struct hist_data, preemptoff_hist); ++static char *preemptoff_hist_dir = "preemptoff"; ++static DEFINE_PER_CPU(cycles_t, hist_preemptoff_start); ++static DEFINE_PER_CPU(int, hist_preemptoff_counting); ++#endif ++ ++#if defined(CONFIG_PREEMPT_OFF_HIST) && defined(CONFIG_INTERRUPT_OFF_HIST) ++static DEFINE_PER_CPU(struct hist_data, preemptirqsoff_hist); ++static char *preemptirqsoff_hist_dir = "preemptirqsoff"; ++static DEFINE_PER_CPU(cycles_t, hist_preemptirqsoff_start); ++static DEFINE_PER_CPU(int, hist_preemptirqsoff_counting); ++#endif ++ ++#if defined(CONFIG_PREEMPT_OFF_HIST) || defined(CONFIG_INTERRUPT_OFF_HIST) ++static notrace void probe_preemptirqsoff_hist(void *v, int reason, int start); ++static struct enable_data preemptirqsoff_enabled_data = { ++ .latency_type = PREEMPTIRQSOFF_LATENCY, ++ .enabled = 0, ++}; ++#endif ++ ++#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \ ++ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) ++struct maxlatproc_data { ++ char comm[FIELD_SIZEOF(struct task_struct, comm)]; ++ char current_comm[FIELD_SIZEOF(struct task_struct, comm)]; ++ int pid; ++ int current_pid; ++ int prio; ++ int current_prio; ++ long latency; ++ long timeroffset; ++ cycle_t timestamp; ++}; ++#endif ++ ++#ifdef CONFIG_WAKEUP_LATENCY_HIST ++static DEFINE_PER_CPU(struct hist_data, wakeup_latency_hist); ++static DEFINE_PER_CPU(struct hist_data, wakeup_latency_hist_sharedprio); ++static char *wakeup_latency_hist_dir = "wakeup"; ++static char *wakeup_latency_hist_dir_sharedprio = "sharedprio"; ++static notrace void probe_wakeup_latency_hist_start(void *v, ++ struct task_struct *p, int success); ++static notrace void probe_wakeup_latency_hist_stop(void *v, ++ struct task_struct *prev, struct task_struct *next); ++static notrace void probe_sched_migrate_task(void *, ++ struct task_struct *task, int cpu); ++static struct enable_data wakeup_latency_enabled_data = { ++ .latency_type = WAKEUP_LATENCY, ++ .enabled = 0, ++}; ++static DEFINE_PER_CPU(struct maxlatproc_data, wakeup_maxlatproc); ++static DEFINE_PER_CPU(struct maxlatproc_data, wakeup_maxlatproc_sharedprio); ++static DEFINE_PER_CPU(struct task_struct *, wakeup_task); ++static DEFINE_PER_CPU(int, wakeup_sharedprio); ++static unsigned long wakeup_pid; ++#endif ++ ++#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST ++static DEFINE_PER_CPU(struct hist_data, missed_timer_offsets); ++static char *missed_timer_offsets_dir = "missed_timer_offsets"; ++static notrace void probe_hrtimer_interrupt(void *v, int cpu, ++ long long offset, struct task_struct *curr, struct task_struct *task); ++static struct enable_data missed_timer_offsets_enabled_data = { ++ .latency_type = MISSED_TIMER_OFFSETS, ++ .enabled = 0, ++}; ++static DEFINE_PER_CPU(struct maxlatproc_data, missed_timer_offsets_maxlatproc); ++static unsigned long missed_timer_offsets_pid; ++#endif ++ ++#if defined(CONFIG_WAKEUP_LATENCY_HIST) && \ ++ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) ++static DEFINE_PER_CPU(struct hist_data, timerandwakeup_latency_hist); ++static char *timerandwakeup_latency_hist_dir = "timerandwakeup"; ++static struct enable_data timerandwakeup_enabled_data = { ++ .latency_type = TIMERANDWAKEUP_LATENCY, ++ .enabled = 0, ++}; ++static DEFINE_PER_CPU(struct maxlatproc_data, timerandwakeup_maxlatproc); ++#endif ++ ++void notrace latency_hist(int latency_type, int cpu, long latency, ++ long timeroffset, cycle_t stop, ++ struct task_struct *p) ++{ ++ struct hist_data *my_hist; ++#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \ ++ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) ++ struct maxlatproc_data *mp = NULL; ++#endif ++ ++ if (!cpu_possible(cpu) || latency_type < 0 || ++ latency_type >= MAX_LATENCY_TYPE) ++ return; ++ ++ switch (latency_type) { ++#ifdef CONFIG_INTERRUPT_OFF_HIST ++ case IRQSOFF_LATENCY: ++ my_hist = &per_cpu(irqsoff_hist, cpu); ++ break; ++#endif ++#ifdef CONFIG_PREEMPT_OFF_HIST ++ case PREEMPTOFF_LATENCY: ++ my_hist = &per_cpu(preemptoff_hist, cpu); ++ break; ++#endif ++#if defined(CONFIG_PREEMPT_OFF_HIST) && defined(CONFIG_INTERRUPT_OFF_HIST) ++ case PREEMPTIRQSOFF_LATENCY: ++ my_hist = &per_cpu(preemptirqsoff_hist, cpu); ++ break; ++#endif ++#ifdef CONFIG_WAKEUP_LATENCY_HIST ++ case WAKEUP_LATENCY: ++ my_hist = &per_cpu(wakeup_latency_hist, cpu); ++ mp = &per_cpu(wakeup_maxlatproc, cpu); ++ break; ++ case WAKEUP_LATENCY_SHAREDPRIO: ++ my_hist = &per_cpu(wakeup_latency_hist_sharedprio, cpu); ++ mp = &per_cpu(wakeup_maxlatproc_sharedprio, cpu); ++ break; ++#endif ++#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST ++ case MISSED_TIMER_OFFSETS: ++ my_hist = &per_cpu(missed_timer_offsets, cpu); ++ mp = &per_cpu(missed_timer_offsets_maxlatproc, cpu); ++ break; ++#endif ++#if defined(CONFIG_WAKEUP_LATENCY_HIST) && \ ++ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) ++ case TIMERANDWAKEUP_LATENCY: ++ my_hist = &per_cpu(timerandwakeup_latency_hist, cpu); ++ mp = &per_cpu(timerandwakeup_maxlatproc, cpu); ++ break; ++#endif ++ ++ default: ++ return; ++ } ++ ++ latency += my_hist->offset; ++ ++ if (atomic_read(&my_hist->hist_mode) == 0) ++ return; ++ ++ if (latency < 0 || latency >= MAX_ENTRY_NUM) { ++ if (latency < 0) ++ my_hist->below_hist_bound_samples++; ++ else ++ my_hist->above_hist_bound_samples++; ++ } else ++ my_hist->hist_array[latency]++; ++ ++ if (unlikely(latency > my_hist->max_lat || ++ my_hist->min_lat == LONG_MAX)) { ++#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \ ++ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) ++ if (latency_type == WAKEUP_LATENCY || ++ latency_type == WAKEUP_LATENCY_SHAREDPRIO || ++ latency_type == MISSED_TIMER_OFFSETS || ++ latency_type == TIMERANDWAKEUP_LATENCY) { ++ strncpy(mp->comm, p->comm, sizeof(mp->comm)); ++ strncpy(mp->current_comm, current->comm, ++ sizeof(mp->current_comm)); ++ mp->pid = task_pid_nr(p); ++ mp->current_pid = task_pid_nr(current); ++ mp->prio = p->prio; ++ mp->current_prio = current->prio; ++ mp->latency = latency; ++ mp->timeroffset = timeroffset; ++ mp->timestamp = stop; ++ } ++#endif ++ my_hist->max_lat = latency; ++ } ++ if (unlikely(latency < my_hist->min_lat)) ++ my_hist->min_lat = latency; ++ my_hist->total_samples++; ++ my_hist->accumulate_lat += latency; ++} ++ ++static void *l_start(struct seq_file *m, loff_t *pos) ++{ ++ loff_t *index_ptr = NULL; ++ loff_t index = *pos; ++ struct hist_data *my_hist = m->private; ++ ++ if (index == 0) { ++ char minstr[32], avgstr[32], maxstr[32]; ++ ++ atomic_dec(&my_hist->hist_mode); ++ ++ if (likely(my_hist->total_samples)) { ++ long avg = (long) div64_s64(my_hist->accumulate_lat, ++ my_hist->total_samples); ++ snprintf(minstr, sizeof(minstr), "%ld", ++ my_hist->min_lat - my_hist->offset); ++ snprintf(avgstr, sizeof(avgstr), "%ld", ++ avg - my_hist->offset); ++ snprintf(maxstr, sizeof(maxstr), "%ld", ++ my_hist->max_lat - my_hist->offset); ++ } else { ++ strcpy(minstr, ""); ++ strcpy(avgstr, minstr); ++ strcpy(maxstr, minstr); ++ } ++ ++ seq_printf(m, "#Minimum latency: %s microseconds\n" ++ "#Average latency: %s microseconds\n" ++ "#Maximum latency: %s microseconds\n" ++ "#Total samples: %llu\n" ++ "#There are %llu samples lower than %ld" ++ " microseconds.\n" ++ "#There are %llu samples greater or equal" ++ " than %ld microseconds.\n" ++ "#usecs\t%16s\n", ++ minstr, avgstr, maxstr, ++ my_hist->total_samples, ++ my_hist->below_hist_bound_samples, ++ -my_hist->offset, ++ my_hist->above_hist_bound_samples, ++ MAX_ENTRY_NUM - my_hist->offset, ++ "samples"); ++ } ++ if (index < MAX_ENTRY_NUM) { ++ index_ptr = kmalloc(sizeof(loff_t), GFP_KERNEL); ++ if (index_ptr) ++ *index_ptr = index; ++ } ++ ++ return index_ptr; ++} ++ ++static void *l_next(struct seq_file *m, void *p, loff_t *pos) ++{ ++ loff_t *index_ptr = p; ++ struct hist_data *my_hist = m->private; ++ ++ if (++*pos >= MAX_ENTRY_NUM) { ++ atomic_inc(&my_hist->hist_mode); ++ return NULL; ++ } ++ *index_ptr = *pos; ++ return index_ptr; ++} ++ ++static void l_stop(struct seq_file *m, void *p) ++{ ++ kfree(p); ++} ++ ++static int l_show(struct seq_file *m, void *p) ++{ ++ int index = *(loff_t *) p; ++ struct hist_data *my_hist = m->private; ++ ++ seq_printf(m, "%6ld\t%16llu\n", index - my_hist->offset, ++ my_hist->hist_array[index]); ++ return 0; ++} ++ ++static const struct seq_operations latency_hist_seq_op = { ++ .start = l_start, ++ .next = l_next, ++ .stop = l_stop, ++ .show = l_show ++}; ++ ++static int latency_hist_open(struct inode *inode, struct file *file) ++{ ++ int ret; ++ ++ ret = seq_open(file, &latency_hist_seq_op); ++ if (!ret) { ++ struct seq_file *seq = file->private_data; ++ seq->private = inode->i_private; ++ } ++ return ret; ++} ++ ++static const struct file_operations latency_hist_fops = { ++ .open = latency_hist_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = seq_release, ++}; ++ ++#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \ ++ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) ++static void clear_maxlatprocdata(struct maxlatproc_data *mp) ++{ ++ mp->comm[0] = mp->current_comm[0] = '\0'; ++ mp->prio = mp->current_prio = mp->pid = mp->current_pid = ++ mp->latency = mp->timeroffset = -1; ++ mp->timestamp = 0; ++} ++#endif ++ ++static void hist_reset(struct hist_data *hist) ++{ ++ atomic_dec(&hist->hist_mode); ++ ++ memset(hist->hist_array, 0, sizeof(hist->hist_array)); ++ hist->below_hist_bound_samples = 0ULL; ++ hist->above_hist_bound_samples = 0ULL; ++ hist->min_lat = LONG_MAX; ++ hist->max_lat = LONG_MIN; ++ hist->total_samples = 0ULL; ++ hist->accumulate_lat = 0LL; ++ ++ atomic_inc(&hist->hist_mode); ++} ++ ++static ssize_t ++latency_hist_reset(struct file *file, const char __user *a, ++ size_t size, loff_t *off) ++{ ++ int cpu; ++ struct hist_data *hist = NULL; ++#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \ ++ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) ++ struct maxlatproc_data *mp = NULL; ++#endif ++ off_t latency_type = (off_t) file->private_data; ++ ++ for_each_online_cpu(cpu) { ++ ++ switch (latency_type) { ++#ifdef CONFIG_PREEMPT_OFF_HIST ++ case PREEMPTOFF_LATENCY: ++ hist = &per_cpu(preemptoff_hist, cpu); ++ break; ++#endif ++#ifdef CONFIG_INTERRUPT_OFF_HIST ++ case IRQSOFF_LATENCY: ++ hist = &per_cpu(irqsoff_hist, cpu); ++ break; ++#endif ++#if defined(CONFIG_INTERRUPT_OFF_HIST) && defined(CONFIG_PREEMPT_OFF_HIST) ++ case PREEMPTIRQSOFF_LATENCY: ++ hist = &per_cpu(preemptirqsoff_hist, cpu); ++ break; ++#endif ++#ifdef CONFIG_WAKEUP_LATENCY_HIST ++ case WAKEUP_LATENCY: ++ hist = &per_cpu(wakeup_latency_hist, cpu); ++ mp = &per_cpu(wakeup_maxlatproc, cpu); ++ break; ++ case WAKEUP_LATENCY_SHAREDPRIO: ++ hist = &per_cpu(wakeup_latency_hist_sharedprio, cpu); ++ mp = &per_cpu(wakeup_maxlatproc_sharedprio, cpu); ++ break; ++#endif ++#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST ++ case MISSED_TIMER_OFFSETS: ++ hist = &per_cpu(missed_timer_offsets, cpu); ++ mp = &per_cpu(missed_timer_offsets_maxlatproc, cpu); ++ break; ++#endif ++#if defined(CONFIG_WAKEUP_LATENCY_HIST) && \ ++ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) ++ case TIMERANDWAKEUP_LATENCY: ++ hist = &per_cpu(timerandwakeup_latency_hist, cpu); ++ mp = &per_cpu(timerandwakeup_maxlatproc, cpu); ++ break; ++#endif ++ } ++ ++ hist_reset(hist); ++#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \ ++ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) ++ if (latency_type == WAKEUP_LATENCY || ++ latency_type == WAKEUP_LATENCY_SHAREDPRIO || ++ latency_type == MISSED_TIMER_OFFSETS || ++ latency_type == TIMERANDWAKEUP_LATENCY) ++ clear_maxlatprocdata(mp); ++#endif ++ } ++ ++ return size; ++} ++ ++#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \ ++ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) ++static ssize_t ++show_pid(struct file *file, char __user *ubuf, size_t cnt, loff_t *ppos) ++{ ++ char buf[64]; ++ int r; ++ unsigned long *this_pid = file->private_data; ++ ++ r = snprintf(buf, sizeof(buf), "%lu\n", *this_pid); ++ return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); ++} ++ ++static ssize_t do_pid(struct file *file, const char __user *ubuf, ++ size_t cnt, loff_t *ppos) ++{ ++ char buf[64]; ++ unsigned long pid; ++ unsigned long *this_pid = file->private_data; ++ ++ if (cnt >= sizeof(buf)) ++ return -EINVAL; ++ ++ if (copy_from_user(&buf, ubuf, cnt)) ++ return -EFAULT; ++ ++ buf[cnt] = '\0'; ++ ++ if (kstrtoul(buf, 10, &pid)) ++ return -EINVAL; ++ ++ *this_pid = pid; ++ ++ return cnt; ++} ++#endif ++ ++#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \ ++ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) ++static ssize_t ++show_maxlatproc(struct file *file, char __user *ubuf, size_t cnt, loff_t *ppos) ++{ ++ int r; ++ struct maxlatproc_data *mp = file->private_data; ++ int strmaxlen = (TASK_COMM_LEN * 2) + (8 * 8); ++ unsigned long long t; ++ unsigned long usecs, secs; ++ char *buf; ++ ++ if (mp->pid == -1 || mp->current_pid == -1) { ++ buf = "(none)\n"; ++ return simple_read_from_buffer(ubuf, cnt, ppos, buf, ++ strlen(buf)); ++ } ++ ++ buf = kmalloc(strmaxlen, GFP_KERNEL); ++ if (buf == NULL) ++ return -ENOMEM; ++ ++ t = ns2usecs(mp->timestamp); ++ usecs = do_div(t, USEC_PER_SEC); ++ secs = (unsigned long) t; ++ r = snprintf(buf, strmaxlen, ++ "%d %d %ld (%ld) %s <- %d %d %s %lu.%06lu\n", mp->pid, ++ MAX_RT_PRIO-1 - mp->prio, mp->latency, mp->timeroffset, mp->comm, ++ mp->current_pid, MAX_RT_PRIO-1 - mp->current_prio, mp->current_comm, ++ secs, usecs); ++ r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r); ++ kfree(buf); ++ return r; ++} ++#endif ++ ++static ssize_t ++show_enable(struct file *file, char __user *ubuf, size_t cnt, loff_t *ppos) ++{ ++ char buf[64]; ++ struct enable_data *ed = file->private_data; ++ int r; ++ ++ r = snprintf(buf, sizeof(buf), "%d\n", ed->enabled); ++ return simple_read_from_buffer(ubuf, cnt, ppos, buf, r); ++} ++ ++static ssize_t ++do_enable(struct file *file, const char __user *ubuf, size_t cnt, loff_t *ppos) ++{ ++ char buf[64]; ++ long enable; ++ struct enable_data *ed = file->private_data; ++ ++ if (cnt >= sizeof(buf)) ++ return -EINVAL; ++ ++ if (copy_from_user(&buf, ubuf, cnt)) ++ return -EFAULT; ++ ++ buf[cnt] = 0; ++ ++ if (kstrtoul(buf, 10, &enable)) ++ return -EINVAL; ++ ++ if ((enable && ed->enabled) || (!enable && !ed->enabled)) ++ return cnt; ++ ++ if (enable) { ++ int ret; ++ ++ switch (ed->latency_type) { ++#if defined(CONFIG_INTERRUPT_OFF_HIST) || defined(CONFIG_PREEMPT_OFF_HIST) ++ case PREEMPTIRQSOFF_LATENCY: ++ ret = register_trace_preemptirqsoff_hist( ++ probe_preemptirqsoff_hist, NULL); ++ if (ret) { ++ pr_info("wakeup trace: Couldn't assign " ++ "probe_preemptirqsoff_hist " ++ "to trace_preemptirqsoff_hist\n"); ++ return ret; ++ } ++ break; ++#endif ++#ifdef CONFIG_WAKEUP_LATENCY_HIST ++ case WAKEUP_LATENCY: ++ ret = register_trace_sched_wakeup( ++ probe_wakeup_latency_hist_start, NULL); ++ if (ret) { ++ pr_info("wakeup trace: Couldn't assign " ++ "probe_wakeup_latency_hist_start " ++ "to trace_sched_wakeup\n"); ++ return ret; ++ } ++ ret = register_trace_sched_wakeup_new( ++ probe_wakeup_latency_hist_start, NULL); ++ if (ret) { ++ pr_info("wakeup trace: Couldn't assign " ++ "probe_wakeup_latency_hist_start " ++ "to trace_sched_wakeup_new\n"); ++ unregister_trace_sched_wakeup( ++ probe_wakeup_latency_hist_start, NULL); ++ return ret; ++ } ++ ret = register_trace_sched_switch( ++ probe_wakeup_latency_hist_stop, NULL); ++ if (ret) { ++ pr_info("wakeup trace: Couldn't assign " ++ "probe_wakeup_latency_hist_stop " ++ "to trace_sched_switch\n"); ++ unregister_trace_sched_wakeup( ++ probe_wakeup_latency_hist_start, NULL); ++ unregister_trace_sched_wakeup_new( ++ probe_wakeup_latency_hist_start, NULL); ++ return ret; ++ } ++ ret = register_trace_sched_migrate_task( ++ probe_sched_migrate_task, NULL); ++ if (ret) { ++ pr_info("wakeup trace: Couldn't assign " ++ "probe_sched_migrate_task " ++ "to trace_sched_migrate_task\n"); ++ unregister_trace_sched_wakeup( ++ probe_wakeup_latency_hist_start, NULL); ++ unregister_trace_sched_wakeup_new( ++ probe_wakeup_latency_hist_start, NULL); ++ unregister_trace_sched_switch( ++ probe_wakeup_latency_hist_stop, NULL); ++ return ret; ++ } ++ break; ++#endif ++#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST ++ case MISSED_TIMER_OFFSETS: ++ ret = register_trace_hrtimer_interrupt( ++ probe_hrtimer_interrupt, NULL); ++ if (ret) { ++ pr_info("wakeup trace: Couldn't assign " ++ "probe_hrtimer_interrupt " ++ "to trace_hrtimer_interrupt\n"); ++ return ret; ++ } ++ break; ++#endif ++#if defined(CONFIG_WAKEUP_LATENCY_HIST) && \ ++ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) ++ case TIMERANDWAKEUP_LATENCY: ++ if (!wakeup_latency_enabled_data.enabled || ++ !missed_timer_offsets_enabled_data.enabled) ++ return -EINVAL; ++ break; ++#endif ++ default: ++ break; ++ } ++ } else { ++ switch (ed->latency_type) { ++#if defined(CONFIG_INTERRUPT_OFF_HIST) || defined(CONFIG_PREEMPT_OFF_HIST) ++ case PREEMPTIRQSOFF_LATENCY: ++ { ++ int cpu; ++ ++ unregister_trace_preemptirqsoff_hist( ++ probe_preemptirqsoff_hist, NULL); ++ for_each_online_cpu(cpu) { ++#ifdef CONFIG_INTERRUPT_OFF_HIST ++ per_cpu(hist_irqsoff_counting, ++ cpu) = 0; ++#endif ++#ifdef CONFIG_PREEMPT_OFF_HIST ++ per_cpu(hist_preemptoff_counting, ++ cpu) = 0; ++#endif ++#if defined(CONFIG_INTERRUPT_OFF_HIST) && defined(CONFIG_PREEMPT_OFF_HIST) ++ per_cpu(hist_preemptirqsoff_counting, ++ cpu) = 0; ++#endif ++ } ++ } ++ break; ++#endif ++#ifdef CONFIG_WAKEUP_LATENCY_HIST ++ case WAKEUP_LATENCY: ++ { ++ int cpu; ++ ++ unregister_trace_sched_wakeup( ++ probe_wakeup_latency_hist_start, NULL); ++ unregister_trace_sched_wakeup_new( ++ probe_wakeup_latency_hist_start, NULL); ++ unregister_trace_sched_switch( ++ probe_wakeup_latency_hist_stop, NULL); ++ unregister_trace_sched_migrate_task( ++ probe_sched_migrate_task, NULL); ++ ++ for_each_online_cpu(cpu) { ++ per_cpu(wakeup_task, cpu) = NULL; ++ per_cpu(wakeup_sharedprio, cpu) = 0; ++ } ++ } ++#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST ++ timerandwakeup_enabled_data.enabled = 0; ++#endif ++ break; ++#endif ++#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST ++ case MISSED_TIMER_OFFSETS: ++ unregister_trace_hrtimer_interrupt( ++ probe_hrtimer_interrupt, NULL); ++#ifdef CONFIG_WAKEUP_LATENCY_HIST ++ timerandwakeup_enabled_data.enabled = 0; ++#endif ++ break; ++#endif ++ default: ++ break; ++ } ++ } ++ ed->enabled = enable; ++ return cnt; ++} ++ ++static const struct file_operations latency_hist_reset_fops = { ++ .open = tracing_open_generic, ++ .write = latency_hist_reset, ++}; ++ ++static const struct file_operations enable_fops = { ++ .open = tracing_open_generic, ++ .read = show_enable, ++ .write = do_enable, ++}; ++ ++#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \ ++ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) ++static const struct file_operations pid_fops = { ++ .open = tracing_open_generic, ++ .read = show_pid, ++ .write = do_pid, ++}; ++ ++static const struct file_operations maxlatproc_fops = { ++ .open = tracing_open_generic, ++ .read = show_maxlatproc, ++}; ++#endif ++ ++#if defined(CONFIG_INTERRUPT_OFF_HIST) || defined(CONFIG_PREEMPT_OFF_HIST) ++static notrace void probe_preemptirqsoff_hist(void *v, int reason, ++ int starthist) ++{ ++ int cpu = raw_smp_processor_id(); ++ int time_set = 0; ++ ++ if (starthist) { ++ cycle_t uninitialized_var(start); ++ ++ if (!preempt_count() && !irqs_disabled()) ++ return; ++ ++#ifdef CONFIG_INTERRUPT_OFF_HIST ++ if ((reason == IRQS_OFF || reason == TRACE_START) && ++ !per_cpu(hist_irqsoff_counting, cpu)) { ++ per_cpu(hist_irqsoff_counting, cpu) = 1; ++ start = ftrace_now(cpu); ++ time_set++; ++ per_cpu(hist_irqsoff_start, cpu) = start; ++ } ++#endif ++ ++#ifdef CONFIG_PREEMPT_OFF_HIST ++ if ((reason == PREEMPT_OFF || reason == TRACE_START) && ++ !per_cpu(hist_preemptoff_counting, cpu)) { ++ per_cpu(hist_preemptoff_counting, cpu) = 1; ++ if (!(time_set++)) ++ start = ftrace_now(cpu); ++ per_cpu(hist_preemptoff_start, cpu) = start; ++ } ++#endif ++ ++#if defined(CONFIG_INTERRUPT_OFF_HIST) && defined(CONFIG_PREEMPT_OFF_HIST) ++ if (per_cpu(hist_irqsoff_counting, cpu) && ++ per_cpu(hist_preemptoff_counting, cpu) && ++ !per_cpu(hist_preemptirqsoff_counting, cpu)) { ++ per_cpu(hist_preemptirqsoff_counting, cpu) = 1; ++ if (!time_set) ++ start = ftrace_now(cpu); ++ per_cpu(hist_preemptirqsoff_start, cpu) = start; ++ } ++#endif ++ } else { ++ cycle_t uninitialized_var(stop); ++ ++#ifdef CONFIG_INTERRUPT_OFF_HIST ++ if ((reason == IRQS_ON || reason == TRACE_STOP) && ++ per_cpu(hist_irqsoff_counting, cpu)) { ++ cycle_t start = per_cpu(hist_irqsoff_start, cpu); ++ ++ stop = ftrace_now(cpu); ++ time_set++; ++ if (start) { ++ long latency = ((long) (stop - start)) / ++ NSECS_PER_USECS; ++ ++ latency_hist(IRQSOFF_LATENCY, cpu, latency, 0, ++ stop, NULL); ++ } ++ per_cpu(hist_irqsoff_counting, cpu) = 0; ++ } ++#endif ++ ++#ifdef CONFIG_PREEMPT_OFF_HIST ++ if ((reason == PREEMPT_ON || reason == TRACE_STOP) && ++ per_cpu(hist_preemptoff_counting, cpu)) { ++ cycle_t start = per_cpu(hist_preemptoff_start, cpu); ++ ++ if (!(time_set++)) ++ stop = ftrace_now(cpu); ++ if (start) { ++ long latency = ((long) (stop - start)) / ++ NSECS_PER_USECS; ++ ++ latency_hist(PREEMPTOFF_LATENCY, cpu, latency, ++ 0, stop, NULL); ++ } ++ per_cpu(hist_preemptoff_counting, cpu) = 0; ++ } ++#endif ++ ++#if defined(CONFIG_INTERRUPT_OFF_HIST) && defined(CONFIG_PREEMPT_OFF_HIST) ++ if ((!per_cpu(hist_irqsoff_counting, cpu) || ++ !per_cpu(hist_preemptoff_counting, cpu)) && ++ per_cpu(hist_preemptirqsoff_counting, cpu)) { ++ cycle_t start = per_cpu(hist_preemptirqsoff_start, cpu); ++ ++ if (!time_set) ++ stop = ftrace_now(cpu); ++ if (start) { ++ long latency = ((long) (stop - start)) / ++ NSECS_PER_USECS; ++ ++ latency_hist(PREEMPTIRQSOFF_LATENCY, cpu, ++ latency, 0, stop, NULL); ++ } ++ per_cpu(hist_preemptirqsoff_counting, cpu) = 0; ++ } ++#endif ++ } ++} ++#endif ++ ++#ifdef CONFIG_WAKEUP_LATENCY_HIST ++static DEFINE_RAW_SPINLOCK(wakeup_lock); ++static notrace void probe_sched_migrate_task(void *v, struct task_struct *task, ++ int cpu) ++{ ++ int old_cpu = task_cpu(task); ++ ++ if (cpu != old_cpu) { ++ unsigned long flags; ++ struct task_struct *cpu_wakeup_task; ++ ++ raw_spin_lock_irqsave(&wakeup_lock, flags); ++ ++ cpu_wakeup_task = per_cpu(wakeup_task, old_cpu); ++ if (task == cpu_wakeup_task) { ++ put_task_struct(cpu_wakeup_task); ++ per_cpu(wakeup_task, old_cpu) = NULL; ++ cpu_wakeup_task = per_cpu(wakeup_task, cpu) = task; ++ get_task_struct(cpu_wakeup_task); ++ } ++ ++ raw_spin_unlock_irqrestore(&wakeup_lock, flags); ++ } ++} ++ ++static notrace void probe_wakeup_latency_hist_start(void *v, ++ struct task_struct *p, int success) ++{ ++ unsigned long flags; ++ struct task_struct *curr = current; ++ int cpu = task_cpu(p); ++ struct task_struct *cpu_wakeup_task; ++ ++ raw_spin_lock_irqsave(&wakeup_lock, flags); ++ ++ cpu_wakeup_task = per_cpu(wakeup_task, cpu); ++ ++ if (wakeup_pid) { ++ if ((cpu_wakeup_task && p->prio == cpu_wakeup_task->prio) || ++ p->prio == curr->prio) ++ per_cpu(wakeup_sharedprio, cpu) = 1; ++ if (likely(wakeup_pid != task_pid_nr(p))) ++ goto out; ++ } else { ++ if (likely(!rt_task(p)) || ++ (cpu_wakeup_task && p->prio > cpu_wakeup_task->prio) || ++ p->prio > curr->prio) ++ goto out; ++ if ((cpu_wakeup_task && p->prio == cpu_wakeup_task->prio) || ++ p->prio == curr->prio) ++ per_cpu(wakeup_sharedprio, cpu) = 1; ++ } ++ ++ if (cpu_wakeup_task) ++ put_task_struct(cpu_wakeup_task); ++ cpu_wakeup_task = per_cpu(wakeup_task, cpu) = p; ++ get_task_struct(cpu_wakeup_task); ++ cpu_wakeup_task->preempt_timestamp_hist = ++ ftrace_now(raw_smp_processor_id()); ++out: ++ raw_spin_unlock_irqrestore(&wakeup_lock, flags); ++} ++ ++static notrace void probe_wakeup_latency_hist_stop(void *v, ++ struct task_struct *prev, struct task_struct *next) ++{ ++ unsigned long flags; ++ int cpu = task_cpu(next); ++ long latency; ++ cycle_t stop; ++ struct task_struct *cpu_wakeup_task; ++ ++ raw_spin_lock_irqsave(&wakeup_lock, flags); ++ ++ cpu_wakeup_task = per_cpu(wakeup_task, cpu); ++ ++ if (cpu_wakeup_task == NULL) ++ goto out; ++ ++ /* Already running? */ ++ if (unlikely(current == cpu_wakeup_task)) ++ goto out_reset; ++ ++ if (next != cpu_wakeup_task) { ++ if (next->prio < cpu_wakeup_task->prio) ++ goto out_reset; ++ ++ if (next->prio == cpu_wakeup_task->prio) ++ per_cpu(wakeup_sharedprio, cpu) = 1; ++ ++ goto out; ++ } ++ ++ if (current->prio == cpu_wakeup_task->prio) ++ per_cpu(wakeup_sharedprio, cpu) = 1; ++ ++ /* ++ * The task we are waiting for is about to be switched to. ++ * Calculate latency and store it in histogram. ++ */ ++ stop = ftrace_now(raw_smp_processor_id()); ++ ++ latency = ((long) (stop - next->preempt_timestamp_hist)) / ++ NSECS_PER_USECS; ++ ++ if (per_cpu(wakeup_sharedprio, cpu)) { ++ latency_hist(WAKEUP_LATENCY_SHAREDPRIO, cpu, latency, 0, stop, ++ next); ++ per_cpu(wakeup_sharedprio, cpu) = 0; ++ } else { ++ latency_hist(WAKEUP_LATENCY, cpu, latency, 0, stop, next); ++#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST ++ if (timerandwakeup_enabled_data.enabled) { ++ latency_hist(TIMERANDWAKEUP_LATENCY, cpu, ++ next->timer_offset + latency, next->timer_offset, ++ stop, next); ++ } ++#endif ++ } ++ ++out_reset: ++#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST ++ next->timer_offset = 0; ++#endif ++ put_task_struct(cpu_wakeup_task); ++ per_cpu(wakeup_task, cpu) = NULL; ++out: ++ raw_spin_unlock_irqrestore(&wakeup_lock, flags); ++} ++#endif ++ ++#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST ++static notrace void probe_hrtimer_interrupt(void *v, int cpu, ++ long long latency_ns, struct task_struct *curr, ++ struct task_struct *task) ++{ ++ if (latency_ns <= 0 && task != NULL && rt_task(task) && ++ (task->prio < curr->prio || ++ (task->prio == curr->prio && ++ !cpumask_test_cpu(cpu, &task->cpus_allowed)))) { ++ long latency; ++ cycle_t now; ++ ++ if (missed_timer_offsets_pid) { ++ if (likely(missed_timer_offsets_pid != ++ task_pid_nr(task))) ++ return; ++ } ++ ++ now = ftrace_now(cpu); ++ latency = (long) div_s64(-latency_ns, NSECS_PER_USECS); ++ latency_hist(MISSED_TIMER_OFFSETS, cpu, latency, latency, now, ++ task); ++#ifdef CONFIG_WAKEUP_LATENCY_HIST ++ task->timer_offset = latency; ++#endif ++ } ++} ++#endif ++ ++static __init int latency_hist_init(void) ++{ ++ struct dentry *latency_hist_root = NULL; ++ struct dentry *dentry; ++#ifdef CONFIG_WAKEUP_LATENCY_HIST ++ struct dentry *dentry_sharedprio; ++#endif ++ struct dentry *entry; ++ struct dentry *enable_root; ++ int i = 0; ++ struct hist_data *my_hist; ++ char name[64]; ++ char *cpufmt = "CPU%d"; ++#if defined(CONFIG_WAKEUP_LATENCY_HIST) || \ ++ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) ++ char *cpufmt_maxlatproc = "max_latency-CPU%d"; ++ struct maxlatproc_data *mp = NULL; ++#endif ++ ++ dentry = tracing_init_dentry(); ++ latency_hist_root = debugfs_create_dir(latency_hist_dir_root, dentry); ++ enable_root = debugfs_create_dir("enable", latency_hist_root); ++ ++#ifdef CONFIG_INTERRUPT_OFF_HIST ++ dentry = debugfs_create_dir(irqsoff_hist_dir, latency_hist_root); ++ for_each_possible_cpu(i) { ++ sprintf(name, cpufmt, i); ++ entry = debugfs_create_file(name, 0444, dentry, ++ &per_cpu(irqsoff_hist, i), &latency_hist_fops); ++ my_hist = &per_cpu(irqsoff_hist, i); ++ atomic_set(&my_hist->hist_mode, 1); ++ my_hist->min_lat = LONG_MAX; ++ } ++ entry = debugfs_create_file("reset", 0644, dentry, ++ (void *)IRQSOFF_LATENCY, &latency_hist_reset_fops); ++#endif ++ ++#ifdef CONFIG_PREEMPT_OFF_HIST ++ dentry = debugfs_create_dir(preemptoff_hist_dir, ++ latency_hist_root); ++ for_each_possible_cpu(i) { ++ sprintf(name, cpufmt, i); ++ entry = debugfs_create_file(name, 0444, dentry, ++ &per_cpu(preemptoff_hist, i), &latency_hist_fops); ++ my_hist = &per_cpu(preemptoff_hist, i); ++ atomic_set(&my_hist->hist_mode, 1); ++ my_hist->min_lat = LONG_MAX; ++ } ++ entry = debugfs_create_file("reset", 0644, dentry, ++ (void *)PREEMPTOFF_LATENCY, &latency_hist_reset_fops); ++#endif ++ ++#if defined(CONFIG_INTERRUPT_OFF_HIST) && defined(CONFIG_PREEMPT_OFF_HIST) ++ dentry = debugfs_create_dir(preemptirqsoff_hist_dir, ++ latency_hist_root); ++ for_each_possible_cpu(i) { ++ sprintf(name, cpufmt, i); ++ entry = debugfs_create_file(name, 0444, dentry, ++ &per_cpu(preemptirqsoff_hist, i), &latency_hist_fops); ++ my_hist = &per_cpu(preemptirqsoff_hist, i); ++ atomic_set(&my_hist->hist_mode, 1); ++ my_hist->min_lat = LONG_MAX; ++ } ++ entry = debugfs_create_file("reset", 0644, dentry, ++ (void *)PREEMPTIRQSOFF_LATENCY, &latency_hist_reset_fops); ++#endif ++ ++#if defined(CONFIG_INTERRUPT_OFF_HIST) || defined(CONFIG_PREEMPT_OFF_HIST) ++ entry = debugfs_create_file("preemptirqsoff", 0644, ++ enable_root, (void *)&preemptirqsoff_enabled_data, ++ &enable_fops); ++#endif ++ ++#ifdef CONFIG_WAKEUP_LATENCY_HIST ++ dentry = debugfs_create_dir(wakeup_latency_hist_dir, ++ latency_hist_root); ++ dentry_sharedprio = debugfs_create_dir( ++ wakeup_latency_hist_dir_sharedprio, dentry); ++ for_each_possible_cpu(i) { ++ sprintf(name, cpufmt, i); ++ ++ entry = debugfs_create_file(name, 0444, dentry, ++ &per_cpu(wakeup_latency_hist, i), ++ &latency_hist_fops); ++ my_hist = &per_cpu(wakeup_latency_hist, i); ++ atomic_set(&my_hist->hist_mode, 1); ++ my_hist->min_lat = LONG_MAX; ++ ++ entry = debugfs_create_file(name, 0444, dentry_sharedprio, ++ &per_cpu(wakeup_latency_hist_sharedprio, i), ++ &latency_hist_fops); ++ my_hist = &per_cpu(wakeup_latency_hist_sharedprio, i); ++ atomic_set(&my_hist->hist_mode, 1); ++ my_hist->min_lat = LONG_MAX; ++ ++ sprintf(name, cpufmt_maxlatproc, i); ++ ++ mp = &per_cpu(wakeup_maxlatproc, i); ++ entry = debugfs_create_file(name, 0444, dentry, mp, ++ &maxlatproc_fops); ++ clear_maxlatprocdata(mp); ++ ++ mp = &per_cpu(wakeup_maxlatproc_sharedprio, i); ++ entry = debugfs_create_file(name, 0444, dentry_sharedprio, mp, ++ &maxlatproc_fops); ++ clear_maxlatprocdata(mp); ++ } ++ entry = debugfs_create_file("pid", 0644, dentry, ++ (void *)&wakeup_pid, &pid_fops); ++ entry = debugfs_create_file("reset", 0644, dentry, ++ (void *)WAKEUP_LATENCY, &latency_hist_reset_fops); ++ entry = debugfs_create_file("reset", 0644, dentry_sharedprio, ++ (void *)WAKEUP_LATENCY_SHAREDPRIO, &latency_hist_reset_fops); ++ entry = debugfs_create_file("wakeup", 0644, ++ enable_root, (void *)&wakeup_latency_enabled_data, ++ &enable_fops); ++#endif ++ ++#ifdef CONFIG_MISSED_TIMER_OFFSETS_HIST ++ dentry = debugfs_create_dir(missed_timer_offsets_dir, ++ latency_hist_root); ++ for_each_possible_cpu(i) { ++ sprintf(name, cpufmt, i); ++ entry = debugfs_create_file(name, 0444, dentry, ++ &per_cpu(missed_timer_offsets, i), &latency_hist_fops); ++ my_hist = &per_cpu(missed_timer_offsets, i); ++ atomic_set(&my_hist->hist_mode, 1); ++ my_hist->min_lat = LONG_MAX; ++ ++ sprintf(name, cpufmt_maxlatproc, i); ++ mp = &per_cpu(missed_timer_offsets_maxlatproc, i); ++ entry = debugfs_create_file(name, 0444, dentry, mp, ++ &maxlatproc_fops); ++ clear_maxlatprocdata(mp); ++ } ++ entry = debugfs_create_file("pid", 0644, dentry, ++ (void *)&missed_timer_offsets_pid, &pid_fops); ++ entry = debugfs_create_file("reset", 0644, dentry, ++ (void *)MISSED_TIMER_OFFSETS, &latency_hist_reset_fops); ++ entry = debugfs_create_file("missed_timer_offsets", 0644, ++ enable_root, (void *)&missed_timer_offsets_enabled_data, ++ &enable_fops); ++#endif ++ ++#if defined(CONFIG_WAKEUP_LATENCY_HIST) && \ ++ defined(CONFIG_MISSED_TIMER_OFFSETS_HIST) ++ dentry = debugfs_create_dir(timerandwakeup_latency_hist_dir, ++ latency_hist_root); ++ for_each_possible_cpu(i) { ++ sprintf(name, cpufmt, i); ++ entry = debugfs_create_file(name, 0444, dentry, ++ &per_cpu(timerandwakeup_latency_hist, i), ++ &latency_hist_fops); ++ my_hist = &per_cpu(timerandwakeup_latency_hist, i); ++ atomic_set(&my_hist->hist_mode, 1); ++ my_hist->min_lat = LONG_MAX; ++ ++ sprintf(name, cpufmt_maxlatproc, i); ++ mp = &per_cpu(timerandwakeup_maxlatproc, i); ++ entry = debugfs_create_file(name, 0444, dentry, mp, ++ &maxlatproc_fops); ++ clear_maxlatprocdata(mp); ++ } ++ entry = debugfs_create_file("reset", 0644, dentry, ++ (void *)TIMERANDWAKEUP_LATENCY, &latency_hist_reset_fops); ++ entry = debugfs_create_file("timerandwakeup", 0644, ++ enable_root, (void *)&timerandwakeup_enabled_data, ++ &enable_fops); ++#endif ++ return 0; ++} ++ ++device_initcall(latency_hist_init); +diff -Nur linux-4.1.10.orig/kernel/trace/Makefile linux-4.1.10/kernel/trace/Makefile +--- linux-4.1.10.orig/kernel/trace/Makefile 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/trace/Makefile 2015-10-07 18:00:08.000000000 +0200 +@@ -36,6 +36,10 @@ + obj-$(CONFIG_IRQSOFF_TRACER) += trace_irqsoff.o + obj-$(CONFIG_PREEMPT_TRACER) += trace_irqsoff.o + obj-$(CONFIG_SCHED_TRACER) += trace_sched_wakeup.o ++obj-$(CONFIG_INTERRUPT_OFF_HIST) += latency_hist.o ++obj-$(CONFIG_PREEMPT_OFF_HIST) += latency_hist.o ++obj-$(CONFIG_WAKEUP_LATENCY_HIST) += latency_hist.o ++obj-$(CONFIG_MISSED_TIMER_OFFSETS_HIST) += latency_hist.o + obj-$(CONFIG_NOP_TRACER) += trace_nop.o + obj-$(CONFIG_STACK_TRACER) += trace_stack.o + obj-$(CONFIG_MMIOTRACE) += trace_mmiotrace.o +diff -Nur linux-4.1.10.orig/kernel/trace/trace.c linux-4.1.10/kernel/trace/trace.c +--- linux-4.1.10.orig/kernel/trace/trace.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/trace/trace.c 2015-10-07 18:00:08.000000000 +0200 +@@ -1630,6 +1630,7 @@ + struct task_struct *tsk = current; + + entry->preempt_count = pc & 0xff; ++ entry->preempt_lazy_count = preempt_lazy_count(); + entry->pid = (tsk) ? tsk->pid : 0; + entry->flags = + #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT +@@ -1639,8 +1640,11 @@ + #endif + ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) | + ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) | +- (tif_need_resched() ? TRACE_FLAG_NEED_RESCHED : 0) | ++ (tif_need_resched_now() ? TRACE_FLAG_NEED_RESCHED : 0) | ++ (need_resched_lazy() ? TRACE_FLAG_NEED_RESCHED_LAZY : 0) | + (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED : 0); ++ ++ entry->migrate_disable = (tsk) ? __migrate_disabled(tsk) & 0xFF : 0; + } + EXPORT_SYMBOL_GPL(tracing_generic_entry_update); + +@@ -2558,14 +2562,17 @@ + + static void print_lat_help_header(struct seq_file *m) + { +- seq_puts(m, "# _------=> CPU# \n" +- "# / _-----=> irqs-off \n" +- "# | / _----=> need-resched \n" +- "# || / _---=> hardirq/softirq \n" +- "# ||| / _--=> preempt-depth \n" +- "# |||| / delay \n" +- "# cmd pid ||||| time | caller \n" +- "# \\ / ||||| \\ | / \n"); ++ seq_puts(m, "# _--------=> CPU# \n" ++ "# / _-------=> irqs-off \n" ++ "# | / _------=> need-resched \n" ++ "# || / _-----=> need-resched_lazy \n" ++ "# ||| / _----=> hardirq/softirq \n" ++ "# |||| / _---=> preempt-depth \n" ++ "# ||||| / _--=> preempt-lazy-depth\n" ++ "# |||||| / _-=> migrate-disable \n" ++ "# ||||||| / delay \n" ++ "# cmd pid |||||||| time | caller \n" ++ "# \\ / |||||||| \\ | / \n"); + } + + static void print_event_info(struct trace_buffer *buf, struct seq_file *m) +@@ -2591,11 +2598,14 @@ + print_event_info(buf, m); + seq_puts(m, "# _-----=> irqs-off\n" + "# / _----=> need-resched\n" +- "# | / _---=> hardirq/softirq\n" +- "# || / _--=> preempt-depth\n" +- "# ||| / delay\n" +- "# TASK-PID CPU# |||| TIMESTAMP FUNCTION\n" +- "# | | | |||| | |\n"); ++ "# |/ _-----=> need-resched_lazy\n" ++ "# || / _---=> hardirq/softirq\n" ++ "# ||| / _--=> preempt-depth\n" ++ "# |||| /_--=> preempt-lazy-depth\n" ++ "# ||||| _-=> migrate-disable \n" ++ "# ||||| / delay\n" ++ "# TASK-PID CPU# |||||| TIMESTAMP FUNCTION\n" ++ "# | | | |||||| | |\n"); + } + + void +diff -Nur linux-4.1.10.orig/kernel/trace/trace_events.c linux-4.1.10/kernel/trace/trace_events.c +--- linux-4.1.10.orig/kernel/trace/trace_events.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/trace/trace_events.c 2015-10-07 18:00:08.000000000 +0200 +@@ -162,6 +162,8 @@ + __common_field(unsigned char, flags); + __common_field(unsigned char, preempt_count); + __common_field(int, pid); ++ __common_field(unsigned short, migrate_disable); ++ __common_field(unsigned short, padding); + + return ret; + } +diff -Nur linux-4.1.10.orig/kernel/trace/trace.h linux-4.1.10/kernel/trace/trace.h +--- linux-4.1.10.orig/kernel/trace/trace.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/trace/trace.h 2015-10-07 18:00:08.000000000 +0200 +@@ -120,6 +120,7 @@ + * NEED_RESCHED - reschedule is requested + * HARDIRQ - inside an interrupt handler + * SOFTIRQ - inside a softirq handler ++ * NEED_RESCHED_LAZY - lazy reschedule is requested + */ + enum trace_flag_type { + TRACE_FLAG_IRQS_OFF = 0x01, +@@ -128,6 +129,7 @@ + TRACE_FLAG_HARDIRQ = 0x08, + TRACE_FLAG_SOFTIRQ = 0x10, + TRACE_FLAG_PREEMPT_RESCHED = 0x20, ++ TRACE_FLAG_NEED_RESCHED_LAZY = 0x40, + }; + + #define TRACE_BUF_SIZE 1024 +diff -Nur linux-4.1.10.orig/kernel/trace/trace_irqsoff.c linux-4.1.10/kernel/trace/trace_irqsoff.c +--- linux-4.1.10.orig/kernel/trace/trace_irqsoff.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/trace/trace_irqsoff.c 2015-10-07 18:00:08.000000000 +0200 +@@ -13,6 +13,7 @@ + #include + #include + #include ++#include + + #include "trace.h" + +@@ -433,11 +434,13 @@ + { + if (preempt_trace() || irq_trace()) + start_critical_timing(CALLER_ADDR0, CALLER_ADDR1); ++ trace_preemptirqsoff_hist(TRACE_START, 1); + } + EXPORT_SYMBOL_GPL(start_critical_timings); + + void stop_critical_timings(void) + { ++ trace_preemptirqsoff_hist(TRACE_STOP, 0); + if (preempt_trace() || irq_trace()) + stop_critical_timing(CALLER_ADDR0, CALLER_ADDR1); + } +@@ -447,6 +450,7 @@ + #ifdef CONFIG_PROVE_LOCKING + void time_hardirqs_on(unsigned long a0, unsigned long a1) + { ++ trace_preemptirqsoff_hist_rcuidle(IRQS_ON, 0); + if (!preempt_trace() && irq_trace()) + stop_critical_timing(a0, a1); + } +@@ -455,6 +459,7 @@ + { + if (!preempt_trace() && irq_trace()) + start_critical_timing(a0, a1); ++ trace_preemptirqsoff_hist_rcuidle(IRQS_OFF, 1); + } + + #else /* !CONFIG_PROVE_LOCKING */ +@@ -480,6 +485,7 @@ + */ + void trace_hardirqs_on(void) + { ++ trace_preemptirqsoff_hist(IRQS_ON, 0); + if (!preempt_trace() && irq_trace()) + stop_critical_timing(CALLER_ADDR0, CALLER_ADDR1); + } +@@ -489,11 +495,13 @@ + { + if (!preempt_trace() && irq_trace()) + start_critical_timing(CALLER_ADDR0, CALLER_ADDR1); ++ trace_preemptirqsoff_hist(IRQS_OFF, 1); + } + EXPORT_SYMBOL(trace_hardirqs_off); + + __visible void trace_hardirqs_on_caller(unsigned long caller_addr) + { ++ trace_preemptirqsoff_hist(IRQS_ON, 0); + if (!preempt_trace() && irq_trace()) + stop_critical_timing(CALLER_ADDR0, caller_addr); + } +@@ -503,6 +511,7 @@ + { + if (!preempt_trace() && irq_trace()) + start_critical_timing(CALLER_ADDR0, caller_addr); ++ trace_preemptirqsoff_hist(IRQS_OFF, 1); + } + EXPORT_SYMBOL(trace_hardirqs_off_caller); + +@@ -512,12 +521,14 @@ + #ifdef CONFIG_PREEMPT_TRACER + void trace_preempt_on(unsigned long a0, unsigned long a1) + { ++ trace_preemptirqsoff_hist(PREEMPT_ON, 0); + if (preempt_trace() && !irq_trace()) + stop_critical_timing(a0, a1); + } + + void trace_preempt_off(unsigned long a0, unsigned long a1) + { ++ trace_preemptirqsoff_hist(PREEMPT_ON, 1); + if (preempt_trace() && !irq_trace()) + start_critical_timing(a0, a1); + } +diff -Nur linux-4.1.10.orig/kernel/trace/trace_output.c linux-4.1.10/kernel/trace/trace_output.c +--- linux-4.1.10.orig/kernel/trace/trace_output.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/trace/trace_output.c 2015-10-07 18:00:08.000000000 +0200 +@@ -430,6 +430,7 @@ + { + char hardsoft_irq; + char need_resched; ++ char need_resched_lazy; + char irqs_off; + int hardirq; + int softirq; +@@ -457,6 +458,8 @@ + need_resched = '.'; + break; + } ++ need_resched_lazy = ++ (entry->flags & TRACE_FLAG_NEED_RESCHED_LAZY) ? 'L' : '.'; + + hardsoft_irq = + (hardirq && softirq) ? 'H' : +@@ -464,14 +467,25 @@ + softirq ? 's' : + '.'; + +- trace_seq_printf(s, "%c%c%c", +- irqs_off, need_resched, hardsoft_irq); ++ trace_seq_printf(s, "%c%c%c%c", ++ irqs_off, need_resched, need_resched_lazy, ++ hardsoft_irq); + + if (entry->preempt_count) + trace_seq_printf(s, "%x", entry->preempt_count); + else + trace_seq_putc(s, '.'); + ++ if (entry->preempt_lazy_count) ++ trace_seq_printf(s, "%x", entry->preempt_lazy_count); ++ else ++ trace_seq_putc(s, '.'); ++ ++ if (entry->migrate_disable) ++ trace_seq_printf(s, "%x", entry->migrate_disable); ++ else ++ trace_seq_putc(s, '.'); ++ + return !trace_seq_has_overflowed(s); + } + +diff -Nur linux-4.1.10.orig/kernel/user.c linux-4.1.10/kernel/user.c +--- linux-4.1.10.orig/kernel/user.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/user.c 2015-10-07 18:00:08.000000000 +0200 +@@ -161,11 +161,11 @@ + if (!up) + return; + +- local_irq_save(flags); ++ local_irq_save_nort(flags); + if (atomic_dec_and_lock(&up->__count, &uidhash_lock)) + free_user(up, flags); + else +- local_irq_restore(flags); ++ local_irq_restore_nort(flags); + } + + struct user_struct *alloc_uid(kuid_t uid) +diff -Nur linux-4.1.10.orig/kernel/watchdog.c linux-4.1.10/kernel/watchdog.c +--- linux-4.1.10.orig/kernel/watchdog.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/watchdog.c 2015-10-07 18:00:08.000000000 +0200 +@@ -262,6 +262,8 @@ + + #ifdef CONFIG_HARDLOCKUP_DETECTOR + ++static DEFINE_RAW_SPINLOCK(watchdog_output_lock); ++ + static struct perf_event_attr wd_hw_attr = { + .type = PERF_TYPE_HARDWARE, + .config = PERF_COUNT_HW_CPU_CYCLES, +@@ -295,13 +297,21 @@ + /* only print hardlockups once */ + if (__this_cpu_read(hard_watchdog_warn) == true) + return; ++ /* ++ * If early-printk is enabled then make sure we do not ++ * lock up in printk() and kill console logging: ++ */ ++ printk_kill(); + +- if (hardlockup_panic) ++ if (hardlockup_panic) { + panic("Watchdog detected hard LOCKUP on cpu %d", + this_cpu); +- else ++ } else { ++ raw_spin_lock(&watchdog_output_lock); + WARN(1, "Watchdog detected hard LOCKUP on cpu %d", + this_cpu); ++ raw_spin_unlock(&watchdog_output_lock); ++ } + + __this_cpu_write(hard_watchdog_warn, true); + return; +@@ -444,6 +454,7 @@ + /* kick off the timer for the hardlockup detector */ + hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + hrtimer->function = watchdog_timer_fn; ++ hrtimer->irqsafe = 1; + + /* Enable the perf event */ + watchdog_nmi_enable(cpu); +diff -Nur linux-4.1.10.orig/kernel/workqueue.c linux-4.1.10/kernel/workqueue.c +--- linux-4.1.10.orig/kernel/workqueue.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/workqueue.c 2015-10-07 18:00:08.000000000 +0200 +@@ -48,6 +48,8 @@ + #include + #include + #include ++#include ++#include + + #include "workqueue_internal.h" + +@@ -121,15 +123,20 @@ + * cpu or grabbing pool->lock is enough for read access. If + * POOL_DISASSOCIATED is set, it's identical to L. + * ++ * On RT we need the extra protection via rt_lock_idle_list() for ++ * the list manipulations against read access from ++ * wq_worker_sleeping(). All other places are nicely serialized via ++ * pool->lock. ++ * + * A: pool->attach_mutex protected. + * + * PL: wq_pool_mutex protected. + * +- * PR: wq_pool_mutex protected for writes. Sched-RCU protected for reads. ++ * PR: wq_pool_mutex protected for writes. RCU protected for reads. + * + * WQ: wq->mutex protected. + * +- * WR: wq->mutex protected for writes. Sched-RCU protected for reads. ++ * WR: wq->mutex protected for writes. RCU protected for reads. + * + * MD: wq_mayday_lock protected. + */ +@@ -178,7 +185,7 @@ + atomic_t nr_running ____cacheline_aligned_in_smp; + + /* +- * Destruction of pool is sched-RCU protected to allow dereferences ++ * Destruction of pool is RCU protected to allow dereferences + * from get_work_pool(). + */ + struct rcu_head rcu; +@@ -207,7 +214,7 @@ + /* + * Release of unbound pwq is punted to system_wq. See put_pwq() + * and pwq_unbound_release_workfn() for details. pool_workqueue +- * itself is also sched-RCU protected so that the first pwq can be ++ * itself is also RCU protected so that the first pwq can be + * determined without grabbing wq->mutex. + */ + struct work_struct unbound_release_work; +@@ -329,6 +336,8 @@ + struct workqueue_struct *system_freezable_power_efficient_wq __read_mostly; + EXPORT_SYMBOL_GPL(system_freezable_power_efficient_wq); + ++static DEFINE_LOCAL_IRQ_LOCK(pendingb_lock); ++ + static int worker_thread(void *__worker); + static void copy_workqueue_attrs(struct workqueue_attrs *to, + const struct workqueue_attrs *from); +@@ -338,14 +347,14 @@ + #include + + #define assert_rcu_or_pool_mutex() \ +- rcu_lockdep_assert(rcu_read_lock_sched_held() || \ ++ rcu_lockdep_assert(rcu_read_lock_held() || \ + lockdep_is_held(&wq_pool_mutex), \ +- "sched RCU or wq_pool_mutex should be held") ++ "RCU or wq_pool_mutex should be held") + + #define assert_rcu_or_wq_mutex(wq) \ +- rcu_lockdep_assert(rcu_read_lock_sched_held() || \ ++ rcu_lockdep_assert(rcu_read_lock_held() || \ + lockdep_is_held(&wq->mutex), \ +- "sched RCU or wq->mutex should be held") ++ "RCU or wq->mutex should be held") + + #define for_each_cpu_worker_pool(pool, cpu) \ + for ((pool) = &per_cpu(cpu_worker_pools, cpu)[0]; \ +@@ -357,7 +366,7 @@ + * @pool: iteration cursor + * @pi: integer used for iteration + * +- * This must be called either with wq_pool_mutex held or sched RCU read ++ * This must be called either with wq_pool_mutex held or RCU read + * locked. If the pool needs to be used beyond the locking in effect, the + * caller is responsible for guaranteeing that the pool stays online. + * +@@ -389,7 +398,7 @@ + * @pwq: iteration cursor + * @wq: the target workqueue + * +- * This must be called either with wq->mutex held or sched RCU read locked. ++ * This must be called either with wq->mutex held or RCU read locked. + * If the pwq needs to be used beyond the locking in effect, the caller is + * responsible for guaranteeing that the pwq stays online. + * +@@ -401,6 +410,31 @@ + if (({ assert_rcu_or_wq_mutex(wq); false; })) { } \ + else + ++#ifdef CONFIG_PREEMPT_RT_BASE ++static inline void rt_lock_idle_list(struct worker_pool *pool) ++{ ++ preempt_disable(); ++} ++static inline void rt_unlock_idle_list(struct worker_pool *pool) ++{ ++ preempt_enable(); ++} ++static inline void sched_lock_idle_list(struct worker_pool *pool) { } ++static inline void sched_unlock_idle_list(struct worker_pool *pool) { } ++#else ++static inline void rt_lock_idle_list(struct worker_pool *pool) { } ++static inline void rt_unlock_idle_list(struct worker_pool *pool) { } ++static inline void sched_lock_idle_list(struct worker_pool *pool) ++{ ++ spin_lock_irq(&pool->lock); ++} ++static inline void sched_unlock_idle_list(struct worker_pool *pool) ++{ ++ spin_unlock_irq(&pool->lock); ++} ++#endif ++ ++ + #ifdef CONFIG_DEBUG_OBJECTS_WORK + + static struct debug_obj_descr work_debug_descr; +@@ -551,7 +585,7 @@ + * @wq: the target workqueue + * @node: the node ID + * +- * This must be called either with pwq_lock held or sched RCU read locked. ++ * This must be called either with pwq_lock held or RCU read locked. + * If the pwq needs to be used beyond the locking in effect, the caller is + * responsible for guaranteeing that the pwq stays online. + * +@@ -655,8 +689,8 @@ + * @work: the work item of interest + * + * Pools are created and destroyed under wq_pool_mutex, and allows read +- * access under sched-RCU read lock. As such, this function should be +- * called under wq_pool_mutex or with preemption disabled. ++ * access under RCU read lock. As such, this function should be ++ * called under wq_pool_mutex or inside of a rcu_read_lock() region. + * + * All fields of the returned pool are accessible as long as the above + * mentioned locking is in effect. If the returned pool needs to be used +@@ -793,51 +827,44 @@ + */ + static void wake_up_worker(struct worker_pool *pool) + { +- struct worker *worker = first_idle_worker(pool); ++ struct worker *worker; ++ ++ rt_lock_idle_list(pool); ++ ++ worker = first_idle_worker(pool); + + if (likely(worker)) + wake_up_process(worker->task); ++ ++ rt_unlock_idle_list(pool); + } + + /** +- * wq_worker_waking_up - a worker is waking up +- * @task: task waking up +- * @cpu: CPU @task is waking up to ++ * wq_worker_running - a worker is running again ++ * @task: task returning from sleep + * +- * This function is called during try_to_wake_up() when a worker is +- * being awoken. +- * +- * CONTEXT: +- * spin_lock_irq(rq->lock) ++ * This function is called when a worker returns from schedule() + */ +-void wq_worker_waking_up(struct task_struct *task, int cpu) ++void wq_worker_running(struct task_struct *task) + { + struct worker *worker = kthread_data(task); + +- if (!(worker->flags & WORKER_NOT_RUNNING)) { +- WARN_ON_ONCE(worker->pool->cpu != cpu); ++ if (!worker->sleeping) ++ return; ++ if (!(worker->flags & WORKER_NOT_RUNNING)) + atomic_inc(&worker->pool->nr_running); +- } ++ worker->sleeping = 0; + } + + /** + * wq_worker_sleeping - a worker is going to sleep + * @task: task going to sleep +- * @cpu: CPU in question, must be the current CPU number +- * +- * This function is called during schedule() when a busy worker is +- * going to sleep. Worker on the same cpu can be woken up by +- * returning pointer to its task. +- * +- * CONTEXT: +- * spin_lock_irq(rq->lock) +- * +- * Return: +- * Worker task on @cpu to wake up, %NULL if none. ++ * This function is called from schedule() when a busy worker is ++ * going to sleep. + */ +-struct task_struct *wq_worker_sleeping(struct task_struct *task, int cpu) ++void wq_worker_sleeping(struct task_struct *task) + { +- struct worker *worker = kthread_data(task), *to_wakeup = NULL; ++ struct worker *worker = kthread_data(task); + struct worker_pool *pool; + + /* +@@ -846,29 +873,26 @@ + * checking NOT_RUNNING. + */ + if (worker->flags & WORKER_NOT_RUNNING) +- return NULL; ++ return; + + pool = worker->pool; + +- /* this can only happen on the local cpu */ +- if (WARN_ON_ONCE(cpu != raw_smp_processor_id() || pool->cpu != cpu)) +- return NULL; ++ if (WARN_ON_ONCE(worker->sleeping)) ++ return; ++ ++ worker->sleeping = 1; + + /* + * The counterpart of the following dec_and_test, implied mb, + * worklist not empty test sequence is in insert_work(). + * Please read comment there. +- * +- * NOT_RUNNING is clear. This means that we're bound to and +- * running on the local cpu w/ rq lock held and preemption +- * disabled, which in turn means that none else could be +- * manipulating idle_list, so dereferencing idle_list without pool +- * lock is safe. + */ + if (atomic_dec_and_test(&pool->nr_running) && +- !list_empty(&pool->worklist)) +- to_wakeup = first_idle_worker(pool); +- return to_wakeup ? to_wakeup->task : NULL; ++ !list_empty(&pool->worklist)) { ++ sched_lock_idle_list(pool); ++ wake_up_worker(pool); ++ sched_unlock_idle_list(pool); ++ } + } + + /** +@@ -1062,12 +1086,12 @@ + { + if (pwq) { + /* +- * As both pwqs and pools are sched-RCU protected, the ++ * As both pwqs and pools are RCU protected, the + * following lock operations are safe. + */ +- spin_lock_irq(&pwq->pool->lock); ++ local_spin_lock_irq(pendingb_lock, &pwq->pool->lock); + put_pwq(pwq); +- spin_unlock_irq(&pwq->pool->lock); ++ local_spin_unlock_irq(pendingb_lock, &pwq->pool->lock); + } + } + +@@ -1169,7 +1193,7 @@ + struct worker_pool *pool; + struct pool_workqueue *pwq; + +- local_irq_save(*flags); ++ local_lock_irqsave(pendingb_lock, *flags); + + /* try to steal the timer if it exists */ + if (is_dwork) { +@@ -1188,6 +1212,7 @@ + if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) + return 0; + ++ rcu_read_lock(); + /* + * The queueing is in progress, or it is already queued. Try to + * steal it from ->worklist without clearing WORK_STRUCT_PENDING. +@@ -1226,14 +1251,16 @@ + set_work_pool_and_keep_pending(work, pool->id); + + spin_unlock(&pool->lock); ++ rcu_read_unlock(); + return 1; + } + spin_unlock(&pool->lock); + fail: +- local_irq_restore(*flags); ++ rcu_read_unlock(); ++ local_unlock_irqrestore(pendingb_lock, *flags); + if (work_is_canceling(work)) + return -ENOENT; +- cpu_relax(); ++ cpu_chill(); + return -EAGAIN; + } + +@@ -1302,7 +1329,7 @@ + * queued or lose PENDING. Grabbing PENDING and queueing should + * happen with IRQ disabled. + */ +- WARN_ON_ONCE(!irqs_disabled()); ++ WARN_ON_ONCE_NONRT(!irqs_disabled()); + + debug_work_activate(work); + +@@ -1310,6 +1337,8 @@ + if (unlikely(wq->flags & __WQ_DRAINING) && + WARN_ON_ONCE(!is_chained_work(wq))) + return; ++ ++ rcu_read_lock(); + retry: + if (req_cpu == WORK_CPU_UNBOUND) + cpu = raw_smp_processor_id(); +@@ -1366,10 +1395,8 @@ + /* pwq determined, queue */ + trace_workqueue_queue_work(req_cpu, pwq, work); + +- if (WARN_ON(!list_empty(&work->entry))) { +- spin_unlock(&pwq->pool->lock); +- return; +- } ++ if (WARN_ON(!list_empty(&work->entry))) ++ goto out; + + pwq->nr_in_flight[pwq->work_color]++; + work_flags = work_color_to_flags(pwq->work_color); +@@ -1385,7 +1412,9 @@ + + insert_work(pwq, work, worklist, work_flags); + ++out: + spin_unlock(&pwq->pool->lock); ++ rcu_read_unlock(); + } + + /** +@@ -1405,14 +1434,14 @@ + bool ret = false; + unsigned long flags; + +- local_irq_save(flags); ++ local_lock_irqsave(pendingb_lock,flags); + + if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) { + __queue_work(cpu, wq, work); + ret = true; + } + +- local_irq_restore(flags); ++ local_unlock_irqrestore(pendingb_lock, flags); + return ret; + } + EXPORT_SYMBOL(queue_work_on); +@@ -1479,14 +1508,14 @@ + unsigned long flags; + + /* read the comment in __queue_work() */ +- local_irq_save(flags); ++ local_lock_irqsave(pendingb_lock, flags); + + if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) { + __queue_delayed_work(cpu, wq, dwork, delay); + ret = true; + } + +- local_irq_restore(flags); ++ local_unlock_irqrestore(pendingb_lock, flags); + return ret; + } + EXPORT_SYMBOL(queue_delayed_work_on); +@@ -1521,7 +1550,7 @@ + + if (likely(ret >= 0)) { + __queue_delayed_work(cpu, wq, dwork, delay); +- local_irq_restore(flags); ++ local_unlock_irqrestore(pendingb_lock, flags); + } + + /* -ENOENT from try_to_grab_pending() becomes %true */ +@@ -1554,7 +1583,9 @@ + worker->last_active = jiffies; + + /* idle_list is LIFO */ ++ rt_lock_idle_list(pool); + list_add(&worker->entry, &pool->idle_list); ++ rt_unlock_idle_list(pool); + + if (too_many_workers(pool) && !timer_pending(&pool->idle_timer)) + mod_timer(&pool->idle_timer, jiffies + IDLE_WORKER_TIMEOUT); +@@ -1587,7 +1618,9 @@ + return; + worker_clr_flags(worker, WORKER_IDLE); + pool->nr_idle--; ++ rt_lock_idle_list(pool); + list_del_init(&worker->entry); ++ rt_unlock_idle_list(pool); + } + + static struct worker *alloc_worker(int node) +@@ -1755,7 +1788,9 @@ + pool->nr_workers--; + pool->nr_idle--; + ++ rt_lock_idle_list(pool); + list_del_init(&worker->entry); ++ rt_unlock_idle_list(pool); + worker->flags |= WORKER_DIE; + wake_up_process(worker->task); + } +@@ -2672,14 +2707,14 @@ + + might_sleep(); + +- local_irq_disable(); ++ rcu_read_lock(); + pool = get_work_pool(work); + if (!pool) { +- local_irq_enable(); ++ rcu_read_unlock(); + return false; + } + +- spin_lock(&pool->lock); ++ spin_lock_irq(&pool->lock); + /* see the comment in try_to_grab_pending() with the same code */ + pwq = get_work_pwq(work); + if (pwq) { +@@ -2706,10 +2741,11 @@ + else + lock_map_acquire_read(&pwq->wq->lockdep_map); + lock_map_release(&pwq->wq->lockdep_map); +- ++ rcu_read_unlock(); + return true; + already_gone: + spin_unlock_irq(&pool->lock); ++ rcu_read_unlock(); + return false; + } + +@@ -2796,7 +2832,7 @@ + + /* tell other tasks trying to grab @work to back off */ + mark_work_canceling(work); +- local_irq_restore(flags); ++ local_unlock_irqrestore(pendingb_lock, flags); + + flush_work(work); + clear_work_data(work); +@@ -2851,10 +2887,10 @@ + */ + bool flush_delayed_work(struct delayed_work *dwork) + { +- local_irq_disable(); ++ local_lock_irq(pendingb_lock); + if (del_timer_sync(&dwork->timer)) + __queue_work(dwork->cpu, dwork->wq, &dwork->work); +- local_irq_enable(); ++ local_unlock_irq(pendingb_lock); + return flush_work(&dwork->work); + } + EXPORT_SYMBOL(flush_delayed_work); +@@ -2889,7 +2925,7 @@ + + set_work_pool_and_clear_pending(&dwork->work, + get_work_pool_id(&dwork->work)); +- local_irq_restore(flags); ++ local_unlock_irqrestore(pendingb_lock, flags); + return ret; + } + EXPORT_SYMBOL(cancel_delayed_work); +@@ -3147,7 +3183,7 @@ + * put_unbound_pool - put a worker_pool + * @pool: worker_pool to put + * +- * Put @pool. If its refcnt reaches zero, it gets destroyed in sched-RCU ++ * Put @pool. If its refcnt reaches zero, it gets destroyed in RCU + * safe manner. get_unbound_pool() calls this function on its failure path + * and this function should be able to release pools which went through, + * successfully or not, init_worker_pool(). +@@ -3201,8 +3237,8 @@ + del_timer_sync(&pool->idle_timer); + del_timer_sync(&pool->mayday_timer); + +- /* sched-RCU protected to allow dereferences from get_work_pool() */ +- call_rcu_sched(&pool->rcu, rcu_free_pool); ++ /* RCU protected to allow dereferences from get_work_pool() */ ++ call_rcu(&pool->rcu, rcu_free_pool); + } + + /** +@@ -3307,14 +3343,14 @@ + put_unbound_pool(pool); + mutex_unlock(&wq_pool_mutex); + +- call_rcu_sched(&pwq->rcu, rcu_free_pwq); ++ call_rcu(&pwq->rcu, rcu_free_pwq); + + /* + * If we're the last pwq going away, @wq is already dead and no one + * is gonna access it anymore. Schedule RCU free. + */ + if (is_last) +- call_rcu_sched(&wq->rcu, rcu_free_wq); ++ call_rcu(&wq->rcu, rcu_free_wq); + } + + /** +@@ -3920,7 +3956,7 @@ + * The base ref is never dropped on per-cpu pwqs. Directly + * schedule RCU free. + */ +- call_rcu_sched(&wq->rcu, rcu_free_wq); ++ call_rcu(&wq->rcu, rcu_free_wq); + } else { + /* + * We're the sole accessor of @wq at this point. Directly +@@ -4013,7 +4049,8 @@ + struct pool_workqueue *pwq; + bool ret; + +- rcu_read_lock_sched(); ++ rcu_read_lock(); ++ preempt_disable(); + + if (cpu == WORK_CPU_UNBOUND) + cpu = smp_processor_id(); +@@ -4024,7 +4061,8 @@ + pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu)); + + ret = !list_empty(&pwq->delayed_works); +- rcu_read_unlock_sched(); ++ preempt_enable(); ++ rcu_read_unlock(); + + return ret; + } +@@ -4050,15 +4088,15 @@ + if (work_pending(work)) + ret |= WORK_BUSY_PENDING; + +- local_irq_save(flags); ++ rcu_read_lock(); + pool = get_work_pool(work); + if (pool) { +- spin_lock(&pool->lock); ++ spin_lock_irqsave(&pool->lock, flags); + if (find_worker_executing_work(pool, work)) + ret |= WORK_BUSY_RUNNING; +- spin_unlock(&pool->lock); ++ spin_unlock_irqrestore(&pool->lock, flags); + } +- local_irq_restore(flags); ++ rcu_read_unlock(); + + return ret; + } +@@ -4247,7 +4285,7 @@ + unsigned long flags; + int pi; + +- rcu_read_lock_sched(); ++ rcu_read_lock(); + + pr_info("Showing busy workqueues and worker pools:\n"); + +@@ -4298,7 +4336,7 @@ + spin_unlock_irqrestore(&pool->lock, flags); + } + +- rcu_read_unlock_sched(); ++ rcu_read_unlock(); + } + + /* +@@ -4648,16 +4686,16 @@ + * nr_active is monotonically decreasing. It's safe + * to peek without lock. + */ +- rcu_read_lock_sched(); ++ rcu_read_lock(); + for_each_pwq(pwq, wq) { + WARN_ON_ONCE(pwq->nr_active < 0); + if (pwq->nr_active) { + busy = true; +- rcu_read_unlock_sched(); ++ rcu_read_unlock(); + goto out_unlock; + } + } +- rcu_read_unlock_sched(); ++ rcu_read_unlock(); + } + out_unlock: + mutex_unlock(&wq_pool_mutex); +@@ -4771,7 +4809,8 @@ + const char *delim = ""; + int node, written = 0; + +- rcu_read_lock_sched(); ++ get_online_cpus(); ++ rcu_read_lock(); + for_each_node(node) { + written += scnprintf(buf + written, PAGE_SIZE - written, + "%s%d:%d", delim, node, +@@ -4779,7 +4818,8 @@ + delim = " "; + } + written += scnprintf(buf + written, PAGE_SIZE - written, "\n"); +- rcu_read_unlock_sched(); ++ rcu_read_unlock(); ++ put_online_cpus(); + + return written; + } +diff -Nur linux-4.1.10.orig/kernel/workqueue_internal.h linux-4.1.10/kernel/workqueue_internal.h +--- linux-4.1.10.orig/kernel/workqueue_internal.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/kernel/workqueue_internal.h 2015-10-07 18:00:08.000000000 +0200 +@@ -43,6 +43,7 @@ + unsigned long last_active; /* L: last active timestamp */ + unsigned int flags; /* X: flags */ + int id; /* I: worker id */ ++ int sleeping; /* None */ + + /* + * Opaque string set with work_set_desc(). Printed out with task +@@ -68,7 +69,7 @@ + * Scheduler hooks for concurrency managed workqueue. Only to be used from + * sched/core.c and workqueue.c. + */ +-void wq_worker_waking_up(struct task_struct *task, int cpu); +-struct task_struct *wq_worker_sleeping(struct task_struct *task, int cpu); ++void wq_worker_running(struct task_struct *task); ++void wq_worker_sleeping(struct task_struct *task); + + #endif /* _KERNEL_WORKQUEUE_INTERNAL_H */ +diff -Nur linux-4.1.10.orig/lib/debugobjects.c linux-4.1.10/lib/debugobjects.c +--- linux-4.1.10.orig/lib/debugobjects.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/lib/debugobjects.c 2015-10-07 18:00:08.000000000 +0200 +@@ -309,7 +309,10 @@ + struct debug_obj *obj; + unsigned long flags; + +- fill_pool(); ++#ifdef CONFIG_PREEMPT_RT_FULL ++ if (preempt_count() == 0 && !irqs_disabled()) ++#endif ++ fill_pool(); + + db = get_bucket((unsigned long) addr); + +diff -Nur linux-4.1.10.orig/lib/dump_stack.c linux-4.1.10/lib/dump_stack.c +--- linux-4.1.10.orig/lib/dump_stack.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/lib/dump_stack.c 2015-10-07 18:00:08.000000000 +0200 +@@ -33,7 +33,7 @@ + * Permit this cpu to perform nested stack dumps while serialising + * against other CPUs + */ +- preempt_disable(); ++ migrate_disable(); + + retry: + cpu = smp_processor_id(); +@@ -52,7 +52,7 @@ + if (!was_locked) + atomic_set(&dump_lock, -1); + +- preempt_enable(); ++ migrate_enable(); + } + #else + asmlinkage __visible void dump_stack(void) +diff -Nur linux-4.1.10.orig/lib/idr.c linux-4.1.10/lib/idr.c +--- linux-4.1.10.orig/lib/idr.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/lib/idr.c 2015-10-07 18:00:08.000000000 +0200 +@@ -30,6 +30,7 @@ + #include + #include + #include ++#include + + #define MAX_IDR_SHIFT (sizeof(int) * 8 - 1) + #define MAX_IDR_BIT (1U << MAX_IDR_SHIFT) +@@ -366,6 +367,35 @@ + idr_mark_full(pa, id); + } + ++#ifdef CONFIG_PREEMPT_RT_FULL ++static DEFINE_LOCAL_IRQ_LOCK(idr_lock); ++ ++static inline void idr_preload_lock(void) ++{ ++ local_lock(idr_lock); ++} ++ ++static inline void idr_preload_unlock(void) ++{ ++ local_unlock(idr_lock); ++} ++ ++void idr_preload_end(void) ++{ ++ idr_preload_unlock(); ++} ++EXPORT_SYMBOL(idr_preload_end); ++#else ++static inline void idr_preload_lock(void) ++{ ++ preempt_disable(); ++} ++ ++static inline void idr_preload_unlock(void) ++{ ++ preempt_enable(); ++} ++#endif + + /** + * idr_preload - preload for idr_alloc() +@@ -401,7 +431,7 @@ + WARN_ON_ONCE(in_interrupt()); + might_sleep_if(gfp_mask & __GFP_WAIT); + +- preempt_disable(); ++ idr_preload_lock(); + + /* + * idr_alloc() is likely to succeed w/o full idr_layer buffer and +@@ -413,9 +443,9 @@ + while (__this_cpu_read(idr_preload_cnt) < MAX_IDR_FREE) { + struct idr_layer *new; + +- preempt_enable(); ++ idr_preload_unlock(); + new = kmem_cache_zalloc(idr_layer_cache, gfp_mask); +- preempt_disable(); ++ idr_preload_lock(); + if (!new) + break; + +diff -Nur linux-4.1.10.orig/lib/Kconfig linux-4.1.10/lib/Kconfig +--- linux-4.1.10.orig/lib/Kconfig 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/lib/Kconfig 2015-10-07 18:00:08.000000000 +0200 +@@ -391,6 +391,7 @@ + + config CPUMASK_OFFSTACK + bool "Force CPU masks off stack" if DEBUG_PER_CPU_MAPS ++ depends on !PREEMPT_RT_FULL + help + Use dynamic allocation for cpumask_var_t, instead of putting + them on the stack. This is a bit more expensive, but avoids +diff -Nur linux-4.1.10.orig/lib/locking-selftest.c linux-4.1.10/lib/locking-selftest.c +--- linux-4.1.10.orig/lib/locking-selftest.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/lib/locking-selftest.c 2015-10-07 18:00:08.000000000 +0200 +@@ -590,6 +590,8 @@ + #include "locking-selftest-spin-hardirq.h" + GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_hard_spin) + ++#ifndef CONFIG_PREEMPT_RT_FULL ++ + #include "locking-selftest-rlock-hardirq.h" + GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_hard_rlock) + +@@ -605,9 +607,12 @@ + #include "locking-selftest-wlock-softirq.h" + GENERATE_PERMUTATIONS_2_EVENTS(irqsafe1_soft_wlock) + ++#endif ++ + #undef E1 + #undef E2 + ++#ifndef CONFIG_PREEMPT_RT_FULL + /* + * Enabling hardirqs with a softirq-safe lock held: + */ +@@ -640,6 +645,8 @@ + #undef E1 + #undef E2 + ++#endif ++ + /* + * Enabling irqs with an irq-safe lock held: + */ +@@ -663,6 +670,8 @@ + #include "locking-selftest-spin-hardirq.h" + GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_hard_spin) + ++#ifndef CONFIG_PREEMPT_RT_FULL ++ + #include "locking-selftest-rlock-hardirq.h" + GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_hard_rlock) + +@@ -678,6 +687,8 @@ + #include "locking-selftest-wlock-softirq.h" + GENERATE_PERMUTATIONS_2_EVENTS(irqsafe2B_soft_wlock) + ++#endif ++ + #undef E1 + #undef E2 + +@@ -709,6 +720,8 @@ + #include "locking-selftest-spin-hardirq.h" + GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_hard_spin) + ++#ifndef CONFIG_PREEMPT_RT_FULL ++ + #include "locking-selftest-rlock-hardirq.h" + GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_hard_rlock) + +@@ -724,6 +737,8 @@ + #include "locking-selftest-wlock-softirq.h" + GENERATE_PERMUTATIONS_3_EVENTS(irqsafe3_soft_wlock) + ++#endif ++ + #undef E1 + #undef E2 + #undef E3 +@@ -757,6 +772,8 @@ + #include "locking-selftest-spin-hardirq.h" + GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_hard_spin) + ++#ifndef CONFIG_PREEMPT_RT_FULL ++ + #include "locking-selftest-rlock-hardirq.h" + GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_hard_rlock) + +@@ -772,10 +789,14 @@ + #include "locking-selftest-wlock-softirq.h" + GENERATE_PERMUTATIONS_3_EVENTS(irqsafe4_soft_wlock) + ++#endif ++ + #undef E1 + #undef E2 + #undef E3 + ++#ifndef CONFIG_PREEMPT_RT_FULL ++ + /* + * read-lock / write-lock irq inversion. + * +@@ -838,6 +859,10 @@ + #undef E2 + #undef E3 + ++#endif ++ ++#ifndef CONFIG_PREEMPT_RT_FULL ++ + /* + * read-lock / write-lock recursion that is actually safe. + */ +@@ -876,6 +901,8 @@ + #undef E2 + #undef E3 + ++#endif ++ + /* + * read-lock / write-lock recursion that is unsafe. + */ +@@ -1858,6 +1885,7 @@ + + printk(" --------------------------------------------------------------------------\n"); + ++#ifndef CONFIG_PREEMPT_RT_FULL + /* + * irq-context testcases: + */ +@@ -1870,6 +1898,28 @@ + + DO_TESTCASE_6x2("irq read-recursion", irq_read_recursion); + // DO_TESTCASE_6x2B("irq read-recursion #2", irq_read_recursion2); ++#else ++ /* On -rt, we only do hardirq context test for raw spinlock */ ++ DO_TESTCASE_1B("hard-irqs-on + irq-safe-A", irqsafe1_hard_spin, 12); ++ DO_TESTCASE_1B("hard-irqs-on + irq-safe-A", irqsafe1_hard_spin, 21); ++ ++ DO_TESTCASE_1B("hard-safe-A + irqs-on", irqsafe2B_hard_spin, 12); ++ DO_TESTCASE_1B("hard-safe-A + irqs-on", irqsafe2B_hard_spin, 21); ++ ++ DO_TESTCASE_1B("hard-safe-A + unsafe-B #1", irqsafe3_hard_spin, 123); ++ DO_TESTCASE_1B("hard-safe-A + unsafe-B #1", irqsafe3_hard_spin, 132); ++ DO_TESTCASE_1B("hard-safe-A + unsafe-B #1", irqsafe3_hard_spin, 213); ++ DO_TESTCASE_1B("hard-safe-A + unsafe-B #1", irqsafe3_hard_spin, 231); ++ DO_TESTCASE_1B("hard-safe-A + unsafe-B #1", irqsafe3_hard_spin, 312); ++ DO_TESTCASE_1B("hard-safe-A + unsafe-B #1", irqsafe3_hard_spin, 321); ++ ++ DO_TESTCASE_1B("hard-safe-A + unsafe-B #2", irqsafe4_hard_spin, 123); ++ DO_TESTCASE_1B("hard-safe-A + unsafe-B #2", irqsafe4_hard_spin, 132); ++ DO_TESTCASE_1B("hard-safe-A + unsafe-B #2", irqsafe4_hard_spin, 213); ++ DO_TESTCASE_1B("hard-safe-A + unsafe-B #2", irqsafe4_hard_spin, 231); ++ DO_TESTCASE_1B("hard-safe-A + unsafe-B #2", irqsafe4_hard_spin, 312); ++ DO_TESTCASE_1B("hard-safe-A + unsafe-B #2", irqsafe4_hard_spin, 321); ++#endif + + ww_tests(); + +diff -Nur linux-4.1.10.orig/lib/percpu_ida.c linux-4.1.10/lib/percpu_ida.c +--- linux-4.1.10.orig/lib/percpu_ida.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/lib/percpu_ida.c 2015-10-07 18:00:08.000000000 +0200 +@@ -26,6 +26,9 @@ + #include + #include + #include ++#include ++ ++static DEFINE_LOCAL_IRQ_LOCK(irq_off_lock); + + struct percpu_ida_cpu { + /* +@@ -148,13 +151,13 @@ + unsigned long flags; + int tag; + +- local_irq_save(flags); ++ local_lock_irqsave(irq_off_lock, flags); + tags = this_cpu_ptr(pool->tag_cpu); + + /* Fastpath */ + tag = alloc_local_tag(tags); + if (likely(tag >= 0)) { +- local_irq_restore(flags); ++ local_unlock_irqrestore(irq_off_lock, flags); + return tag; + } + +@@ -173,6 +176,7 @@ + + if (!tags->nr_free) + alloc_global_tags(pool, tags); ++ + if (!tags->nr_free) + steal_tags(pool, tags); + +@@ -184,7 +188,7 @@ + } + + spin_unlock(&pool->lock); +- local_irq_restore(flags); ++ local_unlock_irqrestore(irq_off_lock, flags); + + if (tag >= 0 || state == TASK_RUNNING) + break; +@@ -196,7 +200,7 @@ + + schedule(); + +- local_irq_save(flags); ++ local_lock_irqsave(irq_off_lock, flags); + tags = this_cpu_ptr(pool->tag_cpu); + } + if (state != TASK_RUNNING) +@@ -221,7 +225,7 @@ + + BUG_ON(tag >= pool->nr_tags); + +- local_irq_save(flags); ++ local_lock_irqsave(irq_off_lock, flags); + tags = this_cpu_ptr(pool->tag_cpu); + + spin_lock(&tags->lock); +@@ -253,7 +257,7 @@ + spin_unlock(&pool->lock); + } + +- local_irq_restore(flags); ++ local_unlock_irqrestore(irq_off_lock, flags); + } + EXPORT_SYMBOL_GPL(percpu_ida_free); + +@@ -345,7 +349,7 @@ + struct percpu_ida_cpu *remote; + unsigned cpu, i, err = 0; + +- local_irq_save(flags); ++ local_lock_irqsave(irq_off_lock, flags); + for_each_possible_cpu(cpu) { + remote = per_cpu_ptr(pool->tag_cpu, cpu); + spin_lock(&remote->lock); +@@ -367,7 +371,7 @@ + } + spin_unlock(&pool->lock); + out: +- local_irq_restore(flags); ++ local_unlock_irqrestore(irq_off_lock, flags); + return err; + } + EXPORT_SYMBOL_GPL(percpu_ida_for_each_free); +diff -Nur linux-4.1.10.orig/lib/radix-tree.c linux-4.1.10/lib/radix-tree.c +--- linux-4.1.10.orig/lib/radix-tree.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/lib/radix-tree.c 2015-10-07 18:00:08.000000000 +0200 +@@ -195,12 +195,13 @@ + * succeed in getting a node here (and never reach + * kmem_cache_alloc) + */ +- rtp = this_cpu_ptr(&radix_tree_preloads); ++ rtp = &get_cpu_var(radix_tree_preloads); + if (rtp->nr) { + ret = rtp->nodes[rtp->nr - 1]; + rtp->nodes[rtp->nr - 1] = NULL; + rtp->nr--; + } ++ put_cpu_var(radix_tree_preloads); + /* + * Update the allocation stack trace as this is more useful + * for debugging. +@@ -240,6 +241,7 @@ + call_rcu(&node->rcu_head, radix_tree_node_rcu_free); + } + ++#ifndef CONFIG_PREEMPT_RT_FULL + /* + * Load up this CPU's radix_tree_node buffer with sufficient objects to + * ensure that the addition of a single element in the tree cannot fail. On +@@ -305,6 +307,7 @@ + return 0; + } + EXPORT_SYMBOL(radix_tree_maybe_preload); ++#endif + + /* + * Return the maximum key which can be store into a +diff -Nur linux-4.1.10.orig/lib/scatterlist.c linux-4.1.10/lib/scatterlist.c +--- linux-4.1.10.orig/lib/scatterlist.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/lib/scatterlist.c 2015-10-07 18:00:08.000000000 +0200 +@@ -592,7 +592,7 @@ + flush_kernel_dcache_page(miter->page); + + if (miter->__flags & SG_MITER_ATOMIC) { +- WARN_ON_ONCE(preemptible()); ++ WARN_ON_ONCE(!pagefault_disabled()); + kunmap_atomic(miter->addr); + } else + kunmap(miter->page); +@@ -637,7 +637,7 @@ + if (!sg_miter_skip(&miter, skip)) + return false; + +- local_irq_save(flags); ++ local_irq_save_nort(flags); + + while (sg_miter_next(&miter) && offset < buflen) { + unsigned int len; +@@ -654,7 +654,7 @@ + + sg_miter_stop(&miter); + +- local_irq_restore(flags); ++ local_irq_restore_nort(flags); + return offset; + } + +diff -Nur linux-4.1.10.orig/lib/smp_processor_id.c linux-4.1.10/lib/smp_processor_id.c +--- linux-4.1.10.orig/lib/smp_processor_id.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/lib/smp_processor_id.c 2015-10-07 18:00:08.000000000 +0200 +@@ -39,8 +39,9 @@ + if (!printk_ratelimit()) + goto out_enable; + +- printk(KERN_ERR "BUG: using %s%s() in preemptible [%08x] code: %s/%d\n", +- what1, what2, preempt_count() - 1, current->comm, current->pid); ++ printk(KERN_ERR "BUG: using %s%s() in preemptible [%08x %08x] code: %s/%d\n", ++ what1, what2, preempt_count() - 1, __migrate_disabled(current), ++ current->comm, current->pid); + + print_symbol("caller is %s\n", (long)__builtin_return_address(0)); + dump_stack(); +diff -Nur linux-4.1.10.orig/lib/strnlen_user.c linux-4.1.10/lib/strnlen_user.c +--- linux-4.1.10.orig/lib/strnlen_user.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/lib/strnlen_user.c 2015-10-07 18:00:08.000000000 +0200 +@@ -85,7 +85,8 @@ + * @str: The string to measure. + * @count: Maximum count (including NUL character) + * +- * Context: User context only. This function may sleep. ++ * Context: User context only. This function may sleep if pagefaults are ++ * enabled. + * + * Get the size of a NUL-terminated string in user space. + * +@@ -121,7 +122,8 @@ + * strlen_user: - Get the size of a user string INCLUDING final NUL. + * @str: The string to measure. + * +- * Context: User context only. This function may sleep. ++ * Context: User context only. This function may sleep if pagefaults are ++ * enabled. + * + * Get the size of a NUL-terminated string in user space. + * +diff -Nur linux-4.1.10.orig/localversion-rt linux-4.1.10/localversion-rt +--- linux-4.1.10.orig/localversion-rt 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/localversion-rt 2015-10-07 18:00:08.000000000 +0200 +@@ -0,0 +1 @@ ++-rt8 +diff -Nur linux-4.1.10.orig/mm/compaction.c linux-4.1.10/mm/compaction.c +--- linux-4.1.10.orig/mm/compaction.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/mm/compaction.c 2015-10-07 18:00:08.000000000 +0200 +@@ -1406,10 +1406,12 @@ + cc->migrate_pfn & ~((1UL << cc->order) - 1); + + if (last_migrated_pfn < current_block_start) { +- cpu = get_cpu(); ++ cpu = get_cpu_light(); ++ local_lock_irq(swapvec_lock); + lru_add_drain_cpu(cpu); ++ local_unlock_irq(swapvec_lock); + drain_local_pages(zone); +- put_cpu(); ++ put_cpu_light(); + /* No more flushing until we migrate again */ + last_migrated_pfn = 0; + } +diff -Nur linux-4.1.10.orig/mm/filemap.c linux-4.1.10/mm/filemap.c +--- linux-4.1.10.orig/mm/filemap.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/mm/filemap.c 2015-10-07 18:00:08.000000000 +0200 +@@ -167,7 +167,9 @@ + if (!workingset_node_pages(node) && + list_empty(&node->private_list)) { + node->private_data = mapping; +- list_lru_add(&workingset_shadow_nodes, &node->private_list); ++ local_lock(workingset_shadow_lock); ++ list_lru_add(&__workingset_shadow_nodes, &node->private_list); ++ local_unlock(workingset_shadow_lock); + } + } + +@@ -533,9 +535,12 @@ + * node->private_list is protected by + * mapping->tree_lock. + */ +- if (!list_empty(&node->private_list)) +- list_lru_del(&workingset_shadow_nodes, ++ if (!list_empty(&node->private_list)) { ++ local_lock(workingset_shadow_lock); ++ list_lru_del(&__workingset_shadow_nodes, + &node->private_list); ++ local_unlock(workingset_shadow_lock); ++ } + } + return 0; + } +diff -Nur linux-4.1.10.orig/mm/highmem.c linux-4.1.10/mm/highmem.c +--- linux-4.1.10.orig/mm/highmem.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/mm/highmem.c 2015-10-07 18:00:08.000000000 +0200 +@@ -29,10 +29,11 @@ + #include + #include + +- ++#ifndef CONFIG_PREEMPT_RT_FULL + #if defined(CONFIG_HIGHMEM) || defined(CONFIG_X86_32) + DEFINE_PER_CPU(int, __kmap_atomic_idx); + #endif ++#endif + + /* + * Virtual_count is not a pure "count". +@@ -107,8 +108,9 @@ + unsigned long totalhigh_pages __read_mostly; + EXPORT_SYMBOL(totalhigh_pages); + +- ++#ifndef CONFIG_PREEMPT_RT_FULL + EXPORT_PER_CPU_SYMBOL(__kmap_atomic_idx); ++#endif + + unsigned int nr_free_highpages (void) + { +diff -Nur linux-4.1.10.orig/mm/Kconfig linux-4.1.10/mm/Kconfig +--- linux-4.1.10.orig/mm/Kconfig 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/mm/Kconfig 2015-10-07 18:00:08.000000000 +0200 +@@ -409,7 +409,7 @@ + + config TRANSPARENT_HUGEPAGE + bool "Transparent Hugepage Support" +- depends on HAVE_ARCH_TRANSPARENT_HUGEPAGE ++ depends on HAVE_ARCH_TRANSPARENT_HUGEPAGE && !PREEMPT_RT_FULL + select COMPACTION + help + Transparent Hugepages allows the kernel to use huge pages and +diff -Nur linux-4.1.10.orig/mm/memcontrol.c linux-4.1.10/mm/memcontrol.c +--- linux-4.1.10.orig/mm/memcontrol.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/mm/memcontrol.c 2015-10-07 18:00:08.000000000 +0200 +@@ -66,6 +66,8 @@ + #include + #include + #include ++#include ++ + #include "slab.h" + + #include +@@ -85,6 +87,7 @@ + #define do_swap_account 0 + #endif + ++static DEFINE_LOCAL_IRQ_LOCK(event_lock); + static const char * const mem_cgroup_stat_names[] = { + "cache", + "rss", +@@ -2124,14 +2127,17 @@ + */ + static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages) + { +- struct memcg_stock_pcp *stock = &get_cpu_var(memcg_stock); ++ struct memcg_stock_pcp *stock; ++ int cpu = get_cpu_light(); ++ ++ stock = &per_cpu(memcg_stock, cpu); + + if (stock->cached != memcg) { /* reset if necessary */ + drain_stock(stock); + stock->cached = memcg; + } + stock->nr_pages += nr_pages; +- put_cpu_var(memcg_stock); ++ put_cpu_light(); + } + + /* +@@ -2147,7 +2153,7 @@ + return; + /* Notify other cpus that system-wide "drain" is running */ + get_online_cpus(); +- curcpu = get_cpu(); ++ curcpu = get_cpu_light(); + for_each_online_cpu(cpu) { + struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu); + struct mem_cgroup *memcg; +@@ -2164,7 +2170,7 @@ + schedule_work_on(cpu, &stock->work); + } + } +- put_cpu(); ++ put_cpu_light(); + put_online_cpus(); + mutex_unlock(&percpu_charge_mutex); + } +@@ -4801,12 +4807,12 @@ + + ret = 0; + +- local_irq_disable(); ++ local_lock_irq(event_lock); + mem_cgroup_charge_statistics(to, page, nr_pages); + memcg_check_events(to, page); + mem_cgroup_charge_statistics(from, page, -nr_pages); + memcg_check_events(from, page); +- local_irq_enable(); ++ local_unlock_irq(event_lock); + out_unlock: + unlock_page(page); + out: +@@ -5543,10 +5549,10 @@ + VM_BUG_ON_PAGE(!PageTransHuge(page), page); + } + +- local_irq_disable(); ++ local_lock_irq(event_lock); + mem_cgroup_charge_statistics(memcg, page, nr_pages); + memcg_check_events(memcg, page); +- local_irq_enable(); ++ local_unlock_irq(event_lock); + + if (do_swap_account && PageSwapCache(page)) { + swp_entry_t entry = { .val = page_private(page) }; +@@ -5602,14 +5608,14 @@ + memcg_oom_recover(memcg); + } + +- local_irq_save(flags); ++ local_lock_irqsave(event_lock, flags); + __this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_RSS], nr_anon); + __this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_CACHE], nr_file); + __this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_RSS_HUGE], nr_huge); + __this_cpu_add(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGOUT], pgpgout); + __this_cpu_add(memcg->stat->nr_page_events, nr_pages); + memcg_check_events(memcg, dummy_page); +- local_irq_restore(flags); ++ local_unlock_irqrestore(event_lock, flags); + + if (!mem_cgroup_is_root(memcg)) + css_put_many(&memcg->css, nr_pages); +@@ -5813,6 +5819,7 @@ + { + struct mem_cgroup *memcg; + unsigned short oldid; ++ unsigned long flags; + + VM_BUG_ON_PAGE(PageLRU(page), page); + VM_BUG_ON_PAGE(page_count(page), page); +@@ -5835,9 +5842,11 @@ + if (!mem_cgroup_is_root(memcg)) + page_counter_uncharge(&memcg->memory, 1); + ++ local_lock_irqsave(event_lock, flags); + /* Caller disabled preemption with mapping->tree_lock */ + mem_cgroup_charge_statistics(memcg, page, -1); + memcg_check_events(memcg, page); ++ local_unlock_irqrestore(event_lock, flags); + } + + /** +diff -Nur linux-4.1.10.orig/mm/memory.c linux-4.1.10/mm/memory.c +--- linux-4.1.10.orig/mm/memory.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/mm/memory.c 2015-10-07 18:00:08.000000000 +0200 +@@ -3743,7 +3743,7 @@ + } + + #if defined(CONFIG_PROVE_LOCKING) || defined(CONFIG_DEBUG_ATOMIC_SLEEP) +-void might_fault(void) ++void __might_fault(const char *file, int line) + { + /* + * Some code (nfs/sunrpc) uses socket ops on kernel memory while +@@ -3753,21 +3753,15 @@ + */ + if (segment_eq(get_fs(), KERNEL_DS)) + return; +- +- /* +- * it would be nicer only to annotate paths which are not under +- * pagefault_disable, however that requires a larger audit and +- * providing helpers like get_user_atomic. +- */ +- if (in_atomic()) ++ if (pagefault_disabled()) + return; +- +- __might_sleep(__FILE__, __LINE__, 0); +- ++ __might_sleep(file, line, 0); ++#if defined(CONFIG_DEBUG_ATOMIC_SLEEP) + if (current->mm) + might_lock_read(¤t->mm->mmap_sem); ++#endif + } +-EXPORT_SYMBOL(might_fault); ++EXPORT_SYMBOL(__might_fault); + #endif + + #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS) +diff -Nur linux-4.1.10.orig/mm/mmu_context.c linux-4.1.10/mm/mmu_context.c +--- linux-4.1.10.orig/mm/mmu_context.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/mm/mmu_context.c 2015-10-07 18:00:08.000000000 +0200 +@@ -23,6 +23,7 @@ + struct task_struct *tsk = current; + + task_lock(tsk); ++ preempt_disable_rt(); + active_mm = tsk->active_mm; + if (active_mm != mm) { + atomic_inc(&mm->mm_count); +@@ -30,6 +31,7 @@ + } + tsk->mm = mm; + switch_mm(active_mm, mm, tsk); ++ preempt_enable_rt(); + task_unlock(tsk); + #ifdef finish_arch_post_lock_switch + finish_arch_post_lock_switch(); +diff -Nur linux-4.1.10.orig/mm/page_alloc.c linux-4.1.10/mm/page_alloc.c +--- linux-4.1.10.orig/mm/page_alloc.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/mm/page_alloc.c 2015-10-07 18:00:08.000000000 +0200 +@@ -60,6 +60,7 @@ + #include + #include + #include ++#include + #include + + #include +@@ -233,6 +234,18 @@ + EXPORT_SYMBOL(nr_online_nodes); + #endif + ++static DEFINE_LOCAL_IRQ_LOCK(pa_lock); ++ ++#ifdef CONFIG_PREEMPT_RT_BASE ++# define cpu_lock_irqsave(cpu, flags) \ ++ local_lock_irqsave_on(pa_lock, flags, cpu) ++# define cpu_unlock_irqrestore(cpu, flags) \ ++ local_unlock_irqrestore_on(pa_lock, flags, cpu) ++#else ++# define cpu_lock_irqsave(cpu, flags) local_irq_save(flags) ++# define cpu_unlock_irqrestore(cpu, flags) local_irq_restore(flags) ++#endif ++ + int page_group_by_mobility_disabled __read_mostly; + + void set_pageblock_migratetype(struct page *page, int migratetype) +@@ -681,7 +694,7 @@ + } + + /* +- * Frees a number of pages from the PCP lists ++ * Frees a number of pages which have been collected from the pcp lists. + * Assumes all pages on list are in same zone, and of same order. + * count is the number of pages to free. + * +@@ -692,18 +705,51 @@ + * pinned" detection logic. + */ + static void free_pcppages_bulk(struct zone *zone, int count, +- struct per_cpu_pages *pcp) ++ struct list_head *list) + { +- int migratetype = 0; +- int batch_free = 0; + int to_free = count; + unsigned long nr_scanned; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&zone->lock, flags); + +- spin_lock(&zone->lock); + nr_scanned = zone_page_state(zone, NR_PAGES_SCANNED); + if (nr_scanned) + __mod_zone_page_state(zone, NR_PAGES_SCANNED, -nr_scanned); + ++ while (!list_empty(list)) { ++ struct page *page = list_first_entry(list, struct page, lru); ++ int mt; /* migratetype of the to-be-freed page */ ++ ++ /* must delete as __free_one_page list manipulates */ ++ list_del(&page->lru); ++ ++ mt = get_freepage_migratetype(page); ++ if (unlikely(has_isolate_pageblock(zone))) ++ mt = get_pageblock_migratetype(page); ++ ++ /* MIGRATE_MOVABLE list may include MIGRATE_RESERVEs */ ++ __free_one_page(page, page_to_pfn(page), zone, 0, mt); ++ trace_mm_page_pcpu_drain(page, 0, mt); ++ to_free--; ++ } ++ WARN_ON(to_free != 0); ++ spin_unlock_irqrestore(&zone->lock, flags); ++} ++ ++/* ++ * Moves a number of pages from the PCP lists to free list which ++ * is freed outside of the locked region. ++ * ++ * Assumes all pages on list are in same zone, and of same order. ++ * count is the number of pages to free. ++ */ ++static void isolate_pcp_pages(int to_free, struct per_cpu_pages *src, ++ struct list_head *dst) ++{ ++ int migratetype = 0; ++ int batch_free = 0; ++ + while (to_free) { + struct page *page; + struct list_head *list; +@@ -719,7 +765,7 @@ + batch_free++; + if (++migratetype == MIGRATE_PCPTYPES) + migratetype = 0; +- list = &pcp->lists[migratetype]; ++ list = &src->lists[migratetype]; + } while (list_empty(list)); + + /* This is the only non-empty list. Free them all. */ +@@ -727,21 +773,11 @@ + batch_free = to_free; + + do { +- int mt; /* migratetype of the to-be-freed page */ +- +- page = list_entry(list->prev, struct page, lru); +- /* must delete as __free_one_page list manipulates */ ++ page = list_last_entry(list, struct page, lru); + list_del(&page->lru); +- mt = get_freepage_migratetype(page); +- if (unlikely(has_isolate_pageblock(zone))) +- mt = get_pageblock_migratetype(page); +- +- /* MIGRATE_MOVABLE list may include MIGRATE_RESERVEs */ +- __free_one_page(page, page_to_pfn(page), zone, 0, mt); +- trace_mm_page_pcpu_drain(page, 0, mt); ++ list_add(&page->lru, dst); + } while (--to_free && --batch_free && !list_empty(list)); + } +- spin_unlock(&zone->lock); + } + + static void free_one_page(struct zone *zone, +@@ -750,7 +786,9 @@ + int migratetype) + { + unsigned long nr_scanned; +- spin_lock(&zone->lock); ++ unsigned long flags; ++ ++ spin_lock_irqsave(&zone->lock, flags); + nr_scanned = zone_page_state(zone, NR_PAGES_SCANNED); + if (nr_scanned) + __mod_zone_page_state(zone, NR_PAGES_SCANNED, -nr_scanned); +@@ -760,7 +798,7 @@ + migratetype = get_pfnblock_migratetype(page, pfn); + } + __free_one_page(page, pfn, zone, order, migratetype); +- spin_unlock(&zone->lock); ++ spin_unlock_irqrestore(&zone->lock, flags); + } + + static int free_tail_pages_check(struct page *head_page, struct page *page) +@@ -825,11 +863,11 @@ + return; + + migratetype = get_pfnblock_migratetype(page, pfn); +- local_irq_save(flags); ++ local_lock_irqsave(pa_lock, flags); + __count_vm_events(PGFREE, 1 << order); + set_freepage_migratetype(page, migratetype); + free_one_page(page_zone(page), page, pfn, order, migratetype); +- local_irq_restore(flags); ++ local_unlock_irqrestore(pa_lock, flags); + } + + void __init __free_pages_bootmem(struct page *page, unsigned int order) +@@ -1371,16 +1409,18 @@ + void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp) + { + unsigned long flags; ++ LIST_HEAD(dst); + int to_drain, batch; + +- local_irq_save(flags); ++ local_lock_irqsave(pa_lock, flags); + batch = READ_ONCE(pcp->batch); + to_drain = min(pcp->count, batch); + if (to_drain > 0) { +- free_pcppages_bulk(zone, to_drain, pcp); ++ isolate_pcp_pages(to_drain, pcp, &dst); + pcp->count -= to_drain; + } +- local_irq_restore(flags); ++ local_unlock_irqrestore(pa_lock, flags); ++ free_pcppages_bulk(zone, to_drain, &dst); + } + #endif + +@@ -1396,16 +1436,21 @@ + unsigned long flags; + struct per_cpu_pageset *pset; + struct per_cpu_pages *pcp; ++ LIST_HEAD(dst); ++ int count; + +- local_irq_save(flags); ++ cpu_lock_irqsave(cpu, flags); + pset = per_cpu_ptr(zone->pageset, cpu); + + pcp = &pset->pcp; +- if (pcp->count) { +- free_pcppages_bulk(zone, pcp->count, pcp); ++ count = pcp->count; ++ if (count) { ++ isolate_pcp_pages(count, pcp, &dst); + pcp->count = 0; + } +- local_irq_restore(flags); ++ cpu_unlock_irqrestore(cpu, flags); ++ if (count) ++ free_pcppages_bulk(zone, count, &dst); + } + + /* +@@ -1491,8 +1536,17 @@ + else + cpumask_clear_cpu(cpu, &cpus_with_pcps); + } ++#ifndef CONFIG_PREEMPT_RT_BASE + on_each_cpu_mask(&cpus_with_pcps, (smp_call_func_t) drain_local_pages, + zone, 1); ++#else ++ for_each_cpu(cpu, &cpus_with_pcps) { ++ if (zone) ++ drain_pages_zone(cpu, zone); ++ else ++ drain_pages(cpu); ++ } ++#endif + } + + #ifdef CONFIG_HIBERNATION +@@ -1548,7 +1602,7 @@ + + migratetype = get_pfnblock_migratetype(page, pfn); + set_freepage_migratetype(page, migratetype); +- local_irq_save(flags); ++ local_lock_irqsave(pa_lock, flags); + __count_vm_event(PGFREE); + + /* +@@ -1574,12 +1628,17 @@ + pcp->count++; + if (pcp->count >= pcp->high) { + unsigned long batch = READ_ONCE(pcp->batch); +- free_pcppages_bulk(zone, batch, pcp); ++ LIST_HEAD(dst); ++ ++ isolate_pcp_pages(batch, pcp, &dst); + pcp->count -= batch; ++ local_unlock_irqrestore(pa_lock, flags); ++ free_pcppages_bulk(zone, batch, &dst); ++ return; + } + + out: +- local_irq_restore(flags); ++ local_unlock_irqrestore(pa_lock, flags); + } + + /* +@@ -1710,7 +1769,7 @@ + struct per_cpu_pages *pcp; + struct list_head *list; + +- local_irq_save(flags); ++ local_lock_irqsave(pa_lock, flags); + pcp = &this_cpu_ptr(zone->pageset)->pcp; + list = &pcp->lists[migratetype]; + if (list_empty(list)) { +@@ -1742,13 +1801,15 @@ + */ + WARN_ON_ONCE(order > 1); + } +- spin_lock_irqsave(&zone->lock, flags); ++ local_spin_lock_irqsave(pa_lock, &zone->lock, flags); + page = __rmqueue(zone, order, migratetype); +- spin_unlock(&zone->lock); +- if (!page) ++ if (!page) { ++ spin_unlock(&zone->lock); + goto failed; ++ } + __mod_zone_freepage_state(zone, -(1 << order), + get_freepage_migratetype(page)); ++ spin_unlock(&zone->lock); + } + + __mod_zone_page_state(zone, NR_ALLOC_BATCH, -(1 << order)); +@@ -1758,13 +1819,13 @@ + + __count_zone_vm_events(PGALLOC, zone, 1 << order); + zone_statistics(preferred_zone, zone, gfp_flags); +- local_irq_restore(flags); ++ local_unlock_irqrestore(pa_lock, flags); + + VM_BUG_ON_PAGE(bad_range(zone, page), page); + return page; + + failed: +- local_irq_restore(flags); ++ local_unlock_irqrestore(pa_lock, flags); + return NULL; + } + +@@ -5653,6 +5714,7 @@ + void __init page_alloc_init(void) + { + hotcpu_notifier(page_alloc_cpu_notify, 0); ++ local_irq_lock_init(pa_lock); + } + + /* +@@ -6547,7 +6609,7 @@ + struct per_cpu_pageset *pset; + + /* avoid races with drain_pages() */ +- local_irq_save(flags); ++ local_lock_irqsave(pa_lock, flags); + if (zone->pageset != &boot_pageset) { + for_each_online_cpu(cpu) { + pset = per_cpu_ptr(zone->pageset, cpu); +@@ -6556,7 +6618,7 @@ + free_percpu(zone->pageset); + zone->pageset = &boot_pageset; + } +- local_irq_restore(flags); ++ local_unlock_irqrestore(pa_lock, flags); + } + + #ifdef CONFIG_MEMORY_HOTREMOVE +diff -Nur linux-4.1.10.orig/mm/page_alloc.c.orig linux-4.1.10/mm/page_alloc.c.orig +--- linux-4.1.10.orig/mm/page_alloc.c.orig 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/mm/page_alloc.c.orig 2015-10-03 13:49:38.000000000 +0200 +@@ -0,0 +1,6636 @@ ++/* ++ * linux/mm/page_alloc.c ++ * ++ * Manages the free list, the system allocates free pages here. ++ * Note that kmalloc() lives in slab.c ++ * ++ * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds ++ * Swap reorganised 29.12.95, Stephen Tweedie ++ * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999 ++ * Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999 ++ * Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999 ++ * Zone balancing, Kanoj Sarcar, SGI, Jan 2000 ++ * Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002 ++ * (lots of bits borrowed from Ingo Molnar & Andrew Morton) ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++#include ++#include ++#include "internal.h" ++ ++/* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */ ++static DEFINE_MUTEX(pcp_batch_high_lock); ++#define MIN_PERCPU_PAGELIST_FRACTION (8) ++ ++#ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID ++DEFINE_PER_CPU(int, numa_node); ++EXPORT_PER_CPU_SYMBOL(numa_node); ++#endif ++ ++#ifdef CONFIG_HAVE_MEMORYLESS_NODES ++/* ++ * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly. ++ * It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined. ++ * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem() ++ * defined in . ++ */ ++DEFINE_PER_CPU(int, _numa_mem_); /* Kernel "local memory" node */ ++EXPORT_PER_CPU_SYMBOL(_numa_mem_); ++int _node_numa_mem_[MAX_NUMNODES]; ++#endif ++ ++/* ++ * Array of node states. ++ */ ++nodemask_t node_states[NR_NODE_STATES] __read_mostly = { ++ [N_POSSIBLE] = NODE_MASK_ALL, ++ [N_ONLINE] = { { [0] = 1UL } }, ++#ifndef CONFIG_NUMA ++ [N_NORMAL_MEMORY] = { { [0] = 1UL } }, ++#ifdef CONFIG_HIGHMEM ++ [N_HIGH_MEMORY] = { { [0] = 1UL } }, ++#endif ++#ifdef CONFIG_MOVABLE_NODE ++ [N_MEMORY] = { { [0] = 1UL } }, ++#endif ++ [N_CPU] = { { [0] = 1UL } }, ++#endif /* NUMA */ ++}; ++EXPORT_SYMBOL(node_states); ++ ++/* Protect totalram_pages and zone->managed_pages */ ++static DEFINE_SPINLOCK(managed_page_count_lock); ++ ++unsigned long totalram_pages __read_mostly; ++unsigned long totalreserve_pages __read_mostly; ++unsigned long totalcma_pages __read_mostly; ++/* ++ * When calculating the number of globally allowed dirty pages, there ++ * is a certain number of per-zone reserves that should not be ++ * considered dirtyable memory. This is the sum of those reserves ++ * over all existing zones that contribute dirtyable memory. ++ */ ++unsigned long dirty_balance_reserve __read_mostly; ++ ++int percpu_pagelist_fraction; ++gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK; ++ ++#ifdef CONFIG_PM_SLEEP ++/* ++ * The following functions are used by the suspend/hibernate code to temporarily ++ * change gfp_allowed_mask in order to avoid using I/O during memory allocations ++ * while devices are suspended. To avoid races with the suspend/hibernate code, ++ * they should always be called with pm_mutex held (gfp_allowed_mask also should ++ * only be modified with pm_mutex held, unless the suspend/hibernate code is ++ * guaranteed not to run in parallel with that modification). ++ */ ++ ++static gfp_t saved_gfp_mask; ++ ++void pm_restore_gfp_mask(void) ++{ ++ WARN_ON(!mutex_is_locked(&pm_mutex)); ++ if (saved_gfp_mask) { ++ gfp_allowed_mask = saved_gfp_mask; ++ saved_gfp_mask = 0; ++ } ++} ++ ++void pm_restrict_gfp_mask(void) ++{ ++ WARN_ON(!mutex_is_locked(&pm_mutex)); ++ WARN_ON(saved_gfp_mask); ++ saved_gfp_mask = gfp_allowed_mask; ++ gfp_allowed_mask &= ~GFP_IOFS; ++} ++ ++bool pm_suspended_storage(void) ++{ ++ if ((gfp_allowed_mask & GFP_IOFS) == GFP_IOFS) ++ return false; ++ return true; ++} ++#endif /* CONFIG_PM_SLEEP */ ++ ++#ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE ++int pageblock_order __read_mostly; ++#endif ++ ++static void __free_pages_ok(struct page *page, unsigned int order); ++ ++/* ++ * results with 256, 32 in the lowmem_reserve sysctl: ++ * 1G machine -> (16M dma, 800M-16M normal, 1G-800M high) ++ * 1G machine -> (16M dma, 784M normal, 224M high) ++ * NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA ++ * HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL ++ * HIGHMEM allocation will leave (224M+784M)/256 of ram reserved in ZONE_DMA ++ * ++ * TBD: should special case ZONE_DMA32 machines here - in those we normally ++ * don't need any ZONE_NORMAL reservation ++ */ ++int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES-1] = { ++#ifdef CONFIG_ZONE_DMA ++ 256, ++#endif ++#ifdef CONFIG_ZONE_DMA32 ++ 256, ++#endif ++#ifdef CONFIG_HIGHMEM ++ 32, ++#endif ++ 32, ++}; ++ ++EXPORT_SYMBOL(totalram_pages); ++ ++static char * const zone_names[MAX_NR_ZONES] = { ++#ifdef CONFIG_ZONE_DMA ++ "DMA", ++#endif ++#ifdef CONFIG_ZONE_DMA32 ++ "DMA32", ++#endif ++ "Normal", ++#ifdef CONFIG_HIGHMEM ++ "HighMem", ++#endif ++ "Movable", ++}; ++ ++int min_free_kbytes = 1024; ++int user_min_free_kbytes = -1; ++ ++static unsigned long __meminitdata nr_kernel_pages; ++static unsigned long __meminitdata nr_all_pages; ++static unsigned long __meminitdata dma_reserve; ++ ++#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP ++static unsigned long __meminitdata arch_zone_lowest_possible_pfn[MAX_NR_ZONES]; ++static unsigned long __meminitdata arch_zone_highest_possible_pfn[MAX_NR_ZONES]; ++static unsigned long __initdata required_kernelcore; ++static unsigned long __initdata required_movablecore; ++static unsigned long __meminitdata zone_movable_pfn[MAX_NUMNODES]; ++ ++/* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */ ++int movable_zone; ++EXPORT_SYMBOL(movable_zone); ++#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ ++ ++#if MAX_NUMNODES > 1 ++int nr_node_ids __read_mostly = MAX_NUMNODES; ++int nr_online_nodes __read_mostly = 1; ++EXPORT_SYMBOL(nr_node_ids); ++EXPORT_SYMBOL(nr_online_nodes); ++#endif ++ ++int page_group_by_mobility_disabled __read_mostly; ++ ++void set_pageblock_migratetype(struct page *page, int migratetype) ++{ ++ if (unlikely(page_group_by_mobility_disabled && ++ migratetype < MIGRATE_PCPTYPES)) ++ migratetype = MIGRATE_UNMOVABLE; ++ ++ set_pageblock_flags_group(page, (unsigned long)migratetype, ++ PB_migrate, PB_migrate_end); ++} ++ ++#ifdef CONFIG_DEBUG_VM ++static int page_outside_zone_boundaries(struct zone *zone, struct page *page) ++{ ++ int ret = 0; ++ unsigned seq; ++ unsigned long pfn = page_to_pfn(page); ++ unsigned long sp, start_pfn; ++ ++ do { ++ seq = zone_span_seqbegin(zone); ++ start_pfn = zone->zone_start_pfn; ++ sp = zone->spanned_pages; ++ if (!zone_spans_pfn(zone, pfn)) ++ ret = 1; ++ } while (zone_span_seqretry(zone, seq)); ++ ++ if (ret) ++ pr_err("page 0x%lx outside node %d zone %s [ 0x%lx - 0x%lx ]\n", ++ pfn, zone_to_nid(zone), zone->name, ++ start_pfn, start_pfn + sp); ++ ++ return ret; ++} ++ ++static int page_is_consistent(struct zone *zone, struct page *page) ++{ ++ if (!pfn_valid_within(page_to_pfn(page))) ++ return 0; ++ if (zone != page_zone(page)) ++ return 0; ++ ++ return 1; ++} ++/* ++ * Temporary debugging check for pages not lying within a given zone. ++ */ ++static int bad_range(struct zone *zone, struct page *page) ++{ ++ if (page_outside_zone_boundaries(zone, page)) ++ return 1; ++ if (!page_is_consistent(zone, page)) ++ return 1; ++ ++ return 0; ++} ++#else ++static inline int bad_range(struct zone *zone, struct page *page) ++{ ++ return 0; ++} ++#endif ++ ++static void bad_page(struct page *page, const char *reason, ++ unsigned long bad_flags) ++{ ++ static unsigned long resume; ++ static unsigned long nr_shown; ++ static unsigned long nr_unshown; ++ ++ /* Don't complain about poisoned pages */ ++ if (PageHWPoison(page)) { ++ page_mapcount_reset(page); /* remove PageBuddy */ ++ return; ++ } ++ ++ /* ++ * Allow a burst of 60 reports, then keep quiet for that minute; ++ * or allow a steady drip of one report per second. ++ */ ++ if (nr_shown == 60) { ++ if (time_before(jiffies, resume)) { ++ nr_unshown++; ++ goto out; ++ } ++ if (nr_unshown) { ++ printk(KERN_ALERT ++ "BUG: Bad page state: %lu messages suppressed\n", ++ nr_unshown); ++ nr_unshown = 0; ++ } ++ nr_shown = 0; ++ } ++ if (nr_shown++ == 0) ++ resume = jiffies + 60 * HZ; ++ ++ printk(KERN_ALERT "BUG: Bad page state in process %s pfn:%05lx\n", ++ current->comm, page_to_pfn(page)); ++ dump_page_badflags(page, reason, bad_flags); ++ ++ print_modules(); ++ dump_stack(); ++out: ++ /* Leave bad fields for debug, except PageBuddy could make trouble */ ++ page_mapcount_reset(page); /* remove PageBuddy */ ++ add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); ++} ++ ++/* ++ * Higher-order pages are called "compound pages". They are structured thusly: ++ * ++ * The first PAGE_SIZE page is called the "head page". ++ * ++ * The remaining PAGE_SIZE pages are called "tail pages". ++ * ++ * All pages have PG_compound set. All tail pages have their ->first_page ++ * pointing at the head page. ++ * ++ * The first tail page's ->lru.next holds the address of the compound page's ++ * put_page() function. Its ->lru.prev holds the order of allocation. ++ * This usage means that zero-order pages may not be compound. ++ */ ++ ++static void free_compound_page(struct page *page) ++{ ++ __free_pages_ok(page, compound_order(page)); ++} ++ ++void prep_compound_page(struct page *page, unsigned long order) ++{ ++ int i; ++ int nr_pages = 1 << order; ++ ++ set_compound_page_dtor(page, free_compound_page); ++ set_compound_order(page, order); ++ __SetPageHead(page); ++ for (i = 1; i < nr_pages; i++) { ++ struct page *p = page + i; ++ set_page_count(p, 0); ++ p->first_page = page; ++ /* Make sure p->first_page is always valid for PageTail() */ ++ smp_wmb(); ++ __SetPageTail(p); ++ } ++} ++ ++static inline void prep_zero_page(struct page *page, unsigned int order, ++ gfp_t gfp_flags) ++{ ++ int i; ++ ++ /* ++ * clear_highpage() will use KM_USER0, so it's a bug to use __GFP_ZERO ++ * and __GFP_HIGHMEM from hard or soft interrupt context. ++ */ ++ VM_BUG_ON((gfp_flags & __GFP_HIGHMEM) && in_interrupt()); ++ for (i = 0; i < (1 << order); i++) ++ clear_highpage(page + i); ++} ++ ++#ifdef CONFIG_DEBUG_PAGEALLOC ++unsigned int _debug_guardpage_minorder; ++bool _debug_pagealloc_enabled __read_mostly; ++bool _debug_guardpage_enabled __read_mostly; ++ ++static int __init early_debug_pagealloc(char *buf) ++{ ++ if (!buf) ++ return -EINVAL; ++ ++ if (strcmp(buf, "on") == 0) ++ _debug_pagealloc_enabled = true; ++ ++ return 0; ++} ++early_param("debug_pagealloc", early_debug_pagealloc); ++ ++static bool need_debug_guardpage(void) ++{ ++ /* If we don't use debug_pagealloc, we don't need guard page */ ++ if (!debug_pagealloc_enabled()) ++ return false; ++ ++ return true; ++} ++ ++static void init_debug_guardpage(void) ++{ ++ if (!debug_pagealloc_enabled()) ++ return; ++ ++ _debug_guardpage_enabled = true; ++} ++ ++struct page_ext_operations debug_guardpage_ops = { ++ .need = need_debug_guardpage, ++ .init = init_debug_guardpage, ++}; ++ ++static int __init debug_guardpage_minorder_setup(char *buf) ++{ ++ unsigned long res; ++ ++ if (kstrtoul(buf, 10, &res) < 0 || res > MAX_ORDER / 2) { ++ printk(KERN_ERR "Bad debug_guardpage_minorder value\n"); ++ return 0; ++ } ++ _debug_guardpage_minorder = res; ++ printk(KERN_INFO "Setting debug_guardpage_minorder to %lu\n", res); ++ return 0; ++} ++__setup("debug_guardpage_minorder=", debug_guardpage_minorder_setup); ++ ++static inline void set_page_guard(struct zone *zone, struct page *page, ++ unsigned int order, int migratetype) ++{ ++ struct page_ext *page_ext; ++ ++ if (!debug_guardpage_enabled()) ++ return; ++ ++ page_ext = lookup_page_ext(page); ++ __set_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags); ++ ++ INIT_LIST_HEAD(&page->lru); ++ set_page_private(page, order); ++ /* Guard pages are not available for any usage */ ++ __mod_zone_freepage_state(zone, -(1 << order), migratetype); ++} ++ ++static inline void clear_page_guard(struct zone *zone, struct page *page, ++ unsigned int order, int migratetype) ++{ ++ struct page_ext *page_ext; ++ ++ if (!debug_guardpage_enabled()) ++ return; ++ ++ page_ext = lookup_page_ext(page); ++ __clear_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags); ++ ++ set_page_private(page, 0); ++ if (!is_migrate_isolate(migratetype)) ++ __mod_zone_freepage_state(zone, (1 << order), migratetype); ++} ++#else ++struct page_ext_operations debug_guardpage_ops = { NULL, }; ++static inline void set_page_guard(struct zone *zone, struct page *page, ++ unsigned int order, int migratetype) {} ++static inline void clear_page_guard(struct zone *zone, struct page *page, ++ unsigned int order, int migratetype) {} ++#endif ++ ++static inline void set_page_order(struct page *page, unsigned int order) ++{ ++ set_page_private(page, order); ++ __SetPageBuddy(page); ++} ++ ++static inline void rmv_page_order(struct page *page) ++{ ++ __ClearPageBuddy(page); ++ set_page_private(page, 0); ++} ++ ++/* ++ * This function checks whether a page is free && is the buddy ++ * we can do coalesce a page and its buddy if ++ * (a) the buddy is not in a hole && ++ * (b) the buddy is in the buddy system && ++ * (c) a page and its buddy have the same order && ++ * (d) a page and its buddy are in the same zone. ++ * ++ * For recording whether a page is in the buddy system, we set ->_mapcount ++ * PAGE_BUDDY_MAPCOUNT_VALUE. ++ * Setting, clearing, and testing _mapcount PAGE_BUDDY_MAPCOUNT_VALUE is ++ * serialized by zone->lock. ++ * ++ * For recording page's order, we use page_private(page). ++ */ ++static inline int page_is_buddy(struct page *page, struct page *buddy, ++ unsigned int order) ++{ ++ if (!pfn_valid_within(page_to_pfn(buddy))) ++ return 0; ++ ++ if (page_is_guard(buddy) && page_order(buddy) == order) { ++ if (page_zone_id(page) != page_zone_id(buddy)) ++ return 0; ++ ++ VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy); ++ ++ return 1; ++ } ++ ++ if (PageBuddy(buddy) && page_order(buddy) == order) { ++ /* ++ * zone check is done late to avoid uselessly ++ * calculating zone/node ids for pages that could ++ * never merge. ++ */ ++ if (page_zone_id(page) != page_zone_id(buddy)) ++ return 0; ++ ++ VM_BUG_ON_PAGE(page_count(buddy) != 0, buddy); ++ ++ return 1; ++ } ++ return 0; ++} ++ ++/* ++ * Freeing function for a buddy system allocator. ++ * ++ * The concept of a buddy system is to maintain direct-mapped table ++ * (containing bit values) for memory blocks of various "orders". ++ * The bottom level table contains the map for the smallest allocatable ++ * units of memory (here, pages), and each level above it describes ++ * pairs of units from the levels below, hence, "buddies". ++ * At a high level, all that happens here is marking the table entry ++ * at the bottom level available, and propagating the changes upward ++ * as necessary, plus some accounting needed to play nicely with other ++ * parts of the VM system. ++ * At each level, we keep a list of pages, which are heads of continuous ++ * free pages of length of (1 << order) and marked with _mapcount ++ * PAGE_BUDDY_MAPCOUNT_VALUE. Page's order is recorded in page_private(page) ++ * field. ++ * So when we are allocating or freeing one, we can derive the state of the ++ * other. That is, if we allocate a small block, and both were ++ * free, the remainder of the region must be split into blocks. ++ * If a block is freed, and its buddy is also free, then this ++ * triggers coalescing into a block of larger size. ++ * ++ * -- nyc ++ */ ++ ++static inline void __free_one_page(struct page *page, ++ unsigned long pfn, ++ struct zone *zone, unsigned int order, ++ int migratetype) ++{ ++ unsigned long page_idx; ++ unsigned long combined_idx; ++ unsigned long uninitialized_var(buddy_idx); ++ struct page *buddy; ++ int max_order = MAX_ORDER; ++ ++ VM_BUG_ON(!zone_is_initialized(zone)); ++ VM_BUG_ON_PAGE(page->flags & PAGE_FLAGS_CHECK_AT_PREP, page); ++ ++ VM_BUG_ON(migratetype == -1); ++ if (is_migrate_isolate(migratetype)) { ++ /* ++ * We restrict max order of merging to prevent merge ++ * between freepages on isolate pageblock and normal ++ * pageblock. Without this, pageblock isolation ++ * could cause incorrect freepage accounting. ++ */ ++ max_order = min(MAX_ORDER, pageblock_order + 1); ++ } else { ++ __mod_zone_freepage_state(zone, 1 << order, migratetype); ++ } ++ ++ page_idx = pfn & ((1 << max_order) - 1); ++ ++ VM_BUG_ON_PAGE(page_idx & ((1 << order) - 1), page); ++ VM_BUG_ON_PAGE(bad_range(zone, page), page); ++ ++ while (order < max_order - 1) { ++ buddy_idx = __find_buddy_index(page_idx, order); ++ buddy = page + (buddy_idx - page_idx); ++ if (!page_is_buddy(page, buddy, order)) ++ break; ++ /* ++ * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page, ++ * merge with it and move up one order. ++ */ ++ if (page_is_guard(buddy)) { ++ clear_page_guard(zone, buddy, order, migratetype); ++ } else { ++ list_del(&buddy->lru); ++ zone->free_area[order].nr_free--; ++ rmv_page_order(buddy); ++ } ++ combined_idx = buddy_idx & page_idx; ++ page = page + (combined_idx - page_idx); ++ page_idx = combined_idx; ++ order++; ++ } ++ set_page_order(page, order); ++ ++ /* ++ * If this is not the largest possible page, check if the buddy ++ * of the next-highest order is free. If it is, it's possible ++ * that pages are being freed that will coalesce soon. In case, ++ * that is happening, add the free page to the tail of the list ++ * so it's less likely to be used soon and more likely to be merged ++ * as a higher order page ++ */ ++ if ((order < MAX_ORDER-2) && pfn_valid_within(page_to_pfn(buddy))) { ++ struct page *higher_page, *higher_buddy; ++ combined_idx = buddy_idx & page_idx; ++ higher_page = page + (combined_idx - page_idx); ++ buddy_idx = __find_buddy_index(combined_idx, order + 1); ++ higher_buddy = higher_page + (buddy_idx - combined_idx); ++ if (page_is_buddy(higher_page, higher_buddy, order + 1)) { ++ list_add_tail(&page->lru, ++ &zone->free_area[order].free_list[migratetype]); ++ goto out; ++ } ++ } ++ ++ list_add(&page->lru, &zone->free_area[order].free_list[migratetype]); ++out: ++ zone->free_area[order].nr_free++; ++} ++ ++static inline int free_pages_check(struct page *page) ++{ ++ const char *bad_reason = NULL; ++ unsigned long bad_flags = 0; ++ ++ if (unlikely(page_mapcount(page))) ++ bad_reason = "nonzero mapcount"; ++ if (unlikely(page->mapping != NULL)) ++ bad_reason = "non-NULL mapping"; ++ if (unlikely(atomic_read(&page->_count) != 0)) ++ bad_reason = "nonzero _count"; ++ if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_FREE)) { ++ bad_reason = "PAGE_FLAGS_CHECK_AT_FREE flag(s) set"; ++ bad_flags = PAGE_FLAGS_CHECK_AT_FREE; ++ } ++#ifdef CONFIG_MEMCG ++ if (unlikely(page->mem_cgroup)) ++ bad_reason = "page still charged to cgroup"; ++#endif ++ if (unlikely(bad_reason)) { ++ bad_page(page, bad_reason, bad_flags); ++ return 1; ++ } ++ page_cpupid_reset_last(page); ++ if (page->flags & PAGE_FLAGS_CHECK_AT_PREP) ++ page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP; ++ return 0; ++} ++ ++/* ++ * Frees a number of pages from the PCP lists ++ * Assumes all pages on list are in same zone, and of same order. ++ * count is the number of pages to free. ++ * ++ * If the zone was previously in an "all pages pinned" state then look to ++ * see if this freeing clears that state. ++ * ++ * And clear the zone's pages_scanned counter, to hold off the "all pages are ++ * pinned" detection logic. ++ */ ++static void free_pcppages_bulk(struct zone *zone, int count, ++ struct per_cpu_pages *pcp) ++{ ++ int migratetype = 0; ++ int batch_free = 0; ++ int to_free = count; ++ unsigned long nr_scanned; ++ ++ spin_lock(&zone->lock); ++ nr_scanned = zone_page_state(zone, NR_PAGES_SCANNED); ++ if (nr_scanned) ++ __mod_zone_page_state(zone, NR_PAGES_SCANNED, -nr_scanned); ++ ++ while (to_free) { ++ struct page *page; ++ struct list_head *list; ++ ++ /* ++ * Remove pages from lists in a round-robin fashion. A ++ * batch_free count is maintained that is incremented when an ++ * empty list is encountered. This is so more pages are freed ++ * off fuller lists instead of spinning excessively around empty ++ * lists ++ */ ++ do { ++ batch_free++; ++ if (++migratetype == MIGRATE_PCPTYPES) ++ migratetype = 0; ++ list = &pcp->lists[migratetype]; ++ } while (list_empty(list)); ++ ++ /* This is the only non-empty list. Free them all. */ ++ if (batch_free == MIGRATE_PCPTYPES) ++ batch_free = to_free; ++ ++ do { ++ int mt; /* migratetype of the to-be-freed page */ ++ ++ page = list_entry(list->prev, struct page, lru); ++ /* must delete as __free_one_page list manipulates */ ++ list_del(&page->lru); ++ mt = get_freepage_migratetype(page); ++ if (unlikely(has_isolate_pageblock(zone))) ++ mt = get_pageblock_migratetype(page); ++ ++ /* MIGRATE_MOVABLE list may include MIGRATE_RESERVEs */ ++ __free_one_page(page, page_to_pfn(page), zone, 0, mt); ++ trace_mm_page_pcpu_drain(page, 0, mt); ++ } while (--to_free && --batch_free && !list_empty(list)); ++ } ++ spin_unlock(&zone->lock); ++} ++ ++static void free_one_page(struct zone *zone, ++ struct page *page, unsigned long pfn, ++ unsigned int order, ++ int migratetype) ++{ ++ unsigned long nr_scanned; ++ spin_lock(&zone->lock); ++ nr_scanned = zone_page_state(zone, NR_PAGES_SCANNED); ++ if (nr_scanned) ++ __mod_zone_page_state(zone, NR_PAGES_SCANNED, -nr_scanned); ++ ++ if (unlikely(has_isolate_pageblock(zone) || ++ is_migrate_isolate(migratetype))) { ++ migratetype = get_pfnblock_migratetype(page, pfn); ++ } ++ __free_one_page(page, pfn, zone, order, migratetype); ++ spin_unlock(&zone->lock); ++} ++ ++static int free_tail_pages_check(struct page *head_page, struct page *page) ++{ ++ if (!IS_ENABLED(CONFIG_DEBUG_VM)) ++ return 0; ++ if (unlikely(!PageTail(page))) { ++ bad_page(page, "PageTail not set", 0); ++ return 1; ++ } ++ if (unlikely(page->first_page != head_page)) { ++ bad_page(page, "first_page not consistent", 0); ++ return 1; ++ } ++ return 0; ++} ++ ++static bool free_pages_prepare(struct page *page, unsigned int order) ++{ ++ bool compound = PageCompound(page); ++ int i, bad = 0; ++ ++ VM_BUG_ON_PAGE(PageTail(page), page); ++ VM_BUG_ON_PAGE(compound && compound_order(page) != order, page); ++ ++ trace_mm_page_free(page, order); ++ kmemcheck_free_shadow(page, order); ++ kasan_free_pages(page, order); ++ ++ if (PageAnon(page)) ++ page->mapping = NULL; ++ bad += free_pages_check(page); ++ for (i = 1; i < (1 << order); i++) { ++ if (compound) ++ bad += free_tail_pages_check(page, page + i); ++ bad += free_pages_check(page + i); ++ } ++ if (bad) ++ return false; ++ ++ reset_page_owner(page, order); ++ ++ if (!PageHighMem(page)) { ++ debug_check_no_locks_freed(page_address(page), ++ PAGE_SIZE << order); ++ debug_check_no_obj_freed(page_address(page), ++ PAGE_SIZE << order); ++ } ++ arch_free_page(page, order); ++ kernel_map_pages(page, 1 << order, 0); ++ ++ return true; ++} ++ ++static void __free_pages_ok(struct page *page, unsigned int order) ++{ ++ unsigned long flags; ++ int migratetype; ++ unsigned long pfn = page_to_pfn(page); ++ ++ if (!free_pages_prepare(page, order)) ++ return; ++ ++ migratetype = get_pfnblock_migratetype(page, pfn); ++ local_irq_save(flags); ++ __count_vm_events(PGFREE, 1 << order); ++ set_freepage_migratetype(page, migratetype); ++ free_one_page(page_zone(page), page, pfn, order, migratetype); ++ local_irq_restore(flags); ++} ++ ++void __init __free_pages_bootmem(struct page *page, unsigned int order) ++{ ++ unsigned int nr_pages = 1 << order; ++ struct page *p = page; ++ unsigned int loop; ++ ++ prefetchw(p); ++ for (loop = 0; loop < (nr_pages - 1); loop++, p++) { ++ prefetchw(p + 1); ++ __ClearPageReserved(p); ++ set_page_count(p, 0); ++ } ++ __ClearPageReserved(p); ++ set_page_count(p, 0); ++ ++ page_zone(page)->managed_pages += nr_pages; ++ set_page_refcounted(page); ++ __free_pages(page, order); ++} ++ ++#ifdef CONFIG_CMA ++/* Free whole pageblock and set its migration type to MIGRATE_CMA. */ ++void __init init_cma_reserved_pageblock(struct page *page) ++{ ++ unsigned i = pageblock_nr_pages; ++ struct page *p = page; ++ ++ do { ++ __ClearPageReserved(p); ++ set_page_count(p, 0); ++ } while (++p, --i); ++ ++ set_pageblock_migratetype(page, MIGRATE_CMA); ++ ++ if (pageblock_order >= MAX_ORDER) { ++ i = pageblock_nr_pages; ++ p = page; ++ do { ++ set_page_refcounted(p); ++ __free_pages(p, MAX_ORDER - 1); ++ p += MAX_ORDER_NR_PAGES; ++ } while (i -= MAX_ORDER_NR_PAGES); ++ } else { ++ set_page_refcounted(page); ++ __free_pages(page, pageblock_order); ++ } ++ ++ adjust_managed_page_count(page, pageblock_nr_pages); ++} ++#endif ++ ++/* ++ * The order of subdivision here is critical for the IO subsystem. ++ * Please do not alter this order without good reasons and regression ++ * testing. Specifically, as large blocks of memory are subdivided, ++ * the order in which smaller blocks are delivered depends on the order ++ * they're subdivided in this function. This is the primary factor ++ * influencing the order in which pages are delivered to the IO ++ * subsystem according to empirical testing, and this is also justified ++ * by considering the behavior of a buddy system containing a single ++ * large block of memory acted on by a series of small allocations. ++ * This behavior is a critical factor in sglist merging's success. ++ * ++ * -- nyc ++ */ ++static inline void expand(struct zone *zone, struct page *page, ++ int low, int high, struct free_area *area, ++ int migratetype) ++{ ++ unsigned long size = 1 << high; ++ ++ while (high > low) { ++ area--; ++ high--; ++ size >>= 1; ++ VM_BUG_ON_PAGE(bad_range(zone, &page[size]), &page[size]); ++ ++ if (IS_ENABLED(CONFIG_DEBUG_PAGEALLOC) && ++ debug_guardpage_enabled() && ++ high < debug_guardpage_minorder()) { ++ /* ++ * Mark as guard pages (or page), that will allow to ++ * merge back to allocator when buddy will be freed. ++ * Corresponding page table entries will not be touched, ++ * pages will stay not present in virtual address space ++ */ ++ set_page_guard(zone, &page[size], high, migratetype); ++ continue; ++ } ++ list_add(&page[size].lru, &area->free_list[migratetype]); ++ area->nr_free++; ++ set_page_order(&page[size], high); ++ } ++} ++ ++/* ++ * This page is about to be returned from the page allocator ++ */ ++static inline int check_new_page(struct page *page) ++{ ++ const char *bad_reason = NULL; ++ unsigned long bad_flags = 0; ++ ++ if (unlikely(page_mapcount(page))) ++ bad_reason = "nonzero mapcount"; ++ if (unlikely(page->mapping != NULL)) ++ bad_reason = "non-NULL mapping"; ++ if (unlikely(atomic_read(&page->_count) != 0)) ++ bad_reason = "nonzero _count"; ++ if (unlikely(page->flags & PAGE_FLAGS_CHECK_AT_PREP)) { ++ bad_reason = "PAGE_FLAGS_CHECK_AT_PREP flag set"; ++ bad_flags = PAGE_FLAGS_CHECK_AT_PREP; ++ } ++#ifdef CONFIG_MEMCG ++ if (unlikely(page->mem_cgroup)) ++ bad_reason = "page still charged to cgroup"; ++#endif ++ if (unlikely(bad_reason)) { ++ bad_page(page, bad_reason, bad_flags); ++ return 1; ++ } ++ return 0; ++} ++ ++static int prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags, ++ int alloc_flags) ++{ ++ int i; ++ ++ for (i = 0; i < (1 << order); i++) { ++ struct page *p = page + i; ++ if (unlikely(check_new_page(p))) ++ return 1; ++ } ++ ++ set_page_private(page, 0); ++ set_page_refcounted(page); ++ ++ arch_alloc_page(page, order); ++ kernel_map_pages(page, 1 << order, 1); ++ kasan_alloc_pages(page, order); ++ ++ if (gfp_flags & __GFP_ZERO) ++ prep_zero_page(page, order, gfp_flags); ++ ++ if (order && (gfp_flags & __GFP_COMP)) ++ prep_compound_page(page, order); ++ ++ set_page_owner(page, order, gfp_flags); ++ ++ /* ++ * page is set pfmemalloc when ALLOC_NO_WATERMARKS was necessary to ++ * allocate the page. The expectation is that the caller is taking ++ * steps that will free more memory. The caller should avoid the page ++ * being used for !PFMEMALLOC purposes. ++ */ ++ if (alloc_flags & ALLOC_NO_WATERMARKS) ++ set_page_pfmemalloc(page); ++ else ++ clear_page_pfmemalloc(page); ++ ++ return 0; ++} ++ ++/* ++ * Go through the free lists for the given migratetype and remove ++ * the smallest available page from the freelists ++ */ ++static inline ++struct page *__rmqueue_smallest(struct zone *zone, unsigned int order, ++ int migratetype) ++{ ++ unsigned int current_order; ++ struct free_area *area; ++ struct page *page; ++ ++ /* Find a page of the appropriate size in the preferred list */ ++ for (current_order = order; current_order < MAX_ORDER; ++current_order) { ++ area = &(zone->free_area[current_order]); ++ if (list_empty(&area->free_list[migratetype])) ++ continue; ++ ++ page = list_entry(area->free_list[migratetype].next, ++ struct page, lru); ++ list_del(&page->lru); ++ rmv_page_order(page); ++ area->nr_free--; ++ expand(zone, page, order, current_order, area, migratetype); ++ set_freepage_migratetype(page, migratetype); ++ return page; ++ } ++ ++ return NULL; ++} ++ ++ ++/* ++ * This array describes the order lists are fallen back to when ++ * the free lists for the desirable migrate type are depleted ++ */ ++static int fallbacks[MIGRATE_TYPES][4] = { ++ [MIGRATE_UNMOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE, MIGRATE_RESERVE }, ++ [MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, MIGRATE_RESERVE }, ++ [MIGRATE_MOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_RESERVE }, ++#ifdef CONFIG_CMA ++ [MIGRATE_CMA] = { MIGRATE_RESERVE }, /* Never used */ ++#endif ++ [MIGRATE_RESERVE] = { MIGRATE_RESERVE }, /* Never used */ ++#ifdef CONFIG_MEMORY_ISOLATION ++ [MIGRATE_ISOLATE] = { MIGRATE_RESERVE }, /* Never used */ ++#endif ++}; ++ ++#ifdef CONFIG_CMA ++static struct page *__rmqueue_cma_fallback(struct zone *zone, ++ unsigned int order) ++{ ++ return __rmqueue_smallest(zone, order, MIGRATE_CMA); ++} ++#else ++static inline struct page *__rmqueue_cma_fallback(struct zone *zone, ++ unsigned int order) { return NULL; } ++#endif ++ ++/* ++ * Move the free pages in a range to the free lists of the requested type. ++ * Note that start_page and end_pages are not aligned on a pageblock ++ * boundary. If alignment is required, use move_freepages_block() ++ */ ++int move_freepages(struct zone *zone, ++ struct page *start_page, struct page *end_page, ++ int migratetype) ++{ ++ struct page *page; ++ unsigned long order; ++ int pages_moved = 0; ++ ++#ifndef CONFIG_HOLES_IN_ZONE ++ /* ++ * page_zone is not safe to call in this context when ++ * CONFIG_HOLES_IN_ZONE is set. This bug check is probably redundant ++ * anyway as we check zone boundaries in move_freepages_block(). ++ * Remove at a later date when no bug reports exist related to ++ * grouping pages by mobility ++ */ ++ VM_BUG_ON(page_zone(start_page) != page_zone(end_page)); ++#endif ++ ++ for (page = start_page; page <= end_page;) { ++ /* Make sure we are not inadvertently changing nodes */ ++ VM_BUG_ON_PAGE(page_to_nid(page) != zone_to_nid(zone), page); ++ ++ if (!pfn_valid_within(page_to_pfn(page))) { ++ page++; ++ continue; ++ } ++ ++ if (!PageBuddy(page)) { ++ page++; ++ continue; ++ } ++ ++ order = page_order(page); ++ list_move(&page->lru, ++ &zone->free_area[order].free_list[migratetype]); ++ set_freepage_migratetype(page, migratetype); ++ page += 1 << order; ++ pages_moved += 1 << order; ++ } ++ ++ return pages_moved; ++} ++ ++int move_freepages_block(struct zone *zone, struct page *page, ++ int migratetype) ++{ ++ unsigned long start_pfn, end_pfn; ++ struct page *start_page, *end_page; ++ ++ start_pfn = page_to_pfn(page); ++ start_pfn = start_pfn & ~(pageblock_nr_pages-1); ++ start_page = pfn_to_page(start_pfn); ++ end_page = start_page + pageblock_nr_pages - 1; ++ end_pfn = start_pfn + pageblock_nr_pages - 1; ++ ++ /* Do not cross zone boundaries */ ++ if (!zone_spans_pfn(zone, start_pfn)) ++ start_page = page; ++ if (!zone_spans_pfn(zone, end_pfn)) ++ return 0; ++ ++ return move_freepages(zone, start_page, end_page, migratetype); ++} ++ ++static void change_pageblock_range(struct page *pageblock_page, ++ int start_order, int migratetype) ++{ ++ int nr_pageblocks = 1 << (start_order - pageblock_order); ++ ++ while (nr_pageblocks--) { ++ set_pageblock_migratetype(pageblock_page, migratetype); ++ pageblock_page += pageblock_nr_pages; ++ } ++} ++ ++/* ++ * When we are falling back to another migratetype during allocation, try to ++ * steal extra free pages from the same pageblocks to satisfy further ++ * allocations, instead of polluting multiple pageblocks. ++ * ++ * If we are stealing a relatively large buddy page, it is likely there will ++ * be more free pages in the pageblock, so try to steal them all. For ++ * reclaimable and unmovable allocations, we steal regardless of page size, ++ * as fragmentation caused by those allocations polluting movable pageblocks ++ * is worse than movable allocations stealing from unmovable and reclaimable ++ * pageblocks. ++ */ ++static bool can_steal_fallback(unsigned int order, int start_mt) ++{ ++ /* ++ * Leaving this order check is intended, although there is ++ * relaxed order check in next check. The reason is that ++ * we can actually steal whole pageblock if this condition met, ++ * but, below check doesn't guarantee it and that is just heuristic ++ * so could be changed anytime. ++ */ ++ if (order >= pageblock_order) ++ return true; ++ ++ if (order >= pageblock_order / 2 || ++ start_mt == MIGRATE_RECLAIMABLE || ++ start_mt == MIGRATE_UNMOVABLE || ++ page_group_by_mobility_disabled) ++ return true; ++ ++ return false; ++} ++ ++/* ++ * This function implements actual steal behaviour. If order is large enough, ++ * we can steal whole pageblock. If not, we first move freepages in this ++ * pageblock and check whether half of pages are moved or not. If half of ++ * pages are moved, we can change migratetype of pageblock and permanently ++ * use it's pages as requested migratetype in the future. ++ */ ++static void steal_suitable_fallback(struct zone *zone, struct page *page, ++ int start_type) ++{ ++ int current_order = page_order(page); ++ int pages; ++ ++ /* Take ownership for orders >= pageblock_order */ ++ if (current_order >= pageblock_order) { ++ change_pageblock_range(page, current_order, start_type); ++ return; ++ } ++ ++ pages = move_freepages_block(zone, page, start_type); ++ ++ /* Claim the whole block if over half of it is free */ ++ if (pages >= (1 << (pageblock_order-1)) || ++ page_group_by_mobility_disabled) ++ set_pageblock_migratetype(page, start_type); ++} ++ ++/* ++ * Check whether there is a suitable fallback freepage with requested order. ++ * If only_stealable is true, this function returns fallback_mt only if ++ * we can steal other freepages all together. This would help to reduce ++ * fragmentation due to mixed migratetype pages in one pageblock. ++ */ ++int find_suitable_fallback(struct free_area *area, unsigned int order, ++ int migratetype, bool only_stealable, bool *can_steal) ++{ ++ int i; ++ int fallback_mt; ++ ++ if (area->nr_free == 0) ++ return -1; ++ ++ *can_steal = false; ++ for (i = 0;; i++) { ++ fallback_mt = fallbacks[migratetype][i]; ++ if (fallback_mt == MIGRATE_RESERVE) ++ break; ++ ++ if (list_empty(&area->free_list[fallback_mt])) ++ continue; ++ ++ if (can_steal_fallback(order, migratetype)) ++ *can_steal = true; ++ ++ if (!only_stealable) ++ return fallback_mt; ++ ++ if (*can_steal) ++ return fallback_mt; ++ } ++ ++ return -1; ++} ++ ++/* Remove an element from the buddy allocator from the fallback list */ ++static inline struct page * ++__rmqueue_fallback(struct zone *zone, unsigned int order, int start_migratetype) ++{ ++ struct free_area *area; ++ unsigned int current_order; ++ struct page *page; ++ int fallback_mt; ++ bool can_steal; ++ ++ /* Find the largest possible block of pages in the other list */ ++ for (current_order = MAX_ORDER-1; ++ current_order >= order && current_order <= MAX_ORDER-1; ++ --current_order) { ++ area = &(zone->free_area[current_order]); ++ fallback_mt = find_suitable_fallback(area, current_order, ++ start_migratetype, false, &can_steal); ++ if (fallback_mt == -1) ++ continue; ++ ++ page = list_entry(area->free_list[fallback_mt].next, ++ struct page, lru); ++ if (can_steal) ++ steal_suitable_fallback(zone, page, start_migratetype); ++ ++ /* Remove the page from the freelists */ ++ area->nr_free--; ++ list_del(&page->lru); ++ rmv_page_order(page); ++ ++ expand(zone, page, order, current_order, area, ++ start_migratetype); ++ /* ++ * The freepage_migratetype may differ from pageblock's ++ * migratetype depending on the decisions in ++ * try_to_steal_freepages(). This is OK as long as it ++ * does not differ for MIGRATE_CMA pageblocks. For CMA ++ * we need to make sure unallocated pages flushed from ++ * pcp lists are returned to the correct freelist. ++ */ ++ set_freepage_migratetype(page, start_migratetype); ++ ++ trace_mm_page_alloc_extfrag(page, order, current_order, ++ start_migratetype, fallback_mt); ++ ++ return page; ++ } ++ ++ return NULL; ++} ++ ++/* ++ * Do the hard work of removing an element from the buddy allocator. ++ * Call me with the zone->lock already held. ++ */ ++static struct page *__rmqueue(struct zone *zone, unsigned int order, ++ int migratetype) ++{ ++ struct page *page; ++ ++retry_reserve: ++ page = __rmqueue_smallest(zone, order, migratetype); ++ ++ if (unlikely(!page) && migratetype != MIGRATE_RESERVE) { ++ if (migratetype == MIGRATE_MOVABLE) ++ page = __rmqueue_cma_fallback(zone, order); ++ ++ if (!page) ++ page = __rmqueue_fallback(zone, order, migratetype); ++ ++ /* ++ * Use MIGRATE_RESERVE rather than fail an allocation. goto ++ * is used because __rmqueue_smallest is an inline function ++ * and we want just one call site ++ */ ++ if (!page) { ++ migratetype = MIGRATE_RESERVE; ++ goto retry_reserve; ++ } ++ } ++ ++ trace_mm_page_alloc_zone_locked(page, order, migratetype); ++ return page; ++} ++ ++/* ++ * Obtain a specified number of elements from the buddy allocator, all under ++ * a single hold of the lock, for efficiency. Add them to the supplied list. ++ * Returns the number of new pages which were placed at *list. ++ */ ++static int rmqueue_bulk(struct zone *zone, unsigned int order, ++ unsigned long count, struct list_head *list, ++ int migratetype, bool cold) ++{ ++ int i; ++ ++ spin_lock(&zone->lock); ++ for (i = 0; i < count; ++i) { ++ struct page *page = __rmqueue(zone, order, migratetype); ++ if (unlikely(page == NULL)) ++ break; ++ ++ /* ++ * Split buddy pages returned by expand() are received here ++ * in physical page order. The page is added to the callers and ++ * list and the list head then moves forward. From the callers ++ * perspective, the linked list is ordered by page number in ++ * some conditions. This is useful for IO devices that can ++ * merge IO requests if the physical pages are ordered ++ * properly. ++ */ ++ if (likely(!cold)) ++ list_add(&page->lru, list); ++ else ++ list_add_tail(&page->lru, list); ++ list = &page->lru; ++ if (is_migrate_cma(get_freepage_migratetype(page))) ++ __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, ++ -(1 << order)); ++ } ++ __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order)); ++ spin_unlock(&zone->lock); ++ return i; ++} ++ ++#ifdef CONFIG_NUMA ++/* ++ * Called from the vmstat counter updater to drain pagesets of this ++ * currently executing processor on remote nodes after they have ++ * expired. ++ * ++ * Note that this function must be called with the thread pinned to ++ * a single processor. ++ */ ++void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp) ++{ ++ unsigned long flags; ++ int to_drain, batch; ++ ++ local_irq_save(flags); ++ batch = READ_ONCE(pcp->batch); ++ to_drain = min(pcp->count, batch); ++ if (to_drain > 0) { ++ free_pcppages_bulk(zone, to_drain, pcp); ++ pcp->count -= to_drain; ++ } ++ local_irq_restore(flags); ++} ++#endif ++ ++/* ++ * Drain pcplists of the indicated processor and zone. ++ * ++ * The processor must either be the current processor and the ++ * thread pinned to the current processor or a processor that ++ * is not online. ++ */ ++static void drain_pages_zone(unsigned int cpu, struct zone *zone) ++{ ++ unsigned long flags; ++ struct per_cpu_pageset *pset; ++ struct per_cpu_pages *pcp; ++ ++ local_irq_save(flags); ++ pset = per_cpu_ptr(zone->pageset, cpu); ++ ++ pcp = &pset->pcp; ++ if (pcp->count) { ++ free_pcppages_bulk(zone, pcp->count, pcp); ++ pcp->count = 0; ++ } ++ local_irq_restore(flags); ++} ++ ++/* ++ * Drain pcplists of all zones on the indicated processor. ++ * ++ * The processor must either be the current processor and the ++ * thread pinned to the current processor or a processor that ++ * is not online. ++ */ ++static void drain_pages(unsigned int cpu) ++{ ++ struct zone *zone; ++ ++ for_each_populated_zone(zone) { ++ drain_pages_zone(cpu, zone); ++ } ++} ++ ++/* ++ * Spill all of this CPU's per-cpu pages back into the buddy allocator. ++ * ++ * The CPU has to be pinned. When zone parameter is non-NULL, spill just ++ * the single zone's pages. ++ */ ++void drain_local_pages(struct zone *zone) ++{ ++ int cpu = smp_processor_id(); ++ ++ if (zone) ++ drain_pages_zone(cpu, zone); ++ else ++ drain_pages(cpu); ++} ++ ++/* ++ * Spill all the per-cpu pages from all CPUs back into the buddy allocator. ++ * ++ * When zone parameter is non-NULL, spill just the single zone's pages. ++ * ++ * Note that this code is protected against sending an IPI to an offline ++ * CPU but does not guarantee sending an IPI to newly hotplugged CPUs: ++ * on_each_cpu_mask() blocks hotplug and won't talk to offlined CPUs but ++ * nothing keeps CPUs from showing up after we populated the cpumask and ++ * before the call to on_each_cpu_mask(). ++ */ ++void drain_all_pages(struct zone *zone) ++{ ++ int cpu; ++ ++ /* ++ * Allocate in the BSS so we wont require allocation in ++ * direct reclaim path for CONFIG_CPUMASK_OFFSTACK=y ++ */ ++ static cpumask_t cpus_with_pcps; ++ ++ /* ++ * We don't care about racing with CPU hotplug event ++ * as offline notification will cause the notified ++ * cpu to drain that CPU pcps and on_each_cpu_mask ++ * disables preemption as part of its processing ++ */ ++ for_each_online_cpu(cpu) { ++ struct per_cpu_pageset *pcp; ++ struct zone *z; ++ bool has_pcps = false; ++ ++ if (zone) { ++ pcp = per_cpu_ptr(zone->pageset, cpu); ++ if (pcp->pcp.count) ++ has_pcps = true; ++ } else { ++ for_each_populated_zone(z) { ++ pcp = per_cpu_ptr(z->pageset, cpu); ++ if (pcp->pcp.count) { ++ has_pcps = true; ++ break; ++ } ++ } ++ } ++ ++ if (has_pcps) ++ cpumask_set_cpu(cpu, &cpus_with_pcps); ++ else ++ cpumask_clear_cpu(cpu, &cpus_with_pcps); ++ } ++ on_each_cpu_mask(&cpus_with_pcps, (smp_call_func_t) drain_local_pages, ++ zone, 1); ++} ++ ++#ifdef CONFIG_HIBERNATION ++ ++void mark_free_pages(struct zone *zone) ++{ ++ unsigned long pfn, max_zone_pfn; ++ unsigned long flags; ++ unsigned int order, t; ++ struct list_head *curr; ++ ++ if (zone_is_empty(zone)) ++ return; ++ ++ spin_lock_irqsave(&zone->lock, flags); ++ ++ max_zone_pfn = zone_end_pfn(zone); ++ for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) ++ if (pfn_valid(pfn)) { ++ struct page *page = pfn_to_page(pfn); ++ ++ if (!swsusp_page_is_forbidden(page)) ++ swsusp_unset_page_free(page); ++ } ++ ++ for_each_migratetype_order(order, t) { ++ list_for_each(curr, &zone->free_area[order].free_list[t]) { ++ unsigned long i; ++ ++ pfn = page_to_pfn(list_entry(curr, struct page, lru)); ++ for (i = 0; i < (1UL << order); i++) ++ swsusp_set_page_free(pfn_to_page(pfn + i)); ++ } ++ } ++ spin_unlock_irqrestore(&zone->lock, flags); ++} ++#endif /* CONFIG_PM */ ++ ++/* ++ * Free a 0-order page ++ * cold == true ? free a cold page : free a hot page ++ */ ++void free_hot_cold_page(struct page *page, bool cold) ++{ ++ struct zone *zone = page_zone(page); ++ struct per_cpu_pages *pcp; ++ unsigned long flags; ++ unsigned long pfn = page_to_pfn(page); ++ int migratetype; ++ ++ if (!free_pages_prepare(page, 0)) ++ return; ++ ++ migratetype = get_pfnblock_migratetype(page, pfn); ++ set_freepage_migratetype(page, migratetype); ++ local_irq_save(flags); ++ __count_vm_event(PGFREE); ++ ++ /* ++ * We only track unmovable, reclaimable and movable on pcp lists. ++ * Free ISOLATE pages back to the allocator because they are being ++ * offlined but treat RESERVE as movable pages so we can get those ++ * areas back if necessary. Otherwise, we may have to free ++ * excessively into the page allocator ++ */ ++ if (migratetype >= MIGRATE_PCPTYPES) { ++ if (unlikely(is_migrate_isolate(migratetype))) { ++ free_one_page(zone, page, pfn, 0, migratetype); ++ goto out; ++ } ++ migratetype = MIGRATE_MOVABLE; ++ } ++ ++ pcp = &this_cpu_ptr(zone->pageset)->pcp; ++ if (!cold) ++ list_add(&page->lru, &pcp->lists[migratetype]); ++ else ++ list_add_tail(&page->lru, &pcp->lists[migratetype]); ++ pcp->count++; ++ if (pcp->count >= pcp->high) { ++ unsigned long batch = READ_ONCE(pcp->batch); ++ free_pcppages_bulk(zone, batch, pcp); ++ pcp->count -= batch; ++ } ++ ++out: ++ local_irq_restore(flags); ++} ++ ++/* ++ * Free a list of 0-order pages ++ */ ++void free_hot_cold_page_list(struct list_head *list, bool cold) ++{ ++ struct page *page, *next; ++ ++ list_for_each_entry_safe(page, next, list, lru) { ++ trace_mm_page_free_batched(page, cold); ++ free_hot_cold_page(page, cold); ++ } ++} ++ ++/* ++ * split_page takes a non-compound higher-order page, and splits it into ++ * n (1<lru); ++ zone->free_area[order].nr_free--; ++ rmv_page_order(page); ++ ++ /* Set the pageblock if the isolated page is at least a pageblock */ ++ if (order >= pageblock_order - 1) { ++ struct page *endpage = page + (1 << order) - 1; ++ for (; page < endpage; page += pageblock_nr_pages) { ++ int mt = get_pageblock_migratetype(page); ++ if (!is_migrate_isolate(mt) && !is_migrate_cma(mt)) ++ set_pageblock_migratetype(page, ++ MIGRATE_MOVABLE); ++ } ++ } ++ ++ set_page_owner(page, order, 0); ++ return 1UL << order; ++} ++ ++/* ++ * Similar to split_page except the page is already free. As this is only ++ * being used for migration, the migratetype of the block also changes. ++ * As this is called with interrupts disabled, the caller is responsible ++ * for calling arch_alloc_page() and kernel_map_page() after interrupts ++ * are enabled. ++ * ++ * Note: this is probably too low level an operation for use in drivers. ++ * Please consult with lkml before using this in your driver. ++ */ ++int split_free_page(struct page *page) ++{ ++ unsigned int order; ++ int nr_pages; ++ ++ order = page_order(page); ++ ++ nr_pages = __isolate_free_page(page, order); ++ if (!nr_pages) ++ return 0; ++ ++ /* Split into individual pages */ ++ set_page_refcounted(page); ++ split_page(page, order); ++ return nr_pages; ++} ++ ++/* ++ * Allocate a page from the given zone. Use pcplists for order-0 allocations. ++ */ ++static inline ++struct page *buffered_rmqueue(struct zone *preferred_zone, ++ struct zone *zone, unsigned int order, ++ gfp_t gfp_flags, int migratetype) ++{ ++ unsigned long flags; ++ struct page *page; ++ bool cold = ((gfp_flags & __GFP_COLD) != 0); ++ ++ if (likely(order == 0)) { ++ struct per_cpu_pages *pcp; ++ struct list_head *list; ++ ++ local_irq_save(flags); ++ pcp = &this_cpu_ptr(zone->pageset)->pcp; ++ list = &pcp->lists[migratetype]; ++ if (list_empty(list)) { ++ pcp->count += rmqueue_bulk(zone, 0, ++ pcp->batch, list, ++ migratetype, cold); ++ if (unlikely(list_empty(list))) ++ goto failed; ++ } ++ ++ if (cold) ++ page = list_entry(list->prev, struct page, lru); ++ else ++ page = list_entry(list->next, struct page, lru); ++ ++ list_del(&page->lru); ++ pcp->count--; ++ } else { ++ if (unlikely(gfp_flags & __GFP_NOFAIL)) { ++ /* ++ * __GFP_NOFAIL is not to be used in new code. ++ * ++ * All __GFP_NOFAIL callers should be fixed so that they ++ * properly detect and handle allocation failures. ++ * ++ * We most definitely don't want callers attempting to ++ * allocate greater than order-1 page units with ++ * __GFP_NOFAIL. ++ */ ++ WARN_ON_ONCE(order > 1); ++ } ++ spin_lock_irqsave(&zone->lock, flags); ++ page = __rmqueue(zone, order, migratetype); ++ spin_unlock(&zone->lock); ++ if (!page) ++ goto failed; ++ __mod_zone_freepage_state(zone, -(1 << order), ++ get_freepage_migratetype(page)); ++ } ++ ++ __mod_zone_page_state(zone, NR_ALLOC_BATCH, -(1 << order)); ++ if (atomic_long_read(&zone->vm_stat[NR_ALLOC_BATCH]) <= 0 && ++ !test_bit(ZONE_FAIR_DEPLETED, &zone->flags)) ++ set_bit(ZONE_FAIR_DEPLETED, &zone->flags); ++ ++ __count_zone_vm_events(PGALLOC, zone, 1 << order); ++ zone_statistics(preferred_zone, zone, gfp_flags); ++ local_irq_restore(flags); ++ ++ VM_BUG_ON_PAGE(bad_range(zone, page), page); ++ return page; ++ ++failed: ++ local_irq_restore(flags); ++ return NULL; ++} ++ ++#ifdef CONFIG_FAIL_PAGE_ALLOC ++ ++static struct { ++ struct fault_attr attr; ++ ++ u32 ignore_gfp_highmem; ++ u32 ignore_gfp_wait; ++ u32 min_order; ++} fail_page_alloc = { ++ .attr = FAULT_ATTR_INITIALIZER, ++ .ignore_gfp_wait = 1, ++ .ignore_gfp_highmem = 1, ++ .min_order = 1, ++}; ++ ++static int __init setup_fail_page_alloc(char *str) ++{ ++ return setup_fault_attr(&fail_page_alloc.attr, str); ++} ++__setup("fail_page_alloc=", setup_fail_page_alloc); ++ ++static bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) ++{ ++ if (order < fail_page_alloc.min_order) ++ return false; ++ if (gfp_mask & __GFP_NOFAIL) ++ return false; ++ if (fail_page_alloc.ignore_gfp_highmem && (gfp_mask & __GFP_HIGHMEM)) ++ return false; ++ if (fail_page_alloc.ignore_gfp_wait && (gfp_mask & __GFP_WAIT)) ++ return false; ++ ++ return should_fail(&fail_page_alloc.attr, 1 << order); ++} ++ ++#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS ++ ++static int __init fail_page_alloc_debugfs(void) ++{ ++ umode_t mode = S_IFREG | S_IRUSR | S_IWUSR; ++ struct dentry *dir; ++ ++ dir = fault_create_debugfs_attr("fail_page_alloc", NULL, ++ &fail_page_alloc.attr); ++ if (IS_ERR(dir)) ++ return PTR_ERR(dir); ++ ++ if (!debugfs_create_bool("ignore-gfp-wait", mode, dir, ++ &fail_page_alloc.ignore_gfp_wait)) ++ goto fail; ++ if (!debugfs_create_bool("ignore-gfp-highmem", mode, dir, ++ &fail_page_alloc.ignore_gfp_highmem)) ++ goto fail; ++ if (!debugfs_create_u32("min-order", mode, dir, ++ &fail_page_alloc.min_order)) ++ goto fail; ++ ++ return 0; ++fail: ++ debugfs_remove_recursive(dir); ++ ++ return -ENOMEM; ++} ++ ++late_initcall(fail_page_alloc_debugfs); ++ ++#endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */ ++ ++#else /* CONFIG_FAIL_PAGE_ALLOC */ ++ ++static inline bool should_fail_alloc_page(gfp_t gfp_mask, unsigned int order) ++{ ++ return false; ++} ++ ++#endif /* CONFIG_FAIL_PAGE_ALLOC */ ++ ++/* ++ * Return true if free pages are above 'mark'. This takes into account the order ++ * of the allocation. ++ */ ++static bool __zone_watermark_ok(struct zone *z, unsigned int order, ++ unsigned long mark, int classzone_idx, int alloc_flags, ++ long free_pages) ++{ ++ /* free_pages may go negative - that's OK */ ++ long min = mark; ++ int o; ++ long free_cma = 0; ++ ++ free_pages -= (1 << order) - 1; ++ if (alloc_flags & ALLOC_HIGH) ++ min -= min / 2; ++ if (alloc_flags & ALLOC_HARDER) ++ min -= min / 4; ++#ifdef CONFIG_CMA ++ /* If allocation can't use CMA areas don't use free CMA pages */ ++ if (!(alloc_flags & ALLOC_CMA)) ++ free_cma = zone_page_state(z, NR_FREE_CMA_PAGES); ++#endif ++ ++ if (free_pages - free_cma <= min + z->lowmem_reserve[classzone_idx]) ++ return false; ++ for (o = 0; o < order; o++) { ++ /* At the next order, this order's pages become unavailable */ ++ free_pages -= z->free_area[o].nr_free << o; ++ ++ /* Require fewer higher order pages to be free */ ++ min >>= 1; ++ ++ if (free_pages <= min) ++ return false; ++ } ++ return true; ++} ++ ++bool zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark, ++ int classzone_idx, int alloc_flags) ++{ ++ return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags, ++ zone_page_state(z, NR_FREE_PAGES)); ++} ++ ++bool zone_watermark_ok_safe(struct zone *z, unsigned int order, ++ unsigned long mark, int classzone_idx, int alloc_flags) ++{ ++ long free_pages = zone_page_state(z, NR_FREE_PAGES); ++ ++ if (z->percpu_drift_mark && free_pages < z->percpu_drift_mark) ++ free_pages = zone_page_state_snapshot(z, NR_FREE_PAGES); ++ ++ return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags, ++ free_pages); ++} ++ ++#ifdef CONFIG_NUMA ++/* ++ * zlc_setup - Setup for "zonelist cache". Uses cached zone data to ++ * skip over zones that are not allowed by the cpuset, or that have ++ * been recently (in last second) found to be nearly full. See further ++ * comments in mmzone.h. Reduces cache footprint of zonelist scans ++ * that have to skip over a lot of full or unallowed zones. ++ * ++ * If the zonelist cache is present in the passed zonelist, then ++ * returns a pointer to the allowed node mask (either the current ++ * tasks mems_allowed, or node_states[N_MEMORY].) ++ * ++ * If the zonelist cache is not available for this zonelist, does ++ * nothing and returns NULL. ++ * ++ * If the fullzones BITMAP in the zonelist cache is stale (more than ++ * a second since last zap'd) then we zap it out (clear its bits.) ++ * ++ * We hold off even calling zlc_setup, until after we've checked the ++ * first zone in the zonelist, on the theory that most allocations will ++ * be satisfied from that first zone, so best to examine that zone as ++ * quickly as we can. ++ */ ++static nodemask_t *zlc_setup(struct zonelist *zonelist, int alloc_flags) ++{ ++ struct zonelist_cache *zlc; /* cached zonelist speedup info */ ++ nodemask_t *allowednodes; /* zonelist_cache approximation */ ++ ++ zlc = zonelist->zlcache_ptr; ++ if (!zlc) ++ return NULL; ++ ++ if (time_after(jiffies, zlc->last_full_zap + HZ)) { ++ bitmap_zero(zlc->fullzones, MAX_ZONES_PER_ZONELIST); ++ zlc->last_full_zap = jiffies; ++ } ++ ++ allowednodes = !in_interrupt() && (alloc_flags & ALLOC_CPUSET) ? ++ &cpuset_current_mems_allowed : ++ &node_states[N_MEMORY]; ++ return allowednodes; ++} ++ ++/* ++ * Given 'z' scanning a zonelist, run a couple of quick checks to see ++ * if it is worth looking at further for free memory: ++ * 1) Check that the zone isn't thought to be full (doesn't have its ++ * bit set in the zonelist_cache fullzones BITMAP). ++ * 2) Check that the zones node (obtained from the zonelist_cache ++ * z_to_n[] mapping) is allowed in the passed in allowednodes mask. ++ * Return true (non-zero) if zone is worth looking at further, or ++ * else return false (zero) if it is not. ++ * ++ * This check -ignores- the distinction between various watermarks, ++ * such as GFP_HIGH, GFP_ATOMIC, PF_MEMALLOC, ... If a zone is ++ * found to be full for any variation of these watermarks, it will ++ * be considered full for up to one second by all requests, unless ++ * we are so low on memory on all allowed nodes that we are forced ++ * into the second scan of the zonelist. ++ * ++ * In the second scan we ignore this zonelist cache and exactly ++ * apply the watermarks to all zones, even it is slower to do so. ++ * We are low on memory in the second scan, and should leave no stone ++ * unturned looking for a free page. ++ */ ++static int zlc_zone_worth_trying(struct zonelist *zonelist, struct zoneref *z, ++ nodemask_t *allowednodes) ++{ ++ struct zonelist_cache *zlc; /* cached zonelist speedup info */ ++ int i; /* index of *z in zonelist zones */ ++ int n; /* node that zone *z is on */ ++ ++ zlc = zonelist->zlcache_ptr; ++ if (!zlc) ++ return 1; ++ ++ i = z - zonelist->_zonerefs; ++ n = zlc->z_to_n[i]; ++ ++ /* This zone is worth trying if it is allowed but not full */ ++ return node_isset(n, *allowednodes) && !test_bit(i, zlc->fullzones); ++} ++ ++/* ++ * Given 'z' scanning a zonelist, set the corresponding bit in ++ * zlc->fullzones, so that subsequent attempts to allocate a page ++ * from that zone don't waste time re-examining it. ++ */ ++static void zlc_mark_zone_full(struct zonelist *zonelist, struct zoneref *z) ++{ ++ struct zonelist_cache *zlc; /* cached zonelist speedup info */ ++ int i; /* index of *z in zonelist zones */ ++ ++ zlc = zonelist->zlcache_ptr; ++ if (!zlc) ++ return; ++ ++ i = z - zonelist->_zonerefs; ++ ++ set_bit(i, zlc->fullzones); ++} ++ ++/* ++ * clear all zones full, called after direct reclaim makes progress so that ++ * a zone that was recently full is not skipped over for up to a second ++ */ ++static void zlc_clear_zones_full(struct zonelist *zonelist) ++{ ++ struct zonelist_cache *zlc; /* cached zonelist speedup info */ ++ ++ zlc = zonelist->zlcache_ptr; ++ if (!zlc) ++ return; ++ ++ bitmap_zero(zlc->fullzones, MAX_ZONES_PER_ZONELIST); ++} ++ ++static bool zone_local(struct zone *local_zone, struct zone *zone) ++{ ++ return local_zone->node == zone->node; ++} ++ ++static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone) ++{ ++ return node_distance(zone_to_nid(local_zone), zone_to_nid(zone)) < ++ RECLAIM_DISTANCE; ++} ++ ++#else /* CONFIG_NUMA */ ++ ++static nodemask_t *zlc_setup(struct zonelist *zonelist, int alloc_flags) ++{ ++ return NULL; ++} ++ ++static int zlc_zone_worth_trying(struct zonelist *zonelist, struct zoneref *z, ++ nodemask_t *allowednodes) ++{ ++ return 1; ++} ++ ++static void zlc_mark_zone_full(struct zonelist *zonelist, struct zoneref *z) ++{ ++} ++ ++static void zlc_clear_zones_full(struct zonelist *zonelist) ++{ ++} ++ ++static bool zone_local(struct zone *local_zone, struct zone *zone) ++{ ++ return true; ++} ++ ++static bool zone_allows_reclaim(struct zone *local_zone, struct zone *zone) ++{ ++ return true; ++} ++ ++#endif /* CONFIG_NUMA */ ++ ++static void reset_alloc_batches(struct zone *preferred_zone) ++{ ++ struct zone *zone = preferred_zone->zone_pgdat->node_zones; ++ ++ do { ++ mod_zone_page_state(zone, NR_ALLOC_BATCH, ++ high_wmark_pages(zone) - low_wmark_pages(zone) - ++ atomic_long_read(&zone->vm_stat[NR_ALLOC_BATCH])); ++ clear_bit(ZONE_FAIR_DEPLETED, &zone->flags); ++ } while (zone++ != preferred_zone); ++} ++ ++/* ++ * get_page_from_freelist goes through the zonelist trying to allocate ++ * a page. ++ */ ++static struct page * ++get_page_from_freelist(gfp_t gfp_mask, unsigned int order, int alloc_flags, ++ const struct alloc_context *ac) ++{ ++ struct zonelist *zonelist = ac->zonelist; ++ struct zoneref *z; ++ struct page *page = NULL; ++ struct zone *zone; ++ nodemask_t *allowednodes = NULL;/* zonelist_cache approximation */ ++ int zlc_active = 0; /* set if using zonelist_cache */ ++ int did_zlc_setup = 0; /* just call zlc_setup() one time */ ++ bool consider_zone_dirty = (alloc_flags & ALLOC_WMARK_LOW) && ++ (gfp_mask & __GFP_WRITE); ++ int nr_fair_skipped = 0; ++ bool zonelist_rescan; ++ ++zonelist_scan: ++ zonelist_rescan = false; ++ ++ /* ++ * Scan zonelist, looking for a zone with enough free. ++ * See also __cpuset_node_allowed() comment in kernel/cpuset.c. ++ */ ++ for_each_zone_zonelist_nodemask(zone, z, zonelist, ac->high_zoneidx, ++ ac->nodemask) { ++ unsigned long mark; ++ ++ if (IS_ENABLED(CONFIG_NUMA) && zlc_active && ++ !zlc_zone_worth_trying(zonelist, z, allowednodes)) ++ continue; ++ if (cpusets_enabled() && ++ (alloc_flags & ALLOC_CPUSET) && ++ !cpuset_zone_allowed(zone, gfp_mask)) ++ continue; ++ /* ++ * Distribute pages in proportion to the individual ++ * zone size to ensure fair page aging. The zone a ++ * page was allocated in should have no effect on the ++ * time the page has in memory before being reclaimed. ++ */ ++ if (alloc_flags & ALLOC_FAIR) { ++ if (!zone_local(ac->preferred_zone, zone)) ++ break; ++ if (test_bit(ZONE_FAIR_DEPLETED, &zone->flags)) { ++ nr_fair_skipped++; ++ continue; ++ } ++ } ++ /* ++ * When allocating a page cache page for writing, we ++ * want to get it from a zone that is within its dirty ++ * limit, such that no single zone holds more than its ++ * proportional share of globally allowed dirty pages. ++ * The dirty limits take into account the zone's ++ * lowmem reserves and high watermark so that kswapd ++ * should be able to balance it without having to ++ * write pages from its LRU list. ++ * ++ * This may look like it could increase pressure on ++ * lower zones by failing allocations in higher zones ++ * before they are full. But the pages that do spill ++ * over are limited as the lower zones are protected ++ * by this very same mechanism. It should not become ++ * a practical burden to them. ++ * ++ * XXX: For now, allow allocations to potentially ++ * exceed the per-zone dirty limit in the slowpath ++ * (ALLOC_WMARK_LOW unset) before going into reclaim, ++ * which is important when on a NUMA setup the allowed ++ * zones are together not big enough to reach the ++ * global limit. The proper fix for these situations ++ * will require awareness of zones in the ++ * dirty-throttling and the flusher threads. ++ */ ++ if (consider_zone_dirty && !zone_dirty_ok(zone)) ++ continue; ++ ++ mark = zone->watermark[alloc_flags & ALLOC_WMARK_MASK]; ++ if (!zone_watermark_ok(zone, order, mark, ++ ac->classzone_idx, alloc_flags)) { ++ int ret; ++ ++ /* Checked here to keep the fast path fast */ ++ BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK); ++ if (alloc_flags & ALLOC_NO_WATERMARKS) ++ goto try_this_zone; ++ ++ if (IS_ENABLED(CONFIG_NUMA) && ++ !did_zlc_setup && nr_online_nodes > 1) { ++ /* ++ * we do zlc_setup if there are multiple nodes ++ * and before considering the first zone allowed ++ * by the cpuset. ++ */ ++ allowednodes = zlc_setup(zonelist, alloc_flags); ++ zlc_active = 1; ++ did_zlc_setup = 1; ++ } ++ ++ if (zone_reclaim_mode == 0 || ++ !zone_allows_reclaim(ac->preferred_zone, zone)) ++ goto this_zone_full; ++ ++ /* ++ * As we may have just activated ZLC, check if the first ++ * eligible zone has failed zone_reclaim recently. ++ */ ++ if (IS_ENABLED(CONFIG_NUMA) && zlc_active && ++ !zlc_zone_worth_trying(zonelist, z, allowednodes)) ++ continue; ++ ++ ret = zone_reclaim(zone, gfp_mask, order); ++ switch (ret) { ++ case ZONE_RECLAIM_NOSCAN: ++ /* did not scan */ ++ continue; ++ case ZONE_RECLAIM_FULL: ++ /* scanned but unreclaimable */ ++ continue; ++ default: ++ /* did we reclaim enough */ ++ if (zone_watermark_ok(zone, order, mark, ++ ac->classzone_idx, alloc_flags)) ++ goto try_this_zone; ++ ++ /* ++ * Failed to reclaim enough to meet watermark. ++ * Only mark the zone full if checking the min ++ * watermark or if we failed to reclaim just ++ * 1<preferred_zone, zone, order, ++ gfp_mask, ac->migratetype); ++ if (page) { ++ if (prep_new_page(page, order, gfp_mask, alloc_flags)) ++ goto try_this_zone; ++ return page; ++ } ++this_zone_full: ++ if (IS_ENABLED(CONFIG_NUMA) && zlc_active) ++ zlc_mark_zone_full(zonelist, z); ++ } ++ ++ /* ++ * The first pass makes sure allocations are spread fairly within the ++ * local node. However, the local node might have free pages left ++ * after the fairness batches are exhausted, and remote zones haven't ++ * even been considered yet. Try once more without fairness, and ++ * include remote zones now, before entering the slowpath and waking ++ * kswapd: prefer spilling to a remote zone over swapping locally. ++ */ ++ if (alloc_flags & ALLOC_FAIR) { ++ alloc_flags &= ~ALLOC_FAIR; ++ if (nr_fair_skipped) { ++ zonelist_rescan = true; ++ reset_alloc_batches(ac->preferred_zone); ++ } ++ if (nr_online_nodes > 1) ++ zonelist_rescan = true; ++ } ++ ++ if (unlikely(IS_ENABLED(CONFIG_NUMA) && zlc_active)) { ++ /* Disable zlc cache for second zonelist scan */ ++ zlc_active = 0; ++ zonelist_rescan = true; ++ } ++ ++ if (zonelist_rescan) ++ goto zonelist_scan; ++ ++ return NULL; ++} ++ ++/* ++ * Large machines with many possible nodes should not always dump per-node ++ * meminfo in irq context. ++ */ ++static inline bool should_suppress_show_mem(void) ++{ ++ bool ret = false; ++ ++#if NODES_SHIFT > 8 ++ ret = in_interrupt(); ++#endif ++ return ret; ++} ++ ++static DEFINE_RATELIMIT_STATE(nopage_rs, ++ DEFAULT_RATELIMIT_INTERVAL, ++ DEFAULT_RATELIMIT_BURST); ++ ++void warn_alloc_failed(gfp_t gfp_mask, int order, const char *fmt, ...) ++{ ++ unsigned int filter = SHOW_MEM_FILTER_NODES; ++ ++ if ((gfp_mask & __GFP_NOWARN) || !__ratelimit(&nopage_rs) || ++ debug_guardpage_minorder() > 0) ++ return; ++ ++ /* ++ * This documents exceptions given to allocations in certain ++ * contexts that are allowed to allocate outside current's set ++ * of allowed nodes. ++ */ ++ if (!(gfp_mask & __GFP_NOMEMALLOC)) ++ if (test_thread_flag(TIF_MEMDIE) || ++ (current->flags & (PF_MEMALLOC | PF_EXITING))) ++ filter &= ~SHOW_MEM_FILTER_NODES; ++ if (in_interrupt() || !(gfp_mask & __GFP_WAIT)) ++ filter &= ~SHOW_MEM_FILTER_NODES; ++ ++ if (fmt) { ++ struct va_format vaf; ++ va_list args; ++ ++ va_start(args, fmt); ++ ++ vaf.fmt = fmt; ++ vaf.va = &args; ++ ++ pr_warn("%pV", &vaf); ++ ++ va_end(args); ++ } ++ ++ pr_warn("%s: page allocation failure: order:%d, mode:0x%x\n", ++ current->comm, order, gfp_mask); ++ ++ dump_stack(); ++ if (!should_suppress_show_mem()) ++ show_mem(filter); ++} ++ ++static inline int ++should_alloc_retry(gfp_t gfp_mask, unsigned int order, ++ unsigned long did_some_progress, ++ unsigned long pages_reclaimed) ++{ ++ /* Do not loop if specifically requested */ ++ if (gfp_mask & __GFP_NORETRY) ++ return 0; ++ ++ /* Always retry if specifically requested */ ++ if (gfp_mask & __GFP_NOFAIL) ++ return 1; ++ ++ /* ++ * Suspend converts GFP_KERNEL to __GFP_WAIT which can prevent reclaim ++ * making forward progress without invoking OOM. Suspend also disables ++ * storage devices so kswapd will not help. Bail if we are suspending. ++ */ ++ if (!did_some_progress && pm_suspended_storage()) ++ return 0; ++ ++ /* ++ * In this implementation, order <= PAGE_ALLOC_COSTLY_ORDER ++ * means __GFP_NOFAIL, but that may not be true in other ++ * implementations. ++ */ ++ if (order <= PAGE_ALLOC_COSTLY_ORDER) ++ return 1; ++ ++ /* ++ * For order > PAGE_ALLOC_COSTLY_ORDER, if __GFP_REPEAT is ++ * specified, then we retry until we no longer reclaim any pages ++ * (above), or we've reclaimed an order of pages at least as ++ * large as the allocation's order. In both cases, if the ++ * allocation still fails, we stop retrying. ++ */ ++ if (gfp_mask & __GFP_REPEAT && pages_reclaimed < (1 << order)) ++ return 1; ++ ++ return 0; ++} ++ ++static inline struct page * ++__alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order, ++ const struct alloc_context *ac, unsigned long *did_some_progress) ++{ ++ struct page *page; ++ ++ *did_some_progress = 0; ++ ++ /* ++ * Acquire the per-zone oom lock for each zone. If that ++ * fails, somebody else is making progress for us. ++ */ ++ if (!oom_zonelist_trylock(ac->zonelist, gfp_mask)) { ++ *did_some_progress = 1; ++ schedule_timeout_uninterruptible(1); ++ return NULL; ++ } ++ ++ /* ++ * Go through the zonelist yet one more time, keep very high watermark ++ * here, this is only to catch a parallel oom killing, we must fail if ++ * we're still under heavy pressure. ++ */ ++ page = get_page_from_freelist(gfp_mask | __GFP_HARDWALL, order, ++ ALLOC_WMARK_HIGH|ALLOC_CPUSET, ac); ++ if (page) ++ goto out; ++ ++ if (!(gfp_mask & __GFP_NOFAIL)) { ++ /* Coredumps can quickly deplete all memory reserves */ ++ if (current->flags & PF_DUMPCORE) ++ goto out; ++ /* The OOM killer will not help higher order allocs */ ++ if (order > PAGE_ALLOC_COSTLY_ORDER) ++ goto out; ++ /* The OOM killer does not needlessly kill tasks for lowmem */ ++ if (ac->high_zoneidx < ZONE_NORMAL) ++ goto out; ++ /* The OOM killer does not compensate for light reclaim */ ++ if (!(gfp_mask & __GFP_FS)) { ++ /* ++ * XXX: Page reclaim didn't yield anything, ++ * and the OOM killer can't be invoked, but ++ * keep looping as per should_alloc_retry(). ++ */ ++ *did_some_progress = 1; ++ goto out; ++ } ++ /* The OOM killer may not free memory on a specific node */ ++ if (gfp_mask & __GFP_THISNODE) ++ goto out; ++ } ++ /* Exhausted what can be done so it's blamo time */ ++ if (out_of_memory(ac->zonelist, gfp_mask, order, ac->nodemask, false) ++ || WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL)) ++ *did_some_progress = 1; ++out: ++ oom_zonelist_unlock(ac->zonelist, gfp_mask); ++ return page; ++} ++ ++#ifdef CONFIG_COMPACTION ++/* Try memory compaction for high-order allocations before reclaim */ ++static struct page * ++__alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, ++ int alloc_flags, const struct alloc_context *ac, ++ enum migrate_mode mode, int *contended_compaction, ++ bool *deferred_compaction) ++{ ++ unsigned long compact_result; ++ struct page *page; ++ ++ if (!order) ++ return NULL; ++ ++ current->flags |= PF_MEMALLOC; ++ compact_result = try_to_compact_pages(gfp_mask, order, alloc_flags, ac, ++ mode, contended_compaction); ++ current->flags &= ~PF_MEMALLOC; ++ ++ switch (compact_result) { ++ case COMPACT_DEFERRED: ++ *deferred_compaction = true; ++ /* fall-through */ ++ case COMPACT_SKIPPED: ++ return NULL; ++ default: ++ break; ++ } ++ ++ /* ++ * At least in one zone compaction wasn't deferred or skipped, so let's ++ * count a compaction stall ++ */ ++ count_vm_event(COMPACTSTALL); ++ ++ page = get_page_from_freelist(gfp_mask, order, ++ alloc_flags & ~ALLOC_NO_WATERMARKS, ac); ++ ++ if (page) { ++ struct zone *zone = page_zone(page); ++ ++ zone->compact_blockskip_flush = false; ++ compaction_defer_reset(zone, order, true); ++ count_vm_event(COMPACTSUCCESS); ++ return page; ++ } ++ ++ /* ++ * It's bad if compaction run occurs and fails. The most likely reason ++ * is that pages exist, but not enough to satisfy watermarks. ++ */ ++ count_vm_event(COMPACTFAIL); ++ ++ cond_resched(); ++ ++ return NULL; ++} ++#else ++static inline struct page * ++__alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order, ++ int alloc_flags, const struct alloc_context *ac, ++ enum migrate_mode mode, int *contended_compaction, ++ bool *deferred_compaction) ++{ ++ return NULL; ++} ++#endif /* CONFIG_COMPACTION */ ++ ++/* Perform direct synchronous page reclaim */ ++static int ++__perform_reclaim(gfp_t gfp_mask, unsigned int order, ++ const struct alloc_context *ac) ++{ ++ struct reclaim_state reclaim_state; ++ int progress; ++ ++ cond_resched(); ++ ++ /* We now go into synchronous reclaim */ ++ cpuset_memory_pressure_bump(); ++ current->flags |= PF_MEMALLOC; ++ lockdep_set_current_reclaim_state(gfp_mask); ++ reclaim_state.reclaimed_slab = 0; ++ current->reclaim_state = &reclaim_state; ++ ++ progress = try_to_free_pages(ac->zonelist, order, gfp_mask, ++ ac->nodemask); ++ ++ current->reclaim_state = NULL; ++ lockdep_clear_current_reclaim_state(); ++ current->flags &= ~PF_MEMALLOC; ++ ++ cond_resched(); ++ ++ return progress; ++} ++ ++/* The really slow allocator path where we enter direct reclaim */ ++static inline struct page * ++__alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order, ++ int alloc_flags, const struct alloc_context *ac, ++ unsigned long *did_some_progress) ++{ ++ struct page *page = NULL; ++ bool drained = false; ++ ++ *did_some_progress = __perform_reclaim(gfp_mask, order, ac); ++ if (unlikely(!(*did_some_progress))) ++ return NULL; ++ ++ /* After successful reclaim, reconsider all zones for allocation */ ++ if (IS_ENABLED(CONFIG_NUMA)) ++ zlc_clear_zones_full(ac->zonelist); ++ ++retry: ++ page = get_page_from_freelist(gfp_mask, order, ++ alloc_flags & ~ALLOC_NO_WATERMARKS, ac); ++ ++ /* ++ * If an allocation failed after direct reclaim, it could be because ++ * pages are pinned on the per-cpu lists. Drain them and try again ++ */ ++ if (!page && !drained) { ++ drain_all_pages(NULL); ++ drained = true; ++ goto retry; ++ } ++ ++ return page; ++} ++ ++/* ++ * This is called in the allocator slow-path if the allocation request is of ++ * sufficient urgency to ignore watermarks and take other desperate measures ++ */ ++static inline struct page * ++__alloc_pages_high_priority(gfp_t gfp_mask, unsigned int order, ++ const struct alloc_context *ac) ++{ ++ struct page *page; ++ ++ do { ++ page = get_page_from_freelist(gfp_mask, order, ++ ALLOC_NO_WATERMARKS, ac); ++ ++ if (!page && gfp_mask & __GFP_NOFAIL) ++ wait_iff_congested(ac->preferred_zone, BLK_RW_ASYNC, ++ HZ/50); ++ } while (!page && (gfp_mask & __GFP_NOFAIL)); ++ ++ return page; ++} ++ ++static void wake_all_kswapds(unsigned int order, const struct alloc_context *ac) ++{ ++ struct zoneref *z; ++ struct zone *zone; ++ ++ for_each_zone_zonelist_nodemask(zone, z, ac->zonelist, ++ ac->high_zoneidx, ac->nodemask) ++ wakeup_kswapd(zone, order, zone_idx(ac->preferred_zone)); ++} ++ ++static inline int ++gfp_to_alloc_flags(gfp_t gfp_mask) ++{ ++ int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET; ++ const bool atomic = !(gfp_mask & (__GFP_WAIT | __GFP_NO_KSWAPD)); ++ ++ /* __GFP_HIGH is assumed to be the same as ALLOC_HIGH to save a branch. */ ++ BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_HIGH); ++ ++ /* ++ * The caller may dip into page reserves a bit more if the caller ++ * cannot run direct reclaim, or if the caller has realtime scheduling ++ * policy or is asking for __GFP_HIGH memory. GFP_ATOMIC requests will ++ * set both ALLOC_HARDER (atomic == true) and ALLOC_HIGH (__GFP_HIGH). ++ */ ++ alloc_flags |= (__force int) (gfp_mask & __GFP_HIGH); ++ ++ if (atomic) { ++ /* ++ * Not worth trying to allocate harder for __GFP_NOMEMALLOC even ++ * if it can't schedule. ++ */ ++ if (!(gfp_mask & __GFP_NOMEMALLOC)) ++ alloc_flags |= ALLOC_HARDER; ++ /* ++ * Ignore cpuset mems for GFP_ATOMIC rather than fail, see the ++ * comment for __cpuset_node_allowed(). ++ */ ++ alloc_flags &= ~ALLOC_CPUSET; ++ } else if (unlikely(rt_task(current)) && !in_interrupt()) ++ alloc_flags |= ALLOC_HARDER; ++ ++ if (likely(!(gfp_mask & __GFP_NOMEMALLOC))) { ++ if (gfp_mask & __GFP_MEMALLOC) ++ alloc_flags |= ALLOC_NO_WATERMARKS; ++ else if (in_serving_softirq() && (current->flags & PF_MEMALLOC)) ++ alloc_flags |= ALLOC_NO_WATERMARKS; ++ else if (!in_interrupt() && ++ ((current->flags & PF_MEMALLOC) || ++ unlikely(test_thread_flag(TIF_MEMDIE)))) ++ alloc_flags |= ALLOC_NO_WATERMARKS; ++ } ++#ifdef CONFIG_CMA ++ if (gfpflags_to_migratetype(gfp_mask) == MIGRATE_MOVABLE) ++ alloc_flags |= ALLOC_CMA; ++#endif ++ return alloc_flags; ++} ++ ++bool gfp_pfmemalloc_allowed(gfp_t gfp_mask) ++{ ++ return !!(gfp_to_alloc_flags(gfp_mask) & ALLOC_NO_WATERMARKS); ++} ++ ++static inline struct page * ++__alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order, ++ struct alloc_context *ac) ++{ ++ const gfp_t wait = gfp_mask & __GFP_WAIT; ++ struct page *page = NULL; ++ int alloc_flags; ++ unsigned long pages_reclaimed = 0; ++ unsigned long did_some_progress; ++ enum migrate_mode migration_mode = MIGRATE_ASYNC; ++ bool deferred_compaction = false; ++ int contended_compaction = COMPACT_CONTENDED_NONE; ++ ++ /* ++ * In the slowpath, we sanity check order to avoid ever trying to ++ * reclaim >= MAX_ORDER areas which will never succeed. Callers may ++ * be using allocators in order of preference for an area that is ++ * too large. ++ */ ++ if (order >= MAX_ORDER) { ++ WARN_ON_ONCE(!(gfp_mask & __GFP_NOWARN)); ++ return NULL; ++ } ++ ++ /* ++ * If this allocation cannot block and it is for a specific node, then ++ * fail early. There's no need to wakeup kswapd or retry for a ++ * speculative node-specific allocation. ++ */ ++ if (IS_ENABLED(CONFIG_NUMA) && (gfp_mask & __GFP_THISNODE) && !wait) ++ goto nopage; ++ ++retry: ++ if (!(gfp_mask & __GFP_NO_KSWAPD)) ++ wake_all_kswapds(order, ac); ++ ++ /* ++ * OK, we're below the kswapd watermark and have kicked background ++ * reclaim. Now things get more complex, so set up alloc_flags according ++ * to how we want to proceed. ++ */ ++ alloc_flags = gfp_to_alloc_flags(gfp_mask); ++ ++ /* ++ * Find the true preferred zone if the allocation is unconstrained by ++ * cpusets. ++ */ ++ if (!(alloc_flags & ALLOC_CPUSET) && !ac->nodemask) { ++ struct zoneref *preferred_zoneref; ++ preferred_zoneref = first_zones_zonelist(ac->zonelist, ++ ac->high_zoneidx, NULL, &ac->preferred_zone); ++ ac->classzone_idx = zonelist_zone_idx(preferred_zoneref); ++ } ++ ++ /* This is the last chance, in general, before the goto nopage. */ ++ page = get_page_from_freelist(gfp_mask, order, ++ alloc_flags & ~ALLOC_NO_WATERMARKS, ac); ++ if (page) ++ goto got_pg; ++ ++ /* Allocate without watermarks if the context allows */ ++ if (alloc_flags & ALLOC_NO_WATERMARKS) { ++ /* ++ * Ignore mempolicies if ALLOC_NO_WATERMARKS on the grounds ++ * the allocation is high priority and these type of ++ * allocations are system rather than user orientated ++ */ ++ ac->zonelist = node_zonelist(numa_node_id(), gfp_mask); ++ ++ page = __alloc_pages_high_priority(gfp_mask, order, ac); ++ ++ if (page) { ++ goto got_pg; ++ } ++ } ++ ++ /* Atomic allocations - we can't balance anything */ ++ if (!wait) { ++ /* ++ * All existing users of the deprecated __GFP_NOFAIL are ++ * blockable, so warn of any new users that actually allow this ++ * type of allocation to fail. ++ */ ++ WARN_ON_ONCE(gfp_mask & __GFP_NOFAIL); ++ goto nopage; ++ } ++ ++ /* Avoid recursion of direct reclaim */ ++ if (current->flags & PF_MEMALLOC) ++ goto nopage; ++ ++ /* Avoid allocations with no watermarks from looping endlessly */ ++ if (test_thread_flag(TIF_MEMDIE) && !(gfp_mask & __GFP_NOFAIL)) ++ goto nopage; ++ ++ /* ++ * Try direct compaction. The first pass is asynchronous. Subsequent ++ * attempts after direct reclaim are synchronous ++ */ ++ page = __alloc_pages_direct_compact(gfp_mask, order, alloc_flags, ac, ++ migration_mode, ++ &contended_compaction, ++ &deferred_compaction); ++ if (page) ++ goto got_pg; ++ ++ /* Checks for THP-specific high-order allocations */ ++ if ((gfp_mask & GFP_TRANSHUGE) == GFP_TRANSHUGE) { ++ /* ++ * If compaction is deferred for high-order allocations, it is ++ * because sync compaction recently failed. If this is the case ++ * and the caller requested a THP allocation, we do not want ++ * to heavily disrupt the system, so we fail the allocation ++ * instead of entering direct reclaim. ++ */ ++ if (deferred_compaction) ++ goto nopage; ++ ++ /* ++ * In all zones where compaction was attempted (and not ++ * deferred or skipped), lock contention has been detected. ++ * For THP allocation we do not want to disrupt the others ++ * so we fallback to base pages instead. ++ */ ++ if (contended_compaction == COMPACT_CONTENDED_LOCK) ++ goto nopage; ++ ++ /* ++ * If compaction was aborted due to need_resched(), we do not ++ * want to further increase allocation latency, unless it is ++ * khugepaged trying to collapse. ++ */ ++ if (contended_compaction == COMPACT_CONTENDED_SCHED ++ && !(current->flags & PF_KTHREAD)) ++ goto nopage; ++ } ++ ++ /* ++ * It can become very expensive to allocate transparent hugepages at ++ * fault, so use asynchronous memory compaction for THP unless it is ++ * khugepaged trying to collapse. ++ */ ++ if ((gfp_mask & GFP_TRANSHUGE) != GFP_TRANSHUGE || ++ (current->flags & PF_KTHREAD)) ++ migration_mode = MIGRATE_SYNC_LIGHT; ++ ++ /* Try direct reclaim and then allocating */ ++ page = __alloc_pages_direct_reclaim(gfp_mask, order, alloc_flags, ac, ++ &did_some_progress); ++ if (page) ++ goto got_pg; ++ ++ /* Check if we should retry the allocation */ ++ pages_reclaimed += did_some_progress; ++ if (should_alloc_retry(gfp_mask, order, did_some_progress, ++ pages_reclaimed)) { ++ /* ++ * If we fail to make progress by freeing individual ++ * pages, but the allocation wants us to keep going, ++ * start OOM killing tasks. ++ */ ++ if (!did_some_progress) { ++ page = __alloc_pages_may_oom(gfp_mask, order, ac, ++ &did_some_progress); ++ if (page) ++ goto got_pg; ++ if (!did_some_progress) ++ goto nopage; ++ } ++ /* Wait for some write requests to complete then retry */ ++ wait_iff_congested(ac->preferred_zone, BLK_RW_ASYNC, HZ/50); ++ goto retry; ++ } else { ++ /* ++ * High-order allocations do not necessarily loop after ++ * direct reclaim and reclaim/compaction depends on compaction ++ * being called after reclaim so call directly if necessary ++ */ ++ page = __alloc_pages_direct_compact(gfp_mask, order, ++ alloc_flags, ac, migration_mode, ++ &contended_compaction, ++ &deferred_compaction); ++ if (page) ++ goto got_pg; ++ } ++ ++nopage: ++ warn_alloc_failed(gfp_mask, order, NULL); ++got_pg: ++ return page; ++} ++ ++/* ++ * This is the 'heart' of the zoned buddy allocator. ++ */ ++struct page * ++__alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order, ++ struct zonelist *zonelist, nodemask_t *nodemask) ++{ ++ struct zoneref *preferred_zoneref; ++ struct page *page = NULL; ++ unsigned int cpuset_mems_cookie; ++ int alloc_flags = ALLOC_WMARK_LOW|ALLOC_CPUSET|ALLOC_FAIR; ++ gfp_t alloc_mask; /* The gfp_t that was actually used for allocation */ ++ struct alloc_context ac = { ++ .high_zoneidx = gfp_zone(gfp_mask), ++ .nodemask = nodemask, ++ .migratetype = gfpflags_to_migratetype(gfp_mask), ++ }; ++ ++ gfp_mask &= gfp_allowed_mask; ++ ++ lockdep_trace_alloc(gfp_mask); ++ ++ might_sleep_if(gfp_mask & __GFP_WAIT); ++ ++ if (should_fail_alloc_page(gfp_mask, order)) ++ return NULL; ++ ++ /* ++ * Check the zones suitable for the gfp_mask contain at least one ++ * valid zone. It's possible to have an empty zonelist as a result ++ * of __GFP_THISNODE and a memoryless node ++ */ ++ if (unlikely(!zonelist->_zonerefs->zone)) ++ return NULL; ++ ++ if (IS_ENABLED(CONFIG_CMA) && ac.migratetype == MIGRATE_MOVABLE) ++ alloc_flags |= ALLOC_CMA; ++ ++retry_cpuset: ++ cpuset_mems_cookie = read_mems_allowed_begin(); ++ ++ /* We set it here, as __alloc_pages_slowpath might have changed it */ ++ ac.zonelist = zonelist; ++ /* The preferred zone is used for statistics later */ ++ preferred_zoneref = first_zones_zonelist(ac.zonelist, ac.high_zoneidx, ++ ac.nodemask ? : &cpuset_current_mems_allowed, ++ &ac.preferred_zone); ++ if (!ac.preferred_zone) ++ goto out; ++ ac.classzone_idx = zonelist_zone_idx(preferred_zoneref); ++ ++ /* First allocation attempt */ ++ alloc_mask = gfp_mask|__GFP_HARDWALL; ++ page = get_page_from_freelist(alloc_mask, order, alloc_flags, &ac); ++ if (unlikely(!page)) { ++ /* ++ * Runtime PM, block IO and its error handling path ++ * can deadlock because I/O on the device might not ++ * complete. ++ */ ++ alloc_mask = memalloc_noio_flags(gfp_mask); ++ ++ page = __alloc_pages_slowpath(alloc_mask, order, &ac); ++ } ++ ++ if (kmemcheck_enabled && page) ++ kmemcheck_pagealloc_alloc(page, order, gfp_mask); ++ ++ trace_mm_page_alloc(page, order, alloc_mask, ac.migratetype); ++ ++out: ++ /* ++ * When updating a task's mems_allowed, it is possible to race with ++ * parallel threads in such a way that an allocation can fail while ++ * the mask is being updated. If a page allocation is about to fail, ++ * check if the cpuset changed during allocation and if so, retry. ++ */ ++ if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie))) ++ goto retry_cpuset; ++ ++ return page; ++} ++EXPORT_SYMBOL(__alloc_pages_nodemask); ++ ++/* ++ * Common helper functions. ++ */ ++unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order) ++{ ++ struct page *page; ++ ++ /* ++ * __get_free_pages() returns a 32-bit address, which cannot represent ++ * a highmem page ++ */ ++ VM_BUG_ON((gfp_mask & __GFP_HIGHMEM) != 0); ++ ++ page = alloc_pages(gfp_mask, order); ++ if (!page) ++ return 0; ++ return (unsigned long) page_address(page); ++} ++EXPORT_SYMBOL(__get_free_pages); ++ ++unsigned long get_zeroed_page(gfp_t gfp_mask) ++{ ++ return __get_free_pages(gfp_mask | __GFP_ZERO, 0); ++} ++EXPORT_SYMBOL(get_zeroed_page); ++ ++void __free_pages(struct page *page, unsigned int order) ++{ ++ if (put_page_testzero(page)) { ++ if (order == 0) ++ free_hot_cold_page(page, false); ++ else ++ __free_pages_ok(page, order); ++ } ++} ++ ++EXPORT_SYMBOL(__free_pages); ++ ++void free_pages(unsigned long addr, unsigned int order) ++{ ++ if (addr != 0) { ++ VM_BUG_ON(!virt_addr_valid((void *)addr)); ++ __free_pages(virt_to_page((void *)addr), order); ++ } ++} ++ ++EXPORT_SYMBOL(free_pages); ++ ++/* ++ * alloc_kmem_pages charges newly allocated pages to the kmem resource counter ++ * of the current memory cgroup. ++ * ++ * It should be used when the caller would like to use kmalloc, but since the ++ * allocation is large, it has to fall back to the page allocator. ++ */ ++struct page *alloc_kmem_pages(gfp_t gfp_mask, unsigned int order) ++{ ++ struct page *page; ++ struct mem_cgroup *memcg = NULL; ++ ++ if (!memcg_kmem_newpage_charge(gfp_mask, &memcg, order)) ++ return NULL; ++ page = alloc_pages(gfp_mask, order); ++ memcg_kmem_commit_charge(page, memcg, order); ++ return page; ++} ++ ++struct page *alloc_kmem_pages_node(int nid, gfp_t gfp_mask, unsigned int order) ++{ ++ struct page *page; ++ struct mem_cgroup *memcg = NULL; ++ ++ if (!memcg_kmem_newpage_charge(gfp_mask, &memcg, order)) ++ return NULL; ++ page = alloc_pages_node(nid, gfp_mask, order); ++ memcg_kmem_commit_charge(page, memcg, order); ++ return page; ++} ++ ++/* ++ * __free_kmem_pages and free_kmem_pages will free pages allocated with ++ * alloc_kmem_pages. ++ */ ++void __free_kmem_pages(struct page *page, unsigned int order) ++{ ++ memcg_kmem_uncharge_pages(page, order); ++ __free_pages(page, order); ++} ++ ++void free_kmem_pages(unsigned long addr, unsigned int order) ++{ ++ if (addr != 0) { ++ VM_BUG_ON(!virt_addr_valid((void *)addr)); ++ __free_kmem_pages(virt_to_page((void *)addr), order); ++ } ++} ++ ++static void *make_alloc_exact(unsigned long addr, unsigned order, size_t size) ++{ ++ if (addr) { ++ unsigned long alloc_end = addr + (PAGE_SIZE << order); ++ unsigned long used = addr + PAGE_ALIGN(size); ++ ++ split_page(virt_to_page((void *)addr), order); ++ while (used < alloc_end) { ++ free_page(used); ++ used += PAGE_SIZE; ++ } ++ } ++ return (void *)addr; ++} ++ ++/** ++ * alloc_pages_exact - allocate an exact number physically-contiguous pages. ++ * @size: the number of bytes to allocate ++ * @gfp_mask: GFP flags for the allocation ++ * ++ * This function is similar to alloc_pages(), except that it allocates the ++ * minimum number of pages to satisfy the request. alloc_pages() can only ++ * allocate memory in power-of-two pages. ++ * ++ * This function is also limited by MAX_ORDER. ++ * ++ * Memory allocated by this function must be released by free_pages_exact(). ++ */ ++void *alloc_pages_exact(size_t size, gfp_t gfp_mask) ++{ ++ unsigned int order = get_order(size); ++ unsigned long addr; ++ ++ addr = __get_free_pages(gfp_mask, order); ++ return make_alloc_exact(addr, order, size); ++} ++EXPORT_SYMBOL(alloc_pages_exact); ++ ++/** ++ * alloc_pages_exact_nid - allocate an exact number of physically-contiguous ++ * pages on a node. ++ * @nid: the preferred node ID where memory should be allocated ++ * @size: the number of bytes to allocate ++ * @gfp_mask: GFP flags for the allocation ++ * ++ * Like alloc_pages_exact(), but try to allocate on node nid first before falling ++ * back. ++ * Note this is not alloc_pages_exact_node() which allocates on a specific node, ++ * but is not exact. ++ */ ++void * __meminit alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask) ++{ ++ unsigned order = get_order(size); ++ struct page *p = alloc_pages_node(nid, gfp_mask, order); ++ if (!p) ++ return NULL; ++ return make_alloc_exact((unsigned long)page_address(p), order, size); ++} ++ ++/** ++ * free_pages_exact - release memory allocated via alloc_pages_exact() ++ * @virt: the value returned by alloc_pages_exact. ++ * @size: size of allocation, same value as passed to alloc_pages_exact(). ++ * ++ * Release the memory allocated by a previous call to alloc_pages_exact. ++ */ ++void free_pages_exact(void *virt, size_t size) ++{ ++ unsigned long addr = (unsigned long)virt; ++ unsigned long end = addr + PAGE_ALIGN(size); ++ ++ while (addr < end) { ++ free_page(addr); ++ addr += PAGE_SIZE; ++ } ++} ++EXPORT_SYMBOL(free_pages_exact); ++ ++/** ++ * nr_free_zone_pages - count number of pages beyond high watermark ++ * @offset: The zone index of the highest zone ++ * ++ * nr_free_zone_pages() counts the number of counts pages which are beyond the ++ * high watermark within all zones at or below a given zone index. For each ++ * zone, the number of pages is calculated as: ++ * managed_pages - high_pages ++ */ ++static unsigned long nr_free_zone_pages(int offset) ++{ ++ struct zoneref *z; ++ struct zone *zone; ++ ++ /* Just pick one node, since fallback list is circular */ ++ unsigned long sum = 0; ++ ++ struct zonelist *zonelist = node_zonelist(numa_node_id(), GFP_KERNEL); ++ ++ for_each_zone_zonelist(zone, z, zonelist, offset) { ++ unsigned long size = zone->managed_pages; ++ unsigned long high = high_wmark_pages(zone); ++ if (size > high) ++ sum += size - high; ++ } ++ ++ return sum; ++} ++ ++/** ++ * nr_free_buffer_pages - count number of pages beyond high watermark ++ * ++ * nr_free_buffer_pages() counts the number of pages which are beyond the high ++ * watermark within ZONE_DMA and ZONE_NORMAL. ++ */ ++unsigned long nr_free_buffer_pages(void) ++{ ++ return nr_free_zone_pages(gfp_zone(GFP_USER)); ++} ++EXPORT_SYMBOL_GPL(nr_free_buffer_pages); ++ ++/** ++ * nr_free_pagecache_pages - count number of pages beyond high watermark ++ * ++ * nr_free_pagecache_pages() counts the number of pages which are beyond the ++ * high watermark within all zones. ++ */ ++unsigned long nr_free_pagecache_pages(void) ++{ ++ return nr_free_zone_pages(gfp_zone(GFP_HIGHUSER_MOVABLE)); ++} ++ ++static inline void show_node(struct zone *zone) ++{ ++ if (IS_ENABLED(CONFIG_NUMA)) ++ printk("Node %d ", zone_to_nid(zone)); ++} ++ ++void si_meminfo(struct sysinfo *val) ++{ ++ val->totalram = totalram_pages; ++ val->sharedram = global_page_state(NR_SHMEM); ++ val->freeram = global_page_state(NR_FREE_PAGES); ++ val->bufferram = nr_blockdev_pages(); ++ val->totalhigh = totalhigh_pages; ++ val->freehigh = nr_free_highpages(); ++ val->mem_unit = PAGE_SIZE; ++} ++ ++EXPORT_SYMBOL(si_meminfo); ++ ++#ifdef CONFIG_NUMA ++void si_meminfo_node(struct sysinfo *val, int nid) ++{ ++ int zone_type; /* needs to be signed */ ++ unsigned long managed_pages = 0; ++ pg_data_t *pgdat = NODE_DATA(nid); ++ ++ for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) ++ managed_pages += pgdat->node_zones[zone_type].managed_pages; ++ val->totalram = managed_pages; ++ val->sharedram = node_page_state(nid, NR_SHMEM); ++ val->freeram = node_page_state(nid, NR_FREE_PAGES); ++#ifdef CONFIG_HIGHMEM ++ val->totalhigh = pgdat->node_zones[ZONE_HIGHMEM].managed_pages; ++ val->freehigh = zone_page_state(&pgdat->node_zones[ZONE_HIGHMEM], ++ NR_FREE_PAGES); ++#else ++ val->totalhigh = 0; ++ val->freehigh = 0; ++#endif ++ val->mem_unit = PAGE_SIZE; ++} ++#endif ++ ++/* ++ * Determine whether the node should be displayed or not, depending on whether ++ * SHOW_MEM_FILTER_NODES was passed to show_free_areas(). ++ */ ++bool skip_free_areas_node(unsigned int flags, int nid) ++{ ++ bool ret = false; ++ unsigned int cpuset_mems_cookie; ++ ++ if (!(flags & SHOW_MEM_FILTER_NODES)) ++ goto out; ++ ++ do { ++ cpuset_mems_cookie = read_mems_allowed_begin(); ++ ret = !node_isset(nid, cpuset_current_mems_allowed); ++ } while (read_mems_allowed_retry(cpuset_mems_cookie)); ++out: ++ return ret; ++} ++ ++#define K(x) ((x) << (PAGE_SHIFT-10)) ++ ++static void show_migration_types(unsigned char type) ++{ ++ static const char types[MIGRATE_TYPES] = { ++ [MIGRATE_UNMOVABLE] = 'U', ++ [MIGRATE_RECLAIMABLE] = 'E', ++ [MIGRATE_MOVABLE] = 'M', ++ [MIGRATE_RESERVE] = 'R', ++#ifdef CONFIG_CMA ++ [MIGRATE_CMA] = 'C', ++#endif ++#ifdef CONFIG_MEMORY_ISOLATION ++ [MIGRATE_ISOLATE] = 'I', ++#endif ++ }; ++ char tmp[MIGRATE_TYPES + 1]; ++ char *p = tmp; ++ int i; ++ ++ for (i = 0; i < MIGRATE_TYPES; i++) { ++ if (type & (1 << i)) ++ *p++ = types[i]; ++ } ++ ++ *p = '\0'; ++ printk("(%s) ", tmp); ++} ++ ++/* ++ * Show free area list (used inside shift_scroll-lock stuff) ++ * We also calculate the percentage fragmentation. We do this by counting the ++ * memory on each free list with the exception of the first item on the list. ++ * ++ * Bits in @filter: ++ * SHOW_MEM_FILTER_NODES: suppress nodes that are not allowed by current's ++ * cpuset. ++ */ ++void show_free_areas(unsigned int filter) ++{ ++ unsigned long free_pcp = 0; ++ int cpu; ++ struct zone *zone; ++ ++ for_each_populated_zone(zone) { ++ if (skip_free_areas_node(filter, zone_to_nid(zone))) ++ continue; ++ ++ for_each_online_cpu(cpu) ++ free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count; ++ } ++ ++ printk("active_anon:%lu inactive_anon:%lu isolated_anon:%lu\n" ++ " active_file:%lu inactive_file:%lu isolated_file:%lu\n" ++ " unevictable:%lu dirty:%lu writeback:%lu unstable:%lu\n" ++ " slab_reclaimable:%lu slab_unreclaimable:%lu\n" ++ " mapped:%lu shmem:%lu pagetables:%lu bounce:%lu\n" ++ " free:%lu free_pcp:%lu free_cma:%lu\n", ++ global_page_state(NR_ACTIVE_ANON), ++ global_page_state(NR_INACTIVE_ANON), ++ global_page_state(NR_ISOLATED_ANON), ++ global_page_state(NR_ACTIVE_FILE), ++ global_page_state(NR_INACTIVE_FILE), ++ global_page_state(NR_ISOLATED_FILE), ++ global_page_state(NR_UNEVICTABLE), ++ global_page_state(NR_FILE_DIRTY), ++ global_page_state(NR_WRITEBACK), ++ global_page_state(NR_UNSTABLE_NFS), ++ global_page_state(NR_SLAB_RECLAIMABLE), ++ global_page_state(NR_SLAB_UNRECLAIMABLE), ++ global_page_state(NR_FILE_MAPPED), ++ global_page_state(NR_SHMEM), ++ global_page_state(NR_PAGETABLE), ++ global_page_state(NR_BOUNCE), ++ global_page_state(NR_FREE_PAGES), ++ free_pcp, ++ global_page_state(NR_FREE_CMA_PAGES)); ++ ++ for_each_populated_zone(zone) { ++ int i; ++ ++ if (skip_free_areas_node(filter, zone_to_nid(zone))) ++ continue; ++ ++ free_pcp = 0; ++ for_each_online_cpu(cpu) ++ free_pcp += per_cpu_ptr(zone->pageset, cpu)->pcp.count; ++ ++ show_node(zone); ++ printk("%s" ++ " free:%lukB" ++ " min:%lukB" ++ " low:%lukB" ++ " high:%lukB" ++ " active_anon:%lukB" ++ " inactive_anon:%lukB" ++ " active_file:%lukB" ++ " inactive_file:%lukB" ++ " unevictable:%lukB" ++ " isolated(anon):%lukB" ++ " isolated(file):%lukB" ++ " present:%lukB" ++ " managed:%lukB" ++ " mlocked:%lukB" ++ " dirty:%lukB" ++ " writeback:%lukB" ++ " mapped:%lukB" ++ " shmem:%lukB" ++ " slab_reclaimable:%lukB" ++ " slab_unreclaimable:%lukB" ++ " kernel_stack:%lukB" ++ " pagetables:%lukB" ++ " unstable:%lukB" ++ " bounce:%lukB" ++ " free_pcp:%lukB" ++ " local_pcp:%ukB" ++ " free_cma:%lukB" ++ " writeback_tmp:%lukB" ++ " pages_scanned:%lu" ++ " all_unreclaimable? %s" ++ "\n", ++ zone->name, ++ K(zone_page_state(zone, NR_FREE_PAGES)), ++ K(min_wmark_pages(zone)), ++ K(low_wmark_pages(zone)), ++ K(high_wmark_pages(zone)), ++ K(zone_page_state(zone, NR_ACTIVE_ANON)), ++ K(zone_page_state(zone, NR_INACTIVE_ANON)), ++ K(zone_page_state(zone, NR_ACTIVE_FILE)), ++ K(zone_page_state(zone, NR_INACTIVE_FILE)), ++ K(zone_page_state(zone, NR_UNEVICTABLE)), ++ K(zone_page_state(zone, NR_ISOLATED_ANON)), ++ K(zone_page_state(zone, NR_ISOLATED_FILE)), ++ K(zone->present_pages), ++ K(zone->managed_pages), ++ K(zone_page_state(zone, NR_MLOCK)), ++ K(zone_page_state(zone, NR_FILE_DIRTY)), ++ K(zone_page_state(zone, NR_WRITEBACK)), ++ K(zone_page_state(zone, NR_FILE_MAPPED)), ++ K(zone_page_state(zone, NR_SHMEM)), ++ K(zone_page_state(zone, NR_SLAB_RECLAIMABLE)), ++ K(zone_page_state(zone, NR_SLAB_UNRECLAIMABLE)), ++ zone_page_state(zone, NR_KERNEL_STACK) * ++ THREAD_SIZE / 1024, ++ K(zone_page_state(zone, NR_PAGETABLE)), ++ K(zone_page_state(zone, NR_UNSTABLE_NFS)), ++ K(zone_page_state(zone, NR_BOUNCE)), ++ K(free_pcp), ++ K(this_cpu_read(zone->pageset->pcp.count)), ++ K(zone_page_state(zone, NR_FREE_CMA_PAGES)), ++ K(zone_page_state(zone, NR_WRITEBACK_TEMP)), ++ K(zone_page_state(zone, NR_PAGES_SCANNED)), ++ (!zone_reclaimable(zone) ? "yes" : "no") ++ ); ++ printk("lowmem_reserve[]:"); ++ for (i = 0; i < MAX_NR_ZONES; i++) ++ printk(" %ld", zone->lowmem_reserve[i]); ++ printk("\n"); ++ } ++ ++ for_each_populated_zone(zone) { ++ unsigned long nr[MAX_ORDER], flags, order, total = 0; ++ unsigned char types[MAX_ORDER]; ++ ++ if (skip_free_areas_node(filter, zone_to_nid(zone))) ++ continue; ++ show_node(zone); ++ printk("%s: ", zone->name); ++ ++ spin_lock_irqsave(&zone->lock, flags); ++ for (order = 0; order < MAX_ORDER; order++) { ++ struct free_area *area = &zone->free_area[order]; ++ int type; ++ ++ nr[order] = area->nr_free; ++ total += nr[order] << order; ++ ++ types[order] = 0; ++ for (type = 0; type < MIGRATE_TYPES; type++) { ++ if (!list_empty(&area->free_list[type])) ++ types[order] |= 1 << type; ++ } ++ } ++ spin_unlock_irqrestore(&zone->lock, flags); ++ for (order = 0; order < MAX_ORDER; order++) { ++ printk("%lu*%lukB ", nr[order], K(1UL) << order); ++ if (nr[order]) ++ show_migration_types(types[order]); ++ } ++ printk("= %lukB\n", K(total)); ++ } ++ ++ hugetlb_show_meminfo(); ++ ++ printk("%ld total pagecache pages\n", global_page_state(NR_FILE_PAGES)); ++ ++ show_swap_cache_info(); ++} ++ ++static void zoneref_set_zone(struct zone *zone, struct zoneref *zoneref) ++{ ++ zoneref->zone = zone; ++ zoneref->zone_idx = zone_idx(zone); ++} ++ ++/* ++ * Builds allocation fallback zone lists. ++ * ++ * Add all populated zones of a node to the zonelist. ++ */ ++static int build_zonelists_node(pg_data_t *pgdat, struct zonelist *zonelist, ++ int nr_zones) ++{ ++ struct zone *zone; ++ enum zone_type zone_type = MAX_NR_ZONES; ++ ++ do { ++ zone_type--; ++ zone = pgdat->node_zones + zone_type; ++ if (populated_zone(zone)) { ++ zoneref_set_zone(zone, ++ &zonelist->_zonerefs[nr_zones++]); ++ check_highest_zone(zone_type); ++ } ++ } while (zone_type); ++ ++ return nr_zones; ++} ++ ++ ++/* ++ * zonelist_order: ++ * 0 = automatic detection of better ordering. ++ * 1 = order by ([node] distance, -zonetype) ++ * 2 = order by (-zonetype, [node] distance) ++ * ++ * If not NUMA, ZONELIST_ORDER_ZONE and ZONELIST_ORDER_NODE will create ++ * the same zonelist. So only NUMA can configure this param. ++ */ ++#define ZONELIST_ORDER_DEFAULT 0 ++#define ZONELIST_ORDER_NODE 1 ++#define ZONELIST_ORDER_ZONE 2 ++ ++/* zonelist order in the kernel. ++ * set_zonelist_order() will set this to NODE or ZONE. ++ */ ++static int current_zonelist_order = ZONELIST_ORDER_DEFAULT; ++static char zonelist_order_name[3][8] = {"Default", "Node", "Zone"}; ++ ++ ++#ifdef CONFIG_NUMA ++/* The value user specified ....changed by config */ ++static int user_zonelist_order = ZONELIST_ORDER_DEFAULT; ++/* string for sysctl */ ++#define NUMA_ZONELIST_ORDER_LEN 16 ++char numa_zonelist_order[16] = "default"; ++ ++/* ++ * interface for configure zonelist ordering. ++ * command line option "numa_zonelist_order" ++ * = "[dD]efault - default, automatic configuration. ++ * = "[nN]ode - order by node locality, then by zone within node ++ * = "[zZ]one - order by zone, then by locality within zone ++ */ ++ ++static int __parse_numa_zonelist_order(char *s) ++{ ++ if (*s == 'd' || *s == 'D') { ++ user_zonelist_order = ZONELIST_ORDER_DEFAULT; ++ } else if (*s == 'n' || *s == 'N') { ++ user_zonelist_order = ZONELIST_ORDER_NODE; ++ } else if (*s == 'z' || *s == 'Z') { ++ user_zonelist_order = ZONELIST_ORDER_ZONE; ++ } else { ++ printk(KERN_WARNING ++ "Ignoring invalid numa_zonelist_order value: " ++ "%s\n", s); ++ return -EINVAL; ++ } ++ return 0; ++} ++ ++static __init int setup_numa_zonelist_order(char *s) ++{ ++ int ret; ++ ++ if (!s) ++ return 0; ++ ++ ret = __parse_numa_zonelist_order(s); ++ if (ret == 0) ++ strlcpy(numa_zonelist_order, s, NUMA_ZONELIST_ORDER_LEN); ++ ++ return ret; ++} ++early_param("numa_zonelist_order", setup_numa_zonelist_order); ++ ++/* ++ * sysctl handler for numa_zonelist_order ++ */ ++int numa_zonelist_order_handler(struct ctl_table *table, int write, ++ void __user *buffer, size_t *length, ++ loff_t *ppos) ++{ ++ char saved_string[NUMA_ZONELIST_ORDER_LEN]; ++ int ret; ++ static DEFINE_MUTEX(zl_order_mutex); ++ ++ mutex_lock(&zl_order_mutex); ++ if (write) { ++ if (strlen((char *)table->data) >= NUMA_ZONELIST_ORDER_LEN) { ++ ret = -EINVAL; ++ goto out; ++ } ++ strcpy(saved_string, (char *)table->data); ++ } ++ ret = proc_dostring(table, write, buffer, length, ppos); ++ if (ret) ++ goto out; ++ if (write) { ++ int oldval = user_zonelist_order; ++ ++ ret = __parse_numa_zonelist_order((char *)table->data); ++ if (ret) { ++ /* ++ * bogus value. restore saved string ++ */ ++ strncpy((char *)table->data, saved_string, ++ NUMA_ZONELIST_ORDER_LEN); ++ user_zonelist_order = oldval; ++ } else if (oldval != user_zonelist_order) { ++ mutex_lock(&zonelists_mutex); ++ build_all_zonelists(NULL, NULL); ++ mutex_unlock(&zonelists_mutex); ++ } ++ } ++out: ++ mutex_unlock(&zl_order_mutex); ++ return ret; ++} ++ ++ ++#define MAX_NODE_LOAD (nr_online_nodes) ++static int node_load[MAX_NUMNODES]; ++ ++/** ++ * find_next_best_node - find the next node that should appear in a given node's fallback list ++ * @node: node whose fallback list we're appending ++ * @used_node_mask: nodemask_t of already used nodes ++ * ++ * We use a number of factors to determine which is the next node that should ++ * appear on a given node's fallback list. The node should not have appeared ++ * already in @node's fallback list, and it should be the next closest node ++ * according to the distance array (which contains arbitrary distance values ++ * from each node to each node in the system), and should also prefer nodes ++ * with no CPUs, since presumably they'll have very little allocation pressure ++ * on them otherwise. ++ * It returns -1 if no node is found. ++ */ ++static int find_next_best_node(int node, nodemask_t *used_node_mask) ++{ ++ int n, val; ++ int min_val = INT_MAX; ++ int best_node = NUMA_NO_NODE; ++ const struct cpumask *tmp = cpumask_of_node(0); ++ ++ /* Use the local node if we haven't already */ ++ if (!node_isset(node, *used_node_mask)) { ++ node_set(node, *used_node_mask); ++ return node; ++ } ++ ++ for_each_node_state(n, N_MEMORY) { ++ ++ /* Don't want a node to appear more than once */ ++ if (node_isset(n, *used_node_mask)) ++ continue; ++ ++ /* Use the distance array to find the distance */ ++ val = node_distance(node, n); ++ ++ /* Penalize nodes under us ("prefer the next node") */ ++ val += (n < node); ++ ++ /* Give preference to headless and unused nodes */ ++ tmp = cpumask_of_node(n); ++ if (!cpumask_empty(tmp)) ++ val += PENALTY_FOR_NODE_WITH_CPUS; ++ ++ /* Slight preference for less loaded node */ ++ val *= (MAX_NODE_LOAD*MAX_NUMNODES); ++ val += node_load[n]; ++ ++ if (val < min_val) { ++ min_val = val; ++ best_node = n; ++ } ++ } ++ ++ if (best_node >= 0) ++ node_set(best_node, *used_node_mask); ++ ++ return best_node; ++} ++ ++ ++/* ++ * Build zonelists ordered by node and zones within node. ++ * This results in maximum locality--normal zone overflows into local ++ * DMA zone, if any--but risks exhausting DMA zone. ++ */ ++static void build_zonelists_in_node_order(pg_data_t *pgdat, int node) ++{ ++ int j; ++ struct zonelist *zonelist; ++ ++ zonelist = &pgdat->node_zonelists[0]; ++ for (j = 0; zonelist->_zonerefs[j].zone != NULL; j++) ++ ; ++ j = build_zonelists_node(NODE_DATA(node), zonelist, j); ++ zonelist->_zonerefs[j].zone = NULL; ++ zonelist->_zonerefs[j].zone_idx = 0; ++} ++ ++/* ++ * Build gfp_thisnode zonelists ++ */ ++static void build_thisnode_zonelists(pg_data_t *pgdat) ++{ ++ int j; ++ struct zonelist *zonelist; ++ ++ zonelist = &pgdat->node_zonelists[1]; ++ j = build_zonelists_node(pgdat, zonelist, 0); ++ zonelist->_zonerefs[j].zone = NULL; ++ zonelist->_zonerefs[j].zone_idx = 0; ++} ++ ++/* ++ * Build zonelists ordered by zone and nodes within zones. ++ * This results in conserving DMA zone[s] until all Normal memory is ++ * exhausted, but results in overflowing to remote node while memory ++ * may still exist in local DMA zone. ++ */ ++static int node_order[MAX_NUMNODES]; ++ ++static void build_zonelists_in_zone_order(pg_data_t *pgdat, int nr_nodes) ++{ ++ int pos, j, node; ++ int zone_type; /* needs to be signed */ ++ struct zone *z; ++ struct zonelist *zonelist; ++ ++ zonelist = &pgdat->node_zonelists[0]; ++ pos = 0; ++ for (zone_type = MAX_NR_ZONES - 1; zone_type >= 0; zone_type--) { ++ for (j = 0; j < nr_nodes; j++) { ++ node = node_order[j]; ++ z = &NODE_DATA(node)->node_zones[zone_type]; ++ if (populated_zone(z)) { ++ zoneref_set_zone(z, ++ &zonelist->_zonerefs[pos++]); ++ check_highest_zone(zone_type); ++ } ++ } ++ } ++ zonelist->_zonerefs[pos].zone = NULL; ++ zonelist->_zonerefs[pos].zone_idx = 0; ++} ++ ++#if defined(CONFIG_64BIT) ++/* ++ * Devices that require DMA32/DMA are relatively rare and do not justify a ++ * penalty to every machine in case the specialised case applies. Default ++ * to Node-ordering on 64-bit NUMA machines ++ */ ++static int default_zonelist_order(void) ++{ ++ return ZONELIST_ORDER_NODE; ++} ++#else ++/* ++ * On 32-bit, the Normal zone needs to be preserved for allocations accessible ++ * by the kernel. If processes running on node 0 deplete the low memory zone ++ * then reclaim will occur more frequency increasing stalls and potentially ++ * be easier to OOM if a large percentage of the zone is under writeback or ++ * dirty. The problem is significantly worse if CONFIG_HIGHPTE is not set. ++ * Hence, default to zone ordering on 32-bit. ++ */ ++static int default_zonelist_order(void) ++{ ++ return ZONELIST_ORDER_ZONE; ++} ++#endif /* CONFIG_64BIT */ ++ ++static void set_zonelist_order(void) ++{ ++ if (user_zonelist_order == ZONELIST_ORDER_DEFAULT) ++ current_zonelist_order = default_zonelist_order(); ++ else ++ current_zonelist_order = user_zonelist_order; ++} ++ ++static void build_zonelists(pg_data_t *pgdat) ++{ ++ int j, node, load; ++ enum zone_type i; ++ nodemask_t used_mask; ++ int local_node, prev_node; ++ struct zonelist *zonelist; ++ int order = current_zonelist_order; ++ ++ /* initialize zonelists */ ++ for (i = 0; i < MAX_ZONELISTS; i++) { ++ zonelist = pgdat->node_zonelists + i; ++ zonelist->_zonerefs[0].zone = NULL; ++ zonelist->_zonerefs[0].zone_idx = 0; ++ } ++ ++ /* NUMA-aware ordering of nodes */ ++ local_node = pgdat->node_id; ++ load = nr_online_nodes; ++ prev_node = local_node; ++ nodes_clear(used_mask); ++ ++ memset(node_order, 0, sizeof(node_order)); ++ j = 0; ++ ++ while ((node = find_next_best_node(local_node, &used_mask)) >= 0) { ++ /* ++ * We don't want to pressure a particular node. ++ * So adding penalty to the first node in same ++ * distance group to make it round-robin. ++ */ ++ if (node_distance(local_node, node) != ++ node_distance(local_node, prev_node)) ++ node_load[node] = load; ++ ++ prev_node = node; ++ load--; ++ if (order == ZONELIST_ORDER_NODE) ++ build_zonelists_in_node_order(pgdat, node); ++ else ++ node_order[j++] = node; /* remember order */ ++ } ++ ++ if (order == ZONELIST_ORDER_ZONE) { ++ /* calculate node order -- i.e., DMA last! */ ++ build_zonelists_in_zone_order(pgdat, j); ++ } ++ ++ build_thisnode_zonelists(pgdat); ++} ++ ++/* Construct the zonelist performance cache - see further mmzone.h */ ++static void build_zonelist_cache(pg_data_t *pgdat) ++{ ++ struct zonelist *zonelist; ++ struct zonelist_cache *zlc; ++ struct zoneref *z; ++ ++ zonelist = &pgdat->node_zonelists[0]; ++ zonelist->zlcache_ptr = zlc = &zonelist->zlcache; ++ bitmap_zero(zlc->fullzones, MAX_ZONES_PER_ZONELIST); ++ for (z = zonelist->_zonerefs; z->zone; z++) ++ zlc->z_to_n[z - zonelist->_zonerefs] = zonelist_node_idx(z); ++} ++ ++#ifdef CONFIG_HAVE_MEMORYLESS_NODES ++/* ++ * Return node id of node used for "local" allocations. ++ * I.e., first node id of first zone in arg node's generic zonelist. ++ * Used for initializing percpu 'numa_mem', which is used primarily ++ * for kernel allocations, so use GFP_KERNEL flags to locate zonelist. ++ */ ++int local_memory_node(int node) ++{ ++ struct zone *zone; ++ ++ (void)first_zones_zonelist(node_zonelist(node, GFP_KERNEL), ++ gfp_zone(GFP_KERNEL), ++ NULL, ++ &zone); ++ return zone->node; ++} ++#endif ++ ++#else /* CONFIG_NUMA */ ++ ++static void set_zonelist_order(void) ++{ ++ current_zonelist_order = ZONELIST_ORDER_ZONE; ++} ++ ++static void build_zonelists(pg_data_t *pgdat) ++{ ++ int node, local_node; ++ enum zone_type j; ++ struct zonelist *zonelist; ++ ++ local_node = pgdat->node_id; ++ ++ zonelist = &pgdat->node_zonelists[0]; ++ j = build_zonelists_node(pgdat, zonelist, 0); ++ ++ /* ++ * Now we build the zonelist so that it contains the zones ++ * of all the other nodes. ++ * We don't want to pressure a particular node, so when ++ * building the zones for node N, we make sure that the ++ * zones coming right after the local ones are those from ++ * node N+1 (modulo N) ++ */ ++ for (node = local_node + 1; node < MAX_NUMNODES; node++) { ++ if (!node_online(node)) ++ continue; ++ j = build_zonelists_node(NODE_DATA(node), zonelist, j); ++ } ++ for (node = 0; node < local_node; node++) { ++ if (!node_online(node)) ++ continue; ++ j = build_zonelists_node(NODE_DATA(node), zonelist, j); ++ } ++ ++ zonelist->_zonerefs[j].zone = NULL; ++ zonelist->_zonerefs[j].zone_idx = 0; ++} ++ ++/* non-NUMA variant of zonelist performance cache - just NULL zlcache_ptr */ ++static void build_zonelist_cache(pg_data_t *pgdat) ++{ ++ pgdat->node_zonelists[0].zlcache_ptr = NULL; ++} ++ ++#endif /* CONFIG_NUMA */ ++ ++/* ++ * Boot pageset table. One per cpu which is going to be used for all ++ * zones and all nodes. The parameters will be set in such a way ++ * that an item put on a list will immediately be handed over to ++ * the buddy list. This is safe since pageset manipulation is done ++ * with interrupts disabled. ++ * ++ * The boot_pagesets must be kept even after bootup is complete for ++ * unused processors and/or zones. They do play a role for bootstrapping ++ * hotplugged processors. ++ * ++ * zoneinfo_show() and maybe other functions do ++ * not check if the processor is online before following the pageset pointer. ++ * Other parts of the kernel may not check if the zone is available. ++ */ ++static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch); ++static DEFINE_PER_CPU(struct per_cpu_pageset, boot_pageset); ++static void setup_zone_pageset(struct zone *zone); ++ ++/* ++ * Global mutex to protect against size modification of zonelists ++ * as well as to serialize pageset setup for the new populated zone. ++ */ ++DEFINE_MUTEX(zonelists_mutex); ++ ++/* return values int ....just for stop_machine() */ ++static int __build_all_zonelists(void *data) ++{ ++ int nid; ++ int cpu; ++ pg_data_t *self = data; ++ ++#ifdef CONFIG_NUMA ++ memset(node_load, 0, sizeof(node_load)); ++#endif ++ ++ if (self && !node_online(self->node_id)) { ++ build_zonelists(self); ++ build_zonelist_cache(self); ++ } ++ ++ for_each_online_node(nid) { ++ pg_data_t *pgdat = NODE_DATA(nid); ++ ++ build_zonelists(pgdat); ++ build_zonelist_cache(pgdat); ++ } ++ ++ /* ++ * Initialize the boot_pagesets that are going to be used ++ * for bootstrapping processors. The real pagesets for ++ * each zone will be allocated later when the per cpu ++ * allocator is available. ++ * ++ * boot_pagesets are used also for bootstrapping offline ++ * cpus if the system is already booted because the pagesets ++ * are needed to initialize allocators on a specific cpu too. ++ * F.e. the percpu allocator needs the page allocator which ++ * needs the percpu allocator in order to allocate its pagesets ++ * (a chicken-egg dilemma). ++ */ ++ for_each_possible_cpu(cpu) { ++ setup_pageset(&per_cpu(boot_pageset, cpu), 0); ++ ++#ifdef CONFIG_HAVE_MEMORYLESS_NODES ++ /* ++ * We now know the "local memory node" for each node-- ++ * i.e., the node of the first zone in the generic zonelist. ++ * Set up numa_mem percpu variable for on-line cpus. During ++ * boot, only the boot cpu should be on-line; we'll init the ++ * secondary cpus' numa_mem as they come on-line. During ++ * node/memory hotplug, we'll fixup all on-line cpus. ++ */ ++ if (cpu_online(cpu)) ++ set_cpu_numa_mem(cpu, local_memory_node(cpu_to_node(cpu))); ++#endif ++ } ++ ++ return 0; ++} ++ ++static noinline void __init ++build_all_zonelists_init(void) ++{ ++ __build_all_zonelists(NULL); ++ mminit_verify_zonelist(); ++ cpuset_init_current_mems_allowed(); ++} ++ ++/* ++ * Called with zonelists_mutex held always ++ * unless system_state == SYSTEM_BOOTING. ++ * ++ * __ref due to (1) call of __meminit annotated setup_zone_pageset ++ * [we're only called with non-NULL zone through __meminit paths] and ++ * (2) call of __init annotated helper build_all_zonelists_init ++ * [protected by SYSTEM_BOOTING]. ++ */ ++void __ref build_all_zonelists(pg_data_t *pgdat, struct zone *zone) ++{ ++ set_zonelist_order(); ++ ++ if (system_state == SYSTEM_BOOTING) { ++ build_all_zonelists_init(); ++ } else { ++#ifdef CONFIG_MEMORY_HOTPLUG ++ if (zone) ++ setup_zone_pageset(zone); ++#endif ++ /* we have to stop all cpus to guarantee there is no user ++ of zonelist */ ++ stop_machine(__build_all_zonelists, pgdat, NULL); ++ /* cpuset refresh routine should be here */ ++ } ++ vm_total_pages = nr_free_pagecache_pages(); ++ /* ++ * Disable grouping by mobility if the number of pages in the ++ * system is too low to allow the mechanism to work. It would be ++ * more accurate, but expensive to check per-zone. This check is ++ * made on memory-hotadd so a system can start with mobility ++ * disabled and enable it later ++ */ ++ if (vm_total_pages < (pageblock_nr_pages * MIGRATE_TYPES)) ++ page_group_by_mobility_disabled = 1; ++ else ++ page_group_by_mobility_disabled = 0; ++ ++ pr_info("Built %i zonelists in %s order, mobility grouping %s. " ++ "Total pages: %ld\n", ++ nr_online_nodes, ++ zonelist_order_name[current_zonelist_order], ++ page_group_by_mobility_disabled ? "off" : "on", ++ vm_total_pages); ++#ifdef CONFIG_NUMA ++ pr_info("Policy zone: %s\n", zone_names[policy_zone]); ++#endif ++} ++ ++/* ++ * Helper functions to size the waitqueue hash table. ++ * Essentially these want to choose hash table sizes sufficiently ++ * large so that collisions trying to wait on pages are rare. ++ * But in fact, the number of active page waitqueues on typical ++ * systems is ridiculously low, less than 200. So this is even ++ * conservative, even though it seems large. ++ * ++ * The constant PAGES_PER_WAITQUEUE specifies the ratio of pages to ++ * waitqueues, i.e. the size of the waitq table given the number of pages. ++ */ ++#define PAGES_PER_WAITQUEUE 256 ++ ++#ifndef CONFIG_MEMORY_HOTPLUG ++static inline unsigned long wait_table_hash_nr_entries(unsigned long pages) ++{ ++ unsigned long size = 1; ++ ++ pages /= PAGES_PER_WAITQUEUE; ++ ++ while (size < pages) ++ size <<= 1; ++ ++ /* ++ * Once we have dozens or even hundreds of threads sleeping ++ * on IO we've got bigger problems than wait queue collision. ++ * Limit the size of the wait table to a reasonable size. ++ */ ++ size = min(size, 4096UL); ++ ++ return max(size, 4UL); ++} ++#else ++/* ++ * A zone's size might be changed by hot-add, so it is not possible to determine ++ * a suitable size for its wait_table. So we use the maximum size now. ++ * ++ * The max wait table size = 4096 x sizeof(wait_queue_head_t). ie: ++ * ++ * i386 (preemption config) : 4096 x 16 = 64Kbyte. ++ * ia64, x86-64 (no preemption): 4096 x 20 = 80Kbyte. ++ * ia64, x86-64 (preemption) : 4096 x 24 = 96Kbyte. ++ * ++ * The maximum entries are prepared when a zone's memory is (512K + 256) pages ++ * or more by the traditional way. (See above). It equals: ++ * ++ * i386, x86-64, powerpc(4K page size) : = ( 2G + 1M)byte. ++ * ia64(16K page size) : = ( 8G + 4M)byte. ++ * powerpc (64K page size) : = (32G +16M)byte. ++ */ ++static inline unsigned long wait_table_hash_nr_entries(unsigned long pages) ++{ ++ return 4096UL; ++} ++#endif ++ ++/* ++ * This is an integer logarithm so that shifts can be used later ++ * to extract the more random high bits from the multiplicative ++ * hash function before the remainder is taken. ++ */ ++static inline unsigned long wait_table_bits(unsigned long size) ++{ ++ return ffz(~size); ++} ++ ++/* ++ * Check if a pageblock contains reserved pages ++ */ ++static int pageblock_is_reserved(unsigned long start_pfn, unsigned long end_pfn) ++{ ++ unsigned long pfn; ++ ++ for (pfn = start_pfn; pfn < end_pfn; pfn++) { ++ if (!pfn_valid_within(pfn) || PageReserved(pfn_to_page(pfn))) ++ return 1; ++ } ++ return 0; ++} ++ ++/* ++ * Mark a number of pageblocks as MIGRATE_RESERVE. The number ++ * of blocks reserved is based on min_wmark_pages(zone). The memory within ++ * the reserve will tend to store contiguous free pages. Setting min_free_kbytes ++ * higher will lead to a bigger reserve which will get freed as contiguous ++ * blocks as reclaim kicks in ++ */ ++static void setup_zone_migrate_reserve(struct zone *zone) ++{ ++ unsigned long start_pfn, pfn, end_pfn, block_end_pfn; ++ struct page *page; ++ unsigned long block_migratetype; ++ int reserve; ++ int old_reserve; ++ ++ /* ++ * Get the start pfn, end pfn and the number of blocks to reserve ++ * We have to be careful to be aligned to pageblock_nr_pages to ++ * make sure that we always check pfn_valid for the first page in ++ * the block. ++ */ ++ start_pfn = zone->zone_start_pfn; ++ end_pfn = zone_end_pfn(zone); ++ start_pfn = roundup(start_pfn, pageblock_nr_pages); ++ reserve = roundup(min_wmark_pages(zone), pageblock_nr_pages) >> ++ pageblock_order; ++ ++ /* ++ * Reserve blocks are generally in place to help high-order atomic ++ * allocations that are short-lived. A min_free_kbytes value that ++ * would result in more than 2 reserve blocks for atomic allocations ++ * is assumed to be in place to help anti-fragmentation for the ++ * future allocation of hugepages at runtime. ++ */ ++ reserve = min(2, reserve); ++ old_reserve = zone->nr_migrate_reserve_block; ++ ++ /* When memory hot-add, we almost always need to do nothing */ ++ if (reserve == old_reserve) ++ return; ++ zone->nr_migrate_reserve_block = reserve; ++ ++ for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) { ++ if (!pfn_valid(pfn)) ++ continue; ++ page = pfn_to_page(pfn); ++ ++ /* Watch out for overlapping nodes */ ++ if (page_to_nid(page) != zone_to_nid(zone)) ++ continue; ++ ++ block_migratetype = get_pageblock_migratetype(page); ++ ++ /* Only test what is necessary when the reserves are not met */ ++ if (reserve > 0) { ++ /* ++ * Blocks with reserved pages will never free, skip ++ * them. ++ */ ++ block_end_pfn = min(pfn + pageblock_nr_pages, end_pfn); ++ if (pageblock_is_reserved(pfn, block_end_pfn)) ++ continue; ++ ++ /* If this block is reserved, account for it */ ++ if (block_migratetype == MIGRATE_RESERVE) { ++ reserve--; ++ continue; ++ } ++ ++ /* Suitable for reserving if this block is movable */ ++ if (block_migratetype == MIGRATE_MOVABLE) { ++ set_pageblock_migratetype(page, ++ MIGRATE_RESERVE); ++ move_freepages_block(zone, page, ++ MIGRATE_RESERVE); ++ reserve--; ++ continue; ++ } ++ } else if (!old_reserve) { ++ /* ++ * At boot time we don't need to scan the whole zone ++ * for turning off MIGRATE_RESERVE. ++ */ ++ break; ++ } ++ ++ /* ++ * If the reserve is met and this is a previous reserved block, ++ * take it back ++ */ ++ if (block_migratetype == MIGRATE_RESERVE) { ++ set_pageblock_migratetype(page, MIGRATE_MOVABLE); ++ move_freepages_block(zone, page, MIGRATE_MOVABLE); ++ } ++ } ++} ++ ++/* ++ * Initially all pages are reserved - free ones are freed ++ * up by free_all_bootmem() once the early boot process is ++ * done. Non-atomic initialization, single-pass. ++ */ ++void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone, ++ unsigned long start_pfn, enum memmap_context context) ++{ ++ struct page *page; ++ unsigned long end_pfn = start_pfn + size; ++ unsigned long pfn; ++ struct zone *z; ++ ++ if (highest_memmap_pfn < end_pfn - 1) ++ highest_memmap_pfn = end_pfn - 1; ++ ++ z = &NODE_DATA(nid)->node_zones[zone]; ++ for (pfn = start_pfn; pfn < end_pfn; pfn++) { ++ /* ++ * There can be holes in boot-time mem_map[]s ++ * handed to this function. They do not ++ * exist on hotplugged memory. ++ */ ++ if (context == MEMMAP_EARLY) { ++ if (!early_pfn_valid(pfn)) ++ continue; ++ if (!early_pfn_in_nid(pfn, nid)) ++ continue; ++ } ++ page = pfn_to_page(pfn); ++ set_page_links(page, zone, nid, pfn); ++ mminit_verify_page_links(page, zone, nid, pfn); ++ init_page_count(page); ++ page_mapcount_reset(page); ++ page_cpupid_reset_last(page); ++ SetPageReserved(page); ++ /* ++ * Mark the block movable so that blocks are reserved for ++ * movable at startup. This will force kernel allocations ++ * to reserve their blocks rather than leaking throughout ++ * the address space during boot when many long-lived ++ * kernel allocations are made. Later some blocks near ++ * the start are marked MIGRATE_RESERVE by ++ * setup_zone_migrate_reserve() ++ * ++ * bitmap is created for zone's valid pfn range. but memmap ++ * can be created for invalid pages (for alignment) ++ * check here not to call set_pageblock_migratetype() against ++ * pfn out of zone. ++ */ ++ if ((z->zone_start_pfn <= pfn) ++ && (pfn < zone_end_pfn(z)) ++ && !(pfn & (pageblock_nr_pages - 1))) ++ set_pageblock_migratetype(page, MIGRATE_MOVABLE); ++ ++ INIT_LIST_HEAD(&page->lru); ++#ifdef WANT_PAGE_VIRTUAL ++ /* The shift won't overflow because ZONE_NORMAL is below 4G. */ ++ if (!is_highmem_idx(zone)) ++ set_page_address(page, __va(pfn << PAGE_SHIFT)); ++#endif ++ } ++} ++ ++static void __meminit zone_init_free_lists(struct zone *zone) ++{ ++ unsigned int order, t; ++ for_each_migratetype_order(order, t) { ++ INIT_LIST_HEAD(&zone->free_area[order].free_list[t]); ++ zone->free_area[order].nr_free = 0; ++ } ++} ++ ++#ifndef __HAVE_ARCH_MEMMAP_INIT ++#define memmap_init(size, nid, zone, start_pfn) \ ++ memmap_init_zone((size), (nid), (zone), (start_pfn), MEMMAP_EARLY) ++#endif ++ ++static int zone_batchsize(struct zone *zone) ++{ ++#ifdef CONFIG_MMU ++ int batch; ++ ++ /* ++ * The per-cpu-pages pools are set to around 1000th of the ++ * size of the zone. But no more than 1/2 of a meg. ++ * ++ * OK, so we don't know how big the cache is. So guess. ++ */ ++ batch = zone->managed_pages / 1024; ++ if (batch * PAGE_SIZE > 512 * 1024) ++ batch = (512 * 1024) / PAGE_SIZE; ++ batch /= 4; /* We effectively *= 4 below */ ++ if (batch < 1) ++ batch = 1; ++ ++ /* ++ * Clamp the batch to a 2^n - 1 value. Having a power ++ * of 2 value was found to be more likely to have ++ * suboptimal cache aliasing properties in some cases. ++ * ++ * For example if 2 tasks are alternately allocating ++ * batches of pages, one task can end up with a lot ++ * of pages of one half of the possible page colors ++ * and the other with pages of the other colors. ++ */ ++ batch = rounddown_pow_of_two(batch + batch/2) - 1; ++ ++ return batch; ++ ++#else ++ /* The deferral and batching of frees should be suppressed under NOMMU ++ * conditions. ++ * ++ * The problem is that NOMMU needs to be able to allocate large chunks ++ * of contiguous memory as there's no hardware page translation to ++ * assemble apparent contiguous memory from discontiguous pages. ++ * ++ * Queueing large contiguous runs of pages for batching, however, ++ * causes the pages to actually be freed in smaller chunks. As there ++ * can be a significant delay between the individual batches being ++ * recycled, this leads to the once large chunks of space being ++ * fragmented and becoming unavailable for high-order allocations. ++ */ ++ return 0; ++#endif ++} ++ ++/* ++ * pcp->high and pcp->batch values are related and dependent on one another: ++ * ->batch must never be higher then ->high. ++ * The following function updates them in a safe manner without read side ++ * locking. ++ * ++ * Any new users of pcp->batch and pcp->high should ensure they can cope with ++ * those fields changing asynchronously (acording the the above rule). ++ * ++ * mutex_is_locked(&pcp_batch_high_lock) required when calling this function ++ * outside of boot time (or some other assurance that no concurrent updaters ++ * exist). ++ */ ++static void pageset_update(struct per_cpu_pages *pcp, unsigned long high, ++ unsigned long batch) ++{ ++ /* start with a fail safe value for batch */ ++ pcp->batch = 1; ++ smp_wmb(); ++ ++ /* Update high, then batch, in order */ ++ pcp->high = high; ++ smp_wmb(); ++ ++ pcp->batch = batch; ++} ++ ++/* a companion to pageset_set_high() */ ++static void pageset_set_batch(struct per_cpu_pageset *p, unsigned long batch) ++{ ++ pageset_update(&p->pcp, 6 * batch, max(1UL, 1 * batch)); ++} ++ ++static void pageset_init(struct per_cpu_pageset *p) ++{ ++ struct per_cpu_pages *pcp; ++ int migratetype; ++ ++ memset(p, 0, sizeof(*p)); ++ ++ pcp = &p->pcp; ++ pcp->count = 0; ++ for (migratetype = 0; migratetype < MIGRATE_PCPTYPES; migratetype++) ++ INIT_LIST_HEAD(&pcp->lists[migratetype]); ++} ++ ++static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch) ++{ ++ pageset_init(p); ++ pageset_set_batch(p, batch); ++} ++ ++/* ++ * pageset_set_high() sets the high water mark for hot per_cpu_pagelist ++ * to the value high for the pageset p. ++ */ ++static void pageset_set_high(struct per_cpu_pageset *p, ++ unsigned long high) ++{ ++ unsigned long batch = max(1UL, high / 4); ++ if ((high / 4) > (PAGE_SHIFT * 8)) ++ batch = PAGE_SHIFT * 8; ++ ++ pageset_update(&p->pcp, high, batch); ++} ++ ++static void pageset_set_high_and_batch(struct zone *zone, ++ struct per_cpu_pageset *pcp) ++{ ++ if (percpu_pagelist_fraction) ++ pageset_set_high(pcp, ++ (zone->managed_pages / ++ percpu_pagelist_fraction)); ++ else ++ pageset_set_batch(pcp, zone_batchsize(zone)); ++} ++ ++static void __meminit zone_pageset_init(struct zone *zone, int cpu) ++{ ++ struct per_cpu_pageset *pcp = per_cpu_ptr(zone->pageset, cpu); ++ ++ pageset_init(pcp); ++ pageset_set_high_and_batch(zone, pcp); ++} ++ ++static void __meminit setup_zone_pageset(struct zone *zone) ++{ ++ int cpu; ++ zone->pageset = alloc_percpu(struct per_cpu_pageset); ++ for_each_possible_cpu(cpu) ++ zone_pageset_init(zone, cpu); ++} ++ ++/* ++ * Allocate per cpu pagesets and initialize them. ++ * Before this call only boot pagesets were available. ++ */ ++void __init setup_per_cpu_pageset(void) ++{ ++ struct zone *zone; ++ ++ for_each_populated_zone(zone) ++ setup_zone_pageset(zone); ++} ++ ++static noinline __init_refok ++int zone_wait_table_init(struct zone *zone, unsigned long zone_size_pages) ++{ ++ int i; ++ size_t alloc_size; ++ ++ /* ++ * The per-page waitqueue mechanism uses hashed waitqueues ++ * per zone. ++ */ ++ zone->wait_table_hash_nr_entries = ++ wait_table_hash_nr_entries(zone_size_pages); ++ zone->wait_table_bits = ++ wait_table_bits(zone->wait_table_hash_nr_entries); ++ alloc_size = zone->wait_table_hash_nr_entries ++ * sizeof(wait_queue_head_t); ++ ++ if (!slab_is_available()) { ++ zone->wait_table = (wait_queue_head_t *) ++ memblock_virt_alloc_node_nopanic( ++ alloc_size, zone->zone_pgdat->node_id); ++ } else { ++ /* ++ * This case means that a zone whose size was 0 gets new memory ++ * via memory hot-add. ++ * But it may be the case that a new node was hot-added. In ++ * this case vmalloc() will not be able to use this new node's ++ * memory - this wait_table must be initialized to use this new ++ * node itself as well. ++ * To use this new node's memory, further consideration will be ++ * necessary. ++ */ ++ zone->wait_table = vmalloc(alloc_size); ++ } ++ if (!zone->wait_table) ++ return -ENOMEM; ++ ++ for (i = 0; i < zone->wait_table_hash_nr_entries; ++i) ++ init_waitqueue_head(zone->wait_table + i); ++ ++ return 0; ++} ++ ++static __meminit void zone_pcp_init(struct zone *zone) ++{ ++ /* ++ * per cpu subsystem is not up at this point. The following code ++ * relies on the ability of the linker to provide the ++ * offset of a (static) per cpu variable into the per cpu area. ++ */ ++ zone->pageset = &boot_pageset; ++ ++ if (populated_zone(zone)) ++ printk(KERN_DEBUG " %s zone: %lu pages, LIFO batch:%u\n", ++ zone->name, zone->present_pages, ++ zone_batchsize(zone)); ++} ++ ++int __meminit init_currently_empty_zone(struct zone *zone, ++ unsigned long zone_start_pfn, ++ unsigned long size, ++ enum memmap_context context) ++{ ++ struct pglist_data *pgdat = zone->zone_pgdat; ++ int ret; ++ ret = zone_wait_table_init(zone, size); ++ if (ret) ++ return ret; ++ pgdat->nr_zones = zone_idx(zone) + 1; ++ ++ zone->zone_start_pfn = zone_start_pfn; ++ ++ mminit_dprintk(MMINIT_TRACE, "memmap_init", ++ "Initialising map node %d zone %lu pfns %lu -> %lu\n", ++ pgdat->node_id, ++ (unsigned long)zone_idx(zone), ++ zone_start_pfn, (zone_start_pfn + size)); ++ ++ zone_init_free_lists(zone); ++ ++ return 0; ++} ++ ++#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP ++#ifndef CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID ++/* ++ * Required by SPARSEMEM. Given a PFN, return what node the PFN is on. ++ */ ++int __meminit __early_pfn_to_nid(unsigned long pfn) ++{ ++ unsigned long start_pfn, end_pfn; ++ int nid; ++ /* ++ * NOTE: The following SMP-unsafe globals are only used early in boot ++ * when the kernel is running single-threaded. ++ */ ++ static unsigned long __meminitdata last_start_pfn, last_end_pfn; ++ static int __meminitdata last_nid; ++ ++ if (last_start_pfn <= pfn && pfn < last_end_pfn) ++ return last_nid; ++ ++ nid = memblock_search_pfn_nid(pfn, &start_pfn, &end_pfn); ++ if (nid != -1) { ++ last_start_pfn = start_pfn; ++ last_end_pfn = end_pfn; ++ last_nid = nid; ++ } ++ ++ return nid; ++} ++#endif /* CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID */ ++ ++int __meminit early_pfn_to_nid(unsigned long pfn) ++{ ++ int nid; ++ ++ nid = __early_pfn_to_nid(pfn); ++ if (nid >= 0) ++ return nid; ++ /* just returns 0 */ ++ return 0; ++} ++ ++#ifdef CONFIG_NODES_SPAN_OTHER_NODES ++bool __meminit early_pfn_in_nid(unsigned long pfn, int node) ++{ ++ int nid; ++ ++ nid = __early_pfn_to_nid(pfn); ++ if (nid >= 0 && nid != node) ++ return false; ++ return true; ++} ++#endif ++ ++/** ++ * free_bootmem_with_active_regions - Call memblock_free_early_nid for each active range ++ * @nid: The node to free memory on. If MAX_NUMNODES, all nodes are freed. ++ * @max_low_pfn: The highest PFN that will be passed to memblock_free_early_nid ++ * ++ * If an architecture guarantees that all ranges registered contain no holes ++ * and may be freed, this this function may be used instead of calling ++ * memblock_free_early_nid() manually. ++ */ ++void __init free_bootmem_with_active_regions(int nid, unsigned long max_low_pfn) ++{ ++ unsigned long start_pfn, end_pfn; ++ int i, this_nid; ++ ++ for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, &this_nid) { ++ start_pfn = min(start_pfn, max_low_pfn); ++ end_pfn = min(end_pfn, max_low_pfn); ++ ++ if (start_pfn < end_pfn) ++ memblock_free_early_nid(PFN_PHYS(start_pfn), ++ (end_pfn - start_pfn) << PAGE_SHIFT, ++ this_nid); ++ } ++} ++ ++/** ++ * sparse_memory_present_with_active_regions - Call memory_present for each active range ++ * @nid: The node to call memory_present for. If MAX_NUMNODES, all nodes will be used. ++ * ++ * If an architecture guarantees that all ranges registered contain no holes and may ++ * be freed, this function may be used instead of calling memory_present() manually. ++ */ ++void __init sparse_memory_present_with_active_regions(int nid) ++{ ++ unsigned long start_pfn, end_pfn; ++ int i, this_nid; ++ ++ for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, &this_nid) ++ memory_present(this_nid, start_pfn, end_pfn); ++} ++ ++/** ++ * get_pfn_range_for_nid - Return the start and end page frames for a node ++ * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned. ++ * @start_pfn: Passed by reference. On return, it will have the node start_pfn. ++ * @end_pfn: Passed by reference. On return, it will have the node end_pfn. ++ * ++ * It returns the start and end page frame of a node based on information ++ * provided by memblock_set_node(). If called for a node ++ * with no available memory, a warning is printed and the start and end ++ * PFNs will be 0. ++ */ ++void __meminit get_pfn_range_for_nid(unsigned int nid, ++ unsigned long *start_pfn, unsigned long *end_pfn) ++{ ++ unsigned long this_start_pfn, this_end_pfn; ++ int i; ++ ++ *start_pfn = -1UL; ++ *end_pfn = 0; ++ ++ for_each_mem_pfn_range(i, nid, &this_start_pfn, &this_end_pfn, NULL) { ++ *start_pfn = min(*start_pfn, this_start_pfn); ++ *end_pfn = max(*end_pfn, this_end_pfn); ++ } ++ ++ if (*start_pfn == -1UL) ++ *start_pfn = 0; ++} ++ ++/* ++ * This finds a zone that can be used for ZONE_MOVABLE pages. The ++ * assumption is made that zones within a node are ordered in monotonic ++ * increasing memory addresses so that the "highest" populated zone is used ++ */ ++static void __init find_usable_zone_for_movable(void) ++{ ++ int zone_index; ++ for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) { ++ if (zone_index == ZONE_MOVABLE) ++ continue; ++ ++ if (arch_zone_highest_possible_pfn[zone_index] > ++ arch_zone_lowest_possible_pfn[zone_index]) ++ break; ++ } ++ ++ VM_BUG_ON(zone_index == -1); ++ movable_zone = zone_index; ++} ++ ++/* ++ * The zone ranges provided by the architecture do not include ZONE_MOVABLE ++ * because it is sized independent of architecture. Unlike the other zones, ++ * the starting point for ZONE_MOVABLE is not fixed. It may be different ++ * in each node depending on the size of each node and how evenly kernelcore ++ * is distributed. This helper function adjusts the zone ranges ++ * provided by the architecture for a given node by using the end of the ++ * highest usable zone for ZONE_MOVABLE. This preserves the assumption that ++ * zones within a node are in order of monotonic increases memory addresses ++ */ ++static void __meminit adjust_zone_range_for_zone_movable(int nid, ++ unsigned long zone_type, ++ unsigned long node_start_pfn, ++ unsigned long node_end_pfn, ++ unsigned long *zone_start_pfn, ++ unsigned long *zone_end_pfn) ++{ ++ /* Only adjust if ZONE_MOVABLE is on this node */ ++ if (zone_movable_pfn[nid]) { ++ /* Size ZONE_MOVABLE */ ++ if (zone_type == ZONE_MOVABLE) { ++ *zone_start_pfn = zone_movable_pfn[nid]; ++ *zone_end_pfn = min(node_end_pfn, ++ arch_zone_highest_possible_pfn[movable_zone]); ++ ++ /* Adjust for ZONE_MOVABLE starting within this range */ ++ } else if (*zone_start_pfn < zone_movable_pfn[nid] && ++ *zone_end_pfn > zone_movable_pfn[nid]) { ++ *zone_end_pfn = zone_movable_pfn[nid]; ++ ++ /* Check if this whole range is within ZONE_MOVABLE */ ++ } else if (*zone_start_pfn >= zone_movable_pfn[nid]) ++ *zone_start_pfn = *zone_end_pfn; ++ } ++} ++ ++/* ++ * Return the number of pages a zone spans in a node, including holes ++ * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node() ++ */ ++static unsigned long __meminit zone_spanned_pages_in_node(int nid, ++ unsigned long zone_type, ++ unsigned long node_start_pfn, ++ unsigned long node_end_pfn, ++ unsigned long *ignored) ++{ ++ unsigned long zone_start_pfn, zone_end_pfn; ++ ++ /* Get the start and end of the zone */ ++ zone_start_pfn = arch_zone_lowest_possible_pfn[zone_type]; ++ zone_end_pfn = arch_zone_highest_possible_pfn[zone_type]; ++ adjust_zone_range_for_zone_movable(nid, zone_type, ++ node_start_pfn, node_end_pfn, ++ &zone_start_pfn, &zone_end_pfn); ++ ++ /* Check that this node has pages within the zone's required range */ ++ if (zone_end_pfn < node_start_pfn || zone_start_pfn > node_end_pfn) ++ return 0; ++ ++ /* Move the zone boundaries inside the node if necessary */ ++ zone_end_pfn = min(zone_end_pfn, node_end_pfn); ++ zone_start_pfn = max(zone_start_pfn, node_start_pfn); ++ ++ /* Return the spanned pages */ ++ return zone_end_pfn - zone_start_pfn; ++} ++ ++/* ++ * Return the number of holes in a range on a node. If nid is MAX_NUMNODES, ++ * then all holes in the requested range will be accounted for. ++ */ ++unsigned long __meminit __absent_pages_in_range(int nid, ++ unsigned long range_start_pfn, ++ unsigned long range_end_pfn) ++{ ++ unsigned long nr_absent = range_end_pfn - range_start_pfn; ++ unsigned long start_pfn, end_pfn; ++ int i; ++ ++ for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { ++ start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn); ++ end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn); ++ nr_absent -= end_pfn - start_pfn; ++ } ++ return nr_absent; ++} ++ ++/** ++ * absent_pages_in_range - Return number of page frames in holes within a range ++ * @start_pfn: The start PFN to start searching for holes ++ * @end_pfn: The end PFN to stop searching for holes ++ * ++ * It returns the number of pages frames in memory holes within a range. ++ */ ++unsigned long __init absent_pages_in_range(unsigned long start_pfn, ++ unsigned long end_pfn) ++{ ++ return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn); ++} ++ ++/* Return the number of page frames in holes in a zone on a node */ ++static unsigned long __meminit zone_absent_pages_in_node(int nid, ++ unsigned long zone_type, ++ unsigned long node_start_pfn, ++ unsigned long node_end_pfn, ++ unsigned long *ignored) ++{ ++ unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type]; ++ unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type]; ++ unsigned long zone_start_pfn, zone_end_pfn; ++ ++ zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high); ++ zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high); ++ ++ adjust_zone_range_for_zone_movable(nid, zone_type, ++ node_start_pfn, node_end_pfn, ++ &zone_start_pfn, &zone_end_pfn); ++ return __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn); ++} ++ ++#else /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ ++static inline unsigned long __meminit zone_spanned_pages_in_node(int nid, ++ unsigned long zone_type, ++ unsigned long node_start_pfn, ++ unsigned long node_end_pfn, ++ unsigned long *zones_size) ++{ ++ return zones_size[zone_type]; ++} ++ ++static inline unsigned long __meminit zone_absent_pages_in_node(int nid, ++ unsigned long zone_type, ++ unsigned long node_start_pfn, ++ unsigned long node_end_pfn, ++ unsigned long *zholes_size) ++{ ++ if (!zholes_size) ++ return 0; ++ ++ return zholes_size[zone_type]; ++} ++ ++#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ ++ ++static void __meminit calculate_node_totalpages(struct pglist_data *pgdat, ++ unsigned long node_start_pfn, ++ unsigned long node_end_pfn, ++ unsigned long *zones_size, ++ unsigned long *zholes_size) ++{ ++ unsigned long realtotalpages, totalpages = 0; ++ enum zone_type i; ++ ++ for (i = 0; i < MAX_NR_ZONES; i++) ++ totalpages += zone_spanned_pages_in_node(pgdat->node_id, i, ++ node_start_pfn, ++ node_end_pfn, ++ zones_size); ++ pgdat->node_spanned_pages = totalpages; ++ ++ realtotalpages = totalpages; ++ for (i = 0; i < MAX_NR_ZONES; i++) ++ realtotalpages -= ++ zone_absent_pages_in_node(pgdat->node_id, i, ++ node_start_pfn, node_end_pfn, ++ zholes_size); ++ pgdat->node_present_pages = realtotalpages; ++ printk(KERN_DEBUG "On node %d totalpages: %lu\n", pgdat->node_id, ++ realtotalpages); ++} ++ ++#ifndef CONFIG_SPARSEMEM ++/* ++ * Calculate the size of the zone->blockflags rounded to an unsigned long ++ * Start by making sure zonesize is a multiple of pageblock_order by rounding ++ * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally ++ * round what is now in bits to nearest long in bits, then return it in ++ * bytes. ++ */ ++static unsigned long __init usemap_size(unsigned long zone_start_pfn, unsigned long zonesize) ++{ ++ unsigned long usemapsize; ++ ++ zonesize += zone_start_pfn & (pageblock_nr_pages-1); ++ usemapsize = roundup(zonesize, pageblock_nr_pages); ++ usemapsize = usemapsize >> pageblock_order; ++ usemapsize *= NR_PAGEBLOCK_BITS; ++ usemapsize = roundup(usemapsize, 8 * sizeof(unsigned long)); ++ ++ return usemapsize / 8; ++} ++ ++static void __init setup_usemap(struct pglist_data *pgdat, ++ struct zone *zone, ++ unsigned long zone_start_pfn, ++ unsigned long zonesize) ++{ ++ unsigned long usemapsize = usemap_size(zone_start_pfn, zonesize); ++ zone->pageblock_flags = NULL; ++ if (usemapsize) ++ zone->pageblock_flags = ++ memblock_virt_alloc_node_nopanic(usemapsize, ++ pgdat->node_id); ++} ++#else ++static inline void setup_usemap(struct pglist_data *pgdat, struct zone *zone, ++ unsigned long zone_start_pfn, unsigned long zonesize) {} ++#endif /* CONFIG_SPARSEMEM */ ++ ++#ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE ++ ++/* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */ ++void __paginginit set_pageblock_order(void) ++{ ++ unsigned int order; ++ ++ /* Check that pageblock_nr_pages has not already been setup */ ++ if (pageblock_order) ++ return; ++ ++ if (HPAGE_SHIFT > PAGE_SHIFT) ++ order = HUGETLB_PAGE_ORDER; ++ else ++ order = MAX_ORDER - 1; ++ ++ /* ++ * Assume the largest contiguous order of interest is a huge page. ++ * This value may be variable depending on boot parameters on IA64 and ++ * powerpc. ++ */ ++ pageblock_order = order; ++} ++#else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ ++ ++/* ++ * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order() ++ * is unused as pageblock_order is set at compile-time. See ++ * include/linux/pageblock-flags.h for the values of pageblock_order based on ++ * the kernel config ++ */ ++void __paginginit set_pageblock_order(void) ++{ ++} ++ ++#endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */ ++ ++static unsigned long __paginginit calc_memmap_size(unsigned long spanned_pages, ++ unsigned long present_pages) ++{ ++ unsigned long pages = spanned_pages; ++ ++ /* ++ * Provide a more accurate estimation if there are holes within ++ * the zone and SPARSEMEM is in use. If there are holes within the ++ * zone, each populated memory region may cost us one or two extra ++ * memmap pages due to alignment because memmap pages for each ++ * populated regions may not naturally algined on page boundary. ++ * So the (present_pages >> 4) heuristic is a tradeoff for that. ++ */ ++ if (spanned_pages > present_pages + (present_pages >> 4) && ++ IS_ENABLED(CONFIG_SPARSEMEM)) ++ pages = present_pages; ++ ++ return PAGE_ALIGN(pages * sizeof(struct page)) >> PAGE_SHIFT; ++} ++ ++/* ++ * Set up the zone data structures: ++ * - mark all pages reserved ++ * - mark all memory queues empty ++ * - clear the memory bitmaps ++ * ++ * NOTE: pgdat should get zeroed by caller. ++ */ ++static void __paginginit free_area_init_core(struct pglist_data *pgdat, ++ unsigned long node_start_pfn, unsigned long node_end_pfn, ++ unsigned long *zones_size, unsigned long *zholes_size) ++{ ++ enum zone_type j; ++ int nid = pgdat->node_id; ++ unsigned long zone_start_pfn = pgdat->node_start_pfn; ++ int ret; ++ ++ pgdat_resize_init(pgdat); ++#ifdef CONFIG_NUMA_BALANCING ++ spin_lock_init(&pgdat->numabalancing_migrate_lock); ++ pgdat->numabalancing_migrate_nr_pages = 0; ++ pgdat->numabalancing_migrate_next_window = jiffies; ++#endif ++ init_waitqueue_head(&pgdat->kswapd_wait); ++ init_waitqueue_head(&pgdat->pfmemalloc_wait); ++ pgdat_page_ext_init(pgdat); ++ ++ for (j = 0; j < MAX_NR_ZONES; j++) { ++ struct zone *zone = pgdat->node_zones + j; ++ unsigned long size, realsize, freesize, memmap_pages; ++ ++ size = zone_spanned_pages_in_node(nid, j, node_start_pfn, ++ node_end_pfn, zones_size); ++ realsize = freesize = size - zone_absent_pages_in_node(nid, j, ++ node_start_pfn, ++ node_end_pfn, ++ zholes_size); ++ ++ /* ++ * Adjust freesize so that it accounts for how much memory ++ * is used by this zone for memmap. This affects the watermark ++ * and per-cpu initialisations ++ */ ++ memmap_pages = calc_memmap_size(size, realsize); ++ if (!is_highmem_idx(j)) { ++ if (freesize >= memmap_pages) { ++ freesize -= memmap_pages; ++ if (memmap_pages) ++ printk(KERN_DEBUG ++ " %s zone: %lu pages used for memmap\n", ++ zone_names[j], memmap_pages); ++ } else ++ printk(KERN_WARNING ++ " %s zone: %lu pages exceeds freesize %lu\n", ++ zone_names[j], memmap_pages, freesize); ++ } ++ ++ /* Account for reserved pages */ ++ if (j == 0 && freesize > dma_reserve) { ++ freesize -= dma_reserve; ++ printk(KERN_DEBUG " %s zone: %lu pages reserved\n", ++ zone_names[0], dma_reserve); ++ } ++ ++ if (!is_highmem_idx(j)) ++ nr_kernel_pages += freesize; ++ /* Charge for highmem memmap if there are enough kernel pages */ ++ else if (nr_kernel_pages > memmap_pages * 2) ++ nr_kernel_pages -= memmap_pages; ++ nr_all_pages += freesize; ++ ++ zone->spanned_pages = size; ++ zone->present_pages = realsize; ++ /* ++ * Set an approximate value for lowmem here, it will be adjusted ++ * when the bootmem allocator frees pages into the buddy system. ++ * And all highmem pages will be managed by the buddy system. ++ */ ++ zone->managed_pages = is_highmem_idx(j) ? realsize : freesize; ++#ifdef CONFIG_NUMA ++ zone->node = nid; ++ zone->min_unmapped_pages = (freesize*sysctl_min_unmapped_ratio) ++ / 100; ++ zone->min_slab_pages = (freesize * sysctl_min_slab_ratio) / 100; ++#endif ++ zone->name = zone_names[j]; ++ spin_lock_init(&zone->lock); ++ spin_lock_init(&zone->lru_lock); ++ zone_seqlock_init(zone); ++ zone->zone_pgdat = pgdat; ++ zone_pcp_init(zone); ++ ++ /* For bootup, initialized properly in watermark setup */ ++ mod_zone_page_state(zone, NR_ALLOC_BATCH, zone->managed_pages); ++ ++ lruvec_init(&zone->lruvec); ++ if (!size) ++ continue; ++ ++ set_pageblock_order(); ++ setup_usemap(pgdat, zone, zone_start_pfn, size); ++ ret = init_currently_empty_zone(zone, zone_start_pfn, ++ size, MEMMAP_EARLY); ++ BUG_ON(ret); ++ memmap_init(size, nid, j, zone_start_pfn); ++ zone_start_pfn += size; ++ } ++} ++ ++static void __init_refok alloc_node_mem_map(struct pglist_data *pgdat) ++{ ++ /* Skip empty nodes */ ++ if (!pgdat->node_spanned_pages) ++ return; ++ ++#ifdef CONFIG_FLAT_NODE_MEM_MAP ++ /* ia64 gets its own node_mem_map, before this, without bootmem */ ++ if (!pgdat->node_mem_map) { ++ unsigned long size, start, end; ++ struct page *map; ++ ++ /* ++ * The zone's endpoints aren't required to be MAX_ORDER ++ * aligned but the node_mem_map endpoints must be in order ++ * for the buddy allocator to function correctly. ++ */ ++ start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1); ++ end = pgdat_end_pfn(pgdat); ++ end = ALIGN(end, MAX_ORDER_NR_PAGES); ++ size = (end - start) * sizeof(struct page); ++ map = alloc_remap(pgdat->node_id, size); ++ if (!map) ++ map = memblock_virt_alloc_node_nopanic(size, ++ pgdat->node_id); ++ pgdat->node_mem_map = map + (pgdat->node_start_pfn - start); ++ } ++#ifndef CONFIG_NEED_MULTIPLE_NODES ++ /* ++ * With no DISCONTIG, the global mem_map is just set as node 0's ++ */ ++ if (pgdat == NODE_DATA(0)) { ++ mem_map = NODE_DATA(0)->node_mem_map; ++#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP ++ if (page_to_pfn(mem_map) != pgdat->node_start_pfn) ++ mem_map -= (pgdat->node_start_pfn - ARCH_PFN_OFFSET); ++#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ ++ } ++#endif ++#endif /* CONFIG_FLAT_NODE_MEM_MAP */ ++} ++ ++void __paginginit free_area_init_node(int nid, unsigned long *zones_size, ++ unsigned long node_start_pfn, unsigned long *zholes_size) ++{ ++ pg_data_t *pgdat = NODE_DATA(nid); ++ unsigned long start_pfn = 0; ++ unsigned long end_pfn = 0; ++ ++ /* pg_data_t should be reset to zero when it's allocated */ ++ WARN_ON(pgdat->nr_zones || pgdat->classzone_idx); ++ ++ pgdat->node_id = nid; ++ pgdat->node_start_pfn = node_start_pfn; ++#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP ++ get_pfn_range_for_nid(nid, &start_pfn, &end_pfn); ++ pr_info("Initmem setup node %d [mem %#018Lx-%#018Lx]\n", nid, ++ (u64)start_pfn << PAGE_SHIFT, ((u64)end_pfn << PAGE_SHIFT) - 1); ++#endif ++ calculate_node_totalpages(pgdat, start_pfn, end_pfn, ++ zones_size, zholes_size); ++ ++ alloc_node_mem_map(pgdat); ++#ifdef CONFIG_FLAT_NODE_MEM_MAP ++ printk(KERN_DEBUG "free_area_init_node: node %d, pgdat %08lx, node_mem_map %08lx\n", ++ nid, (unsigned long)pgdat, ++ (unsigned long)pgdat->node_mem_map); ++#endif ++ ++ free_area_init_core(pgdat, start_pfn, end_pfn, ++ zones_size, zholes_size); ++} ++ ++#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP ++ ++#if MAX_NUMNODES > 1 ++/* ++ * Figure out the number of possible node ids. ++ */ ++void __init setup_nr_node_ids(void) ++{ ++ unsigned int node; ++ unsigned int highest = 0; ++ ++ for_each_node_mask(node, node_possible_map) ++ highest = node; ++ nr_node_ids = highest + 1; ++} ++#endif ++ ++/** ++ * node_map_pfn_alignment - determine the maximum internode alignment ++ * ++ * This function should be called after node map is populated and sorted. ++ * It calculates the maximum power of two alignment which can distinguish ++ * all the nodes. ++ * ++ * For example, if all nodes are 1GiB and aligned to 1GiB, the return value ++ * would indicate 1GiB alignment with (1 << (30 - PAGE_SHIFT)). If the ++ * nodes are shifted by 256MiB, 256MiB. Note that if only the last node is ++ * shifted, 1GiB is enough and this function will indicate so. ++ * ++ * This is used to test whether pfn -> nid mapping of the chosen memory ++ * model has fine enough granularity to avoid incorrect mapping for the ++ * populated node map. ++ * ++ * Returns the determined alignment in pfn's. 0 if there is no alignment ++ * requirement (single node). ++ */ ++unsigned long __init node_map_pfn_alignment(void) ++{ ++ unsigned long accl_mask = 0, last_end = 0; ++ unsigned long start, end, mask; ++ int last_nid = -1; ++ int i, nid; ++ ++ for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) { ++ if (!start || last_nid < 0 || last_nid == nid) { ++ last_nid = nid; ++ last_end = end; ++ continue; ++ } ++ ++ /* ++ * Start with a mask granular enough to pin-point to the ++ * start pfn and tick off bits one-by-one until it becomes ++ * too coarse to separate the current node from the last. ++ */ ++ mask = ~((1 << __ffs(start)) - 1); ++ while (mask && last_end <= (start & (mask << 1))) ++ mask <<= 1; ++ ++ /* accumulate all internode masks */ ++ accl_mask |= mask; ++ } ++ ++ /* convert mask to number of pages */ ++ return ~accl_mask + 1; ++} ++ ++/* Find the lowest pfn for a node */ ++static unsigned long __init find_min_pfn_for_node(int nid) ++{ ++ unsigned long min_pfn = ULONG_MAX; ++ unsigned long start_pfn; ++ int i; ++ ++ for_each_mem_pfn_range(i, nid, &start_pfn, NULL, NULL) ++ min_pfn = min(min_pfn, start_pfn); ++ ++ if (min_pfn == ULONG_MAX) { ++ printk(KERN_WARNING ++ "Could not find start_pfn for node %d\n", nid); ++ return 0; ++ } ++ ++ return min_pfn; ++} ++ ++/** ++ * find_min_pfn_with_active_regions - Find the minimum PFN registered ++ * ++ * It returns the minimum PFN based on information provided via ++ * memblock_set_node(). ++ */ ++unsigned long __init find_min_pfn_with_active_regions(void) ++{ ++ return find_min_pfn_for_node(MAX_NUMNODES); ++} ++ ++/* ++ * early_calculate_totalpages() ++ * Sum pages in active regions for movable zone. ++ * Populate N_MEMORY for calculating usable_nodes. ++ */ ++static unsigned long __init early_calculate_totalpages(void) ++{ ++ unsigned long totalpages = 0; ++ unsigned long start_pfn, end_pfn; ++ int i, nid; ++ ++ for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { ++ unsigned long pages = end_pfn - start_pfn; ++ ++ totalpages += pages; ++ if (pages) ++ node_set_state(nid, N_MEMORY); ++ } ++ return totalpages; ++} ++ ++/* ++ * Find the PFN the Movable zone begins in each node. Kernel memory ++ * is spread evenly between nodes as long as the nodes have enough ++ * memory. When they don't, some nodes will have more kernelcore than ++ * others ++ */ ++static void __init find_zone_movable_pfns_for_nodes(void) ++{ ++ int i, nid; ++ unsigned long usable_startpfn; ++ unsigned long kernelcore_node, kernelcore_remaining; ++ /* save the state before borrow the nodemask */ ++ nodemask_t saved_node_state = node_states[N_MEMORY]; ++ unsigned long totalpages = early_calculate_totalpages(); ++ int usable_nodes = nodes_weight(node_states[N_MEMORY]); ++ struct memblock_region *r; ++ ++ /* Need to find movable_zone earlier when movable_node is specified. */ ++ find_usable_zone_for_movable(); ++ ++ /* ++ * If movable_node is specified, ignore kernelcore and movablecore ++ * options. ++ */ ++ if (movable_node_is_enabled()) { ++ for_each_memblock(memory, r) { ++ if (!memblock_is_hotpluggable(r)) ++ continue; ++ ++ nid = r->nid; ++ ++ usable_startpfn = PFN_DOWN(r->base); ++ zone_movable_pfn[nid] = zone_movable_pfn[nid] ? ++ min(usable_startpfn, zone_movable_pfn[nid]) : ++ usable_startpfn; ++ } ++ ++ goto out2; ++ } ++ ++ /* ++ * If movablecore=nn[KMG] was specified, calculate what size of ++ * kernelcore that corresponds so that memory usable for ++ * any allocation type is evenly spread. If both kernelcore ++ * and movablecore are specified, then the value of kernelcore ++ * will be used for required_kernelcore if it's greater than ++ * what movablecore would have allowed. ++ */ ++ if (required_movablecore) { ++ unsigned long corepages; ++ ++ /* ++ * Round-up so that ZONE_MOVABLE is at least as large as what ++ * was requested by the user ++ */ ++ required_movablecore = ++ roundup(required_movablecore, MAX_ORDER_NR_PAGES); ++ corepages = totalpages - required_movablecore; ++ ++ required_kernelcore = max(required_kernelcore, corepages); ++ } ++ ++ /* If kernelcore was not specified, there is no ZONE_MOVABLE */ ++ if (!required_kernelcore) ++ goto out; ++ ++ /* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */ ++ usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone]; ++ ++restart: ++ /* Spread kernelcore memory as evenly as possible throughout nodes */ ++ kernelcore_node = required_kernelcore / usable_nodes; ++ for_each_node_state(nid, N_MEMORY) { ++ unsigned long start_pfn, end_pfn; ++ ++ /* ++ * Recalculate kernelcore_node if the division per node ++ * now exceeds what is necessary to satisfy the requested ++ * amount of memory for the kernel ++ */ ++ if (required_kernelcore < kernelcore_node) ++ kernelcore_node = required_kernelcore / usable_nodes; ++ ++ /* ++ * As the map is walked, we track how much memory is usable ++ * by the kernel using kernelcore_remaining. When it is ++ * 0, the rest of the node is usable by ZONE_MOVABLE ++ */ ++ kernelcore_remaining = kernelcore_node; ++ ++ /* Go through each range of PFNs within this node */ ++ for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) { ++ unsigned long size_pages; ++ ++ start_pfn = max(start_pfn, zone_movable_pfn[nid]); ++ if (start_pfn >= end_pfn) ++ continue; ++ ++ /* Account for what is only usable for kernelcore */ ++ if (start_pfn < usable_startpfn) { ++ unsigned long kernel_pages; ++ kernel_pages = min(end_pfn, usable_startpfn) ++ - start_pfn; ++ ++ kernelcore_remaining -= min(kernel_pages, ++ kernelcore_remaining); ++ required_kernelcore -= min(kernel_pages, ++ required_kernelcore); ++ ++ /* Continue if range is now fully accounted */ ++ if (end_pfn <= usable_startpfn) { ++ ++ /* ++ * Push zone_movable_pfn to the end so ++ * that if we have to rebalance ++ * kernelcore across nodes, we will ++ * not double account here ++ */ ++ zone_movable_pfn[nid] = end_pfn; ++ continue; ++ } ++ start_pfn = usable_startpfn; ++ } ++ ++ /* ++ * The usable PFN range for ZONE_MOVABLE is from ++ * start_pfn->end_pfn. Calculate size_pages as the ++ * number of pages used as kernelcore ++ */ ++ size_pages = end_pfn - start_pfn; ++ if (size_pages > kernelcore_remaining) ++ size_pages = kernelcore_remaining; ++ zone_movable_pfn[nid] = start_pfn + size_pages; ++ ++ /* ++ * Some kernelcore has been met, update counts and ++ * break if the kernelcore for this node has been ++ * satisfied ++ */ ++ required_kernelcore -= min(required_kernelcore, ++ size_pages); ++ kernelcore_remaining -= size_pages; ++ if (!kernelcore_remaining) ++ break; ++ } ++ } ++ ++ /* ++ * If there is still required_kernelcore, we do another pass with one ++ * less node in the count. This will push zone_movable_pfn[nid] further ++ * along on the nodes that still have memory until kernelcore is ++ * satisfied ++ */ ++ usable_nodes--; ++ if (usable_nodes && required_kernelcore > usable_nodes) ++ goto restart; ++ ++out2: ++ /* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */ ++ for (nid = 0; nid < MAX_NUMNODES; nid++) ++ zone_movable_pfn[nid] = ++ roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES); ++ ++out: ++ /* restore the node_state */ ++ node_states[N_MEMORY] = saved_node_state; ++} ++ ++/* Any regular or high memory on that node ? */ ++static void check_for_memory(pg_data_t *pgdat, int nid) ++{ ++ enum zone_type zone_type; ++ ++ if (N_MEMORY == N_NORMAL_MEMORY) ++ return; ++ ++ for (zone_type = 0; zone_type <= ZONE_MOVABLE - 1; zone_type++) { ++ struct zone *zone = &pgdat->node_zones[zone_type]; ++ if (populated_zone(zone)) { ++ node_set_state(nid, N_HIGH_MEMORY); ++ if (N_NORMAL_MEMORY != N_HIGH_MEMORY && ++ zone_type <= ZONE_NORMAL) ++ node_set_state(nid, N_NORMAL_MEMORY); ++ break; ++ } ++ } ++} ++ ++/** ++ * free_area_init_nodes - Initialise all pg_data_t and zone data ++ * @max_zone_pfn: an array of max PFNs for each zone ++ * ++ * This will call free_area_init_node() for each active node in the system. ++ * Using the page ranges provided by memblock_set_node(), the size of each ++ * zone in each node and their holes is calculated. If the maximum PFN ++ * between two adjacent zones match, it is assumed that the zone is empty. ++ * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed ++ * that arch_max_dma32_pfn has no pages. It is also assumed that a zone ++ * starts where the previous one ended. For example, ZONE_DMA32 starts ++ * at arch_max_dma_pfn. ++ */ ++void __init free_area_init_nodes(unsigned long *max_zone_pfn) ++{ ++ unsigned long start_pfn, end_pfn; ++ int i, nid; ++ ++ /* Record where the zone boundaries are */ ++ memset(arch_zone_lowest_possible_pfn, 0, ++ sizeof(arch_zone_lowest_possible_pfn)); ++ memset(arch_zone_highest_possible_pfn, 0, ++ sizeof(arch_zone_highest_possible_pfn)); ++ arch_zone_lowest_possible_pfn[0] = find_min_pfn_with_active_regions(); ++ arch_zone_highest_possible_pfn[0] = max_zone_pfn[0]; ++ for (i = 1; i < MAX_NR_ZONES; i++) { ++ if (i == ZONE_MOVABLE) ++ continue; ++ arch_zone_lowest_possible_pfn[i] = ++ arch_zone_highest_possible_pfn[i-1]; ++ arch_zone_highest_possible_pfn[i] = ++ max(max_zone_pfn[i], arch_zone_lowest_possible_pfn[i]); ++ } ++ arch_zone_lowest_possible_pfn[ZONE_MOVABLE] = 0; ++ arch_zone_highest_possible_pfn[ZONE_MOVABLE] = 0; ++ ++ /* Find the PFNs that ZONE_MOVABLE begins at in each node */ ++ memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn)); ++ find_zone_movable_pfns_for_nodes(); ++ ++ /* Print out the zone ranges */ ++ pr_info("Zone ranges:\n"); ++ for (i = 0; i < MAX_NR_ZONES; i++) { ++ if (i == ZONE_MOVABLE) ++ continue; ++ pr_info(" %-8s ", zone_names[i]); ++ if (arch_zone_lowest_possible_pfn[i] == ++ arch_zone_highest_possible_pfn[i]) ++ pr_cont("empty\n"); ++ else ++ pr_cont("[mem %#018Lx-%#018Lx]\n", ++ (u64)arch_zone_lowest_possible_pfn[i] ++ << PAGE_SHIFT, ++ ((u64)arch_zone_highest_possible_pfn[i] ++ << PAGE_SHIFT) - 1); ++ } ++ ++ /* Print out the PFNs ZONE_MOVABLE begins at in each node */ ++ pr_info("Movable zone start for each node\n"); ++ for (i = 0; i < MAX_NUMNODES; i++) { ++ if (zone_movable_pfn[i]) ++ pr_info(" Node %d: %#018Lx\n", i, ++ (u64)zone_movable_pfn[i] << PAGE_SHIFT); ++ } ++ ++ /* Print out the early node map */ ++ pr_info("Early memory node ranges\n"); ++ for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) ++ pr_info(" node %3d: [mem %#018Lx-%#018Lx]\n", nid, ++ (u64)start_pfn << PAGE_SHIFT, ++ ((u64)end_pfn << PAGE_SHIFT) - 1); ++ ++ /* Initialise every node */ ++ mminit_verify_pageflags_layout(); ++ setup_nr_node_ids(); ++ for_each_online_node(nid) { ++ pg_data_t *pgdat = NODE_DATA(nid); ++ free_area_init_node(nid, NULL, ++ find_min_pfn_for_node(nid), NULL); ++ ++ /* Any memory on that node */ ++ if (pgdat->node_present_pages) ++ node_set_state(nid, N_MEMORY); ++ check_for_memory(pgdat, nid); ++ } ++} ++ ++static int __init cmdline_parse_core(char *p, unsigned long *core) ++{ ++ unsigned long long coremem; ++ if (!p) ++ return -EINVAL; ++ ++ coremem = memparse(p, &p); ++ *core = coremem >> PAGE_SHIFT; ++ ++ /* Paranoid check that UL is enough for the coremem value */ ++ WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX); ++ ++ return 0; ++} ++ ++/* ++ * kernelcore=size sets the amount of memory for use for allocations that ++ * cannot be reclaimed or migrated. ++ */ ++static int __init cmdline_parse_kernelcore(char *p) ++{ ++ return cmdline_parse_core(p, &required_kernelcore); ++} ++ ++/* ++ * movablecore=size sets the amount of memory for use for allocations that ++ * can be reclaimed or migrated. ++ */ ++static int __init cmdline_parse_movablecore(char *p) ++{ ++ return cmdline_parse_core(p, &required_movablecore); ++} ++ ++early_param("kernelcore", cmdline_parse_kernelcore); ++early_param("movablecore", cmdline_parse_movablecore); ++ ++#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ ++ ++void adjust_managed_page_count(struct page *page, long count) ++{ ++ spin_lock(&managed_page_count_lock); ++ page_zone(page)->managed_pages += count; ++ totalram_pages += count; ++#ifdef CONFIG_HIGHMEM ++ if (PageHighMem(page)) ++ totalhigh_pages += count; ++#endif ++ spin_unlock(&managed_page_count_lock); ++} ++EXPORT_SYMBOL(adjust_managed_page_count); ++ ++unsigned long free_reserved_area(void *start, void *end, int poison, char *s) ++{ ++ void *pos; ++ unsigned long pages = 0; ++ ++ start = (void *)PAGE_ALIGN((unsigned long)start); ++ end = (void *)((unsigned long)end & PAGE_MASK); ++ for (pos = start; pos < end; pos += PAGE_SIZE, pages++) { ++ if ((unsigned int)poison <= 0xFF) ++ memset(pos, poison, PAGE_SIZE); ++ free_reserved_page(virt_to_page(pos)); ++ } ++ ++ if (pages && s) ++ pr_info("Freeing %s memory: %ldK (%p - %p)\n", ++ s, pages << (PAGE_SHIFT - 10), start, end); ++ ++ return pages; ++} ++EXPORT_SYMBOL(free_reserved_area); ++ ++#ifdef CONFIG_HIGHMEM ++void free_highmem_page(struct page *page) ++{ ++ __free_reserved_page(page); ++ totalram_pages++; ++ page_zone(page)->managed_pages++; ++ totalhigh_pages++; ++} ++#endif ++ ++ ++void __init mem_init_print_info(const char *str) ++{ ++ unsigned long physpages, codesize, datasize, rosize, bss_size; ++ unsigned long init_code_size, init_data_size; ++ ++ physpages = get_num_physpages(); ++ codesize = _etext - _stext; ++ datasize = _edata - _sdata; ++ rosize = __end_rodata - __start_rodata; ++ bss_size = __bss_stop - __bss_start; ++ init_data_size = __init_end - __init_begin; ++ init_code_size = _einittext - _sinittext; ++ ++ /* ++ * Detect special cases and adjust section sizes accordingly: ++ * 1) .init.* may be embedded into .data sections ++ * 2) .init.text.* may be out of [__init_begin, __init_end], ++ * please refer to arch/tile/kernel/vmlinux.lds.S. ++ * 3) .rodata.* may be embedded into .text or .data sections. ++ */ ++#define adj_init_size(start, end, size, pos, adj) \ ++ do { \ ++ if (start <= pos && pos < end && size > adj) \ ++ size -= adj; \ ++ } while (0) ++ ++ adj_init_size(__init_begin, __init_end, init_data_size, ++ _sinittext, init_code_size); ++ adj_init_size(_stext, _etext, codesize, _sinittext, init_code_size); ++ adj_init_size(_sdata, _edata, datasize, __init_begin, init_data_size); ++ adj_init_size(_stext, _etext, codesize, __start_rodata, rosize); ++ adj_init_size(_sdata, _edata, datasize, __start_rodata, rosize); ++ ++#undef adj_init_size ++ ++ pr_info("Memory: %luK/%luK available " ++ "(%luK kernel code, %luK rwdata, %luK rodata, " ++ "%luK init, %luK bss, %luK reserved, %luK cma-reserved" ++#ifdef CONFIG_HIGHMEM ++ ", %luK highmem" ++#endif ++ "%s%s)\n", ++ nr_free_pages() << (PAGE_SHIFT-10), physpages << (PAGE_SHIFT-10), ++ codesize >> 10, datasize >> 10, rosize >> 10, ++ (init_data_size + init_code_size) >> 10, bss_size >> 10, ++ (physpages - totalram_pages - totalcma_pages) << (PAGE_SHIFT-10), ++ totalcma_pages << (PAGE_SHIFT-10), ++#ifdef CONFIG_HIGHMEM ++ totalhigh_pages << (PAGE_SHIFT-10), ++#endif ++ str ? ", " : "", str ? str : ""); ++} ++ ++/** ++ * set_dma_reserve - set the specified number of pages reserved in the first zone ++ * @new_dma_reserve: The number of pages to mark reserved ++ * ++ * The per-cpu batchsize and zone watermarks are determined by present_pages. ++ * In the DMA zone, a significant percentage may be consumed by kernel image ++ * and other unfreeable allocations which can skew the watermarks badly. This ++ * function may optionally be used to account for unfreeable pages in the ++ * first zone (e.g., ZONE_DMA). The effect will be lower watermarks and ++ * smaller per-cpu batchsize. ++ */ ++void __init set_dma_reserve(unsigned long new_dma_reserve) ++{ ++ dma_reserve = new_dma_reserve; ++} ++ ++void __init free_area_init(unsigned long *zones_size) ++{ ++ free_area_init_node(0, zones_size, ++ __pa(PAGE_OFFSET) >> PAGE_SHIFT, NULL); ++} ++ ++static int page_alloc_cpu_notify(struct notifier_block *self, ++ unsigned long action, void *hcpu) ++{ ++ int cpu = (unsigned long)hcpu; ++ ++ if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) { ++ lru_add_drain_cpu(cpu); ++ drain_pages(cpu); ++ ++ /* ++ * Spill the event counters of the dead processor ++ * into the current processors event counters. ++ * This artificially elevates the count of the current ++ * processor. ++ */ ++ vm_events_fold_cpu(cpu); ++ ++ /* ++ * Zero the differential counters of the dead processor ++ * so that the vm statistics are consistent. ++ * ++ * This is only okay since the processor is dead and cannot ++ * race with what we are doing. ++ */ ++ cpu_vm_stats_fold(cpu); ++ } ++ return NOTIFY_OK; ++} ++ ++void __init page_alloc_init(void) ++{ ++ hotcpu_notifier(page_alloc_cpu_notify, 0); ++} ++ ++/* ++ * calculate_totalreserve_pages - called when sysctl_lower_zone_reserve_ratio ++ * or min_free_kbytes changes. ++ */ ++static void calculate_totalreserve_pages(void) ++{ ++ struct pglist_data *pgdat; ++ unsigned long reserve_pages = 0; ++ enum zone_type i, j; ++ ++ for_each_online_pgdat(pgdat) { ++ for (i = 0; i < MAX_NR_ZONES; i++) { ++ struct zone *zone = pgdat->node_zones + i; ++ long max = 0; ++ ++ /* Find valid and maximum lowmem_reserve in the zone */ ++ for (j = i; j < MAX_NR_ZONES; j++) { ++ if (zone->lowmem_reserve[j] > max) ++ max = zone->lowmem_reserve[j]; ++ } ++ ++ /* we treat the high watermark as reserved pages. */ ++ max += high_wmark_pages(zone); ++ ++ if (max > zone->managed_pages) ++ max = zone->managed_pages; ++ reserve_pages += max; ++ /* ++ * Lowmem reserves are not available to ++ * GFP_HIGHUSER page cache allocations and ++ * kswapd tries to balance zones to their high ++ * watermark. As a result, neither should be ++ * regarded as dirtyable memory, to prevent a ++ * situation where reclaim has to clean pages ++ * in order to balance the zones. ++ */ ++ zone->dirty_balance_reserve = max; ++ } ++ } ++ dirty_balance_reserve = reserve_pages; ++ totalreserve_pages = reserve_pages; ++} ++ ++/* ++ * setup_per_zone_lowmem_reserve - called whenever ++ * sysctl_lower_zone_reserve_ratio changes. Ensures that each zone ++ * has a correct pages reserved value, so an adequate number of ++ * pages are left in the zone after a successful __alloc_pages(). ++ */ ++static void setup_per_zone_lowmem_reserve(void) ++{ ++ struct pglist_data *pgdat; ++ enum zone_type j, idx; ++ ++ for_each_online_pgdat(pgdat) { ++ for (j = 0; j < MAX_NR_ZONES; j++) { ++ struct zone *zone = pgdat->node_zones + j; ++ unsigned long managed_pages = zone->managed_pages; ++ ++ zone->lowmem_reserve[j] = 0; ++ ++ idx = j; ++ while (idx) { ++ struct zone *lower_zone; ++ ++ idx--; ++ ++ if (sysctl_lowmem_reserve_ratio[idx] < 1) ++ sysctl_lowmem_reserve_ratio[idx] = 1; ++ ++ lower_zone = pgdat->node_zones + idx; ++ lower_zone->lowmem_reserve[j] = managed_pages / ++ sysctl_lowmem_reserve_ratio[idx]; ++ managed_pages += lower_zone->managed_pages; ++ } ++ } ++ } ++ ++ /* update totalreserve_pages */ ++ calculate_totalreserve_pages(); ++} ++ ++static void __setup_per_zone_wmarks(void) ++{ ++ unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10); ++ unsigned long lowmem_pages = 0; ++ struct zone *zone; ++ unsigned long flags; ++ ++ /* Calculate total number of !ZONE_HIGHMEM pages */ ++ for_each_zone(zone) { ++ if (!is_highmem(zone)) ++ lowmem_pages += zone->managed_pages; ++ } ++ ++ for_each_zone(zone) { ++ u64 tmp; ++ ++ spin_lock_irqsave(&zone->lock, flags); ++ tmp = (u64)pages_min * zone->managed_pages; ++ do_div(tmp, lowmem_pages); ++ if (is_highmem(zone)) { ++ /* ++ * __GFP_HIGH and PF_MEMALLOC allocations usually don't ++ * need highmem pages, so cap pages_min to a small ++ * value here. ++ * ++ * The WMARK_HIGH-WMARK_LOW and (WMARK_LOW-WMARK_MIN) ++ * deltas control asynch page reclaim, and so should ++ * not be capped for highmem. ++ */ ++ unsigned long min_pages; ++ ++ min_pages = zone->managed_pages / 1024; ++ min_pages = clamp(min_pages, SWAP_CLUSTER_MAX, 128UL); ++ zone->watermark[WMARK_MIN] = min_pages; ++ } else { ++ /* ++ * If it's a lowmem zone, reserve a number of pages ++ * proportionate to the zone's size. ++ */ ++ zone->watermark[WMARK_MIN] = tmp; ++ } ++ ++ zone->watermark[WMARK_LOW] = min_wmark_pages(zone) + (tmp >> 2); ++ zone->watermark[WMARK_HIGH] = min_wmark_pages(zone) + (tmp >> 1); ++ ++ __mod_zone_page_state(zone, NR_ALLOC_BATCH, ++ high_wmark_pages(zone) - low_wmark_pages(zone) - ++ atomic_long_read(&zone->vm_stat[NR_ALLOC_BATCH])); ++ ++ setup_zone_migrate_reserve(zone); ++ spin_unlock_irqrestore(&zone->lock, flags); ++ } ++ ++ /* update totalreserve_pages */ ++ calculate_totalreserve_pages(); ++} ++ ++/** ++ * setup_per_zone_wmarks - called when min_free_kbytes changes ++ * or when memory is hot-{added|removed} ++ * ++ * Ensures that the watermark[min,low,high] values for each zone are set ++ * correctly with respect to min_free_kbytes. ++ */ ++void setup_per_zone_wmarks(void) ++{ ++ mutex_lock(&zonelists_mutex); ++ __setup_per_zone_wmarks(); ++ mutex_unlock(&zonelists_mutex); ++} ++ ++/* ++ * The inactive anon list should be small enough that the VM never has to ++ * do too much work, but large enough that each inactive page has a chance ++ * to be referenced again before it is swapped out. ++ * ++ * The inactive_anon ratio is the target ratio of ACTIVE_ANON to ++ * INACTIVE_ANON pages on this zone's LRU, maintained by the ++ * pageout code. A zone->inactive_ratio of 3 means 3:1 or 25% of ++ * the anonymous pages are kept on the inactive list. ++ * ++ * total target max ++ * memory ratio inactive anon ++ * ------------------------------------- ++ * 10MB 1 5MB ++ * 100MB 1 50MB ++ * 1GB 3 250MB ++ * 10GB 10 0.9GB ++ * 100GB 31 3GB ++ * 1TB 101 10GB ++ * 10TB 320 32GB ++ */ ++static void __meminit calculate_zone_inactive_ratio(struct zone *zone) ++{ ++ unsigned int gb, ratio; ++ ++ /* Zone size in gigabytes */ ++ gb = zone->managed_pages >> (30 - PAGE_SHIFT); ++ if (gb) ++ ratio = int_sqrt(10 * gb); ++ else ++ ratio = 1; ++ ++ zone->inactive_ratio = ratio; ++} ++ ++static void __meminit setup_per_zone_inactive_ratio(void) ++{ ++ struct zone *zone; ++ ++ for_each_zone(zone) ++ calculate_zone_inactive_ratio(zone); ++} ++ ++/* ++ * Initialise min_free_kbytes. ++ * ++ * For small machines we want it small (128k min). For large machines ++ * we want it large (64MB max). But it is not linear, because network ++ * bandwidth does not increase linearly with machine size. We use ++ * ++ * min_free_kbytes = 4 * sqrt(lowmem_kbytes), for better accuracy: ++ * min_free_kbytes = sqrt(lowmem_kbytes * 16) ++ * ++ * which yields ++ * ++ * 16MB: 512k ++ * 32MB: 724k ++ * 64MB: 1024k ++ * 128MB: 1448k ++ * 256MB: 2048k ++ * 512MB: 2896k ++ * 1024MB: 4096k ++ * 2048MB: 5792k ++ * 4096MB: 8192k ++ * 8192MB: 11584k ++ * 16384MB: 16384k ++ */ ++int __meminit init_per_zone_wmark_min(void) ++{ ++ unsigned long lowmem_kbytes; ++ int new_min_free_kbytes; ++ ++ lowmem_kbytes = nr_free_buffer_pages() * (PAGE_SIZE >> 10); ++ new_min_free_kbytes = int_sqrt(lowmem_kbytes * 16); ++ ++ if (new_min_free_kbytes > user_min_free_kbytes) { ++ min_free_kbytes = new_min_free_kbytes; ++ if (min_free_kbytes < 128) ++ min_free_kbytes = 128; ++ if (min_free_kbytes > 65536) ++ min_free_kbytes = 65536; ++ } else { ++ pr_warn("min_free_kbytes is not updated to %d because user defined value %d is preferred\n", ++ new_min_free_kbytes, user_min_free_kbytes); ++ } ++ setup_per_zone_wmarks(); ++ refresh_zone_stat_thresholds(); ++ setup_per_zone_lowmem_reserve(); ++ setup_per_zone_inactive_ratio(); ++ return 0; ++} ++module_init(init_per_zone_wmark_min) ++ ++/* ++ * min_free_kbytes_sysctl_handler - just a wrapper around proc_dointvec() so ++ * that we can call two helper functions whenever min_free_kbytes ++ * changes. ++ */ ++int min_free_kbytes_sysctl_handler(struct ctl_table *table, int write, ++ void __user *buffer, size_t *length, loff_t *ppos) ++{ ++ int rc; ++ ++ rc = proc_dointvec_minmax(table, write, buffer, length, ppos); ++ if (rc) ++ return rc; ++ ++ if (write) { ++ user_min_free_kbytes = min_free_kbytes; ++ setup_per_zone_wmarks(); ++ } ++ return 0; ++} ++ ++#ifdef CONFIG_NUMA ++int sysctl_min_unmapped_ratio_sysctl_handler(struct ctl_table *table, int write, ++ void __user *buffer, size_t *length, loff_t *ppos) ++{ ++ struct zone *zone; ++ int rc; ++ ++ rc = proc_dointvec_minmax(table, write, buffer, length, ppos); ++ if (rc) ++ return rc; ++ ++ for_each_zone(zone) ++ zone->min_unmapped_pages = (zone->managed_pages * ++ sysctl_min_unmapped_ratio) / 100; ++ return 0; ++} ++ ++int sysctl_min_slab_ratio_sysctl_handler(struct ctl_table *table, int write, ++ void __user *buffer, size_t *length, loff_t *ppos) ++{ ++ struct zone *zone; ++ int rc; ++ ++ rc = proc_dointvec_minmax(table, write, buffer, length, ppos); ++ if (rc) ++ return rc; ++ ++ for_each_zone(zone) ++ zone->min_slab_pages = (zone->managed_pages * ++ sysctl_min_slab_ratio) / 100; ++ return 0; ++} ++#endif ++ ++/* ++ * lowmem_reserve_ratio_sysctl_handler - just a wrapper around ++ * proc_dointvec() so that we can call setup_per_zone_lowmem_reserve() ++ * whenever sysctl_lowmem_reserve_ratio changes. ++ * ++ * The reserve ratio obviously has absolutely no relation with the ++ * minimum watermarks. The lowmem reserve ratio can only make sense ++ * if in function of the boot time zone sizes. ++ */ ++int lowmem_reserve_ratio_sysctl_handler(struct ctl_table *table, int write, ++ void __user *buffer, size_t *length, loff_t *ppos) ++{ ++ proc_dointvec_minmax(table, write, buffer, length, ppos); ++ setup_per_zone_lowmem_reserve(); ++ return 0; ++} ++ ++/* ++ * percpu_pagelist_fraction - changes the pcp->high for each zone on each ++ * cpu. It is the fraction of total pages in each zone that a hot per cpu ++ * pagelist can have before it gets flushed back to buddy allocator. ++ */ ++int percpu_pagelist_fraction_sysctl_handler(struct ctl_table *table, int write, ++ void __user *buffer, size_t *length, loff_t *ppos) ++{ ++ struct zone *zone; ++ int old_percpu_pagelist_fraction; ++ int ret; ++ ++ mutex_lock(&pcp_batch_high_lock); ++ old_percpu_pagelist_fraction = percpu_pagelist_fraction; ++ ++ ret = proc_dointvec_minmax(table, write, buffer, length, ppos); ++ if (!write || ret < 0) ++ goto out; ++ ++ /* Sanity checking to avoid pcp imbalance */ ++ if (percpu_pagelist_fraction && ++ percpu_pagelist_fraction < MIN_PERCPU_PAGELIST_FRACTION) { ++ percpu_pagelist_fraction = old_percpu_pagelist_fraction; ++ ret = -EINVAL; ++ goto out; ++ } ++ ++ /* No change? */ ++ if (percpu_pagelist_fraction == old_percpu_pagelist_fraction) ++ goto out; ++ ++ for_each_populated_zone(zone) { ++ unsigned int cpu; ++ ++ for_each_possible_cpu(cpu) ++ pageset_set_high_and_batch(zone, ++ per_cpu_ptr(zone->pageset, cpu)); ++ } ++out: ++ mutex_unlock(&pcp_batch_high_lock); ++ return ret; ++} ++ ++int hashdist = HASHDIST_DEFAULT; ++ ++#ifdef CONFIG_NUMA ++static int __init set_hashdist(char *str) ++{ ++ if (!str) ++ return 0; ++ hashdist = simple_strtoul(str, &str, 0); ++ return 1; ++} ++__setup("hashdist=", set_hashdist); ++#endif ++ ++/* ++ * allocate a large system hash table from bootmem ++ * - it is assumed that the hash table must contain an exact power-of-2 ++ * quantity of entries ++ * - limit is the number of hash buckets, not the total allocation size ++ */ ++void *__init alloc_large_system_hash(const char *tablename, ++ unsigned long bucketsize, ++ unsigned long numentries, ++ int scale, ++ int flags, ++ unsigned int *_hash_shift, ++ unsigned int *_hash_mask, ++ unsigned long low_limit, ++ unsigned long high_limit) ++{ ++ unsigned long long max = high_limit; ++ unsigned long log2qty, size; ++ void *table = NULL; ++ ++ /* allow the kernel cmdline to have a say */ ++ if (!numentries) { ++ /* round applicable memory size up to nearest megabyte */ ++ numentries = nr_kernel_pages; ++ ++ /* It isn't necessary when PAGE_SIZE >= 1MB */ ++ if (PAGE_SHIFT < 20) ++ numentries = round_up(numentries, (1<<20)/PAGE_SIZE); ++ ++ /* limit to 1 bucket per 2^scale bytes of low memory */ ++ if (scale > PAGE_SHIFT) ++ numentries >>= (scale - PAGE_SHIFT); ++ else ++ numentries <<= (PAGE_SHIFT - scale); ++ ++ /* Make sure we've got at least a 0-order allocation.. */ ++ if (unlikely(flags & HASH_SMALL)) { ++ /* Makes no sense without HASH_EARLY */ ++ WARN_ON(!(flags & HASH_EARLY)); ++ if (!(numentries >> *_hash_shift)) { ++ numentries = 1UL << *_hash_shift; ++ BUG_ON(!numentries); ++ } ++ } else if (unlikely((numentries * bucketsize) < PAGE_SIZE)) ++ numentries = PAGE_SIZE / bucketsize; ++ } ++ numentries = roundup_pow_of_two(numentries); ++ ++ /* limit allocation size to 1/16 total memory by default */ ++ if (max == 0) { ++ max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4; ++ do_div(max, bucketsize); ++ } ++ max = min(max, 0x80000000ULL); ++ ++ if (numentries < low_limit) ++ numentries = low_limit; ++ if (numentries > max) ++ numentries = max; ++ ++ log2qty = ilog2(numentries); ++ ++ do { ++ size = bucketsize << log2qty; ++ if (flags & HASH_EARLY) ++ table = memblock_virt_alloc_nopanic(size, 0); ++ else if (hashdist) ++ table = __vmalloc(size, GFP_ATOMIC, PAGE_KERNEL); ++ else { ++ /* ++ * If bucketsize is not a power-of-two, we may free ++ * some pages at the end of hash table which ++ * alloc_pages_exact() automatically does ++ */ ++ if (get_order(size) < MAX_ORDER) { ++ table = alloc_pages_exact(size, GFP_ATOMIC); ++ kmemleak_alloc(table, size, 1, GFP_ATOMIC); ++ } ++ } ++ } while (!table && size > PAGE_SIZE && --log2qty); ++ ++ if (!table) ++ panic("Failed to allocate %s hash table\n", tablename); ++ ++ printk(KERN_INFO "%s hash table entries: %ld (order: %d, %lu bytes)\n", ++ tablename, ++ (1UL << log2qty), ++ ilog2(size) - PAGE_SHIFT, ++ size); ++ ++ if (_hash_shift) ++ *_hash_shift = log2qty; ++ if (_hash_mask) ++ *_hash_mask = (1 << log2qty) - 1; ++ ++ return table; ++} ++ ++/* Return a pointer to the bitmap storing bits affecting a block of pages */ ++static inline unsigned long *get_pageblock_bitmap(struct zone *zone, ++ unsigned long pfn) ++{ ++#ifdef CONFIG_SPARSEMEM ++ return __pfn_to_section(pfn)->pageblock_flags; ++#else ++ return zone->pageblock_flags; ++#endif /* CONFIG_SPARSEMEM */ ++} ++ ++static inline int pfn_to_bitidx(struct zone *zone, unsigned long pfn) ++{ ++#ifdef CONFIG_SPARSEMEM ++ pfn &= (PAGES_PER_SECTION-1); ++ return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS; ++#else ++ pfn = pfn - round_down(zone->zone_start_pfn, pageblock_nr_pages); ++ return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS; ++#endif /* CONFIG_SPARSEMEM */ ++} ++ ++/** ++ * get_pfnblock_flags_mask - Return the requested group of flags for the pageblock_nr_pages block of pages ++ * @page: The page within the block of interest ++ * @pfn: The target page frame number ++ * @end_bitidx: The last bit of interest to retrieve ++ * @mask: mask of bits that the caller is interested in ++ * ++ * Return: pageblock_bits flags ++ */ ++unsigned long get_pfnblock_flags_mask(struct page *page, unsigned long pfn, ++ unsigned long end_bitidx, ++ unsigned long mask) ++{ ++ struct zone *zone; ++ unsigned long *bitmap; ++ unsigned long bitidx, word_bitidx; ++ unsigned long word; ++ ++ zone = page_zone(page); ++ bitmap = get_pageblock_bitmap(zone, pfn); ++ bitidx = pfn_to_bitidx(zone, pfn); ++ word_bitidx = bitidx / BITS_PER_LONG; ++ bitidx &= (BITS_PER_LONG-1); ++ ++ word = bitmap[word_bitidx]; ++ bitidx += end_bitidx; ++ return (word >> (BITS_PER_LONG - bitidx - 1)) & mask; ++} ++ ++/** ++ * set_pfnblock_flags_mask - Set the requested group of flags for a pageblock_nr_pages block of pages ++ * @page: The page within the block of interest ++ * @flags: The flags to set ++ * @pfn: The target page frame number ++ * @end_bitidx: The last bit of interest ++ * @mask: mask of bits that the caller is interested in ++ */ ++void set_pfnblock_flags_mask(struct page *page, unsigned long flags, ++ unsigned long pfn, ++ unsigned long end_bitidx, ++ unsigned long mask) ++{ ++ struct zone *zone; ++ unsigned long *bitmap; ++ unsigned long bitidx, word_bitidx; ++ unsigned long old_word, word; ++ ++ BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4); ++ ++ zone = page_zone(page); ++ bitmap = get_pageblock_bitmap(zone, pfn); ++ bitidx = pfn_to_bitidx(zone, pfn); ++ word_bitidx = bitidx / BITS_PER_LONG; ++ bitidx &= (BITS_PER_LONG-1); ++ ++ VM_BUG_ON_PAGE(!zone_spans_pfn(zone, pfn), page); ++ ++ bitidx += end_bitidx; ++ mask <<= (BITS_PER_LONG - bitidx - 1); ++ flags <<= (BITS_PER_LONG - bitidx - 1); ++ ++ word = READ_ONCE(bitmap[word_bitidx]); ++ for (;;) { ++ old_word = cmpxchg(&bitmap[word_bitidx], word, (word & ~mask) | flags); ++ if (word == old_word) ++ break; ++ word = old_word; ++ } ++} ++ ++/* ++ * This function checks whether pageblock includes unmovable pages or not. ++ * If @count is not zero, it is okay to include less @count unmovable pages ++ * ++ * PageLRU check without isolation or lru_lock could race so that ++ * MIGRATE_MOVABLE block might include unmovable pages. It means you can't ++ * expect this function should be exact. ++ */ ++bool has_unmovable_pages(struct zone *zone, struct page *page, int count, ++ bool skip_hwpoisoned_pages) ++{ ++ unsigned long pfn, iter, found; ++ int mt; ++ ++ /* ++ * For avoiding noise data, lru_add_drain_all() should be called ++ * If ZONE_MOVABLE, the zone never contains unmovable pages ++ */ ++ if (zone_idx(zone) == ZONE_MOVABLE) ++ return false; ++ mt = get_pageblock_migratetype(page); ++ if (mt == MIGRATE_MOVABLE || is_migrate_cma(mt)) ++ return false; ++ ++ pfn = page_to_pfn(page); ++ for (found = 0, iter = 0; iter < pageblock_nr_pages; iter++) { ++ unsigned long check = pfn + iter; ++ ++ if (!pfn_valid_within(check)) ++ continue; ++ ++ page = pfn_to_page(check); ++ ++ /* ++ * Hugepages are not in LRU lists, but they're movable. ++ * We need not scan over tail pages bacause we don't ++ * handle each tail page individually in migration. ++ */ ++ if (PageHuge(page)) { ++ iter = round_up(iter + 1, 1<_count is zero at all time. ++ */ ++ if (!atomic_read(&page->_count)) { ++ if (PageBuddy(page)) ++ iter += (1 << page_order(page)) - 1; ++ continue; ++ } ++ ++ /* ++ * The HWPoisoned page may be not in buddy system, and ++ * page_count() is not 0. ++ */ ++ if (skip_hwpoisoned_pages && PageHWPoison(page)) ++ continue; ++ ++ if (!PageLRU(page)) ++ found++; ++ /* ++ * If there are RECLAIMABLE pages, we need to check ++ * it. But now, memory offline itself doesn't call ++ * shrink_node_slabs() and it still to be fixed. ++ */ ++ /* ++ * If the page is not RAM, page_count()should be 0. ++ * we don't need more check. This is an _used_ not-movable page. ++ * ++ * The problematic thing here is PG_reserved pages. PG_reserved ++ * is set to both of a memory hole page and a _used_ kernel ++ * page at boot. ++ */ ++ if (found > count) ++ return true; ++ } ++ return false; ++} ++ ++bool is_pageblock_removable_nolock(struct page *page) ++{ ++ struct zone *zone; ++ unsigned long pfn; ++ ++ /* ++ * We have to be careful here because we are iterating over memory ++ * sections which are not zone aware so we might end up outside of ++ * the zone but still within the section. ++ * We have to take care about the node as well. If the node is offline ++ * its NODE_DATA will be NULL - see page_zone. ++ */ ++ if (!node_online(page_to_nid(page))) ++ return false; ++ ++ zone = page_zone(page); ++ pfn = page_to_pfn(page); ++ if (!zone_spans_pfn(zone, pfn)) ++ return false; ++ ++ return !has_unmovable_pages(zone, page, 0, true); ++} ++ ++#ifdef CONFIG_CMA ++ ++static unsigned long pfn_max_align_down(unsigned long pfn) ++{ ++ return pfn & ~(max_t(unsigned long, MAX_ORDER_NR_PAGES, ++ pageblock_nr_pages) - 1); ++} ++ ++static unsigned long pfn_max_align_up(unsigned long pfn) ++{ ++ return ALIGN(pfn, max_t(unsigned long, MAX_ORDER_NR_PAGES, ++ pageblock_nr_pages)); ++} ++ ++/* [start, end) must belong to a single zone. */ ++static int __alloc_contig_migrate_range(struct compact_control *cc, ++ unsigned long start, unsigned long end) ++{ ++ /* This function is based on compact_zone() from compaction.c. */ ++ unsigned long nr_reclaimed; ++ unsigned long pfn = start; ++ unsigned int tries = 0; ++ int ret = 0; ++ ++ migrate_prep(); ++ ++ while (pfn < end || !list_empty(&cc->migratepages)) { ++ if (fatal_signal_pending(current)) { ++ ret = -EINTR; ++ break; ++ } ++ ++ if (list_empty(&cc->migratepages)) { ++ cc->nr_migratepages = 0; ++ pfn = isolate_migratepages_range(cc, pfn, end); ++ if (!pfn) { ++ ret = -EINTR; ++ break; ++ } ++ tries = 0; ++ } else if (++tries == 5) { ++ ret = ret < 0 ? ret : -EBUSY; ++ break; ++ } ++ ++ nr_reclaimed = reclaim_clean_pages_from_list(cc->zone, ++ &cc->migratepages); ++ cc->nr_migratepages -= nr_reclaimed; ++ ++ ret = migrate_pages(&cc->migratepages, alloc_migrate_target, ++ NULL, 0, cc->mode, MR_CMA); ++ } ++ if (ret < 0) { ++ putback_movable_pages(&cc->migratepages); ++ return ret; ++ } ++ return 0; ++} ++ ++/** ++ * alloc_contig_range() -- tries to allocate given range of pages ++ * @start: start PFN to allocate ++ * @end: one-past-the-last PFN to allocate ++ * @migratetype: migratetype of the underlaying pageblocks (either ++ * #MIGRATE_MOVABLE or #MIGRATE_CMA). All pageblocks ++ * in range must have the same migratetype and it must ++ * be either of the two. ++ * ++ * The PFN range does not have to be pageblock or MAX_ORDER_NR_PAGES ++ * aligned, however it's the caller's responsibility to guarantee that ++ * we are the only thread that changes migrate type of pageblocks the ++ * pages fall in. ++ * ++ * The PFN range must belong to a single zone. ++ * ++ * Returns zero on success or negative error code. On success all ++ * pages which PFN is in [start, end) are allocated for the caller and ++ * need to be freed with free_contig_range(). ++ */ ++int alloc_contig_range(unsigned long start, unsigned long end, ++ unsigned migratetype) ++{ ++ unsigned long outer_start, outer_end; ++ int ret = 0, order; ++ ++ struct compact_control cc = { ++ .nr_migratepages = 0, ++ .order = -1, ++ .zone = page_zone(pfn_to_page(start)), ++ .mode = MIGRATE_SYNC, ++ .ignore_skip_hint = true, ++ }; ++ INIT_LIST_HEAD(&cc.migratepages); ++ ++ /* ++ * What we do here is we mark all pageblocks in range as ++ * MIGRATE_ISOLATE. Because pageblock and max order pages may ++ * have different sizes, and due to the way page allocator ++ * work, we align the range to biggest of the two pages so ++ * that page allocator won't try to merge buddies from ++ * different pageblocks and change MIGRATE_ISOLATE to some ++ * other migration type. ++ * ++ * Once the pageblocks are marked as MIGRATE_ISOLATE, we ++ * migrate the pages from an unaligned range (ie. pages that ++ * we are interested in). This will put all the pages in ++ * range back to page allocator as MIGRATE_ISOLATE. ++ * ++ * When this is done, we take the pages in range from page ++ * allocator removing them from the buddy system. This way ++ * page allocator will never consider using them. ++ * ++ * This lets us mark the pageblocks back as ++ * MIGRATE_CMA/MIGRATE_MOVABLE so that free pages in the ++ * aligned range but not in the unaligned, original range are ++ * put back to page allocator so that buddy can use them. ++ */ ++ ++ ret = start_isolate_page_range(pfn_max_align_down(start), ++ pfn_max_align_up(end), migratetype, ++ false); ++ if (ret) ++ return ret; ++ ++ ret = __alloc_contig_migrate_range(&cc, start, end); ++ if (ret) ++ goto done; ++ ++ /* ++ * Pages from [start, end) are within a MAX_ORDER_NR_PAGES ++ * aligned blocks that are marked as MIGRATE_ISOLATE. What's ++ * more, all pages in [start, end) are free in page allocator. ++ * What we are going to do is to allocate all pages from ++ * [start, end) (that is remove them from page allocator). ++ * ++ * The only problem is that pages at the beginning and at the ++ * end of interesting range may be not aligned with pages that ++ * page allocator holds, ie. they can be part of higher order ++ * pages. Because of this, we reserve the bigger range and ++ * once this is done free the pages we are not interested in. ++ * ++ * We don't have to hold zone->lock here because the pages are ++ * isolated thus they won't get removed from buddy. ++ */ ++ ++ lru_add_drain_all(); ++ drain_all_pages(cc.zone); ++ ++ order = 0; ++ outer_start = start; ++ while (!PageBuddy(pfn_to_page(outer_start))) { ++ if (++order >= MAX_ORDER) { ++ ret = -EBUSY; ++ goto done; ++ } ++ outer_start &= ~0UL << order; ++ } ++ ++ /* Make sure the range is really isolated. */ ++ if (test_pages_isolated(outer_start, end, false)) { ++ pr_info("%s: [%lx, %lx) PFNs busy\n", ++ __func__, outer_start, end); ++ ret = -EBUSY; ++ goto done; ++ } ++ ++ /* Grab isolated pages from freelists. */ ++ outer_end = isolate_freepages_range(&cc, outer_start, end); ++ if (!outer_end) { ++ ret = -EBUSY; ++ goto done; ++ } ++ ++ /* Free head and tail (if any) */ ++ if (start != outer_start) ++ free_contig_range(outer_start, start - outer_start); ++ if (end != outer_end) ++ free_contig_range(end, outer_end - end); ++ ++done: ++ undo_isolate_page_range(pfn_max_align_down(start), ++ pfn_max_align_up(end), migratetype); ++ return ret; ++} ++ ++void free_contig_range(unsigned long pfn, unsigned nr_pages) ++{ ++ unsigned int count = 0; ++ ++ for (; nr_pages--; pfn++) { ++ struct page *page = pfn_to_page(pfn); ++ ++ count += page_count(page) != 1; ++ __free_page(page); ++ } ++ WARN(count != 0, "%d pages are still in use!\n", count); ++} ++#endif ++ ++#ifdef CONFIG_MEMORY_HOTPLUG ++/* ++ * The zone indicated has a new number of managed_pages; batch sizes and percpu ++ * page high values need to be recalulated. ++ */ ++void __meminit zone_pcp_update(struct zone *zone) ++{ ++ unsigned cpu; ++ mutex_lock(&pcp_batch_high_lock); ++ for_each_possible_cpu(cpu) ++ pageset_set_high_and_batch(zone, ++ per_cpu_ptr(zone->pageset, cpu)); ++ mutex_unlock(&pcp_batch_high_lock); ++} ++#endif ++ ++void zone_pcp_reset(struct zone *zone) ++{ ++ unsigned long flags; ++ int cpu; ++ struct per_cpu_pageset *pset; ++ ++ /* avoid races with drain_pages() */ ++ local_irq_save(flags); ++ if (zone->pageset != &boot_pageset) { ++ for_each_online_cpu(cpu) { ++ pset = per_cpu_ptr(zone->pageset, cpu); ++ drain_zonestat(zone, pset); ++ } ++ free_percpu(zone->pageset); ++ zone->pageset = &boot_pageset; ++ } ++ local_irq_restore(flags); ++} ++ ++#ifdef CONFIG_MEMORY_HOTREMOVE ++/* ++ * All pages in the range must be isolated before calling this. ++ */ ++void ++__offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn) ++{ ++ struct page *page; ++ struct zone *zone; ++ unsigned int order, i; ++ unsigned long pfn; ++ unsigned long flags; ++ /* find the first valid pfn */ ++ for (pfn = start_pfn; pfn < end_pfn; pfn++) ++ if (pfn_valid(pfn)) ++ break; ++ if (pfn == end_pfn) ++ return; ++ zone = page_zone(pfn_to_page(pfn)); ++ spin_lock_irqsave(&zone->lock, flags); ++ pfn = start_pfn; ++ while (pfn < end_pfn) { ++ if (!pfn_valid(pfn)) { ++ pfn++; ++ continue; ++ } ++ page = pfn_to_page(pfn); ++ /* ++ * The HWPoisoned page may be not in buddy system, and ++ * page_count() is not 0. ++ */ ++ if (unlikely(!PageBuddy(page) && PageHWPoison(page))) { ++ pfn++; ++ SetPageReserved(page); ++ continue; ++ } ++ ++ BUG_ON(page_count(page)); ++ BUG_ON(!PageBuddy(page)); ++ order = page_order(page); ++#ifdef CONFIG_DEBUG_VM ++ printk(KERN_INFO "remove from free list %lx %d %lx\n", ++ pfn, 1 << order, end_pfn); ++#endif ++ list_del(&page->lru); ++ rmv_page_order(page); ++ zone->free_area[order].nr_free--; ++ for (i = 0; i < (1 << order); i++) ++ SetPageReserved((page+i)); ++ pfn += (1 << order); ++ } ++ spin_unlock_irqrestore(&zone->lock, flags); ++} ++#endif ++ ++#ifdef CONFIG_MEMORY_FAILURE ++bool is_free_buddy_page(struct page *page) ++{ ++ struct zone *zone = page_zone(page); ++ unsigned long pfn = page_to_pfn(page); ++ unsigned long flags; ++ unsigned int order; ++ ++ spin_lock_irqsave(&zone->lock, flags); ++ for (order = 0; order < MAX_ORDER; order++) { ++ struct page *page_head = page - (pfn & ((1 << order) - 1)); ++ ++ if (PageBuddy(page_head) && page_order(page_head) >= order) ++ break; ++ } ++ spin_unlock_irqrestore(&zone->lock, flags); ++ ++ return order < MAX_ORDER; ++} ++#endif +diff -Nur linux-4.1.10.orig/mm/slab.h linux-4.1.10/mm/slab.h +--- linux-4.1.10.orig/mm/slab.h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/mm/slab.h 2015-10-07 18:00:08.000000000 +0200 +@@ -330,7 +330,11 @@ + * The slab lists for all objects. + */ + struct kmem_cache_node { ++#ifdef CONFIG_SLUB ++ raw_spinlock_t list_lock; ++#else + spinlock_t list_lock; ++#endif + + #ifdef CONFIG_SLAB + struct list_head slabs_partial; /* partial list first, better asm code */ +diff -Nur linux-4.1.10.orig/mm/slub.c linux-4.1.10/mm/slub.c +--- linux-4.1.10.orig/mm/slub.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/mm/slub.c 2015-10-07 18:00:08.000000000 +0200 +@@ -1069,7 +1069,7 @@ + { + struct kmem_cache_node *n = get_node(s, page_to_nid(page)); + +- spin_lock_irqsave(&n->list_lock, *flags); ++ raw_spin_lock_irqsave(&n->list_lock, *flags); + slab_lock(page); + + if (!check_slab(s, page)) +@@ -1116,7 +1116,7 @@ + + fail: + slab_unlock(page); +- spin_unlock_irqrestore(&n->list_lock, *flags); ++ raw_spin_unlock_irqrestore(&n->list_lock, *flags); + slab_fix(s, "Object at 0x%p not freed", object); + return NULL; + } +@@ -1242,6 +1242,12 @@ + + #endif /* CONFIG_SLUB_DEBUG */ + ++struct slub_free_list { ++ raw_spinlock_t lock; ++ struct list_head list; ++}; ++static DEFINE_PER_CPU(struct slub_free_list, slub_free_list); ++ + /* + * Hooks for other subsystems that check memory allocations. In a typical + * production configuration these hooks all should produce no code at all. +@@ -1306,6 +1312,17 @@ + kasan_slab_free(s, x); + } + ++static void setup_object(struct kmem_cache *s, struct page *page, ++ void *object) ++{ ++ setup_object_debug(s, page, object); ++ if (unlikely(s->ctor)) { ++ kasan_unpoison_object_data(s, object); ++ s->ctor(object); ++ kasan_poison_object_data(s, object); ++ } ++} ++ + /* + * Slab allocation and freeing + */ +@@ -1336,10 +1353,17 @@ + struct page *page; + struct kmem_cache_order_objects oo = s->oo; + gfp_t alloc_gfp; ++ void *start, *p; ++ int idx, order; ++ bool enableirqs; + + flags &= gfp_allowed_mask; + +- if (flags & __GFP_WAIT) ++ enableirqs = (flags & __GFP_WAIT) != 0; ++#ifdef CONFIG_PREEMPT_RT_FULL ++ enableirqs |= system_state == SYSTEM_RUNNING; ++#endif ++ if (enableirqs) + local_irq_enable(); + + flags |= s->allocflags; +@@ -1359,13 +1383,13 @@ + * Try a lower order alloc if possible + */ + page = alloc_slab_page(s, alloc_gfp, node, oo); +- +- if (page) +- stat(s, ORDER_FALLBACK); ++ if (unlikely(!page)) ++ goto out; ++ stat(s, ORDER_FALLBACK); + } + +- if (kmemcheck_enabled && page +- && !(s->flags & (SLAB_NOTRACK | DEBUG_DEFAULT_FLAGS))) { ++ if (kmemcheck_enabled && ++ !(s->flags & (SLAB_NOTRACK | DEBUG_DEFAULT_FLAGS))) { + int pages = 1 << oo_order(oo); + + kmemcheck_alloc_shadow(page, oo_order(oo), alloc_gfp, node); +@@ -1380,51 +1404,9 @@ + kmemcheck_mark_unallocated_pages(page, pages); + } + +- if (flags & __GFP_WAIT) +- local_irq_disable(); +- if (!page) +- return NULL; +- + page->objects = oo_objects(oo); +- mod_zone_page_state(page_zone(page), +- (s->flags & SLAB_RECLAIM_ACCOUNT) ? +- NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE, +- 1 << oo_order(oo)); +- +- return page; +-} +- +-static void setup_object(struct kmem_cache *s, struct page *page, +- void *object) +-{ +- setup_object_debug(s, page, object); +- if (unlikely(s->ctor)) { +- kasan_unpoison_object_data(s, object); +- s->ctor(object); +- kasan_poison_object_data(s, object); +- } +-} +- +-static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node) +-{ +- struct page *page; +- void *start; +- void *p; +- int order; +- int idx; +- +- if (unlikely(flags & GFP_SLAB_BUG_MASK)) { +- pr_emerg("gfp: %u\n", flags & GFP_SLAB_BUG_MASK); +- BUG(); +- } +- +- page = allocate_slab(s, +- flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node); +- if (!page) +- goto out; + + order = compound_order(page); +- inc_slabs_node(s, page_to_nid(page), page->objects); + page->slab_cache = s; + __SetPageSlab(page); + if (page_is_pfmemalloc(page)) +@@ -1448,10 +1430,34 @@ + page->freelist = start; + page->inuse = page->objects; + page->frozen = 1; ++ + out: ++ if (enableirqs) ++ local_irq_disable(); ++ if (!page) ++ return NULL; ++ ++ mod_zone_page_state(page_zone(page), ++ (s->flags & SLAB_RECLAIM_ACCOUNT) ? ++ NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE, ++ 1 << oo_order(oo)); ++ ++ inc_slabs_node(s, page_to_nid(page), page->objects); ++ + return page; + } + ++static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node) ++{ ++ if (unlikely(flags & GFP_SLAB_BUG_MASK)) { ++ pr_emerg("gfp: %u\n", flags & GFP_SLAB_BUG_MASK); ++ BUG(); ++ } ++ ++ return allocate_slab(s, ++ flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node); ++} ++ + static void __free_slab(struct kmem_cache *s, struct page *page) + { + int order = compound_order(page); +@@ -1483,6 +1489,16 @@ + memcg_uncharge_slab(s, order); + } + ++static void free_delayed(struct list_head *h) ++{ ++ while(!list_empty(h)) { ++ struct page *page = list_first_entry(h, struct page, lru); ++ ++ list_del(&page->lru); ++ __free_slab(page->slab_cache, page); ++ } ++} ++ + #define need_reserve_slab_rcu \ + (sizeof(((struct page *)NULL)->lru) < sizeof(struct rcu_head)) + +@@ -1517,6 +1533,12 @@ + } + + call_rcu(head, rcu_free_slab); ++ } else if (irqs_disabled()) { ++ struct slub_free_list *f = this_cpu_ptr(&slub_free_list); ++ ++ raw_spin_lock(&f->lock); ++ list_add(&page->lru, &f->list); ++ raw_spin_unlock(&f->lock); + } else + __free_slab(s, page); + } +@@ -1630,7 +1652,7 @@ + if (!n || !n->nr_partial) + return NULL; + +- spin_lock(&n->list_lock); ++ raw_spin_lock(&n->list_lock); + list_for_each_entry_safe(page, page2, &n->partial, lru) { + void *t; + +@@ -1655,7 +1677,7 @@ + break; + + } +- spin_unlock(&n->list_lock); ++ raw_spin_unlock(&n->list_lock); + return object; + } + +@@ -1901,7 +1923,7 @@ + * that acquire_slab() will see a slab page that + * is frozen + */ +- spin_lock(&n->list_lock); ++ raw_spin_lock(&n->list_lock); + } + } else { + m = M_FULL; +@@ -1912,7 +1934,7 @@ + * slabs from diagnostic functions will not see + * any frozen slabs. + */ +- spin_lock(&n->list_lock); ++ raw_spin_lock(&n->list_lock); + } + } + +@@ -1947,7 +1969,7 @@ + goto redo; + + if (lock) +- spin_unlock(&n->list_lock); ++ raw_spin_unlock(&n->list_lock); + + if (m == M_FREE) { + stat(s, DEACTIVATE_EMPTY); +@@ -1979,10 +2001,10 @@ + n2 = get_node(s, page_to_nid(page)); + if (n != n2) { + if (n) +- spin_unlock(&n->list_lock); ++ raw_spin_unlock(&n->list_lock); + + n = n2; +- spin_lock(&n->list_lock); ++ raw_spin_lock(&n->list_lock); + } + + do { +@@ -2011,7 +2033,7 @@ + } + + if (n) +- spin_unlock(&n->list_lock); ++ raw_spin_unlock(&n->list_lock); + + while (discard_page) { + page = discard_page; +@@ -2050,14 +2072,21 @@ + pobjects = oldpage->pobjects; + pages = oldpage->pages; + if (drain && pobjects > s->cpu_partial) { ++ struct slub_free_list *f; + unsigned long flags; ++ LIST_HEAD(tofree); + /* + * partial array is full. Move the existing + * set to the per node partial list. + */ + local_irq_save(flags); + unfreeze_partials(s, this_cpu_ptr(s->cpu_slab)); ++ f = this_cpu_ptr(&slub_free_list); ++ raw_spin_lock(&f->lock); ++ list_splice_init(&f->list, &tofree); ++ raw_spin_unlock(&f->lock); + local_irq_restore(flags); ++ free_delayed(&tofree); + oldpage = NULL; + pobjects = 0; + pages = 0; +@@ -2129,7 +2158,22 @@ + + static void flush_all(struct kmem_cache *s) + { ++ LIST_HEAD(tofree); ++ int cpu; ++ + on_each_cpu_cond(has_cpu_slab, flush_cpu_slab, s, 1, GFP_ATOMIC); ++ for_each_online_cpu(cpu) { ++ struct slub_free_list *f; ++ ++ if (!has_cpu_slab(cpu, s)) ++ continue; ++ ++ f = &per_cpu(slub_free_list, cpu); ++ raw_spin_lock_irq(&f->lock); ++ list_splice_init(&f->list, &tofree); ++ raw_spin_unlock_irq(&f->lock); ++ free_delayed(&tofree); ++ } + } + + /* +@@ -2165,10 +2209,10 @@ + unsigned long x = 0; + struct page *page; + +- spin_lock_irqsave(&n->list_lock, flags); ++ raw_spin_lock_irqsave(&n->list_lock, flags); + list_for_each_entry(page, &n->partial, lru) + x += get_count(page); +- spin_unlock_irqrestore(&n->list_lock, flags); ++ raw_spin_unlock_irqrestore(&n->list_lock, flags); + return x; + } + #endif /* CONFIG_SLUB_DEBUG || CONFIG_SYSFS */ +@@ -2305,9 +2349,11 @@ + static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, + unsigned long addr, struct kmem_cache_cpu *c) + { ++ struct slub_free_list *f; + void *freelist; + struct page *page; + unsigned long flags; ++ LIST_HEAD(tofree); + + local_irq_save(flags); + #ifdef CONFIG_PREEMPT +@@ -2375,7 +2421,13 @@ + VM_BUG_ON(!c->page->frozen); + c->freelist = get_freepointer(s, freelist); + c->tid = next_tid(c->tid); ++out: ++ f = this_cpu_ptr(&slub_free_list); ++ raw_spin_lock(&f->lock); ++ list_splice_init(&f->list, &tofree); ++ raw_spin_unlock(&f->lock); + local_irq_restore(flags); ++ free_delayed(&tofree); + return freelist; + + new_slab: +@@ -2392,8 +2444,7 @@ + + if (unlikely(!freelist)) { + slab_out_of_memory(s, gfpflags, node); +- local_irq_restore(flags); +- return NULL; ++ goto out; + } + + page = c->page; +@@ -2408,8 +2459,7 @@ + deactivate_slab(s, page, get_freepointer(s, freelist)); + c->page = NULL; + c->freelist = NULL; +- local_irq_restore(flags); +- return freelist; ++ goto out; + } + + /* +@@ -2593,7 +2643,7 @@ + + do { + if (unlikely(n)) { +- spin_unlock_irqrestore(&n->list_lock, flags); ++ raw_spin_unlock_irqrestore(&n->list_lock, flags); + n = NULL; + } + prior = page->freelist; +@@ -2625,7 +2675,7 @@ + * Otherwise the list_lock will synchronize with + * other processors updating the list of slabs. + */ +- spin_lock_irqsave(&n->list_lock, flags); ++ raw_spin_lock_irqsave(&n->list_lock, flags); + + } + } +@@ -2667,7 +2717,7 @@ + add_partial(n, page, DEACTIVATE_TO_TAIL); + stat(s, FREE_ADD_PARTIAL); + } +- spin_unlock_irqrestore(&n->list_lock, flags); ++ raw_spin_unlock_irqrestore(&n->list_lock, flags); + return; + + slab_empty: +@@ -2682,7 +2732,7 @@ + remove_full(s, n, page); + } + +- spin_unlock_irqrestore(&n->list_lock, flags); ++ raw_spin_unlock_irqrestore(&n->list_lock, flags); + stat(s, FREE_SLAB); + discard_slab(s, page); + } +@@ -2881,7 +2931,7 @@ + init_kmem_cache_node(struct kmem_cache_node *n) + { + n->nr_partial = 0; +- spin_lock_init(&n->list_lock); ++ raw_spin_lock_init(&n->list_lock); + INIT_LIST_HEAD(&n->partial); + #ifdef CONFIG_SLUB_DEBUG + atomic_long_set(&n->nr_slabs, 0); +@@ -3463,7 +3513,7 @@ + for (i = 0; i < SHRINK_PROMOTE_MAX; i++) + INIT_LIST_HEAD(promote + i); + +- spin_lock_irqsave(&n->list_lock, flags); ++ raw_spin_lock_irqsave(&n->list_lock, flags); + + /* + * Build lists of slabs to discard or promote. +@@ -3494,7 +3544,7 @@ + for (i = SHRINK_PROMOTE_MAX - 1; i >= 0; i--) + list_splice(promote + i, &n->partial); + +- spin_unlock_irqrestore(&n->list_lock, flags); ++ raw_spin_unlock_irqrestore(&n->list_lock, flags); + + /* Release empty slabs */ + list_for_each_entry_safe(page, t, &discard, lru) +@@ -3670,6 +3720,12 @@ + { + static __initdata struct kmem_cache boot_kmem_cache, + boot_kmem_cache_node; ++ int cpu; ++ ++ for_each_possible_cpu(cpu) { ++ raw_spin_lock_init(&per_cpu(slub_free_list, cpu).lock); ++ INIT_LIST_HEAD(&per_cpu(slub_free_list, cpu).list); ++ } + + if (debug_guardpage_minorder()) + slub_max_order = 0; +@@ -3912,7 +3968,7 @@ + struct page *page; + unsigned long flags; + +- spin_lock_irqsave(&n->list_lock, flags); ++ raw_spin_lock_irqsave(&n->list_lock, flags); + + list_for_each_entry(page, &n->partial, lru) { + validate_slab_slab(s, page, map); +@@ -3934,7 +3990,7 @@ + s->name, count, atomic_long_read(&n->nr_slabs)); + + out: +- spin_unlock_irqrestore(&n->list_lock, flags); ++ raw_spin_unlock_irqrestore(&n->list_lock, flags); + return count; + } + +@@ -4122,12 +4178,12 @@ + if (!atomic_long_read(&n->nr_slabs)) + continue; + +- spin_lock_irqsave(&n->list_lock, flags); ++ raw_spin_lock_irqsave(&n->list_lock, flags); + list_for_each_entry(page, &n->partial, lru) + process_slab(&t, s, page, alloc, map); + list_for_each_entry(page, &n->full, lru) + process_slab(&t, s, page, alloc, map); +- spin_unlock_irqrestore(&n->list_lock, flags); ++ raw_spin_unlock_irqrestore(&n->list_lock, flags); + } + + for (i = 0; i < t.count; i++) { +diff -Nur linux-4.1.10.orig/mm/slub.c.orig linux-4.1.10/mm/slub.c.orig +--- linux-4.1.10.orig/mm/slub.c.orig 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/mm/slub.c.orig 2015-10-03 13:49:38.000000000 +0200 +@@ -0,0 +1,5344 @@ ++/* ++ * SLUB: A slab allocator that limits cache line use instead of queuing ++ * objects in per cpu and per node lists. ++ * ++ * The allocator synchronizes using per slab locks or atomic operatios ++ * and only uses a centralized lock to manage a pool of partial slabs. ++ * ++ * (C) 2007 SGI, Christoph Lameter ++ * (C) 2011 Linux Foundation, Christoph Lameter ++ */ ++ ++#include ++#include /* struct reclaim_state */ ++#include ++#include ++#include ++#include ++#include ++#include "slab.h" ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++ ++#include "internal.h" ++ ++/* ++ * Lock order: ++ * 1. slab_mutex (Global Mutex) ++ * 2. node->list_lock ++ * 3. slab_lock(page) (Only on some arches and for debugging) ++ * ++ * slab_mutex ++ * ++ * The role of the slab_mutex is to protect the list of all the slabs ++ * and to synchronize major metadata changes to slab cache structures. ++ * ++ * The slab_lock is only used for debugging and on arches that do not ++ * have the ability to do a cmpxchg_double. It only protects the second ++ * double word in the page struct. Meaning ++ * A. page->freelist -> List of object free in a page ++ * B. page->counters -> Counters of objects ++ * C. page->frozen -> frozen state ++ * ++ * If a slab is frozen then it is exempt from list management. It is not ++ * on any list. The processor that froze the slab is the one who can ++ * perform list operations on the page. Other processors may put objects ++ * onto the freelist but the processor that froze the slab is the only ++ * one that can retrieve the objects from the page's freelist. ++ * ++ * The list_lock protects the partial and full list on each node and ++ * the partial slab counter. If taken then no new slabs may be added or ++ * removed from the lists nor make the number of partial slabs be modified. ++ * (Note that the total number of slabs is an atomic value that may be ++ * modified without taking the list lock). ++ * ++ * The list_lock is a centralized lock and thus we avoid taking it as ++ * much as possible. As long as SLUB does not have to handle partial ++ * slabs, operations can continue without any centralized lock. F.e. ++ * allocating a long series of objects that fill up slabs does not require ++ * the list lock. ++ * Interrupts are disabled during allocation and deallocation in order to ++ * make the slab allocator safe to use in the context of an irq. In addition ++ * interrupts are disabled to ensure that the processor does not change ++ * while handling per_cpu slabs, due to kernel preemption. ++ * ++ * SLUB assigns one slab for allocation to each processor. ++ * Allocations only occur from these slabs called cpu slabs. ++ * ++ * Slabs with free elements are kept on a partial list and during regular ++ * operations no list for full slabs is used. If an object in a full slab is ++ * freed then the slab will show up again on the partial lists. ++ * We track full slabs for debugging purposes though because otherwise we ++ * cannot scan all objects. ++ * ++ * Slabs are freed when they become empty. Teardown and setup is ++ * minimal so we rely on the page allocators per cpu caches for ++ * fast frees and allocs. ++ * ++ * Overloading of page flags that are otherwise used for LRU management. ++ * ++ * PageActive The slab is frozen and exempt from list processing. ++ * This means that the slab is dedicated to a purpose ++ * such as satisfying allocations for a specific ++ * processor. Objects may be freed in the slab while ++ * it is frozen but slab_free will then skip the usual ++ * list operations. It is up to the processor holding ++ * the slab to integrate the slab into the slab lists ++ * when the slab is no longer needed. ++ * ++ * One use of this flag is to mark slabs that are ++ * used for allocations. Then such a slab becomes a cpu ++ * slab. The cpu slab may be equipped with an additional ++ * freelist that allows lockless access to ++ * free objects in addition to the regular freelist ++ * that requires the slab lock. ++ * ++ * PageError Slab requires special handling due to debug ++ * options set. This moves slab handling out of ++ * the fast path and disables lockless freelists. ++ */ ++ ++static inline int kmem_cache_debug(struct kmem_cache *s) ++{ ++#ifdef CONFIG_SLUB_DEBUG ++ return unlikely(s->flags & SLAB_DEBUG_FLAGS); ++#else ++ return 0; ++#endif ++} ++ ++static inline bool kmem_cache_has_cpu_partial(struct kmem_cache *s) ++{ ++#ifdef CONFIG_SLUB_CPU_PARTIAL ++ return !kmem_cache_debug(s); ++#else ++ return false; ++#endif ++} ++ ++/* ++ * Issues still to be resolved: ++ * ++ * - Support PAGE_ALLOC_DEBUG. Should be easy to do. ++ * ++ * - Variable sizing of the per node arrays ++ */ ++ ++/* Enable to test recovery from slab corruption on boot */ ++#undef SLUB_RESILIENCY_TEST ++ ++/* Enable to log cmpxchg failures */ ++#undef SLUB_DEBUG_CMPXCHG ++ ++/* ++ * Mininum number of partial slabs. These will be left on the partial ++ * lists even if they are empty. kmem_cache_shrink may reclaim them. ++ */ ++#define MIN_PARTIAL 5 ++ ++/* ++ * Maximum number of desirable partial slabs. ++ * The existence of more partial slabs makes kmem_cache_shrink ++ * sort the partial list by the number of objects in use. ++ */ ++#define MAX_PARTIAL 10 ++ ++#define DEBUG_DEFAULT_FLAGS (SLAB_DEBUG_FREE | SLAB_RED_ZONE | \ ++ SLAB_POISON | SLAB_STORE_USER) ++ ++/* ++ * Debugging flags that require metadata to be stored in the slab. These get ++ * disabled when slub_debug=O is used and a cache's min order increases with ++ * metadata. ++ */ ++#define DEBUG_METADATA_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER) ++ ++#define OO_SHIFT 16 ++#define OO_MASK ((1 << OO_SHIFT) - 1) ++#define MAX_OBJS_PER_PAGE 32767 /* since page.objects is u15 */ ++ ++/* Internal SLUB flags */ ++#define __OBJECT_POISON 0x80000000UL /* Poison object */ ++#define __CMPXCHG_DOUBLE 0x40000000UL /* Use cmpxchg_double */ ++ ++#ifdef CONFIG_SMP ++static struct notifier_block slab_notifier; ++#endif ++ ++/* ++ * Tracking user of a slab. ++ */ ++#define TRACK_ADDRS_COUNT 16 ++struct track { ++ unsigned long addr; /* Called from address */ ++#ifdef CONFIG_STACKTRACE ++ unsigned long addrs[TRACK_ADDRS_COUNT]; /* Called from address */ ++#endif ++ int cpu; /* Was running on cpu */ ++ int pid; /* Pid context */ ++ unsigned long when; /* When did the operation occur */ ++}; ++ ++enum track_item { TRACK_ALLOC, TRACK_FREE }; ++ ++#ifdef CONFIG_SYSFS ++static int sysfs_slab_add(struct kmem_cache *); ++static int sysfs_slab_alias(struct kmem_cache *, const char *); ++static void memcg_propagate_slab_attrs(struct kmem_cache *s); ++#else ++static inline int sysfs_slab_add(struct kmem_cache *s) { return 0; } ++static inline int sysfs_slab_alias(struct kmem_cache *s, const char *p) ++ { return 0; } ++static inline void memcg_propagate_slab_attrs(struct kmem_cache *s) { } ++#endif ++ ++static inline void stat(const struct kmem_cache *s, enum stat_item si) ++{ ++#ifdef CONFIG_SLUB_STATS ++ /* ++ * The rmw is racy on a preemptible kernel but this is acceptable, so ++ * avoid this_cpu_add()'s irq-disable overhead. ++ */ ++ raw_cpu_inc(s->cpu_slab->stat[si]); ++#endif ++} ++ ++/******************************************************************** ++ * Core slab cache functions ++ *******************************************************************/ ++ ++/* Verify that a pointer has an address that is valid within a slab page */ ++static inline int check_valid_pointer(struct kmem_cache *s, ++ struct page *page, const void *object) ++{ ++ void *base; ++ ++ if (!object) ++ return 1; ++ ++ base = page_address(page); ++ if (object < base || object >= base + page->objects * s->size || ++ (object - base) % s->size) { ++ return 0; ++ } ++ ++ return 1; ++} ++ ++static inline void *get_freepointer(struct kmem_cache *s, void *object) ++{ ++ return *(void **)(object + s->offset); ++} ++ ++static void prefetch_freepointer(const struct kmem_cache *s, void *object) ++{ ++ prefetch(object + s->offset); ++} ++ ++static inline void *get_freepointer_safe(struct kmem_cache *s, void *object) ++{ ++ void *p; ++ ++#ifdef CONFIG_DEBUG_PAGEALLOC ++ probe_kernel_read(&p, (void **)(object + s->offset), sizeof(p)); ++#else ++ p = get_freepointer(s, object); ++#endif ++ return p; ++} ++ ++static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp) ++{ ++ *(void **)(object + s->offset) = fp; ++} ++ ++/* Loop over all objects in a slab */ ++#define for_each_object(__p, __s, __addr, __objects) \ ++ for (__p = (__addr); __p < (__addr) + (__objects) * (__s)->size;\ ++ __p += (__s)->size) ++ ++#define for_each_object_idx(__p, __idx, __s, __addr, __objects) \ ++ for (__p = (__addr), __idx = 1; __idx <= __objects;\ ++ __p += (__s)->size, __idx++) ++ ++/* Determine object index from a given position */ ++static inline int slab_index(void *p, struct kmem_cache *s, void *addr) ++{ ++ return (p - addr) / s->size; ++} ++ ++static inline size_t slab_ksize(const struct kmem_cache *s) ++{ ++#ifdef CONFIG_SLUB_DEBUG ++ /* ++ * Debugging requires use of the padding between object ++ * and whatever may come after it. ++ */ ++ if (s->flags & (SLAB_RED_ZONE | SLAB_POISON)) ++ return s->object_size; ++ ++#endif ++ /* ++ * If we have the need to store the freelist pointer ++ * back there or track user information then we can ++ * only use the space before that information. ++ */ ++ if (s->flags & (SLAB_DESTROY_BY_RCU | SLAB_STORE_USER)) ++ return s->inuse; ++ /* ++ * Else we can use all the padding etc for the allocation ++ */ ++ return s->size; ++} ++ ++static inline int order_objects(int order, unsigned long size, int reserved) ++{ ++ return ((PAGE_SIZE << order) - reserved) / size; ++} ++ ++static inline struct kmem_cache_order_objects oo_make(int order, ++ unsigned long size, int reserved) ++{ ++ struct kmem_cache_order_objects x = { ++ (order << OO_SHIFT) + order_objects(order, size, reserved) ++ }; ++ ++ return x; ++} ++ ++static inline int oo_order(struct kmem_cache_order_objects x) ++{ ++ return x.x >> OO_SHIFT; ++} ++ ++static inline int oo_objects(struct kmem_cache_order_objects x) ++{ ++ return x.x & OO_MASK; ++} ++ ++/* ++ * Per slab locking using the pagelock ++ */ ++static __always_inline void slab_lock(struct page *page) ++{ ++ bit_spin_lock(PG_locked, &page->flags); ++} ++ ++static __always_inline void slab_unlock(struct page *page) ++{ ++ __bit_spin_unlock(PG_locked, &page->flags); ++} ++ ++static inline void set_page_slub_counters(struct page *page, unsigned long counters_new) ++{ ++ struct page tmp; ++ tmp.counters = counters_new; ++ /* ++ * page->counters can cover frozen/inuse/objects as well ++ * as page->_count. If we assign to ->counters directly ++ * we run the risk of losing updates to page->_count, so ++ * be careful and only assign to the fields we need. ++ */ ++ page->frozen = tmp.frozen; ++ page->inuse = tmp.inuse; ++ page->objects = tmp.objects; ++} ++ ++/* Interrupts must be disabled (for the fallback code to work right) */ ++static inline bool __cmpxchg_double_slab(struct kmem_cache *s, struct page *page, ++ void *freelist_old, unsigned long counters_old, ++ void *freelist_new, unsigned long counters_new, ++ const char *n) ++{ ++ VM_BUG_ON(!irqs_disabled()); ++#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \ ++ defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE) ++ if (s->flags & __CMPXCHG_DOUBLE) { ++ if (cmpxchg_double(&page->freelist, &page->counters, ++ freelist_old, counters_old, ++ freelist_new, counters_new)) ++ return true; ++ } else ++#endif ++ { ++ slab_lock(page); ++ if (page->freelist == freelist_old && ++ page->counters == counters_old) { ++ page->freelist = freelist_new; ++ set_page_slub_counters(page, counters_new); ++ slab_unlock(page); ++ return true; ++ } ++ slab_unlock(page); ++ } ++ ++ cpu_relax(); ++ stat(s, CMPXCHG_DOUBLE_FAIL); ++ ++#ifdef SLUB_DEBUG_CMPXCHG ++ pr_info("%s %s: cmpxchg double redo ", n, s->name); ++#endif ++ ++ return false; ++} ++ ++static inline bool cmpxchg_double_slab(struct kmem_cache *s, struct page *page, ++ void *freelist_old, unsigned long counters_old, ++ void *freelist_new, unsigned long counters_new, ++ const char *n) ++{ ++#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \ ++ defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE) ++ if (s->flags & __CMPXCHG_DOUBLE) { ++ if (cmpxchg_double(&page->freelist, &page->counters, ++ freelist_old, counters_old, ++ freelist_new, counters_new)) ++ return true; ++ } else ++#endif ++ { ++ unsigned long flags; ++ ++ local_irq_save(flags); ++ slab_lock(page); ++ if (page->freelist == freelist_old && ++ page->counters == counters_old) { ++ page->freelist = freelist_new; ++ set_page_slub_counters(page, counters_new); ++ slab_unlock(page); ++ local_irq_restore(flags); ++ return true; ++ } ++ slab_unlock(page); ++ local_irq_restore(flags); ++ } ++ ++ cpu_relax(); ++ stat(s, CMPXCHG_DOUBLE_FAIL); ++ ++#ifdef SLUB_DEBUG_CMPXCHG ++ pr_info("%s %s: cmpxchg double redo ", n, s->name); ++#endif ++ ++ return false; ++} ++ ++#ifdef CONFIG_SLUB_DEBUG ++/* ++ * Determine a map of object in use on a page. ++ * ++ * Node listlock must be held to guarantee that the page does ++ * not vanish from under us. ++ */ ++static void get_map(struct kmem_cache *s, struct page *page, unsigned long *map) ++{ ++ void *p; ++ void *addr = page_address(page); ++ ++ for (p = page->freelist; p; p = get_freepointer(s, p)) ++ set_bit(slab_index(p, s, addr), map); ++} ++ ++/* ++ * Debug settings: ++ */ ++#ifdef CONFIG_SLUB_DEBUG_ON ++static int slub_debug = DEBUG_DEFAULT_FLAGS; ++#else ++static int slub_debug; ++#endif ++ ++static char *slub_debug_slabs; ++static int disable_higher_order_debug; ++ ++/* ++ * slub is about to manipulate internal object metadata. This memory lies ++ * outside the range of the allocated object, so accessing it would normally ++ * be reported by kasan as a bounds error. metadata_access_enable() is used ++ * to tell kasan that these accesses are OK. ++ */ ++static inline void metadata_access_enable(void) ++{ ++ kasan_disable_current(); ++} ++ ++static inline void metadata_access_disable(void) ++{ ++ kasan_enable_current(); ++} ++ ++/* ++ * Object debugging ++ */ ++static void print_section(char *text, u8 *addr, unsigned int length) ++{ ++ metadata_access_enable(); ++ print_hex_dump(KERN_ERR, text, DUMP_PREFIX_ADDRESS, 16, 1, addr, ++ length, 1); ++ metadata_access_disable(); ++} ++ ++static struct track *get_track(struct kmem_cache *s, void *object, ++ enum track_item alloc) ++{ ++ struct track *p; ++ ++ if (s->offset) ++ p = object + s->offset + sizeof(void *); ++ else ++ p = object + s->inuse; ++ ++ return p + alloc; ++} ++ ++static void set_track(struct kmem_cache *s, void *object, ++ enum track_item alloc, unsigned long addr) ++{ ++ struct track *p = get_track(s, object, alloc); ++ ++ if (addr) { ++#ifdef CONFIG_STACKTRACE ++ struct stack_trace trace; ++ int i; ++ ++ trace.nr_entries = 0; ++ trace.max_entries = TRACK_ADDRS_COUNT; ++ trace.entries = p->addrs; ++ trace.skip = 3; ++ metadata_access_enable(); ++ save_stack_trace(&trace); ++ metadata_access_disable(); ++ ++ /* See rant in lockdep.c */ ++ if (trace.nr_entries != 0 && ++ trace.entries[trace.nr_entries - 1] == ULONG_MAX) ++ trace.nr_entries--; ++ ++ for (i = trace.nr_entries; i < TRACK_ADDRS_COUNT; i++) ++ p->addrs[i] = 0; ++#endif ++ p->addr = addr; ++ p->cpu = smp_processor_id(); ++ p->pid = current->pid; ++ p->when = jiffies; ++ } else ++ memset(p, 0, sizeof(struct track)); ++} ++ ++static void init_tracking(struct kmem_cache *s, void *object) ++{ ++ if (!(s->flags & SLAB_STORE_USER)) ++ return; ++ ++ set_track(s, object, TRACK_FREE, 0UL); ++ set_track(s, object, TRACK_ALLOC, 0UL); ++} ++ ++static void print_track(const char *s, struct track *t) ++{ ++ if (!t->addr) ++ return; ++ ++ pr_err("INFO: %s in %pS age=%lu cpu=%u pid=%d\n", ++ s, (void *)t->addr, jiffies - t->when, t->cpu, t->pid); ++#ifdef CONFIG_STACKTRACE ++ { ++ int i; ++ for (i = 0; i < TRACK_ADDRS_COUNT; i++) ++ if (t->addrs[i]) ++ pr_err("\t%pS\n", (void *)t->addrs[i]); ++ else ++ break; ++ } ++#endif ++} ++ ++static void print_tracking(struct kmem_cache *s, void *object) ++{ ++ if (!(s->flags & SLAB_STORE_USER)) ++ return; ++ ++ print_track("Allocated", get_track(s, object, TRACK_ALLOC)); ++ print_track("Freed", get_track(s, object, TRACK_FREE)); ++} ++ ++static void print_page_info(struct page *page) ++{ ++ pr_err("INFO: Slab 0x%p objects=%u used=%u fp=0x%p flags=0x%04lx\n", ++ page, page->objects, page->inuse, page->freelist, page->flags); ++ ++} ++ ++static void slab_bug(struct kmem_cache *s, char *fmt, ...) ++{ ++ struct va_format vaf; ++ va_list args; ++ ++ va_start(args, fmt); ++ vaf.fmt = fmt; ++ vaf.va = &args; ++ pr_err("=============================================================================\n"); ++ pr_err("BUG %s (%s): %pV\n", s->name, print_tainted(), &vaf); ++ pr_err("-----------------------------------------------------------------------------\n\n"); ++ ++ add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE); ++ va_end(args); ++} ++ ++static void slab_fix(struct kmem_cache *s, char *fmt, ...) ++{ ++ struct va_format vaf; ++ va_list args; ++ ++ va_start(args, fmt); ++ vaf.fmt = fmt; ++ vaf.va = &args; ++ pr_err("FIX %s: %pV\n", s->name, &vaf); ++ va_end(args); ++} ++ ++static void print_trailer(struct kmem_cache *s, struct page *page, u8 *p) ++{ ++ unsigned int off; /* Offset of last byte */ ++ u8 *addr = page_address(page); ++ ++ print_tracking(s, p); ++ ++ print_page_info(page); ++ ++ pr_err("INFO: Object 0x%p @offset=%tu fp=0x%p\n\n", ++ p, p - addr, get_freepointer(s, p)); ++ ++ if (p > addr + 16) ++ print_section("Bytes b4 ", p - 16, 16); ++ ++ print_section("Object ", p, min_t(unsigned long, s->object_size, ++ PAGE_SIZE)); ++ if (s->flags & SLAB_RED_ZONE) ++ print_section("Redzone ", p + s->object_size, ++ s->inuse - s->object_size); ++ ++ if (s->offset) ++ off = s->offset + sizeof(void *); ++ else ++ off = s->inuse; ++ ++ if (s->flags & SLAB_STORE_USER) ++ off += 2 * sizeof(struct track); ++ ++ if (off != s->size) ++ /* Beginning of the filler is the free pointer */ ++ print_section("Padding ", p + off, s->size - off); ++ ++ dump_stack(); ++} ++ ++void object_err(struct kmem_cache *s, struct page *page, ++ u8 *object, char *reason) ++{ ++ slab_bug(s, "%s", reason); ++ print_trailer(s, page, object); ++} ++ ++static void slab_err(struct kmem_cache *s, struct page *page, ++ const char *fmt, ...) ++{ ++ va_list args; ++ char buf[100]; ++ ++ va_start(args, fmt); ++ vsnprintf(buf, sizeof(buf), fmt, args); ++ va_end(args); ++ slab_bug(s, "%s", buf); ++ print_page_info(page); ++ dump_stack(); ++} ++ ++static void init_object(struct kmem_cache *s, void *object, u8 val) ++{ ++ u8 *p = object; ++ ++ if (s->flags & __OBJECT_POISON) { ++ memset(p, POISON_FREE, s->object_size - 1); ++ p[s->object_size - 1] = POISON_END; ++ } ++ ++ if (s->flags & SLAB_RED_ZONE) ++ memset(p + s->object_size, val, s->inuse - s->object_size); ++} ++ ++static void restore_bytes(struct kmem_cache *s, char *message, u8 data, ++ void *from, void *to) ++{ ++ slab_fix(s, "Restoring 0x%p-0x%p=0x%x\n", from, to - 1, data); ++ memset(from, data, to - from); ++} ++ ++static int check_bytes_and_report(struct kmem_cache *s, struct page *page, ++ u8 *object, char *what, ++ u8 *start, unsigned int value, unsigned int bytes) ++{ ++ u8 *fault; ++ u8 *end; ++ ++ metadata_access_enable(); ++ fault = memchr_inv(start, value, bytes); ++ metadata_access_disable(); ++ if (!fault) ++ return 1; ++ ++ end = start + bytes; ++ while (end > fault && end[-1] == value) ++ end--; ++ ++ slab_bug(s, "%s overwritten", what); ++ pr_err("INFO: 0x%p-0x%p. First byte 0x%x instead of 0x%x\n", ++ fault, end - 1, fault[0], value); ++ print_trailer(s, page, object); ++ ++ restore_bytes(s, what, value, fault, end); ++ return 0; ++} ++ ++/* ++ * Object layout: ++ * ++ * object address ++ * Bytes of the object to be managed. ++ * If the freepointer may overlay the object then the free ++ * pointer is the first word of the object. ++ * ++ * Poisoning uses 0x6b (POISON_FREE) and the last byte is ++ * 0xa5 (POISON_END) ++ * ++ * object + s->object_size ++ * Padding to reach word boundary. This is also used for Redzoning. ++ * Padding is extended by another word if Redzoning is enabled and ++ * object_size == inuse. ++ * ++ * We fill with 0xbb (RED_INACTIVE) for inactive objects and with ++ * 0xcc (RED_ACTIVE) for objects in use. ++ * ++ * object + s->inuse ++ * Meta data starts here. ++ * ++ * A. Free pointer (if we cannot overwrite object on free) ++ * B. Tracking data for SLAB_STORE_USER ++ * C. Padding to reach required alignment boundary or at mininum ++ * one word if debugging is on to be able to detect writes ++ * before the word boundary. ++ * ++ * Padding is done using 0x5a (POISON_INUSE) ++ * ++ * object + s->size ++ * Nothing is used beyond s->size. ++ * ++ * If slabcaches are merged then the object_size and inuse boundaries are mostly ++ * ignored. And therefore no slab options that rely on these boundaries ++ * may be used with merged slabcaches. ++ */ ++ ++static int check_pad_bytes(struct kmem_cache *s, struct page *page, u8 *p) ++{ ++ unsigned long off = s->inuse; /* The end of info */ ++ ++ if (s->offset) ++ /* Freepointer is placed after the object. */ ++ off += sizeof(void *); ++ ++ if (s->flags & SLAB_STORE_USER) ++ /* We also have user information there */ ++ off += 2 * sizeof(struct track); ++ ++ if (s->size == off) ++ return 1; ++ ++ return check_bytes_and_report(s, page, p, "Object padding", ++ p + off, POISON_INUSE, s->size - off); ++} ++ ++/* Check the pad bytes at the end of a slab page */ ++static int slab_pad_check(struct kmem_cache *s, struct page *page) ++{ ++ u8 *start; ++ u8 *fault; ++ u8 *end; ++ int length; ++ int remainder; ++ ++ if (!(s->flags & SLAB_POISON)) ++ return 1; ++ ++ start = page_address(page); ++ length = (PAGE_SIZE << compound_order(page)) - s->reserved; ++ end = start + length; ++ remainder = length % s->size; ++ if (!remainder) ++ return 1; ++ ++ metadata_access_enable(); ++ fault = memchr_inv(end - remainder, POISON_INUSE, remainder); ++ metadata_access_disable(); ++ if (!fault) ++ return 1; ++ while (end > fault && end[-1] == POISON_INUSE) ++ end--; ++ ++ slab_err(s, page, "Padding overwritten. 0x%p-0x%p", fault, end - 1); ++ print_section("Padding ", end - remainder, remainder); ++ ++ restore_bytes(s, "slab padding", POISON_INUSE, end - remainder, end); ++ return 0; ++} ++ ++static int check_object(struct kmem_cache *s, struct page *page, ++ void *object, u8 val) ++{ ++ u8 *p = object; ++ u8 *endobject = object + s->object_size; ++ ++ if (s->flags & SLAB_RED_ZONE) { ++ if (!check_bytes_and_report(s, page, object, "Redzone", ++ endobject, val, s->inuse - s->object_size)) ++ return 0; ++ } else { ++ if ((s->flags & SLAB_POISON) && s->object_size < s->inuse) { ++ check_bytes_and_report(s, page, p, "Alignment padding", ++ endobject, POISON_INUSE, ++ s->inuse - s->object_size); ++ } ++ } ++ ++ if (s->flags & SLAB_POISON) { ++ if (val != SLUB_RED_ACTIVE && (s->flags & __OBJECT_POISON) && ++ (!check_bytes_and_report(s, page, p, "Poison", p, ++ POISON_FREE, s->object_size - 1) || ++ !check_bytes_and_report(s, page, p, "Poison", ++ p + s->object_size - 1, POISON_END, 1))) ++ return 0; ++ /* ++ * check_pad_bytes cleans up on its own. ++ */ ++ check_pad_bytes(s, page, p); ++ } ++ ++ if (!s->offset && val == SLUB_RED_ACTIVE) ++ /* ++ * Object and freepointer overlap. Cannot check ++ * freepointer while object is allocated. ++ */ ++ return 1; ++ ++ /* Check free pointer validity */ ++ if (!check_valid_pointer(s, page, get_freepointer(s, p))) { ++ object_err(s, page, p, "Freepointer corrupt"); ++ /* ++ * No choice but to zap it and thus lose the remainder ++ * of the free objects in this slab. May cause ++ * another error because the object count is now wrong. ++ */ ++ set_freepointer(s, p, NULL); ++ return 0; ++ } ++ return 1; ++} ++ ++static int check_slab(struct kmem_cache *s, struct page *page) ++{ ++ int maxobj; ++ ++ VM_BUG_ON(!irqs_disabled()); ++ ++ if (!PageSlab(page)) { ++ slab_err(s, page, "Not a valid slab page"); ++ return 0; ++ } ++ ++ maxobj = order_objects(compound_order(page), s->size, s->reserved); ++ if (page->objects > maxobj) { ++ slab_err(s, page, "objects %u > max %u", ++ page->objects, maxobj); ++ return 0; ++ } ++ if (page->inuse > page->objects) { ++ slab_err(s, page, "inuse %u > max %u", ++ page->inuse, page->objects); ++ return 0; ++ } ++ /* Slab_pad_check fixes things up after itself */ ++ slab_pad_check(s, page); ++ return 1; ++} ++ ++/* ++ * Determine if a certain object on a page is on the freelist. Must hold the ++ * slab lock to guarantee that the chains are in a consistent state. ++ */ ++static int on_freelist(struct kmem_cache *s, struct page *page, void *search) ++{ ++ int nr = 0; ++ void *fp; ++ void *object = NULL; ++ int max_objects; ++ ++ fp = page->freelist; ++ while (fp && nr <= page->objects) { ++ if (fp == search) ++ return 1; ++ if (!check_valid_pointer(s, page, fp)) { ++ if (object) { ++ object_err(s, page, object, ++ "Freechain corrupt"); ++ set_freepointer(s, object, NULL); ++ } else { ++ slab_err(s, page, "Freepointer corrupt"); ++ page->freelist = NULL; ++ page->inuse = page->objects; ++ slab_fix(s, "Freelist cleared"); ++ return 0; ++ } ++ break; ++ } ++ object = fp; ++ fp = get_freepointer(s, object); ++ nr++; ++ } ++ ++ max_objects = order_objects(compound_order(page), s->size, s->reserved); ++ if (max_objects > MAX_OBJS_PER_PAGE) ++ max_objects = MAX_OBJS_PER_PAGE; ++ ++ if (page->objects != max_objects) { ++ slab_err(s, page, "Wrong number of objects. Found %d but " ++ "should be %d", page->objects, max_objects); ++ page->objects = max_objects; ++ slab_fix(s, "Number of objects adjusted."); ++ } ++ if (page->inuse != page->objects - nr) { ++ slab_err(s, page, "Wrong object count. Counter is %d but " ++ "counted were %d", page->inuse, page->objects - nr); ++ page->inuse = page->objects - nr; ++ slab_fix(s, "Object count adjusted."); ++ } ++ return search == NULL; ++} ++ ++static void trace(struct kmem_cache *s, struct page *page, void *object, ++ int alloc) ++{ ++ if (s->flags & SLAB_TRACE) { ++ pr_info("TRACE %s %s 0x%p inuse=%d fp=0x%p\n", ++ s->name, ++ alloc ? "alloc" : "free", ++ object, page->inuse, ++ page->freelist); ++ ++ if (!alloc) ++ print_section("Object ", (void *)object, ++ s->object_size); ++ ++ dump_stack(); ++ } ++} ++ ++/* ++ * Tracking of fully allocated slabs for debugging purposes. ++ */ ++static void add_full(struct kmem_cache *s, ++ struct kmem_cache_node *n, struct page *page) ++{ ++ if (!(s->flags & SLAB_STORE_USER)) ++ return; ++ ++ lockdep_assert_held(&n->list_lock); ++ list_add(&page->lru, &n->full); ++} ++ ++static void remove_full(struct kmem_cache *s, struct kmem_cache_node *n, struct page *page) ++{ ++ if (!(s->flags & SLAB_STORE_USER)) ++ return; ++ ++ lockdep_assert_held(&n->list_lock); ++ list_del(&page->lru); ++} ++ ++/* Tracking of the number of slabs for debugging purposes */ ++static inline unsigned long slabs_node(struct kmem_cache *s, int node) ++{ ++ struct kmem_cache_node *n = get_node(s, node); ++ ++ return atomic_long_read(&n->nr_slabs); ++} ++ ++static inline unsigned long node_nr_slabs(struct kmem_cache_node *n) ++{ ++ return atomic_long_read(&n->nr_slabs); ++} ++ ++static inline void inc_slabs_node(struct kmem_cache *s, int node, int objects) ++{ ++ struct kmem_cache_node *n = get_node(s, node); ++ ++ /* ++ * May be called early in order to allocate a slab for the ++ * kmem_cache_node structure. Solve the chicken-egg ++ * dilemma by deferring the increment of the count during ++ * bootstrap (see early_kmem_cache_node_alloc). ++ */ ++ if (likely(n)) { ++ atomic_long_inc(&n->nr_slabs); ++ atomic_long_add(objects, &n->total_objects); ++ } ++} ++static inline void dec_slabs_node(struct kmem_cache *s, int node, int objects) ++{ ++ struct kmem_cache_node *n = get_node(s, node); ++ ++ atomic_long_dec(&n->nr_slabs); ++ atomic_long_sub(objects, &n->total_objects); ++} ++ ++/* Object debug checks for alloc/free paths */ ++static void setup_object_debug(struct kmem_cache *s, struct page *page, ++ void *object) ++{ ++ if (!(s->flags & (SLAB_STORE_USER|SLAB_RED_ZONE|__OBJECT_POISON))) ++ return; ++ ++ init_object(s, object, SLUB_RED_INACTIVE); ++ init_tracking(s, object); ++} ++ ++static noinline int alloc_debug_processing(struct kmem_cache *s, ++ struct page *page, ++ void *object, unsigned long addr) ++{ ++ if (!check_slab(s, page)) ++ goto bad; ++ ++ if (!check_valid_pointer(s, page, object)) { ++ object_err(s, page, object, "Freelist Pointer check fails"); ++ goto bad; ++ } ++ ++ if (!check_object(s, page, object, SLUB_RED_INACTIVE)) ++ goto bad; ++ ++ /* Success perform special debug activities for allocs */ ++ if (s->flags & SLAB_STORE_USER) ++ set_track(s, object, TRACK_ALLOC, addr); ++ trace(s, page, object, 1); ++ init_object(s, object, SLUB_RED_ACTIVE); ++ return 1; ++ ++bad: ++ if (PageSlab(page)) { ++ /* ++ * If this is a slab page then lets do the best we can ++ * to avoid issues in the future. Marking all objects ++ * as used avoids touching the remaining objects. ++ */ ++ slab_fix(s, "Marking all objects used"); ++ page->inuse = page->objects; ++ page->freelist = NULL; ++ } ++ return 0; ++} ++ ++static noinline struct kmem_cache_node *free_debug_processing( ++ struct kmem_cache *s, struct page *page, void *object, ++ unsigned long addr, unsigned long *flags) ++{ ++ struct kmem_cache_node *n = get_node(s, page_to_nid(page)); ++ ++ spin_lock_irqsave(&n->list_lock, *flags); ++ slab_lock(page); ++ ++ if (!check_slab(s, page)) ++ goto fail; ++ ++ if (!check_valid_pointer(s, page, object)) { ++ slab_err(s, page, "Invalid object pointer 0x%p", object); ++ goto fail; ++ } ++ ++ if (on_freelist(s, page, object)) { ++ object_err(s, page, object, "Object already free"); ++ goto fail; ++ } ++ ++ if (!check_object(s, page, object, SLUB_RED_ACTIVE)) ++ goto out; ++ ++ if (unlikely(s != page->slab_cache)) { ++ if (!PageSlab(page)) { ++ slab_err(s, page, "Attempt to free object(0x%p) " ++ "outside of slab", object); ++ } else if (!page->slab_cache) { ++ pr_err("SLUB : no slab for object 0x%p.\n", ++ object); ++ dump_stack(); ++ } else ++ object_err(s, page, object, ++ "page slab pointer corrupt."); ++ goto fail; ++ } ++ ++ if (s->flags & SLAB_STORE_USER) ++ set_track(s, object, TRACK_FREE, addr); ++ trace(s, page, object, 0); ++ init_object(s, object, SLUB_RED_INACTIVE); ++out: ++ slab_unlock(page); ++ /* ++ * Keep node_lock to preserve integrity ++ * until the object is actually freed ++ */ ++ return n; ++ ++fail: ++ slab_unlock(page); ++ spin_unlock_irqrestore(&n->list_lock, *flags); ++ slab_fix(s, "Object at 0x%p not freed", object); ++ return NULL; ++} ++ ++static int __init setup_slub_debug(char *str) ++{ ++ slub_debug = DEBUG_DEFAULT_FLAGS; ++ if (*str++ != '=' || !*str) ++ /* ++ * No options specified. Switch on full debugging. ++ */ ++ goto out; ++ ++ if (*str == ',') ++ /* ++ * No options but restriction on slabs. This means full ++ * debugging for slabs matching a pattern. ++ */ ++ goto check_slabs; ++ ++ slub_debug = 0; ++ if (*str == '-') ++ /* ++ * Switch off all debugging measures. ++ */ ++ goto out; ++ ++ /* ++ * Determine which debug features should be switched on ++ */ ++ for (; *str && *str != ','; str++) { ++ switch (tolower(*str)) { ++ case 'f': ++ slub_debug |= SLAB_DEBUG_FREE; ++ break; ++ case 'z': ++ slub_debug |= SLAB_RED_ZONE; ++ break; ++ case 'p': ++ slub_debug |= SLAB_POISON; ++ break; ++ case 'u': ++ slub_debug |= SLAB_STORE_USER; ++ break; ++ case 't': ++ slub_debug |= SLAB_TRACE; ++ break; ++ case 'a': ++ slub_debug |= SLAB_FAILSLAB; ++ break; ++ case 'o': ++ /* ++ * Avoid enabling debugging on caches if its minimum ++ * order would increase as a result. ++ */ ++ disable_higher_order_debug = 1; ++ break; ++ default: ++ pr_err("slub_debug option '%c' unknown. skipped\n", ++ *str); ++ } ++ } ++ ++check_slabs: ++ if (*str == ',') ++ slub_debug_slabs = str + 1; ++out: ++ return 1; ++} ++ ++__setup("slub_debug", setup_slub_debug); ++ ++unsigned long kmem_cache_flags(unsigned long object_size, ++ unsigned long flags, const char *name, ++ void (*ctor)(void *)) ++{ ++ /* ++ * Enable debugging if selected on the kernel commandline. ++ */ ++ if (slub_debug && (!slub_debug_slabs || (name && ++ !strncmp(slub_debug_slabs, name, strlen(slub_debug_slabs))))) ++ flags |= slub_debug; ++ ++ return flags; ++} ++#else ++static inline void setup_object_debug(struct kmem_cache *s, ++ struct page *page, void *object) {} ++ ++static inline int alloc_debug_processing(struct kmem_cache *s, ++ struct page *page, void *object, unsigned long addr) { return 0; } ++ ++static inline struct kmem_cache_node *free_debug_processing( ++ struct kmem_cache *s, struct page *page, void *object, ++ unsigned long addr, unsigned long *flags) { return NULL; } ++ ++static inline int slab_pad_check(struct kmem_cache *s, struct page *page) ++ { return 1; } ++static inline int check_object(struct kmem_cache *s, struct page *page, ++ void *object, u8 val) { return 1; } ++static inline void add_full(struct kmem_cache *s, struct kmem_cache_node *n, ++ struct page *page) {} ++static inline void remove_full(struct kmem_cache *s, struct kmem_cache_node *n, ++ struct page *page) {} ++unsigned long kmem_cache_flags(unsigned long object_size, ++ unsigned long flags, const char *name, ++ void (*ctor)(void *)) ++{ ++ return flags; ++} ++#define slub_debug 0 ++ ++#define disable_higher_order_debug 0 ++ ++static inline unsigned long slabs_node(struct kmem_cache *s, int node) ++ { return 0; } ++static inline unsigned long node_nr_slabs(struct kmem_cache_node *n) ++ { return 0; } ++static inline void inc_slabs_node(struct kmem_cache *s, int node, ++ int objects) {} ++static inline void dec_slabs_node(struct kmem_cache *s, int node, ++ int objects) {} ++ ++#endif /* CONFIG_SLUB_DEBUG */ ++ ++/* ++ * Hooks for other subsystems that check memory allocations. In a typical ++ * production configuration these hooks all should produce no code at all. ++ */ ++static inline void kmalloc_large_node_hook(void *ptr, size_t size, gfp_t flags) ++{ ++ kmemleak_alloc(ptr, size, 1, flags); ++ kasan_kmalloc_large(ptr, size); ++} ++ ++static inline void kfree_hook(const void *x) ++{ ++ kmemleak_free(x); ++ kasan_kfree_large(x); ++} ++ ++static inline struct kmem_cache *slab_pre_alloc_hook(struct kmem_cache *s, ++ gfp_t flags) ++{ ++ flags &= gfp_allowed_mask; ++ lockdep_trace_alloc(flags); ++ might_sleep_if(flags & __GFP_WAIT); ++ ++ if (should_failslab(s->object_size, flags, s->flags)) ++ return NULL; ++ ++ return memcg_kmem_get_cache(s, flags); ++} ++ ++static inline void slab_post_alloc_hook(struct kmem_cache *s, ++ gfp_t flags, void *object) ++{ ++ flags &= gfp_allowed_mask; ++ kmemcheck_slab_alloc(s, flags, object, slab_ksize(s)); ++ kmemleak_alloc_recursive(object, s->object_size, 1, s->flags, flags); ++ memcg_kmem_put_cache(s); ++ kasan_slab_alloc(s, object); ++} ++ ++static inline void slab_free_hook(struct kmem_cache *s, void *x) ++{ ++ kmemleak_free_recursive(x, s->flags); ++ ++ /* ++ * Trouble is that we may no longer disable interrupts in the fast path ++ * So in order to make the debug calls that expect irqs to be ++ * disabled we need to disable interrupts temporarily. ++ */ ++#if defined(CONFIG_KMEMCHECK) || defined(CONFIG_LOCKDEP) ++ { ++ unsigned long flags; ++ ++ local_irq_save(flags); ++ kmemcheck_slab_free(s, x, s->object_size); ++ debug_check_no_locks_freed(x, s->object_size); ++ local_irq_restore(flags); ++ } ++#endif ++ if (!(s->flags & SLAB_DEBUG_OBJECTS)) ++ debug_check_no_obj_freed(x, s->object_size); ++ ++ kasan_slab_free(s, x); ++} ++ ++/* ++ * Slab allocation and freeing ++ */ ++static inline struct page *alloc_slab_page(struct kmem_cache *s, ++ gfp_t flags, int node, struct kmem_cache_order_objects oo) ++{ ++ struct page *page; ++ int order = oo_order(oo); ++ ++ flags |= __GFP_NOTRACK; ++ ++ if (memcg_charge_slab(s, flags, order)) ++ return NULL; ++ ++ if (node == NUMA_NO_NODE) ++ page = alloc_pages(flags, order); ++ else ++ page = alloc_pages_exact_node(node, flags, order); ++ ++ if (!page) ++ memcg_uncharge_slab(s, order); ++ ++ return page; ++} ++ ++static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) ++{ ++ struct page *page; ++ struct kmem_cache_order_objects oo = s->oo; ++ gfp_t alloc_gfp; ++ ++ flags &= gfp_allowed_mask; ++ ++ if (flags & __GFP_WAIT) ++ local_irq_enable(); ++ ++ flags |= s->allocflags; ++ ++ /* ++ * Let the initial higher-order allocation fail under memory pressure ++ * so we fall-back to the minimum order allocation. ++ */ ++ alloc_gfp = (flags | __GFP_NOWARN | __GFP_NORETRY) & ~__GFP_NOFAIL; ++ ++ page = alloc_slab_page(s, alloc_gfp, node, oo); ++ if (unlikely(!page)) { ++ oo = s->min; ++ alloc_gfp = flags; ++ /* ++ * Allocation may have failed due to fragmentation. ++ * Try a lower order alloc if possible ++ */ ++ page = alloc_slab_page(s, alloc_gfp, node, oo); ++ ++ if (page) ++ stat(s, ORDER_FALLBACK); ++ } ++ ++ if (kmemcheck_enabled && page ++ && !(s->flags & (SLAB_NOTRACK | DEBUG_DEFAULT_FLAGS))) { ++ int pages = 1 << oo_order(oo); ++ ++ kmemcheck_alloc_shadow(page, oo_order(oo), alloc_gfp, node); ++ ++ /* ++ * Objects from caches that have a constructor don't get ++ * cleared when they're allocated, so we need to do it here. ++ */ ++ if (s->ctor) ++ kmemcheck_mark_uninitialized_pages(page, pages); ++ else ++ kmemcheck_mark_unallocated_pages(page, pages); ++ } ++ ++ if (flags & __GFP_WAIT) ++ local_irq_disable(); ++ if (!page) ++ return NULL; ++ ++ page->objects = oo_objects(oo); ++ mod_zone_page_state(page_zone(page), ++ (s->flags & SLAB_RECLAIM_ACCOUNT) ? ++ NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE, ++ 1 << oo_order(oo)); ++ ++ return page; ++} ++ ++static void setup_object(struct kmem_cache *s, struct page *page, ++ void *object) ++{ ++ setup_object_debug(s, page, object); ++ if (unlikely(s->ctor)) { ++ kasan_unpoison_object_data(s, object); ++ s->ctor(object); ++ kasan_poison_object_data(s, object); ++ } ++} ++ ++static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node) ++{ ++ struct page *page; ++ void *start; ++ void *p; ++ int order; ++ int idx; ++ ++ if (unlikely(flags & GFP_SLAB_BUG_MASK)) { ++ pr_emerg("gfp: %u\n", flags & GFP_SLAB_BUG_MASK); ++ BUG(); ++ } ++ ++ page = allocate_slab(s, ++ flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node); ++ if (!page) ++ goto out; ++ ++ order = compound_order(page); ++ inc_slabs_node(s, page_to_nid(page), page->objects); ++ page->slab_cache = s; ++ __SetPageSlab(page); ++ if (page_is_pfmemalloc(page)) ++ SetPageSlabPfmemalloc(page); ++ ++ start = page_address(page); ++ ++ if (unlikely(s->flags & SLAB_POISON)) ++ memset(start, POISON_INUSE, PAGE_SIZE << order); ++ ++ kasan_poison_slab(page); ++ ++ for_each_object_idx(p, idx, s, start, page->objects) { ++ setup_object(s, page, p); ++ if (likely(idx < page->objects)) ++ set_freepointer(s, p, p + s->size); ++ else ++ set_freepointer(s, p, NULL); ++ } ++ ++ page->freelist = start; ++ page->inuse = page->objects; ++ page->frozen = 1; ++out: ++ return page; ++} ++ ++static void __free_slab(struct kmem_cache *s, struct page *page) ++{ ++ int order = compound_order(page); ++ int pages = 1 << order; ++ ++ if (kmem_cache_debug(s)) { ++ void *p; ++ ++ slab_pad_check(s, page); ++ for_each_object(p, s, page_address(page), ++ page->objects) ++ check_object(s, page, p, SLUB_RED_INACTIVE); ++ } ++ ++ kmemcheck_free_shadow(page, compound_order(page)); ++ ++ mod_zone_page_state(page_zone(page), ++ (s->flags & SLAB_RECLAIM_ACCOUNT) ? ++ NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE, ++ -pages); ++ ++ __ClearPageSlabPfmemalloc(page); ++ __ClearPageSlab(page); ++ ++ page_mapcount_reset(page); ++ if (current->reclaim_state) ++ current->reclaim_state->reclaimed_slab += pages; ++ __free_pages(page, order); ++ memcg_uncharge_slab(s, order); ++} ++ ++#define need_reserve_slab_rcu \ ++ (sizeof(((struct page *)NULL)->lru) < sizeof(struct rcu_head)) ++ ++static void rcu_free_slab(struct rcu_head *h) ++{ ++ struct page *page; ++ ++ if (need_reserve_slab_rcu) ++ page = virt_to_head_page(h); ++ else ++ page = container_of((struct list_head *)h, struct page, lru); ++ ++ __free_slab(page->slab_cache, page); ++} ++ ++static void free_slab(struct kmem_cache *s, struct page *page) ++{ ++ if (unlikely(s->flags & SLAB_DESTROY_BY_RCU)) { ++ struct rcu_head *head; ++ ++ if (need_reserve_slab_rcu) { ++ int order = compound_order(page); ++ int offset = (PAGE_SIZE << order) - s->reserved; ++ ++ VM_BUG_ON(s->reserved != sizeof(*head)); ++ head = page_address(page) + offset; ++ } else { ++ /* ++ * RCU free overloads the RCU head over the LRU ++ */ ++ head = (void *)&page->lru; ++ } ++ ++ call_rcu(head, rcu_free_slab); ++ } else ++ __free_slab(s, page); ++} ++ ++static void discard_slab(struct kmem_cache *s, struct page *page) ++{ ++ dec_slabs_node(s, page_to_nid(page), page->objects); ++ free_slab(s, page); ++} ++ ++/* ++ * Management of partially allocated slabs. ++ */ ++static inline void ++__add_partial(struct kmem_cache_node *n, struct page *page, int tail) ++{ ++ n->nr_partial++; ++ if (tail == DEACTIVATE_TO_TAIL) ++ list_add_tail(&page->lru, &n->partial); ++ else ++ list_add(&page->lru, &n->partial); ++} ++ ++static inline void add_partial(struct kmem_cache_node *n, ++ struct page *page, int tail) ++{ ++ lockdep_assert_held(&n->list_lock); ++ __add_partial(n, page, tail); ++} ++ ++static inline void ++__remove_partial(struct kmem_cache_node *n, struct page *page) ++{ ++ list_del(&page->lru); ++ n->nr_partial--; ++} ++ ++static inline void remove_partial(struct kmem_cache_node *n, ++ struct page *page) ++{ ++ lockdep_assert_held(&n->list_lock); ++ __remove_partial(n, page); ++} ++ ++/* ++ * Remove slab from the partial list, freeze it and ++ * return the pointer to the freelist. ++ * ++ * Returns a list of objects or NULL if it fails. ++ */ ++static inline void *acquire_slab(struct kmem_cache *s, ++ struct kmem_cache_node *n, struct page *page, ++ int mode, int *objects) ++{ ++ void *freelist; ++ unsigned long counters; ++ struct page new; ++ ++ lockdep_assert_held(&n->list_lock); ++ ++ /* ++ * Zap the freelist and set the frozen bit. ++ * The old freelist is the list of objects for the ++ * per cpu allocation list. ++ */ ++ freelist = page->freelist; ++ counters = page->counters; ++ new.counters = counters; ++ *objects = new.objects - new.inuse; ++ if (mode) { ++ new.inuse = page->objects; ++ new.freelist = NULL; ++ } else { ++ new.freelist = freelist; ++ } ++ ++ VM_BUG_ON(new.frozen); ++ new.frozen = 1; ++ ++ if (!__cmpxchg_double_slab(s, page, ++ freelist, counters, ++ new.freelist, new.counters, ++ "acquire_slab")) ++ return NULL; ++ ++ remove_partial(n, page); ++ WARN_ON(!freelist); ++ return freelist; ++} ++ ++static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain); ++static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags); ++ ++/* ++ * Try to allocate a partial slab from a specific node. ++ */ ++static void *get_partial_node(struct kmem_cache *s, struct kmem_cache_node *n, ++ struct kmem_cache_cpu *c, gfp_t flags) ++{ ++ struct page *page, *page2; ++ void *object = NULL; ++ int available = 0; ++ int objects; ++ ++ /* ++ * Racy check. If we mistakenly see no partial slabs then we ++ * just allocate an empty slab. If we mistakenly try to get a ++ * partial slab and there is none available then get_partials() ++ * will return NULL. ++ */ ++ if (!n || !n->nr_partial) ++ return NULL; ++ ++ spin_lock(&n->list_lock); ++ list_for_each_entry_safe(page, page2, &n->partial, lru) { ++ void *t; ++ ++ if (!pfmemalloc_match(page, flags)) ++ continue; ++ ++ t = acquire_slab(s, n, page, object == NULL, &objects); ++ if (!t) ++ break; ++ ++ available += objects; ++ if (!object) { ++ c->page = page; ++ stat(s, ALLOC_FROM_PARTIAL); ++ object = t; ++ } else { ++ put_cpu_partial(s, page, 0); ++ stat(s, CPU_PARTIAL_NODE); ++ } ++ if (!kmem_cache_has_cpu_partial(s) ++ || available > s->cpu_partial / 2) ++ break; ++ ++ } ++ spin_unlock(&n->list_lock); ++ return object; ++} ++ ++/* ++ * Get a page from somewhere. Search in increasing NUMA distances. ++ */ ++static void *get_any_partial(struct kmem_cache *s, gfp_t flags, ++ struct kmem_cache_cpu *c) ++{ ++#ifdef CONFIG_NUMA ++ struct zonelist *zonelist; ++ struct zoneref *z; ++ struct zone *zone; ++ enum zone_type high_zoneidx = gfp_zone(flags); ++ void *object; ++ unsigned int cpuset_mems_cookie; ++ ++ /* ++ * The defrag ratio allows a configuration of the tradeoffs between ++ * inter node defragmentation and node local allocations. A lower ++ * defrag_ratio increases the tendency to do local allocations ++ * instead of attempting to obtain partial slabs from other nodes. ++ * ++ * If the defrag_ratio is set to 0 then kmalloc() always ++ * returns node local objects. If the ratio is higher then kmalloc() ++ * may return off node objects because partial slabs are obtained ++ * from other nodes and filled up. ++ * ++ * If /sys/kernel/slab/xx/defrag_ratio is set to 100 (which makes ++ * defrag_ratio = 1000) then every (well almost) allocation will ++ * first attempt to defrag slab caches on other nodes. This means ++ * scanning over all nodes to look for partial slabs which may be ++ * expensive if we do it every time we are trying to find a slab ++ * with available objects. ++ */ ++ if (!s->remote_node_defrag_ratio || ++ get_cycles() % 1024 > s->remote_node_defrag_ratio) ++ return NULL; ++ ++ do { ++ cpuset_mems_cookie = read_mems_allowed_begin(); ++ zonelist = node_zonelist(mempolicy_slab_node(), flags); ++ for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) { ++ struct kmem_cache_node *n; ++ ++ n = get_node(s, zone_to_nid(zone)); ++ ++ if (n && cpuset_zone_allowed(zone, flags) && ++ n->nr_partial > s->min_partial) { ++ object = get_partial_node(s, n, c, flags); ++ if (object) { ++ /* ++ * Don't check read_mems_allowed_retry() ++ * here - if mems_allowed was updated in ++ * parallel, that was a harmless race ++ * between allocation and the cpuset ++ * update ++ */ ++ return object; ++ } ++ } ++ } ++ } while (read_mems_allowed_retry(cpuset_mems_cookie)); ++#endif ++ return NULL; ++} ++ ++/* ++ * Get a partial page, lock it and return it. ++ */ ++static void *get_partial(struct kmem_cache *s, gfp_t flags, int node, ++ struct kmem_cache_cpu *c) ++{ ++ void *object; ++ int searchnode = node; ++ ++ if (node == NUMA_NO_NODE) ++ searchnode = numa_mem_id(); ++ else if (!node_present_pages(node)) ++ searchnode = node_to_mem_node(node); ++ ++ object = get_partial_node(s, get_node(s, searchnode), c, flags); ++ if (object || node != NUMA_NO_NODE) ++ return object; ++ ++ return get_any_partial(s, flags, c); ++} ++ ++#ifdef CONFIG_PREEMPT ++/* ++ * Calculate the next globally unique transaction for disambiguiation ++ * during cmpxchg. The transactions start with the cpu number and are then ++ * incremented by CONFIG_NR_CPUS. ++ */ ++#define TID_STEP roundup_pow_of_two(CONFIG_NR_CPUS) ++#else ++/* ++ * No preemption supported therefore also no need to check for ++ * different cpus. ++ */ ++#define TID_STEP 1 ++#endif ++ ++static inline unsigned long next_tid(unsigned long tid) ++{ ++ return tid + TID_STEP; ++} ++ ++static inline unsigned int tid_to_cpu(unsigned long tid) ++{ ++ return tid % TID_STEP; ++} ++ ++static inline unsigned long tid_to_event(unsigned long tid) ++{ ++ return tid / TID_STEP; ++} ++ ++static inline unsigned int init_tid(int cpu) ++{ ++ return cpu; ++} ++ ++static inline void note_cmpxchg_failure(const char *n, ++ const struct kmem_cache *s, unsigned long tid) ++{ ++#ifdef SLUB_DEBUG_CMPXCHG ++ unsigned long actual_tid = __this_cpu_read(s->cpu_slab->tid); ++ ++ pr_info("%s %s: cmpxchg redo ", n, s->name); ++ ++#ifdef CONFIG_PREEMPT ++ if (tid_to_cpu(tid) != tid_to_cpu(actual_tid)) ++ pr_warn("due to cpu change %d -> %d\n", ++ tid_to_cpu(tid), tid_to_cpu(actual_tid)); ++ else ++#endif ++ if (tid_to_event(tid) != tid_to_event(actual_tid)) ++ pr_warn("due to cpu running other code. Event %ld->%ld\n", ++ tid_to_event(tid), tid_to_event(actual_tid)); ++ else ++ pr_warn("for unknown reason: actual=%lx was=%lx target=%lx\n", ++ actual_tid, tid, next_tid(tid)); ++#endif ++ stat(s, CMPXCHG_DOUBLE_CPU_FAIL); ++} ++ ++static void init_kmem_cache_cpus(struct kmem_cache *s) ++{ ++ int cpu; ++ ++ for_each_possible_cpu(cpu) ++ per_cpu_ptr(s->cpu_slab, cpu)->tid = init_tid(cpu); ++} ++ ++/* ++ * Remove the cpu slab ++ */ ++static void deactivate_slab(struct kmem_cache *s, struct page *page, ++ void *freelist) ++{ ++ enum slab_modes { M_NONE, M_PARTIAL, M_FULL, M_FREE }; ++ struct kmem_cache_node *n = get_node(s, page_to_nid(page)); ++ int lock = 0; ++ enum slab_modes l = M_NONE, m = M_NONE; ++ void *nextfree; ++ int tail = DEACTIVATE_TO_HEAD; ++ struct page new; ++ struct page old; ++ ++ if (page->freelist) { ++ stat(s, DEACTIVATE_REMOTE_FREES); ++ tail = DEACTIVATE_TO_TAIL; ++ } ++ ++ /* ++ * Stage one: Free all available per cpu objects back ++ * to the page freelist while it is still frozen. Leave the ++ * last one. ++ * ++ * There is no need to take the list->lock because the page ++ * is still frozen. ++ */ ++ while (freelist && (nextfree = get_freepointer(s, freelist))) { ++ void *prior; ++ unsigned long counters; ++ ++ do { ++ prior = page->freelist; ++ counters = page->counters; ++ set_freepointer(s, freelist, prior); ++ new.counters = counters; ++ new.inuse--; ++ VM_BUG_ON(!new.frozen); ++ ++ } while (!__cmpxchg_double_slab(s, page, ++ prior, counters, ++ freelist, new.counters, ++ "drain percpu freelist")); ++ ++ freelist = nextfree; ++ } ++ ++ /* ++ * Stage two: Ensure that the page is unfrozen while the ++ * list presence reflects the actual number of objects ++ * during unfreeze. ++ * ++ * We setup the list membership and then perform a cmpxchg ++ * with the count. If there is a mismatch then the page ++ * is not unfrozen but the page is on the wrong list. ++ * ++ * Then we restart the process which may have to remove ++ * the page from the list that we just put it on again ++ * because the number of objects in the slab may have ++ * changed. ++ */ ++redo: ++ ++ old.freelist = page->freelist; ++ old.counters = page->counters; ++ VM_BUG_ON(!old.frozen); ++ ++ /* Determine target state of the slab */ ++ new.counters = old.counters; ++ if (freelist) { ++ new.inuse--; ++ set_freepointer(s, freelist, old.freelist); ++ new.freelist = freelist; ++ } else ++ new.freelist = old.freelist; ++ ++ new.frozen = 0; ++ ++ if (!new.inuse && n->nr_partial >= s->min_partial) ++ m = M_FREE; ++ else if (new.freelist) { ++ m = M_PARTIAL; ++ if (!lock) { ++ lock = 1; ++ /* ++ * Taking the spinlock removes the possiblity ++ * that acquire_slab() will see a slab page that ++ * is frozen ++ */ ++ spin_lock(&n->list_lock); ++ } ++ } else { ++ m = M_FULL; ++ if (kmem_cache_debug(s) && !lock) { ++ lock = 1; ++ /* ++ * This also ensures that the scanning of full ++ * slabs from diagnostic functions will not see ++ * any frozen slabs. ++ */ ++ spin_lock(&n->list_lock); ++ } ++ } ++ ++ if (l != m) { ++ ++ if (l == M_PARTIAL) ++ ++ remove_partial(n, page); ++ ++ else if (l == M_FULL) ++ ++ remove_full(s, n, page); ++ ++ if (m == M_PARTIAL) { ++ ++ add_partial(n, page, tail); ++ stat(s, tail); ++ ++ } else if (m == M_FULL) { ++ ++ stat(s, DEACTIVATE_FULL); ++ add_full(s, n, page); ++ ++ } ++ } ++ ++ l = m; ++ if (!__cmpxchg_double_slab(s, page, ++ old.freelist, old.counters, ++ new.freelist, new.counters, ++ "unfreezing slab")) ++ goto redo; ++ ++ if (lock) ++ spin_unlock(&n->list_lock); ++ ++ if (m == M_FREE) { ++ stat(s, DEACTIVATE_EMPTY); ++ discard_slab(s, page); ++ stat(s, FREE_SLAB); ++ } ++} ++ ++/* ++ * Unfreeze all the cpu partial slabs. ++ * ++ * This function must be called with interrupts disabled ++ * for the cpu using c (or some other guarantee must be there ++ * to guarantee no concurrent accesses). ++ */ ++static void unfreeze_partials(struct kmem_cache *s, ++ struct kmem_cache_cpu *c) ++{ ++#ifdef CONFIG_SLUB_CPU_PARTIAL ++ struct kmem_cache_node *n = NULL, *n2 = NULL; ++ struct page *page, *discard_page = NULL; ++ ++ while ((page = c->partial)) { ++ struct page new; ++ struct page old; ++ ++ c->partial = page->next; ++ ++ n2 = get_node(s, page_to_nid(page)); ++ if (n != n2) { ++ if (n) ++ spin_unlock(&n->list_lock); ++ ++ n = n2; ++ spin_lock(&n->list_lock); ++ } ++ ++ do { ++ ++ old.freelist = page->freelist; ++ old.counters = page->counters; ++ VM_BUG_ON(!old.frozen); ++ ++ new.counters = old.counters; ++ new.freelist = old.freelist; ++ ++ new.frozen = 0; ++ ++ } while (!__cmpxchg_double_slab(s, page, ++ old.freelist, old.counters, ++ new.freelist, new.counters, ++ "unfreezing slab")); ++ ++ if (unlikely(!new.inuse && n->nr_partial >= s->min_partial)) { ++ page->next = discard_page; ++ discard_page = page; ++ } else { ++ add_partial(n, page, DEACTIVATE_TO_TAIL); ++ stat(s, FREE_ADD_PARTIAL); ++ } ++ } ++ ++ if (n) ++ spin_unlock(&n->list_lock); ++ ++ while (discard_page) { ++ page = discard_page; ++ discard_page = discard_page->next; ++ ++ stat(s, DEACTIVATE_EMPTY); ++ discard_slab(s, page); ++ stat(s, FREE_SLAB); ++ } ++#endif ++} ++ ++/* ++ * Put a page that was just frozen (in __slab_free) into a partial page ++ * slot if available. This is done without interrupts disabled and without ++ * preemption disabled. The cmpxchg is racy and may put the partial page ++ * onto a random cpus partial slot. ++ * ++ * If we did not find a slot then simply move all the partials to the ++ * per node partial list. ++ */ ++static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain) ++{ ++#ifdef CONFIG_SLUB_CPU_PARTIAL ++ struct page *oldpage; ++ int pages; ++ int pobjects; ++ ++ preempt_disable(); ++ do { ++ pages = 0; ++ pobjects = 0; ++ oldpage = this_cpu_read(s->cpu_slab->partial); ++ ++ if (oldpage) { ++ pobjects = oldpage->pobjects; ++ pages = oldpage->pages; ++ if (drain && pobjects > s->cpu_partial) { ++ unsigned long flags; ++ /* ++ * partial array is full. Move the existing ++ * set to the per node partial list. ++ */ ++ local_irq_save(flags); ++ unfreeze_partials(s, this_cpu_ptr(s->cpu_slab)); ++ local_irq_restore(flags); ++ oldpage = NULL; ++ pobjects = 0; ++ pages = 0; ++ stat(s, CPU_PARTIAL_DRAIN); ++ } ++ } ++ ++ pages++; ++ pobjects += page->objects - page->inuse; ++ ++ page->pages = pages; ++ page->pobjects = pobjects; ++ page->next = oldpage; ++ ++ } while (this_cpu_cmpxchg(s->cpu_slab->partial, oldpage, page) ++ != oldpage); ++ if (unlikely(!s->cpu_partial)) { ++ unsigned long flags; ++ ++ local_irq_save(flags); ++ unfreeze_partials(s, this_cpu_ptr(s->cpu_slab)); ++ local_irq_restore(flags); ++ } ++ preempt_enable(); ++#endif ++} ++ ++static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c) ++{ ++ stat(s, CPUSLAB_FLUSH); ++ deactivate_slab(s, c->page, c->freelist); ++ ++ c->tid = next_tid(c->tid); ++ c->page = NULL; ++ c->freelist = NULL; ++} ++ ++/* ++ * Flush cpu slab. ++ * ++ * Called from IPI handler with interrupts disabled. ++ */ ++static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu) ++{ ++ struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu); ++ ++ if (likely(c)) { ++ if (c->page) ++ flush_slab(s, c); ++ ++ unfreeze_partials(s, c); ++ } ++} ++ ++static void flush_cpu_slab(void *d) ++{ ++ struct kmem_cache *s = d; ++ ++ __flush_cpu_slab(s, smp_processor_id()); ++} ++ ++static bool has_cpu_slab(int cpu, void *info) ++{ ++ struct kmem_cache *s = info; ++ struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu); ++ ++ return c->page || c->partial; ++} ++ ++static void flush_all(struct kmem_cache *s) ++{ ++ on_each_cpu_cond(has_cpu_slab, flush_cpu_slab, s, 1, GFP_ATOMIC); ++} ++ ++/* ++ * Check if the objects in a per cpu structure fit numa ++ * locality expectations. ++ */ ++static inline int node_match(struct page *page, int node) ++{ ++#ifdef CONFIG_NUMA ++ if (!page || (node != NUMA_NO_NODE && page_to_nid(page) != node)) ++ return 0; ++#endif ++ return 1; ++} ++ ++#ifdef CONFIG_SLUB_DEBUG ++static int count_free(struct page *page) ++{ ++ return page->objects - page->inuse; ++} ++ ++static inline unsigned long node_nr_objs(struct kmem_cache_node *n) ++{ ++ return atomic_long_read(&n->total_objects); ++} ++#endif /* CONFIG_SLUB_DEBUG */ ++ ++#if defined(CONFIG_SLUB_DEBUG) || defined(CONFIG_SYSFS) ++static unsigned long count_partial(struct kmem_cache_node *n, ++ int (*get_count)(struct page *)) ++{ ++ unsigned long flags; ++ unsigned long x = 0; ++ struct page *page; ++ ++ spin_lock_irqsave(&n->list_lock, flags); ++ list_for_each_entry(page, &n->partial, lru) ++ x += get_count(page); ++ spin_unlock_irqrestore(&n->list_lock, flags); ++ return x; ++} ++#endif /* CONFIG_SLUB_DEBUG || CONFIG_SYSFS */ ++ ++static noinline void ++slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid) ++{ ++#ifdef CONFIG_SLUB_DEBUG ++ static DEFINE_RATELIMIT_STATE(slub_oom_rs, DEFAULT_RATELIMIT_INTERVAL, ++ DEFAULT_RATELIMIT_BURST); ++ int node; ++ struct kmem_cache_node *n; ++ ++ if ((gfpflags & __GFP_NOWARN) || !__ratelimit(&slub_oom_rs)) ++ return; ++ ++ pr_warn("SLUB: Unable to allocate memory on node %d (gfp=0x%x)\n", ++ nid, gfpflags); ++ pr_warn(" cache: %s, object size: %d, buffer size: %d, default order: %d, min order: %d\n", ++ s->name, s->object_size, s->size, oo_order(s->oo), ++ oo_order(s->min)); ++ ++ if (oo_order(s->min) > get_order(s->object_size)) ++ pr_warn(" %s debugging increased min order, use slub_debug=O to disable.\n", ++ s->name); ++ ++ for_each_kmem_cache_node(s, node, n) { ++ unsigned long nr_slabs; ++ unsigned long nr_objs; ++ unsigned long nr_free; ++ ++ nr_free = count_partial(n, count_free); ++ nr_slabs = node_nr_slabs(n); ++ nr_objs = node_nr_objs(n); ++ ++ pr_warn(" node %d: slabs: %ld, objs: %ld, free: %ld\n", ++ node, nr_slabs, nr_objs, nr_free); ++ } ++#endif ++} ++ ++static inline void *new_slab_objects(struct kmem_cache *s, gfp_t flags, ++ int node, struct kmem_cache_cpu **pc) ++{ ++ void *freelist; ++ struct kmem_cache_cpu *c = *pc; ++ struct page *page; ++ ++ freelist = get_partial(s, flags, node, c); ++ ++ if (freelist) ++ return freelist; ++ ++ page = new_slab(s, flags, node); ++ if (page) { ++ c = raw_cpu_ptr(s->cpu_slab); ++ if (c->page) ++ flush_slab(s, c); ++ ++ /* ++ * No other reference to the page yet so we can ++ * muck around with it freely without cmpxchg ++ */ ++ freelist = page->freelist; ++ page->freelist = NULL; ++ ++ stat(s, ALLOC_SLAB); ++ c->page = page; ++ *pc = c; ++ } else ++ freelist = NULL; ++ ++ return freelist; ++} ++ ++static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags) ++{ ++ if (unlikely(PageSlabPfmemalloc(page))) ++ return gfp_pfmemalloc_allowed(gfpflags); ++ ++ return true; ++} ++ ++/* ++ * Check the page->freelist of a page and either transfer the freelist to the ++ * per cpu freelist or deactivate the page. ++ * ++ * The page is still frozen if the return value is not NULL. ++ * ++ * If this function returns NULL then the page has been unfrozen. ++ * ++ * This function must be called with interrupt disabled. ++ */ ++static inline void *get_freelist(struct kmem_cache *s, struct page *page) ++{ ++ struct page new; ++ unsigned long counters; ++ void *freelist; ++ ++ do { ++ freelist = page->freelist; ++ counters = page->counters; ++ ++ new.counters = counters; ++ VM_BUG_ON(!new.frozen); ++ ++ new.inuse = page->objects; ++ new.frozen = freelist != NULL; ++ ++ } while (!__cmpxchg_double_slab(s, page, ++ freelist, counters, ++ NULL, new.counters, ++ "get_freelist")); ++ ++ return freelist; ++} ++ ++/* ++ * Slow path. The lockless freelist is empty or we need to perform ++ * debugging duties. ++ * ++ * Processing is still very fast if new objects have been freed to the ++ * regular freelist. In that case we simply take over the regular freelist ++ * as the lockless freelist and zap the regular freelist. ++ * ++ * If that is not working then we fall back to the partial lists. We take the ++ * first element of the freelist as the object to allocate now and move the ++ * rest of the freelist to the lockless freelist. ++ * ++ * And if we were unable to get a new slab from the partial slab lists then ++ * we need to allocate a new slab. This is the slowest path since it involves ++ * a call to the page allocator and the setup of a new slab. ++ */ ++static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, ++ unsigned long addr, struct kmem_cache_cpu *c) ++{ ++ void *freelist; ++ struct page *page; ++ unsigned long flags; ++ ++ local_irq_save(flags); ++#ifdef CONFIG_PREEMPT ++ /* ++ * We may have been preempted and rescheduled on a different ++ * cpu before disabling interrupts. Need to reload cpu area ++ * pointer. ++ */ ++ c = this_cpu_ptr(s->cpu_slab); ++#endif ++ ++ page = c->page; ++ if (!page) ++ goto new_slab; ++redo: ++ ++ if (unlikely(!node_match(page, node))) { ++ int searchnode = node; ++ ++ if (node != NUMA_NO_NODE && !node_present_pages(node)) ++ searchnode = node_to_mem_node(node); ++ ++ if (unlikely(!node_match(page, searchnode))) { ++ stat(s, ALLOC_NODE_MISMATCH); ++ deactivate_slab(s, page, c->freelist); ++ c->page = NULL; ++ c->freelist = NULL; ++ goto new_slab; ++ } ++ } ++ ++ /* ++ * By rights, we should be searching for a slab page that was ++ * PFMEMALLOC but right now, we are losing the pfmemalloc ++ * information when the page leaves the per-cpu allocator ++ */ ++ if (unlikely(!pfmemalloc_match(page, gfpflags))) { ++ deactivate_slab(s, page, c->freelist); ++ c->page = NULL; ++ c->freelist = NULL; ++ goto new_slab; ++ } ++ ++ /* must check again c->freelist in case of cpu migration or IRQ */ ++ freelist = c->freelist; ++ if (freelist) ++ goto load_freelist; ++ ++ freelist = get_freelist(s, page); ++ ++ if (!freelist) { ++ c->page = NULL; ++ stat(s, DEACTIVATE_BYPASS); ++ goto new_slab; ++ } ++ ++ stat(s, ALLOC_REFILL); ++ ++load_freelist: ++ /* ++ * freelist is pointing to the list of objects to be used. ++ * page is pointing to the page from which the objects are obtained. ++ * That page must be frozen for per cpu allocations to work. ++ */ ++ VM_BUG_ON(!c->page->frozen); ++ c->freelist = get_freepointer(s, freelist); ++ c->tid = next_tid(c->tid); ++ local_irq_restore(flags); ++ return freelist; ++ ++new_slab: ++ ++ if (c->partial) { ++ page = c->page = c->partial; ++ c->partial = page->next; ++ stat(s, CPU_PARTIAL_ALLOC); ++ c->freelist = NULL; ++ goto redo; ++ } ++ ++ freelist = new_slab_objects(s, gfpflags, node, &c); ++ ++ if (unlikely(!freelist)) { ++ slab_out_of_memory(s, gfpflags, node); ++ local_irq_restore(flags); ++ return NULL; ++ } ++ ++ page = c->page; ++ if (likely(!kmem_cache_debug(s) && pfmemalloc_match(page, gfpflags))) ++ goto load_freelist; ++ ++ /* Only entered in the debug case */ ++ if (kmem_cache_debug(s) && ++ !alloc_debug_processing(s, page, freelist, addr)) ++ goto new_slab; /* Slab failed checks. Next slab needed */ ++ ++ deactivate_slab(s, page, get_freepointer(s, freelist)); ++ c->page = NULL; ++ c->freelist = NULL; ++ local_irq_restore(flags); ++ return freelist; ++} ++ ++/* ++ * Inlined fastpath so that allocation functions (kmalloc, kmem_cache_alloc) ++ * have the fastpath folded into their functions. So no function call ++ * overhead for requests that can be satisfied on the fastpath. ++ * ++ * The fastpath works by first checking if the lockless freelist can be used. ++ * If not then __slab_alloc is called for slow processing. ++ * ++ * Otherwise we can simply pick the next object from the lockless free list. ++ */ ++static __always_inline void *slab_alloc_node(struct kmem_cache *s, ++ gfp_t gfpflags, int node, unsigned long addr) ++{ ++ void **object; ++ struct kmem_cache_cpu *c; ++ struct page *page; ++ unsigned long tid; ++ ++ s = slab_pre_alloc_hook(s, gfpflags); ++ if (!s) ++ return NULL; ++redo: ++ /* ++ * Must read kmem_cache cpu data via this cpu ptr. Preemption is ++ * enabled. We may switch back and forth between cpus while ++ * reading from one cpu area. That does not matter as long ++ * as we end up on the original cpu again when doing the cmpxchg. ++ * ++ * We should guarantee that tid and kmem_cache are retrieved on ++ * the same cpu. It could be different if CONFIG_PREEMPT so we need ++ * to check if it is matched or not. ++ */ ++ do { ++ tid = this_cpu_read(s->cpu_slab->tid); ++ c = raw_cpu_ptr(s->cpu_slab); ++ } while (IS_ENABLED(CONFIG_PREEMPT) && ++ unlikely(tid != READ_ONCE(c->tid))); ++ ++ /* ++ * Irqless object alloc/free algorithm used here depends on sequence ++ * of fetching cpu_slab's data. tid should be fetched before anything ++ * on c to guarantee that object and page associated with previous tid ++ * won't be used with current tid. If we fetch tid first, object and ++ * page could be one associated with next tid and our alloc/free ++ * request will be failed. In this case, we will retry. So, no problem. ++ */ ++ barrier(); ++ ++ /* ++ * The transaction ids are globally unique per cpu and per operation on ++ * a per cpu queue. Thus they can be guarantee that the cmpxchg_double ++ * occurs on the right processor and that there was no operation on the ++ * linked list in between. ++ */ ++ ++ object = c->freelist; ++ page = c->page; ++ if (unlikely(!object || !node_match(page, node))) { ++ object = __slab_alloc(s, gfpflags, node, addr, c); ++ stat(s, ALLOC_SLOWPATH); ++ } else { ++ void *next_object = get_freepointer_safe(s, object); ++ ++ /* ++ * The cmpxchg will only match if there was no additional ++ * operation and if we are on the right processor. ++ * ++ * The cmpxchg does the following atomically (without lock ++ * semantics!) ++ * 1. Relocate first pointer to the current per cpu area. ++ * 2. Verify that tid and freelist have not been changed ++ * 3. If they were not changed replace tid and freelist ++ * ++ * Since this is without lock semantics the protection is only ++ * against code executing on this cpu *not* from access by ++ * other cpus. ++ */ ++ if (unlikely(!this_cpu_cmpxchg_double( ++ s->cpu_slab->freelist, s->cpu_slab->tid, ++ object, tid, ++ next_object, next_tid(tid)))) { ++ ++ note_cmpxchg_failure("slab_alloc", s, tid); ++ goto redo; ++ } ++ prefetch_freepointer(s, next_object); ++ stat(s, ALLOC_FASTPATH); ++ } ++ ++ if (unlikely(gfpflags & __GFP_ZERO) && object) ++ memset(object, 0, s->object_size); ++ ++ slab_post_alloc_hook(s, gfpflags, object); ++ ++ return object; ++} ++ ++static __always_inline void *slab_alloc(struct kmem_cache *s, ++ gfp_t gfpflags, unsigned long addr) ++{ ++ return slab_alloc_node(s, gfpflags, NUMA_NO_NODE, addr); ++} ++ ++void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags) ++{ ++ void *ret = slab_alloc(s, gfpflags, _RET_IP_); ++ ++ trace_kmem_cache_alloc(_RET_IP_, ret, s->object_size, ++ s->size, gfpflags); ++ ++ return ret; ++} ++EXPORT_SYMBOL(kmem_cache_alloc); ++ ++#ifdef CONFIG_TRACING ++void *kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size) ++{ ++ void *ret = slab_alloc(s, gfpflags, _RET_IP_); ++ trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags); ++ kasan_kmalloc(s, ret, size); ++ return ret; ++} ++EXPORT_SYMBOL(kmem_cache_alloc_trace); ++#endif ++ ++#ifdef CONFIG_NUMA ++void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node) ++{ ++ void *ret = slab_alloc_node(s, gfpflags, node, _RET_IP_); ++ ++ trace_kmem_cache_alloc_node(_RET_IP_, ret, ++ s->object_size, s->size, gfpflags, node); ++ ++ return ret; ++} ++EXPORT_SYMBOL(kmem_cache_alloc_node); ++ ++#ifdef CONFIG_TRACING ++void *kmem_cache_alloc_node_trace(struct kmem_cache *s, ++ gfp_t gfpflags, ++ int node, size_t size) ++{ ++ void *ret = slab_alloc_node(s, gfpflags, node, _RET_IP_); ++ ++ trace_kmalloc_node(_RET_IP_, ret, ++ size, s->size, gfpflags, node); ++ ++ kasan_kmalloc(s, ret, size); ++ return ret; ++} ++EXPORT_SYMBOL(kmem_cache_alloc_node_trace); ++#endif ++#endif ++ ++/* ++ * Slow path handling. This may still be called frequently since objects ++ * have a longer lifetime than the cpu slabs in most processing loads. ++ * ++ * So we still attempt to reduce cache line usage. Just take the slab ++ * lock and free the item. If there is no additional partial page ++ * handling required then we can return immediately. ++ */ ++static void __slab_free(struct kmem_cache *s, struct page *page, ++ void *x, unsigned long addr) ++{ ++ void *prior; ++ void **object = (void *)x; ++ int was_frozen; ++ struct page new; ++ unsigned long counters; ++ struct kmem_cache_node *n = NULL; ++ unsigned long uninitialized_var(flags); ++ ++ stat(s, FREE_SLOWPATH); ++ ++ if (kmem_cache_debug(s) && ++ !(n = free_debug_processing(s, page, x, addr, &flags))) ++ return; ++ ++ do { ++ if (unlikely(n)) { ++ spin_unlock_irqrestore(&n->list_lock, flags); ++ n = NULL; ++ } ++ prior = page->freelist; ++ counters = page->counters; ++ set_freepointer(s, object, prior); ++ new.counters = counters; ++ was_frozen = new.frozen; ++ new.inuse--; ++ if ((!new.inuse || !prior) && !was_frozen) { ++ ++ if (kmem_cache_has_cpu_partial(s) && !prior) { ++ ++ /* ++ * Slab was on no list before and will be ++ * partially empty ++ * We can defer the list move and instead ++ * freeze it. ++ */ ++ new.frozen = 1; ++ ++ } else { /* Needs to be taken off a list */ ++ ++ n = get_node(s, page_to_nid(page)); ++ /* ++ * Speculatively acquire the list_lock. ++ * If the cmpxchg does not succeed then we may ++ * drop the list_lock without any processing. ++ * ++ * Otherwise the list_lock will synchronize with ++ * other processors updating the list of slabs. ++ */ ++ spin_lock_irqsave(&n->list_lock, flags); ++ ++ } ++ } ++ ++ } while (!cmpxchg_double_slab(s, page, ++ prior, counters, ++ object, new.counters, ++ "__slab_free")); ++ ++ if (likely(!n)) { ++ ++ /* ++ * If we just froze the page then put it onto the ++ * per cpu partial list. ++ */ ++ if (new.frozen && !was_frozen) { ++ put_cpu_partial(s, page, 1); ++ stat(s, CPU_PARTIAL_FREE); ++ } ++ /* ++ * The list lock was not taken therefore no list ++ * activity can be necessary. ++ */ ++ if (was_frozen) ++ stat(s, FREE_FROZEN); ++ return; ++ } ++ ++ if (unlikely(!new.inuse && n->nr_partial >= s->min_partial)) ++ goto slab_empty; ++ ++ /* ++ * Objects left in the slab. If it was not on the partial list before ++ * then add it. ++ */ ++ if (!kmem_cache_has_cpu_partial(s) && unlikely(!prior)) { ++ if (kmem_cache_debug(s)) ++ remove_full(s, n, page); ++ add_partial(n, page, DEACTIVATE_TO_TAIL); ++ stat(s, FREE_ADD_PARTIAL); ++ } ++ spin_unlock_irqrestore(&n->list_lock, flags); ++ return; ++ ++slab_empty: ++ if (prior) { ++ /* ++ * Slab on the partial list. ++ */ ++ remove_partial(n, page); ++ stat(s, FREE_REMOVE_PARTIAL); ++ } else { ++ /* Slab must be on the full list */ ++ remove_full(s, n, page); ++ } ++ ++ spin_unlock_irqrestore(&n->list_lock, flags); ++ stat(s, FREE_SLAB); ++ discard_slab(s, page); ++} ++ ++/* ++ * Fastpath with forced inlining to produce a kfree and kmem_cache_free that ++ * can perform fastpath freeing without additional function calls. ++ * ++ * The fastpath is only possible if we are freeing to the current cpu slab ++ * of this processor. This typically the case if we have just allocated ++ * the item before. ++ * ++ * If fastpath is not possible then fall back to __slab_free where we deal ++ * with all sorts of special processing. ++ */ ++static __always_inline void slab_free(struct kmem_cache *s, ++ struct page *page, void *x, unsigned long addr) ++{ ++ void **object = (void *)x; ++ struct kmem_cache_cpu *c; ++ unsigned long tid; ++ ++ slab_free_hook(s, x); ++ ++redo: ++ /* ++ * Determine the currently cpus per cpu slab. ++ * The cpu may change afterward. However that does not matter since ++ * data is retrieved via this pointer. If we are on the same cpu ++ * during the cmpxchg then the free will succedd. ++ */ ++ do { ++ tid = this_cpu_read(s->cpu_slab->tid); ++ c = raw_cpu_ptr(s->cpu_slab); ++ } while (IS_ENABLED(CONFIG_PREEMPT) && ++ unlikely(tid != READ_ONCE(c->tid))); ++ ++ /* Same with comment on barrier() in slab_alloc_node() */ ++ barrier(); ++ ++ if (likely(page == c->page)) { ++ set_freepointer(s, object, c->freelist); ++ ++ if (unlikely(!this_cpu_cmpxchg_double( ++ s->cpu_slab->freelist, s->cpu_slab->tid, ++ c->freelist, tid, ++ object, next_tid(tid)))) { ++ ++ note_cmpxchg_failure("slab_free", s, tid); ++ goto redo; ++ } ++ stat(s, FREE_FASTPATH); ++ } else ++ __slab_free(s, page, x, addr); ++ ++} ++ ++void kmem_cache_free(struct kmem_cache *s, void *x) ++{ ++ s = cache_from_obj(s, x); ++ if (!s) ++ return; ++ slab_free(s, virt_to_head_page(x), x, _RET_IP_); ++ trace_kmem_cache_free(_RET_IP_, x); ++} ++EXPORT_SYMBOL(kmem_cache_free); ++ ++/* ++ * Object placement in a slab is made very easy because we always start at ++ * offset 0. If we tune the size of the object to the alignment then we can ++ * get the required alignment by putting one properly sized object after ++ * another. ++ * ++ * Notice that the allocation order determines the sizes of the per cpu ++ * caches. Each processor has always one slab available for allocations. ++ * Increasing the allocation order reduces the number of times that slabs ++ * must be moved on and off the partial lists and is therefore a factor in ++ * locking overhead. ++ */ ++ ++/* ++ * Mininum / Maximum order of slab pages. This influences locking overhead ++ * and slab fragmentation. A higher order reduces the number of partial slabs ++ * and increases the number of allocations possible without having to ++ * take the list_lock. ++ */ ++static int slub_min_order; ++static int slub_max_order = PAGE_ALLOC_COSTLY_ORDER; ++static int slub_min_objects; ++ ++/* ++ * Calculate the order of allocation given an slab object size. ++ * ++ * The order of allocation has significant impact on performance and other ++ * system components. Generally order 0 allocations should be preferred since ++ * order 0 does not cause fragmentation in the page allocator. Larger objects ++ * be problematic to put into order 0 slabs because there may be too much ++ * unused space left. We go to a higher order if more than 1/16th of the slab ++ * would be wasted. ++ * ++ * In order to reach satisfactory performance we must ensure that a minimum ++ * number of objects is in one slab. Otherwise we may generate too much ++ * activity on the partial lists which requires taking the list_lock. This is ++ * less a concern for large slabs though which are rarely used. ++ * ++ * slub_max_order specifies the order where we begin to stop considering the ++ * number of objects in a slab as critical. If we reach slub_max_order then ++ * we try to keep the page order as low as possible. So we accept more waste ++ * of space in favor of a small page order. ++ * ++ * Higher order allocations also allow the placement of more objects in a ++ * slab and thereby reduce object handling overhead. If the user has ++ * requested a higher mininum order then we start with that one instead of ++ * the smallest order which will fit the object. ++ */ ++static inline int slab_order(int size, int min_objects, ++ int max_order, int fract_leftover, int reserved) ++{ ++ int order; ++ int rem; ++ int min_order = slub_min_order; ++ ++ if (order_objects(min_order, size, reserved) > MAX_OBJS_PER_PAGE) ++ return get_order(size * MAX_OBJS_PER_PAGE) - 1; ++ ++ for (order = max(min_order, ++ fls(min_objects * size - 1) - PAGE_SHIFT); ++ order <= max_order; order++) { ++ ++ unsigned long slab_size = PAGE_SIZE << order; ++ ++ if (slab_size < min_objects * size + reserved) ++ continue; ++ ++ rem = (slab_size - reserved) % size; ++ ++ if (rem <= slab_size / fract_leftover) ++ break; ++ ++ } ++ ++ return order; ++} ++ ++static inline int calculate_order(int size, int reserved) ++{ ++ int order; ++ int min_objects; ++ int fraction; ++ int max_objects; ++ ++ /* ++ * Attempt to find best configuration for a slab. This ++ * works by first attempting to generate a layout with ++ * the best configuration and backing off gradually. ++ * ++ * First we reduce the acceptable waste in a slab. Then ++ * we reduce the minimum objects required in a slab. ++ */ ++ min_objects = slub_min_objects; ++ if (!min_objects) ++ min_objects = 4 * (fls(nr_cpu_ids) + 1); ++ max_objects = order_objects(slub_max_order, size, reserved); ++ min_objects = min(min_objects, max_objects); ++ ++ while (min_objects > 1) { ++ fraction = 16; ++ while (fraction >= 4) { ++ order = slab_order(size, min_objects, ++ slub_max_order, fraction, reserved); ++ if (order <= slub_max_order) ++ return order; ++ fraction /= 2; ++ } ++ min_objects--; ++ } ++ ++ /* ++ * We were unable to place multiple objects in a slab. Now ++ * lets see if we can place a single object there. ++ */ ++ order = slab_order(size, 1, slub_max_order, 1, reserved); ++ if (order <= slub_max_order) ++ return order; ++ ++ /* ++ * Doh this slab cannot be placed using slub_max_order. ++ */ ++ order = slab_order(size, 1, MAX_ORDER, 1, reserved); ++ if (order < MAX_ORDER) ++ return order; ++ return -ENOSYS; ++} ++ ++static void ++init_kmem_cache_node(struct kmem_cache_node *n) ++{ ++ n->nr_partial = 0; ++ spin_lock_init(&n->list_lock); ++ INIT_LIST_HEAD(&n->partial); ++#ifdef CONFIG_SLUB_DEBUG ++ atomic_long_set(&n->nr_slabs, 0); ++ atomic_long_set(&n->total_objects, 0); ++ INIT_LIST_HEAD(&n->full); ++#endif ++} ++ ++static inline int alloc_kmem_cache_cpus(struct kmem_cache *s) ++{ ++ BUILD_BUG_ON(PERCPU_DYNAMIC_EARLY_SIZE < ++ KMALLOC_SHIFT_HIGH * sizeof(struct kmem_cache_cpu)); ++ ++ /* ++ * Must align to double word boundary for the double cmpxchg ++ * instructions to work; see __pcpu_double_call_return_bool(). ++ */ ++ s->cpu_slab = __alloc_percpu(sizeof(struct kmem_cache_cpu), ++ 2 * sizeof(void *)); ++ ++ if (!s->cpu_slab) ++ return 0; ++ ++ init_kmem_cache_cpus(s); ++ ++ return 1; ++} ++ ++static struct kmem_cache *kmem_cache_node; ++ ++/* ++ * No kmalloc_node yet so do it by hand. We know that this is the first ++ * slab on the node for this slabcache. There are no concurrent accesses ++ * possible. ++ * ++ * Note that this function only works on the kmem_cache_node ++ * when allocating for the kmem_cache_node. This is used for bootstrapping ++ * memory on a fresh node that has no slab structures yet. ++ */ ++static void early_kmem_cache_node_alloc(int node) ++{ ++ struct page *page; ++ struct kmem_cache_node *n; ++ ++ BUG_ON(kmem_cache_node->size < sizeof(struct kmem_cache_node)); ++ ++ page = new_slab(kmem_cache_node, GFP_NOWAIT, node); ++ ++ BUG_ON(!page); ++ if (page_to_nid(page) != node) { ++ pr_err("SLUB: Unable to allocate memory from node %d\n", node); ++ pr_err("SLUB: Allocating a useless per node structure in order to be able to continue\n"); ++ } ++ ++ n = page->freelist; ++ BUG_ON(!n); ++ page->freelist = get_freepointer(kmem_cache_node, n); ++ page->inuse = 1; ++ page->frozen = 0; ++ kmem_cache_node->node[node] = n; ++#ifdef CONFIG_SLUB_DEBUG ++ init_object(kmem_cache_node, n, SLUB_RED_ACTIVE); ++ init_tracking(kmem_cache_node, n); ++#endif ++ kasan_kmalloc(kmem_cache_node, n, sizeof(struct kmem_cache_node)); ++ init_kmem_cache_node(n); ++ inc_slabs_node(kmem_cache_node, node, page->objects); ++ ++ /* ++ * No locks need to be taken here as it has just been ++ * initialized and there is no concurrent access. ++ */ ++ __add_partial(n, page, DEACTIVATE_TO_HEAD); ++} ++ ++static void free_kmem_cache_nodes(struct kmem_cache *s) ++{ ++ int node; ++ struct kmem_cache_node *n; ++ ++ for_each_kmem_cache_node(s, node, n) { ++ kmem_cache_free(kmem_cache_node, n); ++ s->node[node] = NULL; ++ } ++} ++ ++static int init_kmem_cache_nodes(struct kmem_cache *s) ++{ ++ int node; ++ ++ for_each_node_state(node, N_NORMAL_MEMORY) { ++ struct kmem_cache_node *n; ++ ++ if (slab_state == DOWN) { ++ early_kmem_cache_node_alloc(node); ++ continue; ++ } ++ n = kmem_cache_alloc_node(kmem_cache_node, ++ GFP_KERNEL, node); ++ ++ if (!n) { ++ free_kmem_cache_nodes(s); ++ return 0; ++ } ++ ++ s->node[node] = n; ++ init_kmem_cache_node(n); ++ } ++ return 1; ++} ++ ++static void set_min_partial(struct kmem_cache *s, unsigned long min) ++{ ++ if (min < MIN_PARTIAL) ++ min = MIN_PARTIAL; ++ else if (min > MAX_PARTIAL) ++ min = MAX_PARTIAL; ++ s->min_partial = min; ++} ++ ++/* ++ * calculate_sizes() determines the order and the distribution of data within ++ * a slab object. ++ */ ++static int calculate_sizes(struct kmem_cache *s, int forced_order) ++{ ++ unsigned long flags = s->flags; ++ unsigned long size = s->object_size; ++ int order; ++ ++ /* ++ * Round up object size to the next word boundary. We can only ++ * place the free pointer at word boundaries and this determines ++ * the possible location of the free pointer. ++ */ ++ size = ALIGN(size, sizeof(void *)); ++ ++#ifdef CONFIG_SLUB_DEBUG ++ /* ++ * Determine if we can poison the object itself. If the user of ++ * the slab may touch the object after free or before allocation ++ * then we should never poison the object itself. ++ */ ++ if ((flags & SLAB_POISON) && !(flags & SLAB_DESTROY_BY_RCU) && ++ !s->ctor) ++ s->flags |= __OBJECT_POISON; ++ else ++ s->flags &= ~__OBJECT_POISON; ++ ++ ++ /* ++ * If we are Redzoning then check if there is some space between the ++ * end of the object and the free pointer. If not then add an ++ * additional word to have some bytes to store Redzone information. ++ */ ++ if ((flags & SLAB_RED_ZONE) && size == s->object_size) ++ size += sizeof(void *); ++#endif ++ ++ /* ++ * With that we have determined the number of bytes in actual use ++ * by the object. This is the potential offset to the free pointer. ++ */ ++ s->inuse = size; ++ ++ if (((flags & (SLAB_DESTROY_BY_RCU | SLAB_POISON)) || ++ s->ctor)) { ++ /* ++ * Relocate free pointer after the object if it is not ++ * permitted to overwrite the first word of the object on ++ * kmem_cache_free. ++ * ++ * This is the case if we do RCU, have a constructor or ++ * destructor or are poisoning the objects. ++ */ ++ s->offset = size; ++ size += sizeof(void *); ++ } ++ ++#ifdef CONFIG_SLUB_DEBUG ++ if (flags & SLAB_STORE_USER) ++ /* ++ * Need to store information about allocs and frees after ++ * the object. ++ */ ++ size += 2 * sizeof(struct track); ++ ++ if (flags & SLAB_RED_ZONE) ++ /* ++ * Add some empty padding so that we can catch ++ * overwrites from earlier objects rather than let ++ * tracking information or the free pointer be ++ * corrupted if a user writes before the start ++ * of the object. ++ */ ++ size += sizeof(void *); ++#endif ++ ++ /* ++ * SLUB stores one object immediately after another beginning from ++ * offset 0. In order to align the objects we have to simply size ++ * each object to conform to the alignment. ++ */ ++ size = ALIGN(size, s->align); ++ s->size = size; ++ if (forced_order >= 0) ++ order = forced_order; ++ else ++ order = calculate_order(size, s->reserved); ++ ++ if (order < 0) ++ return 0; ++ ++ s->allocflags = 0; ++ if (order) ++ s->allocflags |= __GFP_COMP; ++ ++ if (s->flags & SLAB_CACHE_DMA) ++ s->allocflags |= GFP_DMA; ++ ++ if (s->flags & SLAB_RECLAIM_ACCOUNT) ++ s->allocflags |= __GFP_RECLAIMABLE; ++ ++ /* ++ * Determine the number of objects per slab ++ */ ++ s->oo = oo_make(order, size, s->reserved); ++ s->min = oo_make(get_order(size), size, s->reserved); ++ if (oo_objects(s->oo) > oo_objects(s->max)) ++ s->max = s->oo; ++ ++ return !!oo_objects(s->oo); ++} ++ ++static int kmem_cache_open(struct kmem_cache *s, unsigned long flags) ++{ ++ s->flags = kmem_cache_flags(s->size, flags, s->name, s->ctor); ++ s->reserved = 0; ++ ++ if (need_reserve_slab_rcu && (s->flags & SLAB_DESTROY_BY_RCU)) ++ s->reserved = sizeof(struct rcu_head); ++ ++ if (!calculate_sizes(s, -1)) ++ goto error; ++ if (disable_higher_order_debug) { ++ /* ++ * Disable debugging flags that store metadata if the min slab ++ * order increased. ++ */ ++ if (get_order(s->size) > get_order(s->object_size)) { ++ s->flags &= ~DEBUG_METADATA_FLAGS; ++ s->offset = 0; ++ if (!calculate_sizes(s, -1)) ++ goto error; ++ } ++ } ++ ++#if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \ ++ defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE) ++ if (system_has_cmpxchg_double() && (s->flags & SLAB_DEBUG_FLAGS) == 0) ++ /* Enable fast mode */ ++ s->flags |= __CMPXCHG_DOUBLE; ++#endif ++ ++ /* ++ * The larger the object size is, the more pages we want on the partial ++ * list to avoid pounding the page allocator excessively. ++ */ ++ set_min_partial(s, ilog2(s->size) / 2); ++ ++ /* ++ * cpu_partial determined the maximum number of objects kept in the ++ * per cpu partial lists of a processor. ++ * ++ * Per cpu partial lists mainly contain slabs that just have one ++ * object freed. If they are used for allocation then they can be ++ * filled up again with minimal effort. The slab will never hit the ++ * per node partial lists and therefore no locking will be required. ++ * ++ * This setting also determines ++ * ++ * A) The number of objects from per cpu partial slabs dumped to the ++ * per node list when we reach the limit. ++ * B) The number of objects in cpu partial slabs to extract from the ++ * per node list when we run out of per cpu objects. We only fetch ++ * 50% to keep some capacity around for frees. ++ */ ++ if (!kmem_cache_has_cpu_partial(s)) ++ s->cpu_partial = 0; ++ else if (s->size >= PAGE_SIZE) ++ s->cpu_partial = 2; ++ else if (s->size >= 1024) ++ s->cpu_partial = 6; ++ else if (s->size >= 256) ++ s->cpu_partial = 13; ++ else ++ s->cpu_partial = 30; ++ ++#ifdef CONFIG_NUMA ++ s->remote_node_defrag_ratio = 1000; ++#endif ++ if (!init_kmem_cache_nodes(s)) ++ goto error; ++ ++ if (alloc_kmem_cache_cpus(s)) ++ return 0; ++ ++ free_kmem_cache_nodes(s); ++error: ++ if (flags & SLAB_PANIC) ++ panic("Cannot create slab %s size=%lu realsize=%u " ++ "order=%u offset=%u flags=%lx\n", ++ s->name, (unsigned long)s->size, s->size, ++ oo_order(s->oo), s->offset, flags); ++ return -EINVAL; ++} ++ ++static void list_slab_objects(struct kmem_cache *s, struct page *page, ++ const char *text) ++{ ++#ifdef CONFIG_SLUB_DEBUG ++ void *addr = page_address(page); ++ void *p; ++ unsigned long *map = kzalloc(BITS_TO_LONGS(page->objects) * ++ sizeof(long), GFP_ATOMIC); ++ if (!map) ++ return; ++ slab_err(s, page, text, s->name); ++ slab_lock(page); ++ ++ get_map(s, page, map); ++ for_each_object(p, s, addr, page->objects) { ++ ++ if (!test_bit(slab_index(p, s, addr), map)) { ++ pr_err("INFO: Object 0x%p @offset=%tu\n", p, p - addr); ++ print_tracking(s, p); ++ } ++ } ++ slab_unlock(page); ++ kfree(map); ++#endif ++} ++ ++/* ++ * Attempt to free all partial slabs on a node. ++ * This is called from kmem_cache_close(). We must be the last thread ++ * using the cache and therefore we do not need to lock anymore. ++ */ ++static void free_partial(struct kmem_cache *s, struct kmem_cache_node *n) ++{ ++ struct page *page, *h; ++ ++ list_for_each_entry_safe(page, h, &n->partial, lru) { ++ if (!page->inuse) { ++ __remove_partial(n, page); ++ discard_slab(s, page); ++ } else { ++ list_slab_objects(s, page, ++ "Objects remaining in %s on kmem_cache_close()"); ++ } ++ } ++} ++ ++/* ++ * Release all resources used by a slab cache. ++ */ ++static inline int kmem_cache_close(struct kmem_cache *s) ++{ ++ int node; ++ struct kmem_cache_node *n; ++ ++ flush_all(s); ++ /* Attempt to free all objects */ ++ for_each_kmem_cache_node(s, node, n) { ++ free_partial(s, n); ++ if (n->nr_partial || slabs_node(s, node)) ++ return 1; ++ } ++ free_percpu(s->cpu_slab); ++ free_kmem_cache_nodes(s); ++ return 0; ++} ++ ++int __kmem_cache_shutdown(struct kmem_cache *s) ++{ ++ return kmem_cache_close(s); ++} ++ ++/******************************************************************** ++ * Kmalloc subsystem ++ *******************************************************************/ ++ ++static int __init setup_slub_min_order(char *str) ++{ ++ get_option(&str, &slub_min_order); ++ ++ return 1; ++} ++ ++__setup("slub_min_order=", setup_slub_min_order); ++ ++static int __init setup_slub_max_order(char *str) ++{ ++ get_option(&str, &slub_max_order); ++ slub_max_order = min(slub_max_order, MAX_ORDER - 1); ++ ++ return 1; ++} ++ ++__setup("slub_max_order=", setup_slub_max_order); ++ ++static int __init setup_slub_min_objects(char *str) ++{ ++ get_option(&str, &slub_min_objects); ++ ++ return 1; ++} ++ ++__setup("slub_min_objects=", setup_slub_min_objects); ++ ++void *__kmalloc(size_t size, gfp_t flags) ++{ ++ struct kmem_cache *s; ++ void *ret; ++ ++ if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) ++ return kmalloc_large(size, flags); ++ ++ s = kmalloc_slab(size, flags); ++ ++ if (unlikely(ZERO_OR_NULL_PTR(s))) ++ return s; ++ ++ ret = slab_alloc(s, flags, _RET_IP_); ++ ++ trace_kmalloc(_RET_IP_, ret, size, s->size, flags); ++ ++ kasan_kmalloc(s, ret, size); ++ ++ return ret; ++} ++EXPORT_SYMBOL(__kmalloc); ++ ++#ifdef CONFIG_NUMA ++static void *kmalloc_large_node(size_t size, gfp_t flags, int node) ++{ ++ struct page *page; ++ void *ptr = NULL; ++ ++ flags |= __GFP_COMP | __GFP_NOTRACK; ++ page = alloc_kmem_pages_node(node, flags, get_order(size)); ++ if (page) ++ ptr = page_address(page); ++ ++ kmalloc_large_node_hook(ptr, size, flags); ++ return ptr; ++} ++ ++void *__kmalloc_node(size_t size, gfp_t flags, int node) ++{ ++ struct kmem_cache *s; ++ void *ret; ++ ++ if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) { ++ ret = kmalloc_large_node(size, flags, node); ++ ++ trace_kmalloc_node(_RET_IP_, ret, ++ size, PAGE_SIZE << get_order(size), ++ flags, node); ++ ++ return ret; ++ } ++ ++ s = kmalloc_slab(size, flags); ++ ++ if (unlikely(ZERO_OR_NULL_PTR(s))) ++ return s; ++ ++ ret = slab_alloc_node(s, flags, node, _RET_IP_); ++ ++ trace_kmalloc_node(_RET_IP_, ret, size, s->size, flags, node); ++ ++ kasan_kmalloc(s, ret, size); ++ ++ return ret; ++} ++EXPORT_SYMBOL(__kmalloc_node); ++#endif ++ ++static size_t __ksize(const void *object) ++{ ++ struct page *page; ++ ++ if (unlikely(object == ZERO_SIZE_PTR)) ++ return 0; ++ ++ page = virt_to_head_page(object); ++ ++ if (unlikely(!PageSlab(page))) { ++ WARN_ON(!PageCompound(page)); ++ return PAGE_SIZE << compound_order(page); ++ } ++ ++ return slab_ksize(page->slab_cache); ++} ++ ++size_t ksize(const void *object) ++{ ++ size_t size = __ksize(object); ++ /* We assume that ksize callers could use whole allocated area, ++ so we need unpoison this area. */ ++ kasan_krealloc(object, size); ++ return size; ++} ++EXPORT_SYMBOL(ksize); ++ ++void kfree(const void *x) ++{ ++ struct page *page; ++ void *object = (void *)x; ++ ++ trace_kfree(_RET_IP_, x); ++ ++ if (unlikely(ZERO_OR_NULL_PTR(x))) ++ return; ++ ++ page = virt_to_head_page(x); ++ if (unlikely(!PageSlab(page))) { ++ BUG_ON(!PageCompound(page)); ++ kfree_hook(x); ++ __free_kmem_pages(page, compound_order(page)); ++ return; ++ } ++ slab_free(page->slab_cache, page, object, _RET_IP_); ++} ++EXPORT_SYMBOL(kfree); ++ ++#define SHRINK_PROMOTE_MAX 32 ++ ++/* ++ * kmem_cache_shrink discards empty slabs and promotes the slabs filled ++ * up most to the head of the partial lists. New allocations will then ++ * fill those up and thus they can be removed from the partial lists. ++ * ++ * The slabs with the least items are placed last. This results in them ++ * being allocated from last increasing the chance that the last objects ++ * are freed in them. ++ */ ++int __kmem_cache_shrink(struct kmem_cache *s, bool deactivate) ++{ ++ int node; ++ int i; ++ struct kmem_cache_node *n; ++ struct page *page; ++ struct page *t; ++ struct list_head discard; ++ struct list_head promote[SHRINK_PROMOTE_MAX]; ++ unsigned long flags; ++ int ret = 0; ++ ++ if (deactivate) { ++ /* ++ * Disable empty slabs caching. Used to avoid pinning offline ++ * memory cgroups by kmem pages that can be freed. ++ */ ++ s->cpu_partial = 0; ++ s->min_partial = 0; ++ ++ /* ++ * s->cpu_partial is checked locklessly (see put_cpu_partial), ++ * so we have to make sure the change is visible. ++ */ ++ kick_all_cpus_sync(); ++ } ++ ++ flush_all(s); ++ for_each_kmem_cache_node(s, node, n) { ++ INIT_LIST_HEAD(&discard); ++ for (i = 0; i < SHRINK_PROMOTE_MAX; i++) ++ INIT_LIST_HEAD(promote + i); ++ ++ spin_lock_irqsave(&n->list_lock, flags); ++ ++ /* ++ * Build lists of slabs to discard or promote. ++ * ++ * Note that concurrent frees may occur while we hold the ++ * list_lock. page->inuse here is the upper limit. ++ */ ++ list_for_each_entry_safe(page, t, &n->partial, lru) { ++ int free = page->objects - page->inuse; ++ ++ /* Do not reread page->inuse */ ++ barrier(); ++ ++ /* We do not keep full slabs on the list */ ++ BUG_ON(free <= 0); ++ ++ if (free == page->objects) { ++ list_move(&page->lru, &discard); ++ n->nr_partial--; ++ } else if (free <= SHRINK_PROMOTE_MAX) ++ list_move(&page->lru, promote + free - 1); ++ } ++ ++ /* ++ * Promote the slabs filled up most to the head of the ++ * partial list. ++ */ ++ for (i = SHRINK_PROMOTE_MAX - 1; i >= 0; i--) ++ list_splice(promote + i, &n->partial); ++ ++ spin_unlock_irqrestore(&n->list_lock, flags); ++ ++ /* Release empty slabs */ ++ list_for_each_entry_safe(page, t, &discard, lru) ++ discard_slab(s, page); ++ ++ if (slabs_node(s, node)) ++ ret = 1; ++ } ++ ++ return ret; ++} ++ ++static int slab_mem_going_offline_callback(void *arg) ++{ ++ struct kmem_cache *s; ++ ++ mutex_lock(&slab_mutex); ++ list_for_each_entry(s, &slab_caches, list) ++ __kmem_cache_shrink(s, false); ++ mutex_unlock(&slab_mutex); ++ ++ return 0; ++} ++ ++static void slab_mem_offline_callback(void *arg) ++{ ++ struct kmem_cache_node *n; ++ struct kmem_cache *s; ++ struct memory_notify *marg = arg; ++ int offline_node; ++ ++ offline_node = marg->status_change_nid_normal; ++ ++ /* ++ * If the node still has available memory. we need kmem_cache_node ++ * for it yet. ++ */ ++ if (offline_node < 0) ++ return; ++ ++ mutex_lock(&slab_mutex); ++ list_for_each_entry(s, &slab_caches, list) { ++ n = get_node(s, offline_node); ++ if (n) { ++ /* ++ * if n->nr_slabs > 0, slabs still exist on the node ++ * that is going down. We were unable to free them, ++ * and offline_pages() function shouldn't call this ++ * callback. So, we must fail. ++ */ ++ BUG_ON(slabs_node(s, offline_node)); ++ ++ s->node[offline_node] = NULL; ++ kmem_cache_free(kmem_cache_node, n); ++ } ++ } ++ mutex_unlock(&slab_mutex); ++} ++ ++static int slab_mem_going_online_callback(void *arg) ++{ ++ struct kmem_cache_node *n; ++ struct kmem_cache *s; ++ struct memory_notify *marg = arg; ++ int nid = marg->status_change_nid_normal; ++ int ret = 0; ++ ++ /* ++ * If the node's memory is already available, then kmem_cache_node is ++ * already created. Nothing to do. ++ */ ++ if (nid < 0) ++ return 0; ++ ++ /* ++ * We are bringing a node online. No memory is available yet. We must ++ * allocate a kmem_cache_node structure in order to bring the node ++ * online. ++ */ ++ mutex_lock(&slab_mutex); ++ list_for_each_entry(s, &slab_caches, list) { ++ /* ++ * XXX: kmem_cache_alloc_node will fallback to other nodes ++ * since memory is not yet available from the node that ++ * is brought up. ++ */ ++ n = kmem_cache_alloc(kmem_cache_node, GFP_KERNEL); ++ if (!n) { ++ ret = -ENOMEM; ++ goto out; ++ } ++ init_kmem_cache_node(n); ++ s->node[nid] = n; ++ } ++out: ++ mutex_unlock(&slab_mutex); ++ return ret; ++} ++ ++static int slab_memory_callback(struct notifier_block *self, ++ unsigned long action, void *arg) ++{ ++ int ret = 0; ++ ++ switch (action) { ++ case MEM_GOING_ONLINE: ++ ret = slab_mem_going_online_callback(arg); ++ break; ++ case MEM_GOING_OFFLINE: ++ ret = slab_mem_going_offline_callback(arg); ++ break; ++ case MEM_OFFLINE: ++ case MEM_CANCEL_ONLINE: ++ slab_mem_offline_callback(arg); ++ break; ++ case MEM_ONLINE: ++ case MEM_CANCEL_OFFLINE: ++ break; ++ } ++ if (ret) ++ ret = notifier_from_errno(ret); ++ else ++ ret = NOTIFY_OK; ++ return ret; ++} ++ ++static struct notifier_block slab_memory_callback_nb = { ++ .notifier_call = slab_memory_callback, ++ .priority = SLAB_CALLBACK_PRI, ++}; ++ ++/******************************************************************** ++ * Basic setup of slabs ++ *******************************************************************/ ++ ++/* ++ * Used for early kmem_cache structures that were allocated using ++ * the page allocator. Allocate them properly then fix up the pointers ++ * that may be pointing to the wrong kmem_cache structure. ++ */ ++ ++static struct kmem_cache * __init bootstrap(struct kmem_cache *static_cache) ++{ ++ int node; ++ struct kmem_cache *s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT); ++ struct kmem_cache_node *n; ++ ++ memcpy(s, static_cache, kmem_cache->object_size); ++ ++ /* ++ * This runs very early, and only the boot processor is supposed to be ++ * up. Even if it weren't true, IRQs are not up so we couldn't fire ++ * IPIs around. ++ */ ++ __flush_cpu_slab(s, smp_processor_id()); ++ for_each_kmem_cache_node(s, node, n) { ++ struct page *p; ++ ++ list_for_each_entry(p, &n->partial, lru) ++ p->slab_cache = s; ++ ++#ifdef CONFIG_SLUB_DEBUG ++ list_for_each_entry(p, &n->full, lru) ++ p->slab_cache = s; ++#endif ++ } ++ slab_init_memcg_params(s); ++ list_add(&s->list, &slab_caches); ++ return s; ++} ++ ++void __init kmem_cache_init(void) ++{ ++ static __initdata struct kmem_cache boot_kmem_cache, ++ boot_kmem_cache_node; ++ ++ if (debug_guardpage_minorder()) ++ slub_max_order = 0; ++ ++ kmem_cache_node = &boot_kmem_cache_node; ++ kmem_cache = &boot_kmem_cache; ++ ++ create_boot_cache(kmem_cache_node, "kmem_cache_node", ++ sizeof(struct kmem_cache_node), SLAB_HWCACHE_ALIGN); ++ ++ register_hotmemory_notifier(&slab_memory_callback_nb); ++ ++ /* Able to allocate the per node structures */ ++ slab_state = PARTIAL; ++ ++ create_boot_cache(kmem_cache, "kmem_cache", ++ offsetof(struct kmem_cache, node) + ++ nr_node_ids * sizeof(struct kmem_cache_node *), ++ SLAB_HWCACHE_ALIGN); ++ ++ kmem_cache = bootstrap(&boot_kmem_cache); ++ ++ /* ++ * Allocate kmem_cache_node properly from the kmem_cache slab. ++ * kmem_cache_node is separately allocated so no need to ++ * update any list pointers. ++ */ ++ kmem_cache_node = bootstrap(&boot_kmem_cache_node); ++ ++ /* Now we can use the kmem_cache to allocate kmalloc slabs */ ++ create_kmalloc_caches(0); ++ ++#ifdef CONFIG_SMP ++ register_cpu_notifier(&slab_notifier); ++#endif ++ ++ pr_info("SLUB: HWalign=%d, Order=%d-%d, MinObjects=%d, CPUs=%d, Nodes=%d\n", ++ cache_line_size(), ++ slub_min_order, slub_max_order, slub_min_objects, ++ nr_cpu_ids, nr_node_ids); ++} ++ ++void __init kmem_cache_init_late(void) ++{ ++} ++ ++struct kmem_cache * ++__kmem_cache_alias(const char *name, size_t size, size_t align, ++ unsigned long flags, void (*ctor)(void *)) ++{ ++ struct kmem_cache *s, *c; ++ ++ s = find_mergeable(size, align, flags, name, ctor); ++ if (s) { ++ s->refcount++; ++ ++ /* ++ * Adjust the object sizes so that we clear ++ * the complete object on kzalloc. ++ */ ++ s->object_size = max(s->object_size, (int)size); ++ s->inuse = max_t(int, s->inuse, ALIGN(size, sizeof(void *))); ++ ++ for_each_memcg_cache(c, s) { ++ c->object_size = s->object_size; ++ c->inuse = max_t(int, c->inuse, ++ ALIGN(size, sizeof(void *))); ++ } ++ ++ if (sysfs_slab_alias(s, name)) { ++ s->refcount--; ++ s = NULL; ++ } ++ } ++ ++ return s; ++} ++ ++int __kmem_cache_create(struct kmem_cache *s, unsigned long flags) ++{ ++ int err; ++ ++ err = kmem_cache_open(s, flags); ++ if (err) ++ return err; ++ ++ /* Mutex is not taken during early boot */ ++ if (slab_state <= UP) ++ return 0; ++ ++ memcg_propagate_slab_attrs(s); ++ err = sysfs_slab_add(s); ++ if (err) ++ kmem_cache_close(s); ++ ++ return err; ++} ++ ++#ifdef CONFIG_SMP ++/* ++ * Use the cpu notifier to insure that the cpu slabs are flushed when ++ * necessary. ++ */ ++static int slab_cpuup_callback(struct notifier_block *nfb, ++ unsigned long action, void *hcpu) ++{ ++ long cpu = (long)hcpu; ++ struct kmem_cache *s; ++ unsigned long flags; ++ ++ switch (action) { ++ case CPU_UP_CANCELED: ++ case CPU_UP_CANCELED_FROZEN: ++ case CPU_DEAD: ++ case CPU_DEAD_FROZEN: ++ mutex_lock(&slab_mutex); ++ list_for_each_entry(s, &slab_caches, list) { ++ local_irq_save(flags); ++ __flush_cpu_slab(s, cpu); ++ local_irq_restore(flags); ++ } ++ mutex_unlock(&slab_mutex); ++ break; ++ default: ++ break; ++ } ++ return NOTIFY_OK; ++} ++ ++static struct notifier_block slab_notifier = { ++ .notifier_call = slab_cpuup_callback ++}; ++ ++#endif ++ ++void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, unsigned long caller) ++{ ++ struct kmem_cache *s; ++ void *ret; ++ ++ if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) ++ return kmalloc_large(size, gfpflags); ++ ++ s = kmalloc_slab(size, gfpflags); ++ ++ if (unlikely(ZERO_OR_NULL_PTR(s))) ++ return s; ++ ++ ret = slab_alloc(s, gfpflags, caller); ++ ++ /* Honor the call site pointer we received. */ ++ trace_kmalloc(caller, ret, size, s->size, gfpflags); ++ ++ return ret; ++} ++ ++#ifdef CONFIG_NUMA ++void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags, ++ int node, unsigned long caller) ++{ ++ struct kmem_cache *s; ++ void *ret; ++ ++ if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) { ++ ret = kmalloc_large_node(size, gfpflags, node); ++ ++ trace_kmalloc_node(caller, ret, ++ size, PAGE_SIZE << get_order(size), ++ gfpflags, node); ++ ++ return ret; ++ } ++ ++ s = kmalloc_slab(size, gfpflags); ++ ++ if (unlikely(ZERO_OR_NULL_PTR(s))) ++ return s; ++ ++ ret = slab_alloc_node(s, gfpflags, node, caller); ++ ++ /* Honor the call site pointer we received. */ ++ trace_kmalloc_node(caller, ret, size, s->size, gfpflags, node); ++ ++ return ret; ++} ++#endif ++ ++#ifdef CONFIG_SYSFS ++static int count_inuse(struct page *page) ++{ ++ return page->inuse; ++} ++ ++static int count_total(struct page *page) ++{ ++ return page->objects; ++} ++#endif ++ ++#ifdef CONFIG_SLUB_DEBUG ++static int validate_slab(struct kmem_cache *s, struct page *page, ++ unsigned long *map) ++{ ++ void *p; ++ void *addr = page_address(page); ++ ++ if (!check_slab(s, page) || ++ !on_freelist(s, page, NULL)) ++ return 0; ++ ++ /* Now we know that a valid freelist exists */ ++ bitmap_zero(map, page->objects); ++ ++ get_map(s, page, map); ++ for_each_object(p, s, addr, page->objects) { ++ if (test_bit(slab_index(p, s, addr), map)) ++ if (!check_object(s, page, p, SLUB_RED_INACTIVE)) ++ return 0; ++ } ++ ++ for_each_object(p, s, addr, page->objects) ++ if (!test_bit(slab_index(p, s, addr), map)) ++ if (!check_object(s, page, p, SLUB_RED_ACTIVE)) ++ return 0; ++ return 1; ++} ++ ++static void validate_slab_slab(struct kmem_cache *s, struct page *page, ++ unsigned long *map) ++{ ++ slab_lock(page); ++ validate_slab(s, page, map); ++ slab_unlock(page); ++} ++ ++static int validate_slab_node(struct kmem_cache *s, ++ struct kmem_cache_node *n, unsigned long *map) ++{ ++ unsigned long count = 0; ++ struct page *page; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&n->list_lock, flags); ++ ++ list_for_each_entry(page, &n->partial, lru) { ++ validate_slab_slab(s, page, map); ++ count++; ++ } ++ if (count != n->nr_partial) ++ pr_err("SLUB %s: %ld partial slabs counted but counter=%ld\n", ++ s->name, count, n->nr_partial); ++ ++ if (!(s->flags & SLAB_STORE_USER)) ++ goto out; ++ ++ list_for_each_entry(page, &n->full, lru) { ++ validate_slab_slab(s, page, map); ++ count++; ++ } ++ if (count != atomic_long_read(&n->nr_slabs)) ++ pr_err("SLUB: %s %ld slabs counted but counter=%ld\n", ++ s->name, count, atomic_long_read(&n->nr_slabs)); ++ ++out: ++ spin_unlock_irqrestore(&n->list_lock, flags); ++ return count; ++} ++ ++static long validate_slab_cache(struct kmem_cache *s) ++{ ++ int node; ++ unsigned long count = 0; ++ unsigned long *map = kmalloc(BITS_TO_LONGS(oo_objects(s->max)) * ++ sizeof(unsigned long), GFP_KERNEL); ++ struct kmem_cache_node *n; ++ ++ if (!map) ++ return -ENOMEM; ++ ++ flush_all(s); ++ for_each_kmem_cache_node(s, node, n) ++ count += validate_slab_node(s, n, map); ++ kfree(map); ++ return count; ++} ++/* ++ * Generate lists of code addresses where slabcache objects are allocated ++ * and freed. ++ */ ++ ++struct location { ++ unsigned long count; ++ unsigned long addr; ++ long long sum_time; ++ long min_time; ++ long max_time; ++ long min_pid; ++ long max_pid; ++ DECLARE_BITMAP(cpus, NR_CPUS); ++ nodemask_t nodes; ++}; ++ ++struct loc_track { ++ unsigned long max; ++ unsigned long count; ++ struct location *loc; ++}; ++ ++static void free_loc_track(struct loc_track *t) ++{ ++ if (t->max) ++ free_pages((unsigned long)t->loc, ++ get_order(sizeof(struct location) * t->max)); ++} ++ ++static int alloc_loc_track(struct loc_track *t, unsigned long max, gfp_t flags) ++{ ++ struct location *l; ++ int order; ++ ++ order = get_order(sizeof(struct location) * max); ++ ++ l = (void *)__get_free_pages(flags, order); ++ if (!l) ++ return 0; ++ ++ if (t->count) { ++ memcpy(l, t->loc, sizeof(struct location) * t->count); ++ free_loc_track(t); ++ } ++ t->max = max; ++ t->loc = l; ++ return 1; ++} ++ ++static int add_location(struct loc_track *t, struct kmem_cache *s, ++ const struct track *track) ++{ ++ long start, end, pos; ++ struct location *l; ++ unsigned long caddr; ++ unsigned long age = jiffies - track->when; ++ ++ start = -1; ++ end = t->count; ++ ++ for ( ; ; ) { ++ pos = start + (end - start + 1) / 2; ++ ++ /* ++ * There is nothing at "end". If we end up there ++ * we need to add something to before end. ++ */ ++ if (pos == end) ++ break; ++ ++ caddr = t->loc[pos].addr; ++ if (track->addr == caddr) { ++ ++ l = &t->loc[pos]; ++ l->count++; ++ if (track->when) { ++ l->sum_time += age; ++ if (age < l->min_time) ++ l->min_time = age; ++ if (age > l->max_time) ++ l->max_time = age; ++ ++ if (track->pid < l->min_pid) ++ l->min_pid = track->pid; ++ if (track->pid > l->max_pid) ++ l->max_pid = track->pid; ++ ++ cpumask_set_cpu(track->cpu, ++ to_cpumask(l->cpus)); ++ } ++ node_set(page_to_nid(virt_to_page(track)), l->nodes); ++ return 1; ++ } ++ ++ if (track->addr < caddr) ++ end = pos; ++ else ++ start = pos; ++ } ++ ++ /* ++ * Not found. Insert new tracking element. ++ */ ++ if (t->count >= t->max && !alloc_loc_track(t, 2 * t->max, GFP_ATOMIC)) ++ return 0; ++ ++ l = t->loc + pos; ++ if (pos < t->count) ++ memmove(l + 1, l, ++ (t->count - pos) * sizeof(struct location)); ++ t->count++; ++ l->count = 1; ++ l->addr = track->addr; ++ l->sum_time = age; ++ l->min_time = age; ++ l->max_time = age; ++ l->min_pid = track->pid; ++ l->max_pid = track->pid; ++ cpumask_clear(to_cpumask(l->cpus)); ++ cpumask_set_cpu(track->cpu, to_cpumask(l->cpus)); ++ nodes_clear(l->nodes); ++ node_set(page_to_nid(virt_to_page(track)), l->nodes); ++ return 1; ++} ++ ++static void process_slab(struct loc_track *t, struct kmem_cache *s, ++ struct page *page, enum track_item alloc, ++ unsigned long *map) ++{ ++ void *addr = page_address(page); ++ void *p; ++ ++ bitmap_zero(map, page->objects); ++ get_map(s, page, map); ++ ++ for_each_object(p, s, addr, page->objects) ++ if (!test_bit(slab_index(p, s, addr), map)) ++ add_location(t, s, get_track(s, p, alloc)); ++} ++ ++static int list_locations(struct kmem_cache *s, char *buf, ++ enum track_item alloc) ++{ ++ int len = 0; ++ unsigned long i; ++ struct loc_track t = { 0, 0, NULL }; ++ int node; ++ unsigned long *map = kmalloc(BITS_TO_LONGS(oo_objects(s->max)) * ++ sizeof(unsigned long), GFP_KERNEL); ++ struct kmem_cache_node *n; ++ ++ if (!map || !alloc_loc_track(&t, PAGE_SIZE / sizeof(struct location), ++ GFP_TEMPORARY)) { ++ kfree(map); ++ return sprintf(buf, "Out of memory\n"); ++ } ++ /* Push back cpu slabs */ ++ flush_all(s); ++ ++ for_each_kmem_cache_node(s, node, n) { ++ unsigned long flags; ++ struct page *page; ++ ++ if (!atomic_long_read(&n->nr_slabs)) ++ continue; ++ ++ spin_lock_irqsave(&n->list_lock, flags); ++ list_for_each_entry(page, &n->partial, lru) ++ process_slab(&t, s, page, alloc, map); ++ list_for_each_entry(page, &n->full, lru) ++ process_slab(&t, s, page, alloc, map); ++ spin_unlock_irqrestore(&n->list_lock, flags); ++ } ++ ++ for (i = 0; i < t.count; i++) { ++ struct location *l = &t.loc[i]; ++ ++ if (len > PAGE_SIZE - KSYM_SYMBOL_LEN - 100) ++ break; ++ len += sprintf(buf + len, "%7ld ", l->count); ++ ++ if (l->addr) ++ len += sprintf(buf + len, "%pS", (void *)l->addr); ++ else ++ len += sprintf(buf + len, ""); ++ ++ if (l->sum_time != l->min_time) { ++ len += sprintf(buf + len, " age=%ld/%ld/%ld", ++ l->min_time, ++ (long)div_u64(l->sum_time, l->count), ++ l->max_time); ++ } else ++ len += sprintf(buf + len, " age=%ld", ++ l->min_time); ++ ++ if (l->min_pid != l->max_pid) ++ len += sprintf(buf + len, " pid=%ld-%ld", ++ l->min_pid, l->max_pid); ++ else ++ len += sprintf(buf + len, " pid=%ld", ++ l->min_pid); ++ ++ if (num_online_cpus() > 1 && ++ !cpumask_empty(to_cpumask(l->cpus)) && ++ len < PAGE_SIZE - 60) ++ len += scnprintf(buf + len, PAGE_SIZE - len - 50, ++ " cpus=%*pbl", ++ cpumask_pr_args(to_cpumask(l->cpus))); ++ ++ if (nr_online_nodes > 1 && !nodes_empty(l->nodes) && ++ len < PAGE_SIZE - 60) ++ len += scnprintf(buf + len, PAGE_SIZE - len - 50, ++ " nodes=%*pbl", ++ nodemask_pr_args(&l->nodes)); ++ ++ len += sprintf(buf + len, "\n"); ++ } ++ ++ free_loc_track(&t); ++ kfree(map); ++ if (!t.count) ++ len += sprintf(buf, "No data\n"); ++ return len; ++} ++#endif ++ ++#ifdef SLUB_RESILIENCY_TEST ++static void __init resiliency_test(void) ++{ ++ u8 *p; ++ ++ BUILD_BUG_ON(KMALLOC_MIN_SIZE > 16 || KMALLOC_SHIFT_HIGH < 10); ++ ++ pr_err("SLUB resiliency testing\n"); ++ pr_err("-----------------------\n"); ++ pr_err("A. Corruption after allocation\n"); ++ ++ p = kzalloc(16, GFP_KERNEL); ++ p[16] = 0x12; ++ pr_err("\n1. kmalloc-16: Clobber Redzone/next pointer 0x12->0x%p\n\n", ++ p + 16); ++ ++ validate_slab_cache(kmalloc_caches[4]); ++ ++ /* Hmmm... The next two are dangerous */ ++ p = kzalloc(32, GFP_KERNEL); ++ p[32 + sizeof(void *)] = 0x34; ++ pr_err("\n2. kmalloc-32: Clobber next pointer/next slab 0x34 -> -0x%p\n", ++ p); ++ pr_err("If allocated object is overwritten then not detectable\n\n"); ++ ++ validate_slab_cache(kmalloc_caches[5]); ++ p = kzalloc(64, GFP_KERNEL); ++ p += 64 + (get_cycles() & 0xff) * sizeof(void *); ++ *p = 0x56; ++ pr_err("\n3. kmalloc-64: corrupting random byte 0x56->0x%p\n", ++ p); ++ pr_err("If allocated object is overwritten then not detectable\n\n"); ++ validate_slab_cache(kmalloc_caches[6]); ++ ++ pr_err("\nB. Corruption after free\n"); ++ p = kzalloc(128, GFP_KERNEL); ++ kfree(p); ++ *p = 0x78; ++ pr_err("1. kmalloc-128: Clobber first word 0x78->0x%p\n\n", p); ++ validate_slab_cache(kmalloc_caches[7]); ++ ++ p = kzalloc(256, GFP_KERNEL); ++ kfree(p); ++ p[50] = 0x9a; ++ pr_err("\n2. kmalloc-256: Clobber 50th byte 0x9a->0x%p\n\n", p); ++ validate_slab_cache(kmalloc_caches[8]); ++ ++ p = kzalloc(512, GFP_KERNEL); ++ kfree(p); ++ p[512] = 0xab; ++ pr_err("\n3. kmalloc-512: Clobber redzone 0xab->0x%p\n\n", p); ++ validate_slab_cache(kmalloc_caches[9]); ++} ++#else ++#ifdef CONFIG_SYSFS ++static void resiliency_test(void) {}; ++#endif ++#endif ++ ++#ifdef CONFIG_SYSFS ++enum slab_stat_type { ++ SL_ALL, /* All slabs */ ++ SL_PARTIAL, /* Only partially allocated slabs */ ++ SL_CPU, /* Only slabs used for cpu caches */ ++ SL_OBJECTS, /* Determine allocated objects not slabs */ ++ SL_TOTAL /* Determine object capacity not slabs */ ++}; ++ ++#define SO_ALL (1 << SL_ALL) ++#define SO_PARTIAL (1 << SL_PARTIAL) ++#define SO_CPU (1 << SL_CPU) ++#define SO_OBJECTS (1 << SL_OBJECTS) ++#define SO_TOTAL (1 << SL_TOTAL) ++ ++static ssize_t show_slab_objects(struct kmem_cache *s, ++ char *buf, unsigned long flags) ++{ ++ unsigned long total = 0; ++ int node; ++ int x; ++ unsigned long *nodes; ++ ++ nodes = kzalloc(sizeof(unsigned long) * nr_node_ids, GFP_KERNEL); ++ if (!nodes) ++ return -ENOMEM; ++ ++ if (flags & SO_CPU) { ++ int cpu; ++ ++ for_each_possible_cpu(cpu) { ++ struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, ++ cpu); ++ int node; ++ struct page *page; ++ ++ page = READ_ONCE(c->page); ++ if (!page) ++ continue; ++ ++ node = page_to_nid(page); ++ if (flags & SO_TOTAL) ++ x = page->objects; ++ else if (flags & SO_OBJECTS) ++ x = page->inuse; ++ else ++ x = 1; ++ ++ total += x; ++ nodes[node] += x; ++ ++ page = READ_ONCE(c->partial); ++ if (page) { ++ node = page_to_nid(page); ++ if (flags & SO_TOTAL) ++ WARN_ON_ONCE(1); ++ else if (flags & SO_OBJECTS) ++ WARN_ON_ONCE(1); ++ else ++ x = page->pages; ++ total += x; ++ nodes[node] += x; ++ } ++ } ++ } ++ ++ get_online_mems(); ++#ifdef CONFIG_SLUB_DEBUG ++ if (flags & SO_ALL) { ++ struct kmem_cache_node *n; ++ ++ for_each_kmem_cache_node(s, node, n) { ++ ++ if (flags & SO_TOTAL) ++ x = atomic_long_read(&n->total_objects); ++ else if (flags & SO_OBJECTS) ++ x = atomic_long_read(&n->total_objects) - ++ count_partial(n, count_free); ++ else ++ x = atomic_long_read(&n->nr_slabs); ++ total += x; ++ nodes[node] += x; ++ } ++ ++ } else ++#endif ++ if (flags & SO_PARTIAL) { ++ struct kmem_cache_node *n; ++ ++ for_each_kmem_cache_node(s, node, n) { ++ if (flags & SO_TOTAL) ++ x = count_partial(n, count_total); ++ else if (flags & SO_OBJECTS) ++ x = count_partial(n, count_inuse); ++ else ++ x = n->nr_partial; ++ total += x; ++ nodes[node] += x; ++ } ++ } ++ x = sprintf(buf, "%lu", total); ++#ifdef CONFIG_NUMA ++ for (node = 0; node < nr_node_ids; node++) ++ if (nodes[node]) ++ x += sprintf(buf + x, " N%d=%lu", ++ node, nodes[node]); ++#endif ++ put_online_mems(); ++ kfree(nodes); ++ return x + sprintf(buf + x, "\n"); ++} ++ ++#ifdef CONFIG_SLUB_DEBUG ++static int any_slab_objects(struct kmem_cache *s) ++{ ++ int node; ++ struct kmem_cache_node *n; ++ ++ for_each_kmem_cache_node(s, node, n) ++ if (atomic_long_read(&n->total_objects)) ++ return 1; ++ ++ return 0; ++} ++#endif ++ ++#define to_slab_attr(n) container_of(n, struct slab_attribute, attr) ++#define to_slab(n) container_of(n, struct kmem_cache, kobj) ++ ++struct slab_attribute { ++ struct attribute attr; ++ ssize_t (*show)(struct kmem_cache *s, char *buf); ++ ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count); ++}; ++ ++#define SLAB_ATTR_RO(_name) \ ++ static struct slab_attribute _name##_attr = \ ++ __ATTR(_name, 0400, _name##_show, NULL) ++ ++#define SLAB_ATTR(_name) \ ++ static struct slab_attribute _name##_attr = \ ++ __ATTR(_name, 0600, _name##_show, _name##_store) ++ ++static ssize_t slab_size_show(struct kmem_cache *s, char *buf) ++{ ++ return sprintf(buf, "%d\n", s->size); ++} ++SLAB_ATTR_RO(slab_size); ++ ++static ssize_t align_show(struct kmem_cache *s, char *buf) ++{ ++ return sprintf(buf, "%d\n", s->align); ++} ++SLAB_ATTR_RO(align); ++ ++static ssize_t object_size_show(struct kmem_cache *s, char *buf) ++{ ++ return sprintf(buf, "%d\n", s->object_size); ++} ++SLAB_ATTR_RO(object_size); ++ ++static ssize_t objs_per_slab_show(struct kmem_cache *s, char *buf) ++{ ++ return sprintf(buf, "%d\n", oo_objects(s->oo)); ++} ++SLAB_ATTR_RO(objs_per_slab); ++ ++static ssize_t order_store(struct kmem_cache *s, ++ const char *buf, size_t length) ++{ ++ unsigned long order; ++ int err; ++ ++ err = kstrtoul(buf, 10, &order); ++ if (err) ++ return err; ++ ++ if (order > slub_max_order || order < slub_min_order) ++ return -EINVAL; ++ ++ calculate_sizes(s, order); ++ return length; ++} ++ ++static ssize_t order_show(struct kmem_cache *s, char *buf) ++{ ++ return sprintf(buf, "%d\n", oo_order(s->oo)); ++} ++SLAB_ATTR(order); ++ ++static ssize_t min_partial_show(struct kmem_cache *s, char *buf) ++{ ++ return sprintf(buf, "%lu\n", s->min_partial); ++} ++ ++static ssize_t min_partial_store(struct kmem_cache *s, const char *buf, ++ size_t length) ++{ ++ unsigned long min; ++ int err; ++ ++ err = kstrtoul(buf, 10, &min); ++ if (err) ++ return err; ++ ++ set_min_partial(s, min); ++ return length; ++} ++SLAB_ATTR(min_partial); ++ ++static ssize_t cpu_partial_show(struct kmem_cache *s, char *buf) ++{ ++ return sprintf(buf, "%u\n", s->cpu_partial); ++} ++ ++static ssize_t cpu_partial_store(struct kmem_cache *s, const char *buf, ++ size_t length) ++{ ++ unsigned long objects; ++ int err; ++ ++ err = kstrtoul(buf, 10, &objects); ++ if (err) ++ return err; ++ if (objects && !kmem_cache_has_cpu_partial(s)) ++ return -EINVAL; ++ ++ s->cpu_partial = objects; ++ flush_all(s); ++ return length; ++} ++SLAB_ATTR(cpu_partial); ++ ++static ssize_t ctor_show(struct kmem_cache *s, char *buf) ++{ ++ if (!s->ctor) ++ return 0; ++ return sprintf(buf, "%pS\n", s->ctor); ++} ++SLAB_ATTR_RO(ctor); ++ ++static ssize_t aliases_show(struct kmem_cache *s, char *buf) ++{ ++ return sprintf(buf, "%d\n", s->refcount < 0 ? 0 : s->refcount - 1); ++} ++SLAB_ATTR_RO(aliases); ++ ++static ssize_t partial_show(struct kmem_cache *s, char *buf) ++{ ++ return show_slab_objects(s, buf, SO_PARTIAL); ++} ++SLAB_ATTR_RO(partial); ++ ++static ssize_t cpu_slabs_show(struct kmem_cache *s, char *buf) ++{ ++ return show_slab_objects(s, buf, SO_CPU); ++} ++SLAB_ATTR_RO(cpu_slabs); ++ ++static ssize_t objects_show(struct kmem_cache *s, char *buf) ++{ ++ return show_slab_objects(s, buf, SO_ALL|SO_OBJECTS); ++} ++SLAB_ATTR_RO(objects); ++ ++static ssize_t objects_partial_show(struct kmem_cache *s, char *buf) ++{ ++ return show_slab_objects(s, buf, SO_PARTIAL|SO_OBJECTS); ++} ++SLAB_ATTR_RO(objects_partial); ++ ++static ssize_t slabs_cpu_partial_show(struct kmem_cache *s, char *buf) ++{ ++ int objects = 0; ++ int pages = 0; ++ int cpu; ++ int len; ++ ++ for_each_online_cpu(cpu) { ++ struct page *page = per_cpu_ptr(s->cpu_slab, cpu)->partial; ++ ++ if (page) { ++ pages += page->pages; ++ objects += page->pobjects; ++ } ++ } ++ ++ len = sprintf(buf, "%d(%d)", objects, pages); ++ ++#ifdef CONFIG_SMP ++ for_each_online_cpu(cpu) { ++ struct page *page = per_cpu_ptr(s->cpu_slab, cpu) ->partial; ++ ++ if (page && len < PAGE_SIZE - 20) ++ len += sprintf(buf + len, " C%d=%d(%d)", cpu, ++ page->pobjects, page->pages); ++ } ++#endif ++ return len + sprintf(buf + len, "\n"); ++} ++SLAB_ATTR_RO(slabs_cpu_partial); ++ ++static ssize_t reclaim_account_show(struct kmem_cache *s, char *buf) ++{ ++ return sprintf(buf, "%d\n", !!(s->flags & SLAB_RECLAIM_ACCOUNT)); ++} ++ ++static ssize_t reclaim_account_store(struct kmem_cache *s, ++ const char *buf, size_t length) ++{ ++ s->flags &= ~SLAB_RECLAIM_ACCOUNT; ++ if (buf[0] == '1') ++ s->flags |= SLAB_RECLAIM_ACCOUNT; ++ return length; ++} ++SLAB_ATTR(reclaim_account); ++ ++static ssize_t hwcache_align_show(struct kmem_cache *s, char *buf) ++{ ++ return sprintf(buf, "%d\n", !!(s->flags & SLAB_HWCACHE_ALIGN)); ++} ++SLAB_ATTR_RO(hwcache_align); ++ ++#ifdef CONFIG_ZONE_DMA ++static ssize_t cache_dma_show(struct kmem_cache *s, char *buf) ++{ ++ return sprintf(buf, "%d\n", !!(s->flags & SLAB_CACHE_DMA)); ++} ++SLAB_ATTR_RO(cache_dma); ++#endif ++ ++static ssize_t destroy_by_rcu_show(struct kmem_cache *s, char *buf) ++{ ++ return sprintf(buf, "%d\n", !!(s->flags & SLAB_DESTROY_BY_RCU)); ++} ++SLAB_ATTR_RO(destroy_by_rcu); ++ ++static ssize_t reserved_show(struct kmem_cache *s, char *buf) ++{ ++ return sprintf(buf, "%d\n", s->reserved); ++} ++SLAB_ATTR_RO(reserved); ++ ++#ifdef CONFIG_SLUB_DEBUG ++static ssize_t slabs_show(struct kmem_cache *s, char *buf) ++{ ++ return show_slab_objects(s, buf, SO_ALL); ++} ++SLAB_ATTR_RO(slabs); ++ ++static ssize_t total_objects_show(struct kmem_cache *s, char *buf) ++{ ++ return show_slab_objects(s, buf, SO_ALL|SO_TOTAL); ++} ++SLAB_ATTR_RO(total_objects); ++ ++static ssize_t sanity_checks_show(struct kmem_cache *s, char *buf) ++{ ++ return sprintf(buf, "%d\n", !!(s->flags & SLAB_DEBUG_FREE)); ++} ++ ++static ssize_t sanity_checks_store(struct kmem_cache *s, ++ const char *buf, size_t length) ++{ ++ s->flags &= ~SLAB_DEBUG_FREE; ++ if (buf[0] == '1') { ++ s->flags &= ~__CMPXCHG_DOUBLE; ++ s->flags |= SLAB_DEBUG_FREE; ++ } ++ return length; ++} ++SLAB_ATTR(sanity_checks); ++ ++static ssize_t trace_show(struct kmem_cache *s, char *buf) ++{ ++ return sprintf(buf, "%d\n", !!(s->flags & SLAB_TRACE)); ++} ++ ++static ssize_t trace_store(struct kmem_cache *s, const char *buf, ++ size_t length) ++{ ++ /* ++ * Tracing a merged cache is going to give confusing results ++ * as well as cause other issues like converting a mergeable ++ * cache into an umergeable one. ++ */ ++ if (s->refcount > 1) ++ return -EINVAL; ++ ++ s->flags &= ~SLAB_TRACE; ++ if (buf[0] == '1') { ++ s->flags &= ~__CMPXCHG_DOUBLE; ++ s->flags |= SLAB_TRACE; ++ } ++ return length; ++} ++SLAB_ATTR(trace); ++ ++static ssize_t red_zone_show(struct kmem_cache *s, char *buf) ++{ ++ return sprintf(buf, "%d\n", !!(s->flags & SLAB_RED_ZONE)); ++} ++ ++static ssize_t red_zone_store(struct kmem_cache *s, ++ const char *buf, size_t length) ++{ ++ if (any_slab_objects(s)) ++ return -EBUSY; ++ ++ s->flags &= ~SLAB_RED_ZONE; ++ if (buf[0] == '1') { ++ s->flags &= ~__CMPXCHG_DOUBLE; ++ s->flags |= SLAB_RED_ZONE; ++ } ++ calculate_sizes(s, -1); ++ return length; ++} ++SLAB_ATTR(red_zone); ++ ++static ssize_t poison_show(struct kmem_cache *s, char *buf) ++{ ++ return sprintf(buf, "%d\n", !!(s->flags & SLAB_POISON)); ++} ++ ++static ssize_t poison_store(struct kmem_cache *s, ++ const char *buf, size_t length) ++{ ++ if (any_slab_objects(s)) ++ return -EBUSY; ++ ++ s->flags &= ~SLAB_POISON; ++ if (buf[0] == '1') { ++ s->flags &= ~__CMPXCHG_DOUBLE; ++ s->flags |= SLAB_POISON; ++ } ++ calculate_sizes(s, -1); ++ return length; ++} ++SLAB_ATTR(poison); ++ ++static ssize_t store_user_show(struct kmem_cache *s, char *buf) ++{ ++ return sprintf(buf, "%d\n", !!(s->flags & SLAB_STORE_USER)); ++} ++ ++static ssize_t store_user_store(struct kmem_cache *s, ++ const char *buf, size_t length) ++{ ++ if (any_slab_objects(s)) ++ return -EBUSY; ++ ++ s->flags &= ~SLAB_STORE_USER; ++ if (buf[0] == '1') { ++ s->flags &= ~__CMPXCHG_DOUBLE; ++ s->flags |= SLAB_STORE_USER; ++ } ++ calculate_sizes(s, -1); ++ return length; ++} ++SLAB_ATTR(store_user); ++ ++static ssize_t validate_show(struct kmem_cache *s, char *buf) ++{ ++ return 0; ++} ++ ++static ssize_t validate_store(struct kmem_cache *s, ++ const char *buf, size_t length) ++{ ++ int ret = -EINVAL; ++ ++ if (buf[0] == '1') { ++ ret = validate_slab_cache(s); ++ if (ret >= 0) ++ ret = length; ++ } ++ return ret; ++} ++SLAB_ATTR(validate); ++ ++static ssize_t alloc_calls_show(struct kmem_cache *s, char *buf) ++{ ++ if (!(s->flags & SLAB_STORE_USER)) ++ return -ENOSYS; ++ return list_locations(s, buf, TRACK_ALLOC); ++} ++SLAB_ATTR_RO(alloc_calls); ++ ++static ssize_t free_calls_show(struct kmem_cache *s, char *buf) ++{ ++ if (!(s->flags & SLAB_STORE_USER)) ++ return -ENOSYS; ++ return list_locations(s, buf, TRACK_FREE); ++} ++SLAB_ATTR_RO(free_calls); ++#endif /* CONFIG_SLUB_DEBUG */ ++ ++#ifdef CONFIG_FAILSLAB ++static ssize_t failslab_show(struct kmem_cache *s, char *buf) ++{ ++ return sprintf(buf, "%d\n", !!(s->flags & SLAB_FAILSLAB)); ++} ++ ++static ssize_t failslab_store(struct kmem_cache *s, const char *buf, ++ size_t length) ++{ ++ if (s->refcount > 1) ++ return -EINVAL; ++ ++ s->flags &= ~SLAB_FAILSLAB; ++ if (buf[0] == '1') ++ s->flags |= SLAB_FAILSLAB; ++ return length; ++} ++SLAB_ATTR(failslab); ++#endif ++ ++static ssize_t shrink_show(struct kmem_cache *s, char *buf) ++{ ++ return 0; ++} ++ ++static ssize_t shrink_store(struct kmem_cache *s, ++ const char *buf, size_t length) ++{ ++ if (buf[0] == '1') ++ kmem_cache_shrink(s); ++ else ++ return -EINVAL; ++ return length; ++} ++SLAB_ATTR(shrink); ++ ++#ifdef CONFIG_NUMA ++static ssize_t remote_node_defrag_ratio_show(struct kmem_cache *s, char *buf) ++{ ++ return sprintf(buf, "%d\n", s->remote_node_defrag_ratio / 10); ++} ++ ++static ssize_t remote_node_defrag_ratio_store(struct kmem_cache *s, ++ const char *buf, size_t length) ++{ ++ unsigned long ratio; ++ int err; ++ ++ err = kstrtoul(buf, 10, &ratio); ++ if (err) ++ return err; ++ ++ if (ratio <= 100) ++ s->remote_node_defrag_ratio = ratio * 10; ++ ++ return length; ++} ++SLAB_ATTR(remote_node_defrag_ratio); ++#endif ++ ++#ifdef CONFIG_SLUB_STATS ++static int show_stat(struct kmem_cache *s, char *buf, enum stat_item si) ++{ ++ unsigned long sum = 0; ++ int cpu; ++ int len; ++ int *data = kmalloc(nr_cpu_ids * sizeof(int), GFP_KERNEL); ++ ++ if (!data) ++ return -ENOMEM; ++ ++ for_each_online_cpu(cpu) { ++ unsigned x = per_cpu_ptr(s->cpu_slab, cpu)->stat[si]; ++ ++ data[cpu] = x; ++ sum += x; ++ } ++ ++ len = sprintf(buf, "%lu", sum); ++ ++#ifdef CONFIG_SMP ++ for_each_online_cpu(cpu) { ++ if (data[cpu] && len < PAGE_SIZE - 20) ++ len += sprintf(buf + len, " C%d=%u", cpu, data[cpu]); ++ } ++#endif ++ kfree(data); ++ return len + sprintf(buf + len, "\n"); ++} ++ ++static void clear_stat(struct kmem_cache *s, enum stat_item si) ++{ ++ int cpu; ++ ++ for_each_online_cpu(cpu) ++ per_cpu_ptr(s->cpu_slab, cpu)->stat[si] = 0; ++} ++ ++#define STAT_ATTR(si, text) \ ++static ssize_t text##_show(struct kmem_cache *s, char *buf) \ ++{ \ ++ return show_stat(s, buf, si); \ ++} \ ++static ssize_t text##_store(struct kmem_cache *s, \ ++ const char *buf, size_t length) \ ++{ \ ++ if (buf[0] != '0') \ ++ return -EINVAL; \ ++ clear_stat(s, si); \ ++ return length; \ ++} \ ++SLAB_ATTR(text); \ ++ ++STAT_ATTR(ALLOC_FASTPATH, alloc_fastpath); ++STAT_ATTR(ALLOC_SLOWPATH, alloc_slowpath); ++STAT_ATTR(FREE_FASTPATH, free_fastpath); ++STAT_ATTR(FREE_SLOWPATH, free_slowpath); ++STAT_ATTR(FREE_FROZEN, free_frozen); ++STAT_ATTR(FREE_ADD_PARTIAL, free_add_partial); ++STAT_ATTR(FREE_REMOVE_PARTIAL, free_remove_partial); ++STAT_ATTR(ALLOC_FROM_PARTIAL, alloc_from_partial); ++STAT_ATTR(ALLOC_SLAB, alloc_slab); ++STAT_ATTR(ALLOC_REFILL, alloc_refill); ++STAT_ATTR(ALLOC_NODE_MISMATCH, alloc_node_mismatch); ++STAT_ATTR(FREE_SLAB, free_slab); ++STAT_ATTR(CPUSLAB_FLUSH, cpuslab_flush); ++STAT_ATTR(DEACTIVATE_FULL, deactivate_full); ++STAT_ATTR(DEACTIVATE_EMPTY, deactivate_empty); ++STAT_ATTR(DEACTIVATE_TO_HEAD, deactivate_to_head); ++STAT_ATTR(DEACTIVATE_TO_TAIL, deactivate_to_tail); ++STAT_ATTR(DEACTIVATE_REMOTE_FREES, deactivate_remote_frees); ++STAT_ATTR(DEACTIVATE_BYPASS, deactivate_bypass); ++STAT_ATTR(ORDER_FALLBACK, order_fallback); ++STAT_ATTR(CMPXCHG_DOUBLE_CPU_FAIL, cmpxchg_double_cpu_fail); ++STAT_ATTR(CMPXCHG_DOUBLE_FAIL, cmpxchg_double_fail); ++STAT_ATTR(CPU_PARTIAL_ALLOC, cpu_partial_alloc); ++STAT_ATTR(CPU_PARTIAL_FREE, cpu_partial_free); ++STAT_ATTR(CPU_PARTIAL_NODE, cpu_partial_node); ++STAT_ATTR(CPU_PARTIAL_DRAIN, cpu_partial_drain); ++#endif ++ ++static struct attribute *slab_attrs[] = { ++ &slab_size_attr.attr, ++ &object_size_attr.attr, ++ &objs_per_slab_attr.attr, ++ &order_attr.attr, ++ &min_partial_attr.attr, ++ &cpu_partial_attr.attr, ++ &objects_attr.attr, ++ &objects_partial_attr.attr, ++ &partial_attr.attr, ++ &cpu_slabs_attr.attr, ++ &ctor_attr.attr, ++ &aliases_attr.attr, ++ &align_attr.attr, ++ &hwcache_align_attr.attr, ++ &reclaim_account_attr.attr, ++ &destroy_by_rcu_attr.attr, ++ &shrink_attr.attr, ++ &reserved_attr.attr, ++ &slabs_cpu_partial_attr.attr, ++#ifdef CONFIG_SLUB_DEBUG ++ &total_objects_attr.attr, ++ &slabs_attr.attr, ++ &sanity_checks_attr.attr, ++ &trace_attr.attr, ++ &red_zone_attr.attr, ++ &poison_attr.attr, ++ &store_user_attr.attr, ++ &validate_attr.attr, ++ &alloc_calls_attr.attr, ++ &free_calls_attr.attr, ++#endif ++#ifdef CONFIG_ZONE_DMA ++ &cache_dma_attr.attr, ++#endif ++#ifdef CONFIG_NUMA ++ &remote_node_defrag_ratio_attr.attr, ++#endif ++#ifdef CONFIG_SLUB_STATS ++ &alloc_fastpath_attr.attr, ++ &alloc_slowpath_attr.attr, ++ &free_fastpath_attr.attr, ++ &free_slowpath_attr.attr, ++ &free_frozen_attr.attr, ++ &free_add_partial_attr.attr, ++ &free_remove_partial_attr.attr, ++ &alloc_from_partial_attr.attr, ++ &alloc_slab_attr.attr, ++ &alloc_refill_attr.attr, ++ &alloc_node_mismatch_attr.attr, ++ &free_slab_attr.attr, ++ &cpuslab_flush_attr.attr, ++ &deactivate_full_attr.attr, ++ &deactivate_empty_attr.attr, ++ &deactivate_to_head_attr.attr, ++ &deactivate_to_tail_attr.attr, ++ &deactivate_remote_frees_attr.attr, ++ &deactivate_bypass_attr.attr, ++ &order_fallback_attr.attr, ++ &cmpxchg_double_fail_attr.attr, ++ &cmpxchg_double_cpu_fail_attr.attr, ++ &cpu_partial_alloc_attr.attr, ++ &cpu_partial_free_attr.attr, ++ &cpu_partial_node_attr.attr, ++ &cpu_partial_drain_attr.attr, ++#endif ++#ifdef CONFIG_FAILSLAB ++ &failslab_attr.attr, ++#endif ++ ++ NULL ++}; ++ ++static struct attribute_group slab_attr_group = { ++ .attrs = slab_attrs, ++}; ++ ++static ssize_t slab_attr_show(struct kobject *kobj, ++ struct attribute *attr, ++ char *buf) ++{ ++ struct slab_attribute *attribute; ++ struct kmem_cache *s; ++ int err; ++ ++ attribute = to_slab_attr(attr); ++ s = to_slab(kobj); ++ ++ if (!attribute->show) ++ return -EIO; ++ ++ err = attribute->show(s, buf); ++ ++ return err; ++} ++ ++static ssize_t slab_attr_store(struct kobject *kobj, ++ struct attribute *attr, ++ const char *buf, size_t len) ++{ ++ struct slab_attribute *attribute; ++ struct kmem_cache *s; ++ int err; ++ ++ attribute = to_slab_attr(attr); ++ s = to_slab(kobj); ++ ++ if (!attribute->store) ++ return -EIO; ++ ++ err = attribute->store(s, buf, len); ++#ifdef CONFIG_MEMCG_KMEM ++ if (slab_state >= FULL && err >= 0 && is_root_cache(s)) { ++ struct kmem_cache *c; ++ ++ mutex_lock(&slab_mutex); ++ if (s->max_attr_size < len) ++ s->max_attr_size = len; ++ ++ /* ++ * This is a best effort propagation, so this function's return ++ * value will be determined by the parent cache only. This is ++ * basically because not all attributes will have a well ++ * defined semantics for rollbacks - most of the actions will ++ * have permanent effects. ++ * ++ * Returning the error value of any of the children that fail ++ * is not 100 % defined, in the sense that users seeing the ++ * error code won't be able to know anything about the state of ++ * the cache. ++ * ++ * Only returning the error code for the parent cache at least ++ * has well defined semantics. The cache being written to ++ * directly either failed or succeeded, in which case we loop ++ * through the descendants with best-effort propagation. ++ */ ++ for_each_memcg_cache(c, s) ++ attribute->store(c, buf, len); ++ mutex_unlock(&slab_mutex); ++ } ++#endif ++ return err; ++} ++ ++static void memcg_propagate_slab_attrs(struct kmem_cache *s) ++{ ++#ifdef CONFIG_MEMCG_KMEM ++ int i; ++ char *buffer = NULL; ++ struct kmem_cache *root_cache; ++ ++ if (is_root_cache(s)) ++ return; ++ ++ root_cache = s->memcg_params.root_cache; ++ ++ /* ++ * This mean this cache had no attribute written. Therefore, no point ++ * in copying default values around ++ */ ++ if (!root_cache->max_attr_size) ++ return; ++ ++ for (i = 0; i < ARRAY_SIZE(slab_attrs); i++) { ++ char mbuf[64]; ++ char *buf; ++ struct slab_attribute *attr = to_slab_attr(slab_attrs[i]); ++ ++ if (!attr || !attr->store || !attr->show) ++ continue; ++ ++ /* ++ * It is really bad that we have to allocate here, so we will ++ * do it only as a fallback. If we actually allocate, though, ++ * we can just use the allocated buffer until the end. ++ * ++ * Most of the slub attributes will tend to be very small in ++ * size, but sysfs allows buffers up to a page, so they can ++ * theoretically happen. ++ */ ++ if (buffer) ++ buf = buffer; ++ else if (root_cache->max_attr_size < ARRAY_SIZE(mbuf)) ++ buf = mbuf; ++ else { ++ buffer = (char *) get_zeroed_page(GFP_KERNEL); ++ if (WARN_ON(!buffer)) ++ continue; ++ buf = buffer; ++ } ++ ++ attr->show(root_cache, buf); ++ attr->store(s, buf, strlen(buf)); ++ } ++ ++ if (buffer) ++ free_page((unsigned long)buffer); ++#endif ++} ++ ++static void kmem_cache_release(struct kobject *k) ++{ ++ slab_kmem_cache_release(to_slab(k)); ++} ++ ++static const struct sysfs_ops slab_sysfs_ops = { ++ .show = slab_attr_show, ++ .store = slab_attr_store, ++}; ++ ++static struct kobj_type slab_ktype = { ++ .sysfs_ops = &slab_sysfs_ops, ++ .release = kmem_cache_release, ++}; ++ ++static int uevent_filter(struct kset *kset, struct kobject *kobj) ++{ ++ struct kobj_type *ktype = get_ktype(kobj); ++ ++ if (ktype == &slab_ktype) ++ return 1; ++ return 0; ++} ++ ++static const struct kset_uevent_ops slab_uevent_ops = { ++ .filter = uevent_filter, ++}; ++ ++static struct kset *slab_kset; ++ ++static inline struct kset *cache_kset(struct kmem_cache *s) ++{ ++#ifdef CONFIG_MEMCG_KMEM ++ if (!is_root_cache(s)) ++ return s->memcg_params.root_cache->memcg_kset; ++#endif ++ return slab_kset; ++} ++ ++#define ID_STR_LENGTH 64 ++ ++/* Create a unique string id for a slab cache: ++ * ++ * Format :[flags-]size ++ */ ++static char *create_unique_id(struct kmem_cache *s) ++{ ++ char *name = kmalloc(ID_STR_LENGTH, GFP_KERNEL); ++ char *p = name; ++ ++ BUG_ON(!name); ++ ++ *p++ = ':'; ++ /* ++ * First flags affecting slabcache operations. We will only ++ * get here for aliasable slabs so we do not need to support ++ * too many flags. The flags here must cover all flags that ++ * are matched during merging to guarantee that the id is ++ * unique. ++ */ ++ if (s->flags & SLAB_CACHE_DMA) ++ *p++ = 'd'; ++ if (s->flags & SLAB_RECLAIM_ACCOUNT) ++ *p++ = 'a'; ++ if (s->flags & SLAB_DEBUG_FREE) ++ *p++ = 'F'; ++ if (!(s->flags & SLAB_NOTRACK)) ++ *p++ = 't'; ++ if (p != name + 1) ++ *p++ = '-'; ++ p += sprintf(p, "%07d", s->size); ++ ++ BUG_ON(p > name + ID_STR_LENGTH - 1); ++ return name; ++} ++ ++static int sysfs_slab_add(struct kmem_cache *s) ++{ ++ int err; ++ const char *name; ++ int unmergeable = slab_unmergeable(s); ++ ++ if (unmergeable) { ++ /* ++ * Slabcache can never be merged so we can use the name proper. ++ * This is typically the case for debug situations. In that ++ * case we can catch duplicate names easily. ++ */ ++ sysfs_remove_link(&slab_kset->kobj, s->name); ++ name = s->name; ++ } else { ++ /* ++ * Create a unique name for the slab as a target ++ * for the symlinks. ++ */ ++ name = create_unique_id(s); ++ } ++ ++ s->kobj.kset = cache_kset(s); ++ err = kobject_init_and_add(&s->kobj, &slab_ktype, NULL, "%s", name); ++ if (err) ++ goto out_put_kobj; ++ ++ err = sysfs_create_group(&s->kobj, &slab_attr_group); ++ if (err) ++ goto out_del_kobj; ++ ++#ifdef CONFIG_MEMCG_KMEM ++ if (is_root_cache(s)) { ++ s->memcg_kset = kset_create_and_add("cgroup", NULL, &s->kobj); ++ if (!s->memcg_kset) { ++ err = -ENOMEM; ++ goto out_del_kobj; ++ } ++ } ++#endif ++ ++ kobject_uevent(&s->kobj, KOBJ_ADD); ++ if (!unmergeable) { ++ /* Setup first alias */ ++ sysfs_slab_alias(s, s->name); ++ } ++out: ++ if (!unmergeable) ++ kfree(name); ++ return err; ++out_del_kobj: ++ kobject_del(&s->kobj); ++out_put_kobj: ++ kobject_put(&s->kobj); ++ goto out; ++} ++ ++void sysfs_slab_remove(struct kmem_cache *s) ++{ ++ if (slab_state < FULL) ++ /* ++ * Sysfs has not been setup yet so no need to remove the ++ * cache from sysfs. ++ */ ++ return; ++ ++#ifdef CONFIG_MEMCG_KMEM ++ kset_unregister(s->memcg_kset); ++#endif ++ kobject_uevent(&s->kobj, KOBJ_REMOVE); ++ kobject_del(&s->kobj); ++ kobject_put(&s->kobj); ++} ++ ++/* ++ * Need to buffer aliases during bootup until sysfs becomes ++ * available lest we lose that information. ++ */ ++struct saved_alias { ++ struct kmem_cache *s; ++ const char *name; ++ struct saved_alias *next; ++}; ++ ++static struct saved_alias *alias_list; ++ ++static int sysfs_slab_alias(struct kmem_cache *s, const char *name) ++{ ++ struct saved_alias *al; ++ ++ if (slab_state == FULL) { ++ /* ++ * If we have a leftover link then remove it. ++ */ ++ sysfs_remove_link(&slab_kset->kobj, name); ++ return sysfs_create_link(&slab_kset->kobj, &s->kobj, name); ++ } ++ ++ al = kmalloc(sizeof(struct saved_alias), GFP_KERNEL); ++ if (!al) ++ return -ENOMEM; ++ ++ al->s = s; ++ al->name = name; ++ al->next = alias_list; ++ alias_list = al; ++ return 0; ++} ++ ++static int __init slab_sysfs_init(void) ++{ ++ struct kmem_cache *s; ++ int err; ++ ++ mutex_lock(&slab_mutex); ++ ++ slab_kset = kset_create_and_add("slab", &slab_uevent_ops, kernel_kobj); ++ if (!slab_kset) { ++ mutex_unlock(&slab_mutex); ++ pr_err("Cannot register slab subsystem.\n"); ++ return -ENOSYS; ++ } ++ ++ slab_state = FULL; ++ ++ list_for_each_entry(s, &slab_caches, list) { ++ err = sysfs_slab_add(s); ++ if (err) ++ pr_err("SLUB: Unable to add boot slab %s to sysfs\n", ++ s->name); ++ } ++ ++ while (alias_list) { ++ struct saved_alias *al = alias_list; ++ ++ alias_list = alias_list->next; ++ err = sysfs_slab_alias(al->s, al->name); ++ if (err) ++ pr_err("SLUB: Unable to add boot slab alias %s to sysfs\n", ++ al->name); ++ kfree(al); ++ } ++ ++ mutex_unlock(&slab_mutex); ++ resiliency_test(); ++ return 0; ++} ++ ++__initcall(slab_sysfs_init); ++#endif /* CONFIG_SYSFS */ ++ ++/* ++ * The /proc/slabinfo ABI ++ */ ++#ifdef CONFIG_SLABINFO ++void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo) ++{ ++ unsigned long nr_slabs = 0; ++ unsigned long nr_objs = 0; ++ unsigned long nr_free = 0; ++ int node; ++ struct kmem_cache_node *n; ++ ++ for_each_kmem_cache_node(s, node, n) { ++ nr_slabs += node_nr_slabs(n); ++ nr_objs += node_nr_objs(n); ++ nr_free += count_partial(n, count_free); ++ } ++ ++ sinfo->active_objs = nr_objs - nr_free; ++ sinfo->num_objs = nr_objs; ++ sinfo->active_slabs = nr_slabs; ++ sinfo->num_slabs = nr_slabs; ++ sinfo->objects_per_slab = oo_objects(s->oo); ++ sinfo->cache_order = oo_order(s->oo); ++} ++ ++void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *s) ++{ ++} ++ ++ssize_t slabinfo_write(struct file *file, const char __user *buffer, ++ size_t count, loff_t *ppos) ++{ ++ return -EIO; ++} ++#endif /* CONFIG_SLABINFO */ +diff -Nur linux-4.1.10.orig/mm/swap.c linux-4.1.10/mm/swap.c +--- linux-4.1.10.orig/mm/swap.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/mm/swap.c 2015-10-07 18:00:08.000000000 +0200 +@@ -32,6 +32,7 @@ + #include + #include + #include ++#include + + #include "internal.h" + +@@ -45,6 +46,9 @@ + static DEFINE_PER_CPU(struct pagevec, lru_rotate_pvecs); + static DEFINE_PER_CPU(struct pagevec, lru_deactivate_file_pvecs); + ++static DEFINE_LOCAL_IRQ_LOCK(rotate_lock); ++DEFINE_LOCAL_IRQ_LOCK(swapvec_lock); ++ + /* + * This path almost never happens for VM activity - pages are normally + * freed via pagevecs. But it gets used by networking. +@@ -481,11 +485,11 @@ + unsigned long flags; + + page_cache_get(page); +- local_irq_save(flags); ++ local_lock_irqsave(rotate_lock, flags); + pvec = this_cpu_ptr(&lru_rotate_pvecs); + if (!pagevec_add(pvec, page)) + pagevec_move_tail(pvec); +- local_irq_restore(flags); ++ local_unlock_irqrestore(rotate_lock, flags); + } + } + +@@ -536,12 +540,13 @@ + void activate_page(struct page *page) + { + if (PageLRU(page) && !PageActive(page) && !PageUnevictable(page)) { +- struct pagevec *pvec = &get_cpu_var(activate_page_pvecs); ++ struct pagevec *pvec = &get_locked_var(swapvec_lock, ++ activate_page_pvecs); + + page_cache_get(page); + if (!pagevec_add(pvec, page)) + pagevec_lru_move_fn(pvec, __activate_page, NULL); +- put_cpu_var(activate_page_pvecs); ++ put_locked_var(swapvec_lock, activate_page_pvecs); + } + } + +@@ -567,7 +572,7 @@ + + static void __lru_cache_activate_page(struct page *page) + { +- struct pagevec *pvec = &get_cpu_var(lru_add_pvec); ++ struct pagevec *pvec = &get_locked_var(swapvec_lock, lru_add_pvec); + int i; + + /* +@@ -589,7 +594,7 @@ + } + } + +- put_cpu_var(lru_add_pvec); ++ put_locked_var(swapvec_lock, lru_add_pvec); + } + + /* +@@ -628,13 +633,13 @@ + + static void __lru_cache_add(struct page *page) + { +- struct pagevec *pvec = &get_cpu_var(lru_add_pvec); ++ struct pagevec *pvec = &get_locked_var(swapvec_lock, lru_add_pvec); + + page_cache_get(page); + if (!pagevec_space(pvec)) + __pagevec_lru_add(pvec); + pagevec_add(pvec, page); +- put_cpu_var(lru_add_pvec); ++ put_locked_var(swapvec_lock, lru_add_pvec); + } + + /** +@@ -814,9 +819,9 @@ + unsigned long flags; + + /* No harm done if a racing interrupt already did this */ +- local_irq_save(flags); ++ local_lock_irqsave(rotate_lock, flags); + pagevec_move_tail(pvec); +- local_irq_restore(flags); ++ local_unlock_irqrestore(rotate_lock, flags); + } + + pvec = &per_cpu(lru_deactivate_file_pvecs, cpu); +@@ -844,18 +849,19 @@ + return; + + if (likely(get_page_unless_zero(page))) { +- struct pagevec *pvec = &get_cpu_var(lru_deactivate_file_pvecs); ++ struct pagevec *pvec = &get_locked_var(swapvec_lock, ++ lru_deactivate_file_pvecs); + + if (!pagevec_add(pvec, page)) + pagevec_lru_move_fn(pvec, lru_deactivate_file_fn, NULL); +- put_cpu_var(lru_deactivate_file_pvecs); ++ put_locked_var(swapvec_lock, lru_deactivate_file_pvecs); + } + } + + void lru_add_drain(void) + { +- lru_add_drain_cpu(get_cpu()); +- put_cpu(); ++ lru_add_drain_cpu(local_lock_cpu(swapvec_lock)); ++ local_unlock_cpu(swapvec_lock); + } + + static void lru_add_drain_per_cpu(struct work_struct *dummy) +diff -Nur linux-4.1.10.orig/mm/truncate.c linux-4.1.10/mm/truncate.c +--- linux-4.1.10.orig/mm/truncate.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/mm/truncate.c 2015-10-07 18:00:08.000000000 +0200 +@@ -56,8 +56,11 @@ + * protected by mapping->tree_lock. + */ + if (!workingset_node_shadows(node) && +- !list_empty(&node->private_list)) +- list_lru_del(&workingset_shadow_nodes, &node->private_list); ++ !list_empty(&node->private_list)) { ++ local_lock(workingset_shadow_lock); ++ list_lru_del(&__workingset_shadow_nodes, &node->private_list); ++ local_unlock(workingset_shadow_lock); ++ } + __radix_tree_delete_node(&mapping->page_tree, node); + unlock: + spin_unlock_irq(&mapping->tree_lock); +diff -Nur linux-4.1.10.orig/mm/vmalloc.c linux-4.1.10/mm/vmalloc.c +--- linux-4.1.10.orig/mm/vmalloc.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/mm/vmalloc.c 2015-10-07 18:00:08.000000000 +0200 +@@ -819,7 +819,7 @@ + struct vmap_block *vb; + struct vmap_area *va; + unsigned long vb_idx; +- int node, err; ++ int node, err, cpu; + void *vaddr; + + node = numa_node_id(); +@@ -862,11 +862,12 @@ + BUG_ON(err); + radix_tree_preload_end(); + +- vbq = &get_cpu_var(vmap_block_queue); ++ cpu = get_cpu_light(); ++ vbq = this_cpu_ptr(&vmap_block_queue); + spin_lock(&vbq->lock); + list_add_tail_rcu(&vb->free_list, &vbq->free); + spin_unlock(&vbq->lock); +- put_cpu_var(vmap_block_queue); ++ put_cpu_light(); + + return vaddr; + } +@@ -935,6 +936,7 @@ + struct vmap_block *vb; + void *vaddr = NULL; + unsigned int order; ++ int cpu; + + BUG_ON(size & ~PAGE_MASK); + BUG_ON(size > PAGE_SIZE*VMAP_MAX_ALLOC); +@@ -949,7 +951,8 @@ + order = get_order(size); + + rcu_read_lock(); +- vbq = &get_cpu_var(vmap_block_queue); ++ cpu = get_cpu_light(); ++ vbq = this_cpu_ptr(&vmap_block_queue); + list_for_each_entry_rcu(vb, &vbq->free, free_list) { + unsigned long pages_off; + @@ -972,7 +975,7 @@ break; } @@ -25858,9 +62121,9 @@ diff -Nur linux-4.1.6.orig/mm/vmalloc.c linux-4.1.6/mm/vmalloc.c rcu_read_unlock(); /* Allocate new block if nothing was found */ -diff -Nur linux-4.1.6.orig/mm/vmstat.c linux-4.1.6/mm/vmstat.c ---- linux-4.1.6.orig/mm/vmstat.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/mm/vmstat.c 2015-09-08 23:49:08.441830131 +0200 +diff -Nur linux-4.1.10.orig/mm/vmstat.c linux-4.1.10/mm/vmstat.c +--- linux-4.1.10.orig/mm/vmstat.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/mm/vmstat.c 2015-10-07 18:00:08.000000000 +0200 @@ -226,6 +226,7 @@ long x; long t; @@ -25909,9 +62172,9 @@ diff -Nur linux-4.1.6.orig/mm/vmstat.c linux-4.1.6/mm/vmstat.c } void __dec_zone_page_state(struct page *page, enum zone_stat_item item) -diff -Nur linux-4.1.6.orig/mm/workingset.c linux-4.1.6/mm/workingset.c ---- linux-4.1.6.orig/mm/workingset.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/mm/workingset.c 2015-09-08 23:49:08.441830131 +0200 +diff -Nur linux-4.1.10.orig/mm/workingset.c linux-4.1.10/mm/workingset.c +--- linux-4.1.10.orig/mm/workingset.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/mm/workingset.c 2015-10-07 18:00:08.000000000 +0200 @@ -264,7 +264,8 @@ * point where they would still be useful. */ @@ -25979,9 +62242,9 @@ diff -Nur linux-4.1.6.orig/mm/workingset.c linux-4.1.6/mm/workingset.c err: return ret; } -diff -Nur linux-4.1.6.orig/net/core/dev.c linux-4.1.6/net/core/dev.c ---- linux-4.1.6.orig/net/core/dev.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/net/core/dev.c 2015-09-08 23:49:08.441830131 +0200 +diff -Nur linux-4.1.10.orig/net/core/dev.c linux-4.1.10/net/core/dev.c +--- linux-4.1.10.orig/net/core/dev.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/net/core/dev.c 2015-10-07 18:00:08.000000000 +0200 @@ -184,6 +184,7 @@ static DEFINE_HASHTABLE(napi_hash, 8); @@ -26007,7 +62270,7 @@ diff -Nur linux-4.1.6.orig/net/core/dev.c linux-4.1.6/net/core/dev.c #endif } -@@ -856,7 +857,8 @@ +@@ -852,7 +853,8 @@ strcpy(name, dev->name); rcu_read_unlock(); if (read_seqcount_retry(&devnet_rename_seq, seq)) { @@ -26017,7 +62280,7 @@ diff -Nur linux-4.1.6.orig/net/core/dev.c linux-4.1.6/net/core/dev.c goto retry; } -@@ -1125,20 +1127,17 @@ +@@ -1121,20 +1123,17 @@ if (dev->flags & IFF_UP) return -EBUSY; @@ -26040,237 +62303,7763 @@ diff -Nur linux-4.1.6.orig/net/core/dev.c linux-4.1.6/net/core/dev.c - return err; - } + if (err < 0) -+ goto outunlock; - - if (oldname[0] && !strchr(oldname, '%')) - netdev_info(dev, "renamed from %s\n", oldname); -@@ -1151,11 +1150,12 @@ - if (ret) { - memcpy(dev->name, oldname, IFNAMSIZ); - dev->name_assign_type = old_assign_type; -- write_seqcount_end(&devnet_rename_seq); -- return ret; -+ err = ret; -+ goto outunlock; - } - -- write_seqcount_end(&devnet_rename_seq); -+ __raw_write_seqcount_end(&devnet_rename_seq); -+ mutex_unlock(&devnet_rename_mutex); - - netdev_adjacent_rename_links(dev, oldname); - -@@ -1176,7 +1176,8 @@ - /* err >= 0 after dev_alloc_name() or stores the first errno */ - if (err >= 0) { - err = ret; -- write_seqcount_begin(&devnet_rename_seq); -+ mutex_lock(&devnet_rename_mutex); -+ __raw_write_seqcount_begin(&devnet_rename_seq); - memcpy(dev->name, oldname, IFNAMSIZ); - memcpy(oldname, newname, IFNAMSIZ); - dev->name_assign_type = old_assign_type; -@@ -1189,6 +1190,11 @@ - } - - return err; ++ goto outunlock; + + if (oldname[0] && !strchr(oldname, '%')) + netdev_info(dev, "renamed from %s\n", oldname); +@@ -1147,11 +1146,12 @@ + if (ret) { + memcpy(dev->name, oldname, IFNAMSIZ); + dev->name_assign_type = old_assign_type; +- write_seqcount_end(&devnet_rename_seq); +- return ret; ++ err = ret; ++ goto outunlock; + } + +- write_seqcount_end(&devnet_rename_seq); ++ __raw_write_seqcount_end(&devnet_rename_seq); ++ mutex_unlock(&devnet_rename_mutex); + + netdev_adjacent_rename_links(dev, oldname); + +@@ -1172,7 +1172,8 @@ + /* err >= 0 after dev_alloc_name() or stores the first errno */ + if (err >= 0) { + err = ret; +- write_seqcount_begin(&devnet_rename_seq); ++ mutex_lock(&devnet_rename_mutex); ++ __raw_write_seqcount_begin(&devnet_rename_seq); + memcpy(dev->name, oldname, IFNAMSIZ); + memcpy(oldname, newname, IFNAMSIZ); + dev->name_assign_type = old_assign_type; +@@ -1185,6 +1186,11 @@ + } + + return err; ++ ++outunlock: ++ __raw_write_seqcount_end(&devnet_rename_seq); ++ mutex_unlock(&devnet_rename_mutex); ++ return err; + } + + /** +@@ -2214,6 +2220,7 @@ + sd->output_queue_tailp = &q->next_sched; + raise_softirq_irqoff(NET_TX_SOFTIRQ); + local_irq_restore(flags); ++ preempt_check_resched_rt(); + } + + void __netif_schedule(struct Qdisc *q) +@@ -2295,6 +2302,7 @@ + __this_cpu_write(softnet_data.completion_queue, skb); + raise_softirq_irqoff(NET_TX_SOFTIRQ); + local_irq_restore(flags); ++ preempt_check_resched_rt(); + } + EXPORT_SYMBOL(__dev_kfree_skb_irq); + +@@ -3365,6 +3373,7 @@ + rps_unlock(sd); + + local_irq_restore(flags); ++ preempt_check_resched_rt(); + + atomic_long_inc(&skb->dev->rx_dropped); + kfree_skb(skb); +@@ -3383,7 +3392,7 @@ + struct rps_dev_flow voidflow, *rflow = &voidflow; + int cpu; + +- preempt_disable(); ++ migrate_disable(); + rcu_read_lock(); + + cpu = get_rps_cpu(skb->dev, skb, &rflow); +@@ -3393,13 +3402,13 @@ + ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail); + + rcu_read_unlock(); +- preempt_enable(); ++ migrate_enable(); + } else + #endif + { + unsigned int qtail; +- ret = enqueue_to_backlog(skb, get_cpu(), &qtail); +- put_cpu(); ++ ret = enqueue_to_backlog(skb, get_cpu_light(), &qtail); ++ put_cpu_light(); + } + return ret; + } +@@ -3433,16 +3442,44 @@ + + trace_netif_rx_ni_entry(skb); + +- preempt_disable(); ++ local_bh_disable(); + err = netif_rx_internal(skb); +- if (local_softirq_pending()) +- do_softirq(); +- preempt_enable(); ++ local_bh_enable(); + + return err; + } + EXPORT_SYMBOL(netif_rx_ni); + ++#ifdef CONFIG_PREEMPT_RT_FULL ++/* ++ * RT runs ksoftirqd as a real time thread and the root_lock is a ++ * "sleeping spinlock". If the trylock fails then we can go into an ++ * infinite loop when ksoftirqd preempted the task which actually ++ * holds the lock, because we requeue q and raise NET_TX softirq ++ * causing ksoftirqd to loop forever. ++ * ++ * It's safe to use spin_lock on RT here as softirqs run in thread ++ * context and cannot deadlock against the thread which is holding ++ * root_lock. ++ * ++ * On !RT the trylock might fail, but there we bail out from the ++ * softirq loop after 10 attempts which we can't do on RT. And the ++ * task holding root_lock cannot be preempted, so the only downside of ++ * that trylock is that we need 10 loops to decide that we should have ++ * given up in the first one :) ++ */ ++static inline int take_root_lock(spinlock_t *lock) ++{ ++ spin_lock(lock); ++ return 1; ++} ++#else ++static inline int take_root_lock(spinlock_t *lock) ++{ ++ return spin_trylock(lock); ++} ++#endif ++ + static void net_tx_action(struct softirq_action *h) + { + struct softnet_data *sd = this_cpu_ptr(&softnet_data); +@@ -3484,7 +3521,7 @@ + head = head->next_sched; + + root_lock = qdisc_lock(q); +- if (spin_trylock(root_lock)) { ++ if (take_root_lock(root_lock)) { + smp_mb__before_atomic(); + clear_bit(__QDISC_STATE_SCHED, + &q->state); +@@ -3881,7 +3918,7 @@ + skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) { + if (skb->dev == dev) { + __skb_unlink(skb, &sd->input_pkt_queue); +- kfree_skb(skb); ++ __skb_queue_tail(&sd->tofree_queue, skb); + input_queue_head_incr(sd); + } + } +@@ -3890,10 +3927,13 @@ + skb_queue_walk_safe(&sd->process_queue, skb, tmp) { + if (skb->dev == dev) { + __skb_unlink(skb, &sd->process_queue); +- kfree_skb(skb); ++ __skb_queue_tail(&sd->tofree_queue, skb); + input_queue_head_incr(sd); + } + } ++ ++ if (!skb_queue_empty(&sd->tofree_queue)) ++ raise_softirq_irqoff(NET_RX_SOFTIRQ); + } + + static int napi_gro_complete(struct sk_buff *skb) +@@ -4344,6 +4384,7 @@ + sd->rps_ipi_list = NULL; + + local_irq_enable(); ++ preempt_check_resched_rt(); + + /* Send pending IPI's to kick RPS processing on remote cpus. */ + while (remsd) { +@@ -4357,6 +4398,7 @@ + } else + #endif + local_irq_enable(); ++ preempt_check_resched_rt(); + } + + static bool sd_has_rps_ipi_waiting(struct softnet_data *sd) +@@ -4438,6 +4480,7 @@ + local_irq_save(flags); + ____napi_schedule(this_cpu_ptr(&softnet_data), n); + local_irq_restore(flags); ++ preempt_check_resched_rt(); + } + EXPORT_SYMBOL(__napi_schedule); + +@@ -7167,6 +7210,7 @@ + + raise_softirq_irqoff(NET_TX_SOFTIRQ); + local_irq_enable(); ++ preempt_check_resched_rt(); + + /* Process offline CPU's input_pkt_queue */ + while ((skb = __skb_dequeue(&oldsd->process_queue))) { +@@ -7177,6 +7221,9 @@ + netif_rx_ni(skb); + input_queue_head_incr(oldsd); + } ++ while ((skb = __skb_dequeue(&oldsd->tofree_queue))) { ++ kfree_skb(skb); ++ } + + return NOTIFY_OK; + } +@@ -7478,8 +7525,9 @@ + for_each_possible_cpu(i) { + struct softnet_data *sd = &per_cpu(softnet_data, i); + +- skb_queue_head_init(&sd->input_pkt_queue); +- skb_queue_head_init(&sd->process_queue); ++ skb_queue_head_init_raw(&sd->input_pkt_queue); ++ skb_queue_head_init_raw(&sd->process_queue); ++ skb_queue_head_init_raw(&sd->tofree_queue); + INIT_LIST_HEAD(&sd->poll_list); + sd->output_queue_tailp = &sd->output_queue; + #ifdef CONFIG_RPS +diff -Nur linux-4.1.10.orig/net/core/dev.c.orig linux-4.1.10/net/core/dev.c.orig +--- linux-4.1.10.orig/net/core/dev.c.orig 1970-01-01 01:00:00.000000000 +0100 ++++ linux-4.1.10/net/core/dev.c.orig 2015-10-03 13:49:38.000000000 +0200 +@@ -0,0 +1,7522 @@ ++/* ++ * NET3 Protocol independent device support routines. ++ * ++ * This program is free software; you can redistribute it and/or ++ * modify it under the terms of the GNU General Public License ++ * as published by the Free Software Foundation; either version ++ * 2 of the License, or (at your option) any later version. ++ * ++ * Derived from the non IP parts of dev.c 1.0.19 ++ * Authors: Ross Biro ++ * Fred N. van Kempen, ++ * Mark Evans, ++ * ++ * Additional Authors: ++ * Florian la Roche ++ * Alan Cox ++ * David Hinds ++ * Alexey Kuznetsov ++ * Adam Sulmicki ++ * Pekka Riikonen ++ * ++ * Changes: ++ * D.J. Barrow : Fixed bug where dev->refcnt gets set ++ * to 2 if register_netdev gets called ++ * before net_dev_init & also removed a ++ * few lines of code in the process. ++ * Alan Cox : device private ioctl copies fields back. ++ * Alan Cox : Transmit queue code does relevant ++ * stunts to keep the queue safe. ++ * Alan Cox : Fixed double lock. ++ * Alan Cox : Fixed promisc NULL pointer trap ++ * ???????? : Support the full private ioctl range ++ * Alan Cox : Moved ioctl permission check into ++ * drivers ++ * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI ++ * Alan Cox : 100 backlog just doesn't cut it when ++ * you start doing multicast video 8) ++ * Alan Cox : Rewrote net_bh and list manager. ++ * Alan Cox : Fix ETH_P_ALL echoback lengths. ++ * Alan Cox : Took out transmit every packet pass ++ * Saved a few bytes in the ioctl handler ++ * Alan Cox : Network driver sets packet type before ++ * calling netif_rx. Saves a function ++ * call a packet. ++ * Alan Cox : Hashed net_bh() ++ * Richard Kooijman: Timestamp fixes. ++ * Alan Cox : Wrong field in SIOCGIFDSTADDR ++ * Alan Cox : Device lock protection. ++ * Alan Cox : Fixed nasty side effect of device close ++ * changes. ++ * Rudi Cilibrasi : Pass the right thing to ++ * set_mac_address() ++ * Dave Miller : 32bit quantity for the device lock to ++ * make it work out on a Sparc. ++ * Bjorn Ekwall : Added KERNELD hack. ++ * Alan Cox : Cleaned up the backlog initialise. ++ * Craig Metz : SIOCGIFCONF fix if space for under ++ * 1 device. ++ * Thomas Bogendoerfer : Return ENODEV for dev_open, if there ++ * is no device open function. ++ * Andi Kleen : Fix error reporting for SIOCGIFCONF ++ * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF ++ * Cyrus Durgin : Cleaned for KMOD ++ * Adam Sulmicki : Bug Fix : Network Device Unload ++ * A network device unload needs to purge ++ * the backlog queue. ++ * Paul Rusty Russell : SIOCSIFNAME ++ * Pekka Riikonen : Netdev boot-time settings code ++ * Andrew Morton : Make unregister_netdevice wait ++ * indefinitely on dev->refcnt ++ * J Hadi Salim : - Backlog queue sampling ++ * - netif_rx() feedback ++ */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "net-sysfs.h" ++ ++/* Instead of increasing this, you should create a hash table. */ ++#define MAX_GRO_SKBS 8 ++ ++/* This should be increased if a protocol with a bigger head is added. */ ++#define GRO_MAX_HEAD (MAX_HEADER + 128) ++ ++static DEFINE_SPINLOCK(ptype_lock); ++static DEFINE_SPINLOCK(offload_lock); ++struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly; ++struct list_head ptype_all __read_mostly; /* Taps */ ++static struct list_head offload_base __read_mostly; ++ ++static int netif_rx_internal(struct sk_buff *skb); ++static int call_netdevice_notifiers_info(unsigned long val, ++ struct net_device *dev, ++ struct netdev_notifier_info *info); ++ ++/* ++ * The @dev_base_head list is protected by @dev_base_lock and the rtnl ++ * semaphore. ++ * ++ * Pure readers hold dev_base_lock for reading, or rcu_read_lock() ++ * ++ * Writers must hold the rtnl semaphore while they loop through the ++ * dev_base_head list, and hold dev_base_lock for writing when they do the ++ * actual updates. This allows pure readers to access the list even ++ * while a writer is preparing to update it. ++ * ++ * To put it another way, dev_base_lock is held for writing only to ++ * protect against pure readers; the rtnl semaphore provides the ++ * protection against other writers. ++ * ++ * See, for example usages, register_netdevice() and ++ * unregister_netdevice(), which must be called with the rtnl ++ * semaphore held. ++ */ ++DEFINE_RWLOCK(dev_base_lock); ++EXPORT_SYMBOL(dev_base_lock); ++ ++/* protects napi_hash addition/deletion and napi_gen_id */ ++static DEFINE_SPINLOCK(napi_hash_lock); ++ ++static unsigned int napi_gen_id; ++static DEFINE_HASHTABLE(napi_hash, 8); ++ ++static seqcount_t devnet_rename_seq; ++ ++static inline void dev_base_seq_inc(struct net *net) ++{ ++ while (++net->dev_base_seq == 0); ++} ++ ++static inline struct hlist_head *dev_name_hash(struct net *net, const char *name) ++{ ++ unsigned int hash = full_name_hash(name, strnlen(name, IFNAMSIZ)); ++ ++ return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)]; ++} ++ ++static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex) ++{ ++ return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)]; ++} ++ ++static inline void rps_lock(struct softnet_data *sd) ++{ ++#ifdef CONFIG_RPS ++ spin_lock(&sd->input_pkt_queue.lock); ++#endif ++} ++ ++static inline void rps_unlock(struct softnet_data *sd) ++{ ++#ifdef CONFIG_RPS ++ spin_unlock(&sd->input_pkt_queue.lock); ++#endif ++} ++ ++/* Device list insertion */ ++static void list_netdevice(struct net_device *dev) ++{ ++ struct net *net = dev_net(dev); ++ ++ ASSERT_RTNL(); ++ ++ write_lock_bh(&dev_base_lock); ++ list_add_tail_rcu(&dev->dev_list, &net->dev_base_head); ++ hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name)); ++ hlist_add_head_rcu(&dev->index_hlist, ++ dev_index_hash(net, dev->ifindex)); ++ write_unlock_bh(&dev_base_lock); ++ ++ dev_base_seq_inc(net); ++} ++ ++/* Device list removal ++ * caller must respect a RCU grace period before freeing/reusing dev ++ */ ++static void unlist_netdevice(struct net_device *dev) ++{ ++ ASSERT_RTNL(); ++ ++ /* Unlink dev from the device chain */ ++ write_lock_bh(&dev_base_lock); ++ list_del_rcu(&dev->dev_list); ++ hlist_del_rcu(&dev->name_hlist); ++ hlist_del_rcu(&dev->index_hlist); ++ write_unlock_bh(&dev_base_lock); ++ ++ dev_base_seq_inc(dev_net(dev)); ++} ++ ++/* ++ * Our notifier list ++ */ ++ ++static RAW_NOTIFIER_HEAD(netdev_chain); ++ ++/* ++ * Device drivers call our routines to queue packets here. We empty the ++ * queue in the local softnet handler. ++ */ ++ ++DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data); ++EXPORT_PER_CPU_SYMBOL(softnet_data); ++ ++#ifdef CONFIG_LOCKDEP ++/* ++ * register_netdevice() inits txq->_xmit_lock and sets lockdep class ++ * according to dev->type ++ */ ++static const unsigned short netdev_lock_type[] = ++ {ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25, ++ ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET, ++ ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM, ++ ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP, ++ ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD, ++ ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25, ++ ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP, ++ ARPHRD_RAWHDLC, ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD, ++ ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI, ++ ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE, ++ ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET, ++ ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL, ++ ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM, ++ ARPHRD_IEEE80211_RADIOTAP, ARPHRD_PHONET, ARPHRD_PHONET_PIPE, ++ ARPHRD_IEEE802154, ARPHRD_VOID, ARPHRD_NONE}; ++ ++static const char *const netdev_lock_name[] = ++ {"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25", ++ "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET", ++ "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM", ++ "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP", ++ "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD", ++ "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25", ++ "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP", ++ "_xmit_RAWHDLC", "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD", ++ "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI", ++ "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE", ++ "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET", ++ "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL", ++ "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM", ++ "_xmit_IEEE80211_RADIOTAP", "_xmit_PHONET", "_xmit_PHONET_PIPE", ++ "_xmit_IEEE802154", "_xmit_VOID", "_xmit_NONE"}; ++ ++static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)]; ++static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)]; ++ ++static inline unsigned short netdev_lock_pos(unsigned short dev_type) ++{ ++ int i; ++ ++ for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++) ++ if (netdev_lock_type[i] == dev_type) ++ return i; ++ /* the last key is used by default */ ++ return ARRAY_SIZE(netdev_lock_type) - 1; ++} ++ ++static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock, ++ unsigned short dev_type) ++{ ++ int i; ++ ++ i = netdev_lock_pos(dev_type); ++ lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i], ++ netdev_lock_name[i]); ++} ++ ++static inline void netdev_set_addr_lockdep_class(struct net_device *dev) ++{ ++ int i; ++ ++ i = netdev_lock_pos(dev->type); ++ lockdep_set_class_and_name(&dev->addr_list_lock, ++ &netdev_addr_lock_key[i], ++ netdev_lock_name[i]); ++} ++#else ++static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock, ++ unsigned short dev_type) ++{ ++} ++static inline void netdev_set_addr_lockdep_class(struct net_device *dev) ++{ ++} ++#endif ++ ++/******************************************************************************* ++ ++ Protocol management and registration routines ++ ++*******************************************************************************/ ++ ++/* ++ * Add a protocol ID to the list. Now that the input handler is ++ * smarter we can dispense with all the messy stuff that used to be ++ * here. ++ * ++ * BEWARE!!! Protocol handlers, mangling input packets, ++ * MUST BE last in hash buckets and checking protocol handlers ++ * MUST start from promiscuous ptype_all chain in net_bh. ++ * It is true now, do not change it. ++ * Explanation follows: if protocol handler, mangling packet, will ++ * be the first on list, it is not able to sense, that packet ++ * is cloned and should be copied-on-write, so that it will ++ * change it and subsequent readers will get broken packet. ++ * --ANK (980803) ++ */ ++ ++static inline struct list_head *ptype_head(const struct packet_type *pt) ++{ ++ if (pt->type == htons(ETH_P_ALL)) ++ return pt->dev ? &pt->dev->ptype_all : &ptype_all; ++ else ++ return pt->dev ? &pt->dev->ptype_specific : ++ &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK]; ++} ++ ++/** ++ * dev_add_pack - add packet handler ++ * @pt: packet type declaration ++ * ++ * Add a protocol handler to the networking stack. The passed &packet_type ++ * is linked into kernel lists and may not be freed until it has been ++ * removed from the kernel lists. ++ * ++ * This call does not sleep therefore it can not ++ * guarantee all CPU's that are in middle of receiving packets ++ * will see the new packet type (until the next received packet). ++ */ ++ ++void dev_add_pack(struct packet_type *pt) ++{ ++ struct list_head *head = ptype_head(pt); ++ ++ spin_lock(&ptype_lock); ++ list_add_rcu(&pt->list, head); ++ spin_unlock(&ptype_lock); ++} ++EXPORT_SYMBOL(dev_add_pack); ++ ++/** ++ * __dev_remove_pack - remove packet handler ++ * @pt: packet type declaration ++ * ++ * Remove a protocol handler that was previously added to the kernel ++ * protocol handlers by dev_add_pack(). The passed &packet_type is removed ++ * from the kernel lists and can be freed or reused once this function ++ * returns. ++ * ++ * The packet type might still be in use by receivers ++ * and must not be freed until after all the CPU's have gone ++ * through a quiescent state. ++ */ ++void __dev_remove_pack(struct packet_type *pt) ++{ ++ struct list_head *head = ptype_head(pt); ++ struct packet_type *pt1; ++ ++ spin_lock(&ptype_lock); ++ ++ list_for_each_entry(pt1, head, list) { ++ if (pt == pt1) { ++ list_del_rcu(&pt->list); ++ goto out; ++ } ++ } ++ ++ pr_warn("dev_remove_pack: %p not found\n", pt); ++out: ++ spin_unlock(&ptype_lock); ++} ++EXPORT_SYMBOL(__dev_remove_pack); ++ ++/** ++ * dev_remove_pack - remove packet handler ++ * @pt: packet type declaration ++ * ++ * Remove a protocol handler that was previously added to the kernel ++ * protocol handlers by dev_add_pack(). The passed &packet_type is removed ++ * from the kernel lists and can be freed or reused once this function ++ * returns. ++ * ++ * This call sleeps to guarantee that no CPU is looking at the packet ++ * type after return. ++ */ ++void dev_remove_pack(struct packet_type *pt) ++{ ++ __dev_remove_pack(pt); ++ ++ synchronize_net(); ++} ++EXPORT_SYMBOL(dev_remove_pack); ++ ++ ++/** ++ * dev_add_offload - register offload handlers ++ * @po: protocol offload declaration ++ * ++ * Add protocol offload handlers to the networking stack. The passed ++ * &proto_offload is linked into kernel lists and may not be freed until ++ * it has been removed from the kernel lists. ++ * ++ * This call does not sleep therefore it can not ++ * guarantee all CPU's that are in middle of receiving packets ++ * will see the new offload handlers (until the next received packet). ++ */ ++void dev_add_offload(struct packet_offload *po) ++{ ++ struct list_head *head = &offload_base; ++ ++ spin_lock(&offload_lock); ++ list_add_rcu(&po->list, head); ++ spin_unlock(&offload_lock); ++} ++EXPORT_SYMBOL(dev_add_offload); ++ ++/** ++ * __dev_remove_offload - remove offload handler ++ * @po: packet offload declaration ++ * ++ * Remove a protocol offload handler that was previously added to the ++ * kernel offload handlers by dev_add_offload(). The passed &offload_type ++ * is removed from the kernel lists and can be freed or reused once this ++ * function returns. ++ * ++ * The packet type might still be in use by receivers ++ * and must not be freed until after all the CPU's have gone ++ * through a quiescent state. ++ */ ++static void __dev_remove_offload(struct packet_offload *po) ++{ ++ struct list_head *head = &offload_base; ++ struct packet_offload *po1; ++ ++ spin_lock(&offload_lock); ++ ++ list_for_each_entry(po1, head, list) { ++ if (po == po1) { ++ list_del_rcu(&po->list); ++ goto out; ++ } ++ } ++ ++ pr_warn("dev_remove_offload: %p not found\n", po); ++out: ++ spin_unlock(&offload_lock); ++} ++ ++/** ++ * dev_remove_offload - remove packet offload handler ++ * @po: packet offload declaration ++ * ++ * Remove a packet offload handler that was previously added to the kernel ++ * offload handlers by dev_add_offload(). The passed &offload_type is ++ * removed from the kernel lists and can be freed or reused once this ++ * function returns. ++ * ++ * This call sleeps to guarantee that no CPU is looking at the packet ++ * type after return. ++ */ ++void dev_remove_offload(struct packet_offload *po) ++{ ++ __dev_remove_offload(po); ++ ++ synchronize_net(); ++} ++EXPORT_SYMBOL(dev_remove_offload); ++ ++/****************************************************************************** ++ ++ Device Boot-time Settings Routines ++ ++*******************************************************************************/ ++ ++/* Boot time configuration table */ ++static struct netdev_boot_setup dev_boot_setup[NETDEV_BOOT_SETUP_MAX]; ++ ++/** ++ * netdev_boot_setup_add - add new setup entry ++ * @name: name of the device ++ * @map: configured settings for the device ++ * ++ * Adds new setup entry to the dev_boot_setup list. The function ++ * returns 0 on error and 1 on success. This is a generic routine to ++ * all netdevices. ++ */ ++static int netdev_boot_setup_add(char *name, struct ifmap *map) ++{ ++ struct netdev_boot_setup *s; ++ int i; ++ ++ s = dev_boot_setup; ++ for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) { ++ if (s[i].name[0] == '\0' || s[i].name[0] == ' ') { ++ memset(s[i].name, 0, sizeof(s[i].name)); ++ strlcpy(s[i].name, name, IFNAMSIZ); ++ memcpy(&s[i].map, map, sizeof(s[i].map)); ++ break; ++ } ++ } ++ ++ return i >= NETDEV_BOOT_SETUP_MAX ? 0 : 1; ++} ++ ++/** ++ * netdev_boot_setup_check - check boot time settings ++ * @dev: the netdevice ++ * ++ * Check boot time settings for the device. ++ * The found settings are set for the device to be used ++ * later in the device probing. ++ * Returns 0 if no settings found, 1 if they are. ++ */ ++int netdev_boot_setup_check(struct net_device *dev) ++{ ++ struct netdev_boot_setup *s = dev_boot_setup; ++ int i; ++ ++ for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) { ++ if (s[i].name[0] != '\0' && s[i].name[0] != ' ' && ++ !strcmp(dev->name, s[i].name)) { ++ dev->irq = s[i].map.irq; ++ dev->base_addr = s[i].map.base_addr; ++ dev->mem_start = s[i].map.mem_start; ++ dev->mem_end = s[i].map.mem_end; ++ return 1; ++ } ++ } ++ return 0; ++} ++EXPORT_SYMBOL(netdev_boot_setup_check); ++ ++ ++/** ++ * netdev_boot_base - get address from boot time settings ++ * @prefix: prefix for network device ++ * @unit: id for network device ++ * ++ * Check boot time settings for the base address of device. ++ * The found settings are set for the device to be used ++ * later in the device probing. ++ * Returns 0 if no settings found. ++ */ ++unsigned long netdev_boot_base(const char *prefix, int unit) ++{ ++ const struct netdev_boot_setup *s = dev_boot_setup; ++ char name[IFNAMSIZ]; ++ int i; ++ ++ sprintf(name, "%s%d", prefix, unit); ++ ++ /* ++ * If device already registered then return base of 1 ++ * to indicate not to probe for this interface ++ */ ++ if (__dev_get_by_name(&init_net, name)) ++ return 1; ++ ++ for (i = 0; i < NETDEV_BOOT_SETUP_MAX; i++) ++ if (!strcmp(name, s[i].name)) ++ return s[i].map.base_addr; ++ return 0; ++} ++ ++/* ++ * Saves at boot time configured settings for any netdevice. ++ */ ++int __init netdev_boot_setup(char *str) ++{ ++ int ints[5]; ++ struct ifmap map; ++ ++ str = get_options(str, ARRAY_SIZE(ints), ints); ++ if (!str || !*str) ++ return 0; ++ ++ /* Save settings */ ++ memset(&map, 0, sizeof(map)); ++ if (ints[0] > 0) ++ map.irq = ints[1]; ++ if (ints[0] > 1) ++ map.base_addr = ints[2]; ++ if (ints[0] > 2) ++ map.mem_start = ints[3]; ++ if (ints[0] > 3) ++ map.mem_end = ints[4]; ++ ++ /* Add new entry to the list */ ++ return netdev_boot_setup_add(str, &map); ++} ++ ++__setup("netdev=", netdev_boot_setup); ++ ++/******************************************************************************* ++ ++ Device Interface Subroutines ++ ++*******************************************************************************/ ++ ++/** ++ * dev_get_iflink - get 'iflink' value of a interface ++ * @dev: targeted interface ++ * ++ * Indicates the ifindex the interface is linked to. ++ * Physical interfaces have the same 'ifindex' and 'iflink' values. ++ */ ++ ++int dev_get_iflink(const struct net_device *dev) ++{ ++ if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink) ++ return dev->netdev_ops->ndo_get_iflink(dev); ++ ++ return dev->ifindex; ++} ++EXPORT_SYMBOL(dev_get_iflink); ++ ++/** ++ * __dev_get_by_name - find a device by its name ++ * @net: the applicable net namespace ++ * @name: name to find ++ * ++ * Find an interface by name. Must be called under RTNL semaphore ++ * or @dev_base_lock. If the name is found a pointer to the device ++ * is returned. If the name is not found then %NULL is returned. The ++ * reference counters are not incremented so the caller must be ++ * careful with locks. ++ */ ++ ++struct net_device *__dev_get_by_name(struct net *net, const char *name) ++{ ++ struct net_device *dev; ++ struct hlist_head *head = dev_name_hash(net, name); ++ ++ hlist_for_each_entry(dev, head, name_hlist) ++ if (!strncmp(dev->name, name, IFNAMSIZ)) ++ return dev; ++ ++ return NULL; ++} ++EXPORT_SYMBOL(__dev_get_by_name); ++ ++/** ++ * dev_get_by_name_rcu - find a device by its name ++ * @net: the applicable net namespace ++ * @name: name to find ++ * ++ * Find an interface by name. ++ * If the name is found a pointer to the device is returned. ++ * If the name is not found then %NULL is returned. ++ * The reference counters are not incremented so the caller must be ++ * careful with locks. The caller must hold RCU lock. ++ */ ++ ++struct net_device *dev_get_by_name_rcu(struct net *net, const char *name) ++{ ++ struct net_device *dev; ++ struct hlist_head *head = dev_name_hash(net, name); ++ ++ hlist_for_each_entry_rcu(dev, head, name_hlist) ++ if (!strncmp(dev->name, name, IFNAMSIZ)) ++ return dev; ++ ++ return NULL; ++} ++EXPORT_SYMBOL(dev_get_by_name_rcu); ++ ++/** ++ * dev_get_by_name - find a device by its name ++ * @net: the applicable net namespace ++ * @name: name to find ++ * ++ * Find an interface by name. This can be called from any ++ * context and does its own locking. The returned handle has ++ * the usage count incremented and the caller must use dev_put() to ++ * release it when it is no longer needed. %NULL is returned if no ++ * matching device is found. ++ */ ++ ++struct net_device *dev_get_by_name(struct net *net, const char *name) ++{ ++ struct net_device *dev; ++ ++ rcu_read_lock(); ++ dev = dev_get_by_name_rcu(net, name); ++ if (dev) ++ dev_hold(dev); ++ rcu_read_unlock(); ++ return dev; ++} ++EXPORT_SYMBOL(dev_get_by_name); ++ ++/** ++ * __dev_get_by_index - find a device by its ifindex ++ * @net: the applicable net namespace ++ * @ifindex: index of device ++ * ++ * Search for an interface by index. Returns %NULL if the device ++ * is not found or a pointer to the device. The device has not ++ * had its reference counter increased so the caller must be careful ++ * about locking. The caller must hold either the RTNL semaphore ++ * or @dev_base_lock. ++ */ ++ ++struct net_device *__dev_get_by_index(struct net *net, int ifindex) ++{ ++ struct net_device *dev; ++ struct hlist_head *head = dev_index_hash(net, ifindex); ++ ++ hlist_for_each_entry(dev, head, index_hlist) ++ if (dev->ifindex == ifindex) ++ return dev; ++ ++ return NULL; ++} ++EXPORT_SYMBOL(__dev_get_by_index); ++ ++/** ++ * dev_get_by_index_rcu - find a device by its ifindex ++ * @net: the applicable net namespace ++ * @ifindex: index of device ++ * ++ * Search for an interface by index. Returns %NULL if the device ++ * is not found or a pointer to the device. The device has not ++ * had its reference counter increased so the caller must be careful ++ * about locking. The caller must hold RCU lock. ++ */ ++ ++struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex) ++{ ++ struct net_device *dev; ++ struct hlist_head *head = dev_index_hash(net, ifindex); ++ ++ hlist_for_each_entry_rcu(dev, head, index_hlist) ++ if (dev->ifindex == ifindex) ++ return dev; ++ ++ return NULL; ++} ++EXPORT_SYMBOL(dev_get_by_index_rcu); ++ ++ ++/** ++ * dev_get_by_index - find a device by its ifindex ++ * @net: the applicable net namespace ++ * @ifindex: index of device ++ * ++ * Search for an interface by index. Returns NULL if the device ++ * is not found or a pointer to the device. The device returned has ++ * had a reference added and the pointer is safe until the user calls ++ * dev_put to indicate they have finished with it. ++ */ ++ ++struct net_device *dev_get_by_index(struct net *net, int ifindex) ++{ ++ struct net_device *dev; ++ ++ rcu_read_lock(); ++ dev = dev_get_by_index_rcu(net, ifindex); ++ if (dev) ++ dev_hold(dev); ++ rcu_read_unlock(); ++ return dev; ++} ++EXPORT_SYMBOL(dev_get_by_index); ++ ++/** ++ * netdev_get_name - get a netdevice name, knowing its ifindex. ++ * @net: network namespace ++ * @name: a pointer to the buffer where the name will be stored. ++ * @ifindex: the ifindex of the interface to get the name from. ++ * ++ * The use of raw_seqcount_begin() and cond_resched() before ++ * retrying is required as we want to give the writers a chance ++ * to complete when CONFIG_PREEMPT is not set. ++ */ ++int netdev_get_name(struct net *net, char *name, int ifindex) ++{ ++ struct net_device *dev; ++ unsigned int seq; ++ ++retry: ++ seq = raw_seqcount_begin(&devnet_rename_seq); ++ rcu_read_lock(); ++ dev = dev_get_by_index_rcu(net, ifindex); ++ if (!dev) { ++ rcu_read_unlock(); ++ return -ENODEV; ++ } ++ ++ strcpy(name, dev->name); ++ rcu_read_unlock(); ++ if (read_seqcount_retry(&devnet_rename_seq, seq)) { ++ cond_resched(); ++ goto retry; ++ } ++ ++ return 0; ++} ++ ++/** ++ * dev_getbyhwaddr_rcu - find a device by its hardware address ++ * @net: the applicable net namespace ++ * @type: media type of device ++ * @ha: hardware address ++ * ++ * Search for an interface by MAC address. Returns NULL if the device ++ * is not found or a pointer to the device. ++ * The caller must hold RCU or RTNL. ++ * The returned device has not had its ref count increased ++ * and the caller must therefore be careful about locking ++ * ++ */ ++ ++struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type, ++ const char *ha) ++{ ++ struct net_device *dev; ++ ++ for_each_netdev_rcu(net, dev) ++ if (dev->type == type && ++ !memcmp(dev->dev_addr, ha, dev->addr_len)) ++ return dev; ++ ++ return NULL; ++} ++EXPORT_SYMBOL(dev_getbyhwaddr_rcu); ++ ++struct net_device *__dev_getfirstbyhwtype(struct net *net, unsigned short type) ++{ ++ struct net_device *dev; ++ ++ ASSERT_RTNL(); ++ for_each_netdev(net, dev) ++ if (dev->type == type) ++ return dev; ++ ++ return NULL; ++} ++EXPORT_SYMBOL(__dev_getfirstbyhwtype); ++ ++struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type) ++{ ++ struct net_device *dev, *ret = NULL; ++ ++ rcu_read_lock(); ++ for_each_netdev_rcu(net, dev) ++ if (dev->type == type) { ++ dev_hold(dev); ++ ret = dev; ++ break; ++ } ++ rcu_read_unlock(); ++ return ret; ++} ++EXPORT_SYMBOL(dev_getfirstbyhwtype); ++ ++/** ++ * __dev_get_by_flags - find any device with given flags ++ * @net: the applicable net namespace ++ * @if_flags: IFF_* values ++ * @mask: bitmask of bits in if_flags to check ++ * ++ * Search for any interface with the given flags. Returns NULL if a device ++ * is not found or a pointer to the device. Must be called inside ++ * rtnl_lock(), and result refcount is unchanged. ++ */ ++ ++struct net_device *__dev_get_by_flags(struct net *net, unsigned short if_flags, ++ unsigned short mask) ++{ ++ struct net_device *dev, *ret; ++ ++ ASSERT_RTNL(); ++ ++ ret = NULL; ++ for_each_netdev(net, dev) { ++ if (((dev->flags ^ if_flags) & mask) == 0) { ++ ret = dev; ++ break; ++ } ++ } ++ return ret; ++} ++EXPORT_SYMBOL(__dev_get_by_flags); ++ ++/** ++ * dev_valid_name - check if name is okay for network device ++ * @name: name string ++ * ++ * Network device names need to be valid file names to ++ * to allow sysfs to work. We also disallow any kind of ++ * whitespace. ++ */ ++bool dev_valid_name(const char *name) ++{ ++ if (*name == '\0') ++ return false; ++ if (strlen(name) >= IFNAMSIZ) ++ return false; ++ if (!strcmp(name, ".") || !strcmp(name, "..")) ++ return false; ++ ++ while (*name) { ++ if (*name == '/' || *name == ':' || isspace(*name)) ++ return false; ++ name++; ++ } ++ return true; ++} ++EXPORT_SYMBOL(dev_valid_name); ++ ++/** ++ * __dev_alloc_name - allocate a name for a device ++ * @net: network namespace to allocate the device name in ++ * @name: name format string ++ * @buf: scratch buffer and result name string ++ * ++ * Passed a format string - eg "lt%d" it will try and find a suitable ++ * id. It scans list of devices to build up a free map, then chooses ++ * the first empty slot. The caller must hold the dev_base or rtnl lock ++ * while allocating the name and adding the device in order to avoid ++ * duplicates. ++ * Limited to bits_per_byte * page size devices (ie 32K on most platforms). ++ * Returns the number of the unit assigned or a negative errno code. ++ */ ++ ++static int __dev_alloc_name(struct net *net, const char *name, char *buf) ++{ ++ int i = 0; ++ const char *p; ++ const int max_netdevices = 8*PAGE_SIZE; ++ unsigned long *inuse; ++ struct net_device *d; ++ ++ p = strnchr(name, IFNAMSIZ-1, '%'); ++ if (p) { ++ /* ++ * Verify the string as this thing may have come from ++ * the user. There must be either one "%d" and no other "%" ++ * characters. ++ */ ++ if (p[1] != 'd' || strchr(p + 2, '%')) ++ return -EINVAL; ++ ++ /* Use one page as a bit array of possible slots */ ++ inuse = (unsigned long *) get_zeroed_page(GFP_ATOMIC); ++ if (!inuse) ++ return -ENOMEM; ++ ++ for_each_netdev(net, d) { ++ if (!sscanf(d->name, name, &i)) ++ continue; ++ if (i < 0 || i >= max_netdevices) ++ continue; ++ ++ /* avoid cases where sscanf is not exact inverse of printf */ ++ snprintf(buf, IFNAMSIZ, name, i); ++ if (!strncmp(buf, d->name, IFNAMSIZ)) ++ set_bit(i, inuse); ++ } ++ ++ i = find_first_zero_bit(inuse, max_netdevices); ++ free_page((unsigned long) inuse); ++ } ++ ++ if (buf != name) ++ snprintf(buf, IFNAMSIZ, name, i); ++ if (!__dev_get_by_name(net, buf)) ++ return i; ++ ++ /* It is possible to run out of possible slots ++ * when the name is long and there isn't enough space left ++ * for the digits, or if all bits are used. ++ */ ++ return -ENFILE; ++} ++ ++/** ++ * dev_alloc_name - allocate a name for a device ++ * @dev: device ++ * @name: name format string ++ * ++ * Passed a format string - eg "lt%d" it will try and find a suitable ++ * id. It scans list of devices to build up a free map, then chooses ++ * the first empty slot. The caller must hold the dev_base or rtnl lock ++ * while allocating the name and adding the device in order to avoid ++ * duplicates. ++ * Limited to bits_per_byte * page size devices (ie 32K on most platforms). ++ * Returns the number of the unit assigned or a negative errno code. ++ */ ++ ++int dev_alloc_name(struct net_device *dev, const char *name) ++{ ++ char buf[IFNAMSIZ]; ++ struct net *net; ++ int ret; ++ ++ BUG_ON(!dev_net(dev)); ++ net = dev_net(dev); ++ ret = __dev_alloc_name(net, name, buf); ++ if (ret >= 0) ++ strlcpy(dev->name, buf, IFNAMSIZ); ++ return ret; ++} ++EXPORT_SYMBOL(dev_alloc_name); ++ ++static int dev_alloc_name_ns(struct net *net, ++ struct net_device *dev, ++ const char *name) ++{ ++ char buf[IFNAMSIZ]; ++ int ret; ++ ++ ret = __dev_alloc_name(net, name, buf); ++ if (ret >= 0) ++ strlcpy(dev->name, buf, IFNAMSIZ); ++ return ret; ++} ++ ++static int dev_get_valid_name(struct net *net, ++ struct net_device *dev, ++ const char *name) ++{ ++ BUG_ON(!net); ++ ++ if (!dev_valid_name(name)) ++ return -EINVAL; ++ ++ if (strchr(name, '%')) ++ return dev_alloc_name_ns(net, dev, name); ++ else if (__dev_get_by_name(net, name)) ++ return -EEXIST; ++ else if (dev->name != name) ++ strlcpy(dev->name, name, IFNAMSIZ); ++ ++ return 0; ++} ++ ++/** ++ * dev_change_name - change name of a device ++ * @dev: device ++ * @newname: name (or format string) must be at least IFNAMSIZ ++ * ++ * Change name of a device, can pass format strings "eth%d". ++ * for wildcarding. ++ */ ++int dev_change_name(struct net_device *dev, const char *newname) ++{ ++ unsigned char old_assign_type; ++ char oldname[IFNAMSIZ]; ++ int err = 0; ++ int ret; ++ struct net *net; ++ ++ ASSERT_RTNL(); ++ BUG_ON(!dev_net(dev)); ++ ++ net = dev_net(dev); ++ if (dev->flags & IFF_UP) ++ return -EBUSY; ++ ++ write_seqcount_begin(&devnet_rename_seq); ++ ++ if (strncmp(newname, dev->name, IFNAMSIZ) == 0) { ++ write_seqcount_end(&devnet_rename_seq); ++ return 0; ++ } ++ ++ memcpy(oldname, dev->name, IFNAMSIZ); ++ ++ err = dev_get_valid_name(net, dev, newname); ++ if (err < 0) { ++ write_seqcount_end(&devnet_rename_seq); ++ return err; ++ } ++ ++ if (oldname[0] && !strchr(oldname, '%')) ++ netdev_info(dev, "renamed from %s\n", oldname); ++ ++ old_assign_type = dev->name_assign_type; ++ dev->name_assign_type = NET_NAME_RENAMED; ++ ++rollback: ++ ret = device_rename(&dev->dev, dev->name); ++ if (ret) { ++ memcpy(dev->name, oldname, IFNAMSIZ); ++ dev->name_assign_type = old_assign_type; ++ write_seqcount_end(&devnet_rename_seq); ++ return ret; ++ } ++ ++ write_seqcount_end(&devnet_rename_seq); ++ ++ netdev_adjacent_rename_links(dev, oldname); ++ ++ write_lock_bh(&dev_base_lock); ++ hlist_del_rcu(&dev->name_hlist); ++ write_unlock_bh(&dev_base_lock); ++ ++ synchronize_rcu(); ++ ++ write_lock_bh(&dev_base_lock); ++ hlist_add_head_rcu(&dev->name_hlist, dev_name_hash(net, dev->name)); ++ write_unlock_bh(&dev_base_lock); ++ ++ ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev); ++ ret = notifier_to_errno(ret); ++ ++ if (ret) { ++ /* err >= 0 after dev_alloc_name() or stores the first errno */ ++ if (err >= 0) { ++ err = ret; ++ write_seqcount_begin(&devnet_rename_seq); ++ memcpy(dev->name, oldname, IFNAMSIZ); ++ memcpy(oldname, newname, IFNAMSIZ); ++ dev->name_assign_type = old_assign_type; ++ old_assign_type = NET_NAME_RENAMED; ++ goto rollback; ++ } else { ++ pr_err("%s: name change rollback failed: %d\n", ++ dev->name, ret); ++ } ++ } ++ ++ return err; ++} ++ ++/** ++ * dev_set_alias - change ifalias of a device ++ * @dev: device ++ * @alias: name up to IFALIASZ ++ * @len: limit of bytes to copy from info ++ * ++ * Set ifalias for a device, ++ */ ++int dev_set_alias(struct net_device *dev, const char *alias, size_t len) ++{ ++ char *new_ifalias; ++ ++ ASSERT_RTNL(); ++ ++ if (len >= IFALIASZ) ++ return -EINVAL; ++ ++ if (!len) { ++ kfree(dev->ifalias); ++ dev->ifalias = NULL; ++ return 0; ++ } ++ ++ new_ifalias = krealloc(dev->ifalias, len + 1, GFP_KERNEL); ++ if (!new_ifalias) ++ return -ENOMEM; ++ dev->ifalias = new_ifalias; ++ ++ strlcpy(dev->ifalias, alias, len+1); ++ return len; ++} ++ ++ ++/** ++ * netdev_features_change - device changes features ++ * @dev: device to cause notification ++ * ++ * Called to indicate a device has changed features. ++ */ ++void netdev_features_change(struct net_device *dev) ++{ ++ call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev); ++} ++EXPORT_SYMBOL(netdev_features_change); ++ ++/** ++ * netdev_state_change - device changes state ++ * @dev: device to cause notification ++ * ++ * Called to indicate a device has changed state. This function calls ++ * the notifier chains for netdev_chain and sends a NEWLINK message ++ * to the routing socket. ++ */ ++void netdev_state_change(struct net_device *dev) ++{ ++ if (dev->flags & IFF_UP) { ++ struct netdev_notifier_change_info change_info; ++ ++ change_info.flags_changed = 0; ++ call_netdevice_notifiers_info(NETDEV_CHANGE, dev, ++ &change_info.info); ++ rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL); ++ } ++} ++EXPORT_SYMBOL(netdev_state_change); ++ ++/** ++ * netdev_notify_peers - notify network peers about existence of @dev ++ * @dev: network device ++ * ++ * Generate traffic such that interested network peers are aware of ++ * @dev, such as by generating a gratuitous ARP. This may be used when ++ * a device wants to inform the rest of the network about some sort of ++ * reconfiguration such as a failover event or virtual machine ++ * migration. ++ */ ++void netdev_notify_peers(struct net_device *dev) ++{ ++ rtnl_lock(); ++ call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev); ++ rtnl_unlock(); ++} ++EXPORT_SYMBOL(netdev_notify_peers); ++ ++static int __dev_open(struct net_device *dev) ++{ ++ const struct net_device_ops *ops = dev->netdev_ops; ++ int ret; ++ ++ ASSERT_RTNL(); ++ ++ if (!netif_device_present(dev)) ++ return -ENODEV; ++ ++ /* Block netpoll from trying to do any rx path servicing. ++ * If we don't do this there is a chance ndo_poll_controller ++ * or ndo_poll may be running while we open the device ++ */ ++ netpoll_poll_disable(dev); ++ ++ ret = call_netdevice_notifiers(NETDEV_PRE_UP, dev); ++ ret = notifier_to_errno(ret); ++ if (ret) ++ return ret; ++ ++ set_bit(__LINK_STATE_START, &dev->state); ++ ++ if (ops->ndo_validate_addr) ++ ret = ops->ndo_validate_addr(dev); ++ ++ if (!ret && ops->ndo_open) ++ ret = ops->ndo_open(dev); ++ ++ netpoll_poll_enable(dev); ++ ++ if (ret) ++ clear_bit(__LINK_STATE_START, &dev->state); ++ else { ++ dev->flags |= IFF_UP; ++ dev_set_rx_mode(dev); ++ dev_activate(dev); ++ add_device_randomness(dev->dev_addr, dev->addr_len); ++ } ++ ++ return ret; ++} ++ ++/** ++ * dev_open - prepare an interface for use. ++ * @dev: device to open ++ * ++ * Takes a device from down to up state. The device's private open ++ * function is invoked and then the multicast lists are loaded. Finally ++ * the device is moved into the up state and a %NETDEV_UP message is ++ * sent to the netdev notifier chain. ++ * ++ * Calling this function on an active interface is a nop. On a failure ++ * a negative errno code is returned. ++ */ ++int dev_open(struct net_device *dev) ++{ ++ int ret; ++ ++ if (dev->flags & IFF_UP) ++ return 0; ++ ++ ret = __dev_open(dev); ++ if (ret < 0) ++ return ret; ++ ++ rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL); ++ call_netdevice_notifiers(NETDEV_UP, dev); ++ ++ return ret; ++} ++EXPORT_SYMBOL(dev_open); ++ ++static int __dev_close_many(struct list_head *head) ++{ ++ struct net_device *dev; ++ ++ ASSERT_RTNL(); ++ might_sleep(); ++ ++ list_for_each_entry(dev, head, close_list) { ++ /* Temporarily disable netpoll until the interface is down */ ++ netpoll_poll_disable(dev); ++ ++ call_netdevice_notifiers(NETDEV_GOING_DOWN, dev); ++ ++ clear_bit(__LINK_STATE_START, &dev->state); ++ ++ /* Synchronize to scheduled poll. We cannot touch poll list, it ++ * can be even on different cpu. So just clear netif_running(). ++ * ++ * dev->stop() will invoke napi_disable() on all of it's ++ * napi_struct instances on this device. ++ */ ++ smp_mb__after_atomic(); /* Commit netif_running(). */ ++ } ++ ++ dev_deactivate_many(head); ++ ++ list_for_each_entry(dev, head, close_list) { ++ const struct net_device_ops *ops = dev->netdev_ops; ++ ++ /* ++ * Call the device specific close. This cannot fail. ++ * Only if device is UP ++ * ++ * We allow it to be called even after a DETACH hot-plug ++ * event. ++ */ ++ if (ops->ndo_stop) ++ ops->ndo_stop(dev); ++ ++ dev->flags &= ~IFF_UP; ++ netpoll_poll_enable(dev); ++ } ++ ++ return 0; ++} ++ ++static int __dev_close(struct net_device *dev) ++{ ++ int retval; ++ LIST_HEAD(single); ++ ++ list_add(&dev->close_list, &single); ++ retval = __dev_close_many(&single); ++ list_del(&single); ++ ++ return retval; ++} ++ ++int dev_close_many(struct list_head *head, bool unlink) ++{ ++ struct net_device *dev, *tmp; ++ ++ /* Remove the devices that don't need to be closed */ ++ list_for_each_entry_safe(dev, tmp, head, close_list) ++ if (!(dev->flags & IFF_UP)) ++ list_del_init(&dev->close_list); ++ ++ __dev_close_many(head); ++ ++ list_for_each_entry_safe(dev, tmp, head, close_list) { ++ rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP|IFF_RUNNING, GFP_KERNEL); ++ call_netdevice_notifiers(NETDEV_DOWN, dev); ++ if (unlink) ++ list_del_init(&dev->close_list); ++ } ++ ++ return 0; ++} ++EXPORT_SYMBOL(dev_close_many); ++ ++/** ++ * dev_close - shutdown an interface. ++ * @dev: device to shutdown ++ * ++ * This function moves an active device into down state. A ++ * %NETDEV_GOING_DOWN is sent to the netdev notifier chain. The device ++ * is then deactivated and finally a %NETDEV_DOWN is sent to the notifier ++ * chain. ++ */ ++int dev_close(struct net_device *dev) ++{ ++ if (dev->flags & IFF_UP) { ++ LIST_HEAD(single); ++ ++ list_add(&dev->close_list, &single); ++ dev_close_many(&single, true); ++ list_del(&single); ++ } ++ return 0; ++} ++EXPORT_SYMBOL(dev_close); ++ ++ ++/** ++ * dev_disable_lro - disable Large Receive Offload on a device ++ * @dev: device ++ * ++ * Disable Large Receive Offload (LRO) on a net device. Must be ++ * called under RTNL. This is needed if received packets may be ++ * forwarded to another interface. ++ */ ++void dev_disable_lro(struct net_device *dev) ++{ ++ struct net_device *lower_dev; ++ struct list_head *iter; ++ ++ dev->wanted_features &= ~NETIF_F_LRO; ++ netdev_update_features(dev); ++ ++ if (unlikely(dev->features & NETIF_F_LRO)) ++ netdev_WARN(dev, "failed to disable LRO!\n"); ++ ++ netdev_for_each_lower_dev(dev, lower_dev, iter) ++ dev_disable_lro(lower_dev); ++} ++EXPORT_SYMBOL(dev_disable_lro); ++ ++static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val, ++ struct net_device *dev) ++{ ++ struct netdev_notifier_info info; ++ ++ netdev_notifier_info_init(&info, dev); ++ return nb->notifier_call(nb, val, &info); ++} ++ ++static int dev_boot_phase = 1; ++ ++/** ++ * register_netdevice_notifier - register a network notifier block ++ * @nb: notifier ++ * ++ * Register a notifier to be called when network device events occur. ++ * The notifier passed is linked into the kernel structures and must ++ * not be reused until it has been unregistered. A negative errno code ++ * is returned on a failure. ++ * ++ * When registered all registration and up events are replayed ++ * to the new notifier to allow device to have a race free ++ * view of the network device list. ++ */ ++ ++int register_netdevice_notifier(struct notifier_block *nb) ++{ ++ struct net_device *dev; ++ struct net_device *last; ++ struct net *net; ++ int err; ++ ++ rtnl_lock(); ++ err = raw_notifier_chain_register(&netdev_chain, nb); ++ if (err) ++ goto unlock; ++ if (dev_boot_phase) ++ goto unlock; ++ for_each_net(net) { ++ for_each_netdev(net, dev) { ++ err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev); ++ err = notifier_to_errno(err); ++ if (err) ++ goto rollback; ++ ++ if (!(dev->flags & IFF_UP)) ++ continue; ++ ++ call_netdevice_notifier(nb, NETDEV_UP, dev); ++ } ++ } ++ ++unlock: ++ rtnl_unlock(); ++ return err; ++ ++rollback: ++ last = dev; ++ for_each_net(net) { ++ for_each_netdev(net, dev) { ++ if (dev == last) ++ goto outroll; ++ ++ if (dev->flags & IFF_UP) { ++ call_netdevice_notifier(nb, NETDEV_GOING_DOWN, ++ dev); ++ call_netdevice_notifier(nb, NETDEV_DOWN, dev); ++ } ++ call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev); ++ } ++ } ++ ++outroll: ++ raw_notifier_chain_unregister(&netdev_chain, nb); ++ goto unlock; ++} ++EXPORT_SYMBOL(register_netdevice_notifier); ++ ++/** ++ * unregister_netdevice_notifier - unregister a network notifier block ++ * @nb: notifier ++ * ++ * Unregister a notifier previously registered by ++ * register_netdevice_notifier(). The notifier is unlinked into the ++ * kernel structures and may then be reused. A negative errno code ++ * is returned on a failure. ++ * ++ * After unregistering unregister and down device events are synthesized ++ * for all devices on the device list to the removed notifier to remove ++ * the need for special case cleanup code. ++ */ ++ ++int unregister_netdevice_notifier(struct notifier_block *nb) ++{ ++ struct net_device *dev; ++ struct net *net; ++ int err; ++ ++ rtnl_lock(); ++ err = raw_notifier_chain_unregister(&netdev_chain, nb); ++ if (err) ++ goto unlock; ++ ++ for_each_net(net) { ++ for_each_netdev(net, dev) { ++ if (dev->flags & IFF_UP) { ++ call_netdevice_notifier(nb, NETDEV_GOING_DOWN, ++ dev); ++ call_netdevice_notifier(nb, NETDEV_DOWN, dev); ++ } ++ call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev); ++ } ++ } ++unlock: ++ rtnl_unlock(); ++ return err; ++} ++EXPORT_SYMBOL(unregister_netdevice_notifier); ++ ++/** ++ * call_netdevice_notifiers_info - call all network notifier blocks ++ * @val: value passed unmodified to notifier function ++ * @dev: net_device pointer passed unmodified to notifier function ++ * @info: notifier information data ++ * ++ * Call all network notifier blocks. Parameters and return value ++ * are as for raw_notifier_call_chain(). ++ */ ++ ++static int call_netdevice_notifiers_info(unsigned long val, ++ struct net_device *dev, ++ struct netdev_notifier_info *info) ++{ ++ ASSERT_RTNL(); ++ netdev_notifier_info_init(info, dev); ++ return raw_notifier_call_chain(&netdev_chain, val, info); ++} ++ ++/** ++ * call_netdevice_notifiers - call all network notifier blocks ++ * @val: value passed unmodified to notifier function ++ * @dev: net_device pointer passed unmodified to notifier function ++ * ++ * Call all network notifier blocks. Parameters and return value ++ * are as for raw_notifier_call_chain(). ++ */ ++ ++int call_netdevice_notifiers(unsigned long val, struct net_device *dev) ++{ ++ struct netdev_notifier_info info; ++ ++ return call_netdevice_notifiers_info(val, dev, &info); ++} ++EXPORT_SYMBOL(call_netdevice_notifiers); ++ ++#ifdef CONFIG_NET_CLS_ACT ++static struct static_key ingress_needed __read_mostly; ++ ++void net_inc_ingress_queue(void) ++{ ++ static_key_slow_inc(&ingress_needed); ++} ++EXPORT_SYMBOL_GPL(net_inc_ingress_queue); ++ ++void net_dec_ingress_queue(void) ++{ ++ static_key_slow_dec(&ingress_needed); ++} ++EXPORT_SYMBOL_GPL(net_dec_ingress_queue); ++#endif ++ ++static struct static_key netstamp_needed __read_mostly; ++#ifdef HAVE_JUMP_LABEL ++/* We are not allowed to call static_key_slow_dec() from irq context ++ * If net_disable_timestamp() is called from irq context, defer the ++ * static_key_slow_dec() calls. ++ */ ++static atomic_t netstamp_needed_deferred; ++#endif ++ ++void net_enable_timestamp(void) ++{ ++#ifdef HAVE_JUMP_LABEL ++ int deferred = atomic_xchg(&netstamp_needed_deferred, 0); ++ ++ if (deferred) { ++ while (--deferred) ++ static_key_slow_dec(&netstamp_needed); ++ return; ++ } ++#endif ++ static_key_slow_inc(&netstamp_needed); ++} ++EXPORT_SYMBOL(net_enable_timestamp); ++ ++void net_disable_timestamp(void) ++{ ++#ifdef HAVE_JUMP_LABEL ++ if (in_interrupt()) { ++ atomic_inc(&netstamp_needed_deferred); ++ return; ++ } ++#endif ++ static_key_slow_dec(&netstamp_needed); ++} ++EXPORT_SYMBOL(net_disable_timestamp); ++ ++static inline void net_timestamp_set(struct sk_buff *skb) ++{ ++ skb->tstamp.tv64 = 0; ++ if (static_key_false(&netstamp_needed)) ++ __net_timestamp(skb); ++} ++ ++#define net_timestamp_check(COND, SKB) \ ++ if (static_key_false(&netstamp_needed)) { \ ++ if ((COND) && !(SKB)->tstamp.tv64) \ ++ __net_timestamp(SKB); \ ++ } \ ++ ++bool is_skb_forwardable(struct net_device *dev, struct sk_buff *skb) ++{ ++ unsigned int len; ++ ++ if (!(dev->flags & IFF_UP)) ++ return false; ++ ++ len = dev->mtu + dev->hard_header_len + VLAN_HLEN; ++ if (skb->len <= len) ++ return true; ++ ++ /* if TSO is enabled, we don't care about the length as the packet ++ * could be forwarded without being segmented before ++ */ ++ if (skb_is_gso(skb)) ++ return true; ++ ++ return false; ++} ++EXPORT_SYMBOL_GPL(is_skb_forwardable); ++ ++int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb) ++{ ++ if (skb_orphan_frags(skb, GFP_ATOMIC) || ++ unlikely(!is_skb_forwardable(dev, skb))) { ++ atomic_long_inc(&dev->rx_dropped); ++ kfree_skb(skb); ++ return NET_RX_DROP; ++ } ++ ++ skb_scrub_packet(skb, true); ++ skb->priority = 0; ++ skb->protocol = eth_type_trans(skb, dev); ++ skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN); ++ ++ return 0; ++} ++EXPORT_SYMBOL_GPL(__dev_forward_skb); ++ ++/** ++ * dev_forward_skb - loopback an skb to another netif ++ * ++ * @dev: destination network device ++ * @skb: buffer to forward ++ * ++ * return values: ++ * NET_RX_SUCCESS (no congestion) ++ * NET_RX_DROP (packet was dropped, but freed) ++ * ++ * dev_forward_skb can be used for injecting an skb from the ++ * start_xmit function of one device into the receive queue ++ * of another device. ++ * ++ * The receiving device may be in another namespace, so ++ * we have to clear all information in the skb that could ++ * impact namespace isolation. ++ */ ++int dev_forward_skb(struct net_device *dev, struct sk_buff *skb) ++{ ++ return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb); ++} ++EXPORT_SYMBOL_GPL(dev_forward_skb); ++ ++static inline int deliver_skb(struct sk_buff *skb, ++ struct packet_type *pt_prev, ++ struct net_device *orig_dev) ++{ ++ if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC))) ++ return -ENOMEM; ++ atomic_inc(&skb->users); ++ return pt_prev->func(skb, skb->dev, pt_prev, orig_dev); ++} ++ ++static inline void deliver_ptype_list_skb(struct sk_buff *skb, ++ struct packet_type **pt, ++ struct net_device *orig_dev, ++ __be16 type, ++ struct list_head *ptype_list) ++{ ++ struct packet_type *ptype, *pt_prev = *pt; ++ ++ list_for_each_entry_rcu(ptype, ptype_list, list) { ++ if (ptype->type != type) ++ continue; ++ if (pt_prev) ++ deliver_skb(skb, pt_prev, orig_dev); ++ pt_prev = ptype; ++ } ++ *pt = pt_prev; ++} ++ ++static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb) ++{ ++ if (!ptype->af_packet_priv || !skb->sk) ++ return false; ++ ++ if (ptype->id_match) ++ return ptype->id_match(ptype, skb->sk); ++ else if ((struct sock *)ptype->af_packet_priv == skb->sk) ++ return true; ++ ++ return false; ++} ++ ++/* ++ * Support routine. Sends outgoing frames to any network ++ * taps currently in use. ++ */ ++ ++static void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev) ++{ ++ struct packet_type *ptype; ++ struct sk_buff *skb2 = NULL; ++ struct packet_type *pt_prev = NULL; ++ struct list_head *ptype_list = &ptype_all; ++ ++ rcu_read_lock(); ++again: ++ list_for_each_entry_rcu(ptype, ptype_list, list) { ++ /* Never send packets back to the socket ++ * they originated from - MvS (miquels@drinkel.ow.org) ++ */ ++ if (skb_loop_sk(ptype, skb)) ++ continue; ++ ++ if (pt_prev) { ++ deliver_skb(skb2, pt_prev, skb->dev); ++ pt_prev = ptype; ++ continue; ++ } ++ ++ /* need to clone skb, done only once */ ++ skb2 = skb_clone(skb, GFP_ATOMIC); ++ if (!skb2) ++ goto out_unlock; ++ ++ net_timestamp_set(skb2); ++ ++ /* skb->nh should be correctly ++ * set by sender, so that the second statement is ++ * just protection against buggy protocols. ++ */ ++ skb_reset_mac_header(skb2); ++ ++ if (skb_network_header(skb2) < skb2->data || ++ skb_network_header(skb2) > skb_tail_pointer(skb2)) { ++ net_crit_ratelimited("protocol %04x is buggy, dev %s\n", ++ ntohs(skb2->protocol), ++ dev->name); ++ skb_reset_network_header(skb2); ++ } ++ ++ skb2->transport_header = skb2->network_header; ++ skb2->pkt_type = PACKET_OUTGOING; ++ pt_prev = ptype; ++ } ++ ++ if (ptype_list == &ptype_all) { ++ ptype_list = &dev->ptype_all; ++ goto again; ++ } ++out_unlock: ++ if (pt_prev) ++ pt_prev->func(skb2, skb->dev, pt_prev, skb->dev); ++ rcu_read_unlock(); ++} ++ ++/** ++ * netif_setup_tc - Handle tc mappings on real_num_tx_queues change ++ * @dev: Network device ++ * @txq: number of queues available ++ * ++ * If real_num_tx_queues is changed the tc mappings may no longer be ++ * valid. To resolve this verify the tc mapping remains valid and if ++ * not NULL the mapping. With no priorities mapping to this ++ * offset/count pair it will no longer be used. In the worst case TC0 ++ * is invalid nothing can be done so disable priority mappings. If is ++ * expected that drivers will fix this mapping if they can before ++ * calling netif_set_real_num_tx_queues. ++ */ ++static void netif_setup_tc(struct net_device *dev, unsigned int txq) ++{ ++ int i; ++ struct netdev_tc_txq *tc = &dev->tc_to_txq[0]; ++ ++ /* If TC0 is invalidated disable TC mapping */ ++ if (tc->offset + tc->count > txq) { ++ pr_warn("Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n"); ++ dev->num_tc = 0; ++ return; ++ } ++ ++ /* Invalidated prio to tc mappings set to TC0 */ ++ for (i = 1; i < TC_BITMASK + 1; i++) { ++ int q = netdev_get_prio_tc_map(dev, i); ++ ++ tc = &dev->tc_to_txq[q]; ++ if (tc->offset + tc->count > txq) { ++ pr_warn("Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n", ++ i, q); ++ netdev_set_prio_tc_map(dev, i, 0); ++ } ++ } ++} ++ ++#ifdef CONFIG_XPS ++static DEFINE_MUTEX(xps_map_mutex); ++#define xmap_dereference(P) \ ++ rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex)) ++ ++static struct xps_map *remove_xps_queue(struct xps_dev_maps *dev_maps, ++ int cpu, u16 index) ++{ ++ struct xps_map *map = NULL; ++ int pos; ++ ++ if (dev_maps) ++ map = xmap_dereference(dev_maps->cpu_map[cpu]); ++ ++ for (pos = 0; map && pos < map->len; pos++) { ++ if (map->queues[pos] == index) { ++ if (map->len > 1) { ++ map->queues[pos] = map->queues[--map->len]; ++ } else { ++ RCU_INIT_POINTER(dev_maps->cpu_map[cpu], NULL); ++ kfree_rcu(map, rcu); ++ map = NULL; ++ } ++ break; ++ } ++ } ++ ++ return map; ++} ++ ++static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index) ++{ ++ struct xps_dev_maps *dev_maps; ++ int cpu, i; ++ bool active = false; ++ ++ mutex_lock(&xps_map_mutex); ++ dev_maps = xmap_dereference(dev->xps_maps); ++ ++ if (!dev_maps) ++ goto out_no_maps; ++ ++ for_each_possible_cpu(cpu) { ++ for (i = index; i < dev->num_tx_queues; i++) { ++ if (!remove_xps_queue(dev_maps, cpu, i)) ++ break; ++ } ++ if (i == dev->num_tx_queues) ++ active = true; ++ } ++ ++ if (!active) { ++ RCU_INIT_POINTER(dev->xps_maps, NULL); ++ kfree_rcu(dev_maps, rcu); ++ } ++ ++ for (i = index; i < dev->num_tx_queues; i++) ++ netdev_queue_numa_node_write(netdev_get_tx_queue(dev, i), ++ NUMA_NO_NODE); ++ ++out_no_maps: ++ mutex_unlock(&xps_map_mutex); ++} ++ ++static struct xps_map *expand_xps_map(struct xps_map *map, ++ int cpu, u16 index) ++{ ++ struct xps_map *new_map; ++ int alloc_len = XPS_MIN_MAP_ALLOC; ++ int i, pos; ++ ++ for (pos = 0; map && pos < map->len; pos++) { ++ if (map->queues[pos] != index) ++ continue; ++ return map; ++ } ++ ++ /* Need to add queue to this CPU's existing map */ ++ if (map) { ++ if (pos < map->alloc_len) ++ return map; ++ ++ alloc_len = map->alloc_len * 2; ++ } ++ ++ /* Need to allocate new map to store queue on this CPU's map */ ++ new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL, ++ cpu_to_node(cpu)); ++ if (!new_map) ++ return NULL; ++ ++ for (i = 0; i < pos; i++) ++ new_map->queues[i] = map->queues[i]; ++ new_map->alloc_len = alloc_len; ++ new_map->len = pos; ++ ++ return new_map; ++} ++ ++int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask, ++ u16 index) ++{ ++ struct xps_dev_maps *dev_maps, *new_dev_maps = NULL; ++ struct xps_map *map, *new_map; ++ int maps_sz = max_t(unsigned int, XPS_DEV_MAPS_SIZE, L1_CACHE_BYTES); ++ int cpu, numa_node_id = -2; ++ bool active = false; ++ ++ mutex_lock(&xps_map_mutex); ++ ++ dev_maps = xmap_dereference(dev->xps_maps); ++ ++ /* allocate memory for queue storage */ ++ for_each_online_cpu(cpu) { ++ if (!cpumask_test_cpu(cpu, mask)) ++ continue; ++ ++ if (!new_dev_maps) ++ new_dev_maps = kzalloc(maps_sz, GFP_KERNEL); ++ if (!new_dev_maps) { ++ mutex_unlock(&xps_map_mutex); ++ return -ENOMEM; ++ } ++ ++ map = dev_maps ? xmap_dereference(dev_maps->cpu_map[cpu]) : ++ NULL; ++ ++ map = expand_xps_map(map, cpu, index); ++ if (!map) ++ goto error; ++ ++ RCU_INIT_POINTER(new_dev_maps->cpu_map[cpu], map); ++ } ++ ++ if (!new_dev_maps) ++ goto out_no_new_maps; ++ ++ for_each_possible_cpu(cpu) { ++ if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu)) { ++ /* add queue to CPU maps */ ++ int pos = 0; ++ ++ map = xmap_dereference(new_dev_maps->cpu_map[cpu]); ++ while ((pos < map->len) && (map->queues[pos] != index)) ++ pos++; ++ ++ if (pos == map->len) ++ map->queues[map->len++] = index; ++#ifdef CONFIG_NUMA ++ if (numa_node_id == -2) ++ numa_node_id = cpu_to_node(cpu); ++ else if (numa_node_id != cpu_to_node(cpu)) ++ numa_node_id = -1; ++#endif ++ } else if (dev_maps) { ++ /* fill in the new device map from the old device map */ ++ map = xmap_dereference(dev_maps->cpu_map[cpu]); ++ RCU_INIT_POINTER(new_dev_maps->cpu_map[cpu], map); ++ } ++ ++ } ++ ++ rcu_assign_pointer(dev->xps_maps, new_dev_maps); ++ ++ /* Cleanup old maps */ ++ if (dev_maps) { ++ for_each_possible_cpu(cpu) { ++ new_map = xmap_dereference(new_dev_maps->cpu_map[cpu]); ++ map = xmap_dereference(dev_maps->cpu_map[cpu]); ++ if (map && map != new_map) ++ kfree_rcu(map, rcu); ++ } ++ ++ kfree_rcu(dev_maps, rcu); ++ } ++ ++ dev_maps = new_dev_maps; ++ active = true; ++ ++out_no_new_maps: ++ /* update Tx queue numa node */ ++ netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index), ++ (numa_node_id >= 0) ? numa_node_id : ++ NUMA_NO_NODE); ++ ++ if (!dev_maps) ++ goto out_no_maps; ++ ++ /* removes queue from unused CPUs */ ++ for_each_possible_cpu(cpu) { ++ if (cpumask_test_cpu(cpu, mask) && cpu_online(cpu)) ++ continue; ++ ++ if (remove_xps_queue(dev_maps, cpu, index)) ++ active = true; ++ } ++ ++ /* free map if not active */ ++ if (!active) { ++ RCU_INIT_POINTER(dev->xps_maps, NULL); ++ kfree_rcu(dev_maps, rcu); ++ } ++ ++out_no_maps: ++ mutex_unlock(&xps_map_mutex); ++ ++ return 0; ++error: ++ /* remove any maps that we added */ ++ for_each_possible_cpu(cpu) { ++ new_map = xmap_dereference(new_dev_maps->cpu_map[cpu]); ++ map = dev_maps ? xmap_dereference(dev_maps->cpu_map[cpu]) : ++ NULL; ++ if (new_map && new_map != map) ++ kfree(new_map); ++ } ++ ++ mutex_unlock(&xps_map_mutex); ++ ++ kfree(new_dev_maps); ++ return -ENOMEM; ++} ++EXPORT_SYMBOL(netif_set_xps_queue); ++ ++#endif ++/* ++ * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues ++ * greater then real_num_tx_queues stale skbs on the qdisc must be flushed. ++ */ ++int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq) ++{ ++ int rc; ++ ++ if (txq < 1 || txq > dev->num_tx_queues) ++ return -EINVAL; ++ ++ if (dev->reg_state == NETREG_REGISTERED || ++ dev->reg_state == NETREG_UNREGISTERING) { ++ ASSERT_RTNL(); ++ ++ rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues, ++ txq); ++ if (rc) ++ return rc; ++ ++ if (dev->num_tc) ++ netif_setup_tc(dev, txq); ++ ++ if (txq < dev->real_num_tx_queues) { ++ qdisc_reset_all_tx_gt(dev, txq); ++#ifdef CONFIG_XPS ++ netif_reset_xps_queues_gt(dev, txq); ++#endif ++ } ++ } ++ ++ dev->real_num_tx_queues = txq; ++ return 0; ++} ++EXPORT_SYMBOL(netif_set_real_num_tx_queues); ++ ++#ifdef CONFIG_SYSFS ++/** ++ * netif_set_real_num_rx_queues - set actual number of RX queues used ++ * @dev: Network device ++ * @rxq: Actual number of RX queues ++ * ++ * This must be called either with the rtnl_lock held or before ++ * registration of the net device. Returns 0 on success, or a ++ * negative error code. If called before registration, it always ++ * succeeds. ++ */ ++int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq) ++{ ++ int rc; ++ ++ if (rxq < 1 || rxq > dev->num_rx_queues) ++ return -EINVAL; ++ ++ if (dev->reg_state == NETREG_REGISTERED) { ++ ASSERT_RTNL(); ++ ++ rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues, ++ rxq); ++ if (rc) ++ return rc; ++ } ++ ++ dev->real_num_rx_queues = rxq; ++ return 0; ++} ++EXPORT_SYMBOL(netif_set_real_num_rx_queues); ++#endif ++ ++/** ++ * netif_get_num_default_rss_queues - default number of RSS queues ++ * ++ * This routine should set an upper limit on the number of RSS queues ++ * used by default by multiqueue devices. ++ */ ++int netif_get_num_default_rss_queues(void) ++{ ++ return min_t(int, DEFAULT_MAX_NUM_RSS_QUEUES, num_online_cpus()); ++} ++EXPORT_SYMBOL(netif_get_num_default_rss_queues); ++ ++static inline void __netif_reschedule(struct Qdisc *q) ++{ ++ struct softnet_data *sd; ++ unsigned long flags; ++ ++ local_irq_save(flags); ++ sd = this_cpu_ptr(&softnet_data); ++ q->next_sched = NULL; ++ *sd->output_queue_tailp = q; ++ sd->output_queue_tailp = &q->next_sched; ++ raise_softirq_irqoff(NET_TX_SOFTIRQ); ++ local_irq_restore(flags); ++} ++ ++void __netif_schedule(struct Qdisc *q) ++{ ++ if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state)) ++ __netif_reschedule(q); ++} ++EXPORT_SYMBOL(__netif_schedule); ++ ++struct dev_kfree_skb_cb { ++ enum skb_free_reason reason; ++}; ++ ++static struct dev_kfree_skb_cb *get_kfree_skb_cb(const struct sk_buff *skb) ++{ ++ return (struct dev_kfree_skb_cb *)skb->cb; ++} ++ ++void netif_schedule_queue(struct netdev_queue *txq) ++{ ++ rcu_read_lock(); ++ if (!(txq->state & QUEUE_STATE_ANY_XOFF)) { ++ struct Qdisc *q = rcu_dereference(txq->qdisc); ++ ++ __netif_schedule(q); ++ } ++ rcu_read_unlock(); ++} ++EXPORT_SYMBOL(netif_schedule_queue); ++ ++/** ++ * netif_wake_subqueue - allow sending packets on subqueue ++ * @dev: network device ++ * @queue_index: sub queue index ++ * ++ * Resume individual transmit queue of a device with multiple transmit queues. ++ */ ++void netif_wake_subqueue(struct net_device *dev, u16 queue_index) ++{ ++ struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index); ++ ++ if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &txq->state)) { ++ struct Qdisc *q; ++ ++ rcu_read_lock(); ++ q = rcu_dereference(txq->qdisc); ++ __netif_schedule(q); ++ rcu_read_unlock(); ++ } ++} ++EXPORT_SYMBOL(netif_wake_subqueue); ++ ++void netif_tx_wake_queue(struct netdev_queue *dev_queue) ++{ ++ if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) { ++ struct Qdisc *q; ++ ++ rcu_read_lock(); ++ q = rcu_dereference(dev_queue->qdisc); ++ __netif_schedule(q); ++ rcu_read_unlock(); ++ } ++} ++EXPORT_SYMBOL(netif_tx_wake_queue); ++ ++void __dev_kfree_skb_irq(struct sk_buff *skb, enum skb_free_reason reason) ++{ ++ unsigned long flags; ++ ++ if (likely(atomic_read(&skb->users) == 1)) { ++ smp_rmb(); ++ atomic_set(&skb->users, 0); ++ } else if (likely(!atomic_dec_and_test(&skb->users))) { ++ return; ++ } ++ get_kfree_skb_cb(skb)->reason = reason; ++ local_irq_save(flags); ++ skb->next = __this_cpu_read(softnet_data.completion_queue); ++ __this_cpu_write(softnet_data.completion_queue, skb); ++ raise_softirq_irqoff(NET_TX_SOFTIRQ); ++ local_irq_restore(flags); ++} ++EXPORT_SYMBOL(__dev_kfree_skb_irq); ++ ++void __dev_kfree_skb_any(struct sk_buff *skb, enum skb_free_reason reason) ++{ ++ if (in_irq() || irqs_disabled()) ++ __dev_kfree_skb_irq(skb, reason); ++ else ++ dev_kfree_skb(skb); ++} ++EXPORT_SYMBOL(__dev_kfree_skb_any); ++ ++ ++/** ++ * netif_device_detach - mark device as removed ++ * @dev: network device ++ * ++ * Mark device as removed from system and therefore no longer available. ++ */ ++void netif_device_detach(struct net_device *dev) ++{ ++ if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) && ++ netif_running(dev)) { ++ netif_tx_stop_all_queues(dev); ++ } ++} ++EXPORT_SYMBOL(netif_device_detach); ++ ++/** ++ * netif_device_attach - mark device as attached ++ * @dev: network device ++ * ++ * Mark device as attached from system and restart if needed. ++ */ ++void netif_device_attach(struct net_device *dev) ++{ ++ if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) && ++ netif_running(dev)) { ++ netif_tx_wake_all_queues(dev); ++ __netdev_watchdog_up(dev); ++ } ++} ++EXPORT_SYMBOL(netif_device_attach); ++ ++static void skb_warn_bad_offload(const struct sk_buff *skb) ++{ ++ static const netdev_features_t null_features = 0; ++ struct net_device *dev = skb->dev; ++ const char *driver = ""; ++ ++ if (!net_ratelimit()) ++ return; ++ ++ if (dev && dev->dev.parent) ++ driver = dev_driver_string(dev->dev.parent); ++ ++ WARN(1, "%s: caps=(%pNF, %pNF) len=%d data_len=%d gso_size=%d " ++ "gso_type=%d ip_summed=%d\n", ++ driver, dev ? &dev->features : &null_features, ++ skb->sk ? &skb->sk->sk_route_caps : &null_features, ++ skb->len, skb->data_len, skb_shinfo(skb)->gso_size, ++ skb_shinfo(skb)->gso_type, skb->ip_summed); ++} ++ ++/* ++ * Invalidate hardware checksum when packet is to be mangled, and ++ * complete checksum manually on outgoing path. ++ */ ++int skb_checksum_help(struct sk_buff *skb) ++{ ++ __wsum csum; ++ int ret = 0, offset; ++ ++ if (skb->ip_summed == CHECKSUM_COMPLETE) ++ goto out_set_summed; ++ ++ if (unlikely(skb_shinfo(skb)->gso_size)) { ++ skb_warn_bad_offload(skb); ++ return -EINVAL; ++ } ++ ++ /* Before computing a checksum, we should make sure no frag could ++ * be modified by an external entity : checksum could be wrong. ++ */ ++ if (skb_has_shared_frag(skb)) { ++ ret = __skb_linearize(skb); ++ if (ret) ++ goto out; ++ } ++ ++ offset = skb_checksum_start_offset(skb); ++ BUG_ON(offset >= skb_headlen(skb)); ++ csum = skb_checksum(skb, offset, skb->len - offset, 0); ++ ++ offset += skb->csum_offset; ++ BUG_ON(offset + sizeof(__sum16) > skb_headlen(skb)); ++ ++ if (skb_cloned(skb) && ++ !skb_clone_writable(skb, offset + sizeof(__sum16))) { ++ ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC); ++ if (ret) ++ goto out; ++ } ++ ++ *(__sum16 *)(skb->data + offset) = csum_fold(csum); ++out_set_summed: ++ skb->ip_summed = CHECKSUM_NONE; ++out: ++ return ret; ++} ++EXPORT_SYMBOL(skb_checksum_help); ++ ++__be16 skb_network_protocol(struct sk_buff *skb, int *depth) ++{ ++ __be16 type = skb->protocol; ++ ++ /* Tunnel gso handlers can set protocol to ethernet. */ ++ if (type == htons(ETH_P_TEB)) { ++ struct ethhdr *eth; ++ ++ if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr)))) ++ return 0; ++ ++ eth = (struct ethhdr *)skb_mac_header(skb); ++ type = eth->h_proto; ++ } ++ ++ return __vlan_get_protocol(skb, type, depth); ++} ++ ++/** ++ * skb_mac_gso_segment - mac layer segmentation handler. ++ * @skb: buffer to segment ++ * @features: features for the output path (see dev->features) ++ */ ++struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb, ++ netdev_features_t features) ++{ ++ struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT); ++ struct packet_offload *ptype; ++ int vlan_depth = skb->mac_len; ++ __be16 type = skb_network_protocol(skb, &vlan_depth); ++ ++ if (unlikely(!type)) ++ return ERR_PTR(-EINVAL); ++ ++ __skb_pull(skb, vlan_depth); ++ ++ rcu_read_lock(); ++ list_for_each_entry_rcu(ptype, &offload_base, list) { ++ if (ptype->type == type && ptype->callbacks.gso_segment) { ++ segs = ptype->callbacks.gso_segment(skb, features); ++ break; ++ } ++ } ++ rcu_read_unlock(); ++ ++ __skb_push(skb, skb->data - skb_mac_header(skb)); ++ ++ return segs; ++} ++EXPORT_SYMBOL(skb_mac_gso_segment); ++ ++ ++/* openvswitch calls this on rx path, so we need a different check. ++ */ ++static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path) ++{ ++ if (tx_path) ++ return skb->ip_summed != CHECKSUM_PARTIAL; ++ else ++ return skb->ip_summed == CHECKSUM_NONE; ++} ++ ++/** ++ * __skb_gso_segment - Perform segmentation on skb. ++ * @skb: buffer to segment ++ * @features: features for the output path (see dev->features) ++ * @tx_path: whether it is called in TX path ++ * ++ * This function segments the given skb and returns a list of segments. ++ * ++ * It may return NULL if the skb requires no segmentation. This is ++ * only possible when GSO is used for verifying header integrity. ++ */ ++struct sk_buff *__skb_gso_segment(struct sk_buff *skb, ++ netdev_features_t features, bool tx_path) ++{ ++ if (unlikely(skb_needs_check(skb, tx_path))) { ++ int err; ++ ++ skb_warn_bad_offload(skb); ++ ++ err = skb_cow_head(skb, 0); ++ if (err < 0) ++ return ERR_PTR(err); ++ } ++ ++ SKB_GSO_CB(skb)->mac_offset = skb_headroom(skb); ++ SKB_GSO_CB(skb)->encap_level = 0; ++ ++ skb_reset_mac_header(skb); ++ skb_reset_mac_len(skb); ++ ++ return skb_mac_gso_segment(skb, features); ++} ++EXPORT_SYMBOL(__skb_gso_segment); ++ ++/* Take action when hardware reception checksum errors are detected. */ ++#ifdef CONFIG_BUG ++void netdev_rx_csum_fault(struct net_device *dev) ++{ ++ if (net_ratelimit()) { ++ pr_err("%s: hw csum failure\n", dev ? dev->name : ""); ++ dump_stack(); ++ } ++} ++EXPORT_SYMBOL(netdev_rx_csum_fault); ++#endif ++ ++/* Actually, we should eliminate this check as soon as we know, that: ++ * 1. IOMMU is present and allows to map all the memory. ++ * 2. No high memory really exists on this machine. ++ */ ++ ++static int illegal_highdma(struct net_device *dev, struct sk_buff *skb) ++{ ++#ifdef CONFIG_HIGHMEM ++ int i; ++ if (!(dev->features & NETIF_F_HIGHDMA)) { ++ for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { ++ skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; ++ if (PageHighMem(skb_frag_page(frag))) ++ return 1; ++ } ++ } ++ ++ if (PCI_DMA_BUS_IS_PHYS) { ++ struct device *pdev = dev->dev.parent; ++ ++ if (!pdev) ++ return 0; ++ for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { ++ skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; ++ dma_addr_t addr = page_to_phys(skb_frag_page(frag)); ++ if (!pdev->dma_mask || addr + PAGE_SIZE - 1 > *pdev->dma_mask) ++ return 1; ++ } ++ } ++#endif ++ return 0; ++} ++ ++/* If MPLS offload request, verify we are testing hardware MPLS features ++ * instead of standard features for the netdev. ++ */ ++#if IS_ENABLED(CONFIG_NET_MPLS_GSO) ++static netdev_features_t net_mpls_features(struct sk_buff *skb, ++ netdev_features_t features, ++ __be16 type) ++{ ++ if (eth_p_mpls(type)) ++ features &= skb->dev->mpls_features; ++ ++ return features; ++} ++#else ++static netdev_features_t net_mpls_features(struct sk_buff *skb, ++ netdev_features_t features, ++ __be16 type) ++{ ++ return features; ++} ++#endif ++ ++static netdev_features_t harmonize_features(struct sk_buff *skb, ++ netdev_features_t features) ++{ ++ int tmp; ++ __be16 type; ++ ++ type = skb_network_protocol(skb, &tmp); ++ features = net_mpls_features(skb, features, type); ++ ++ if (skb->ip_summed != CHECKSUM_NONE && ++ !can_checksum_protocol(features, type)) { ++ features &= ~NETIF_F_ALL_CSUM; ++ } else if (illegal_highdma(skb->dev, skb)) { ++ features &= ~NETIF_F_SG; ++ } ++ ++ return features; ++} ++ ++netdev_features_t passthru_features_check(struct sk_buff *skb, ++ struct net_device *dev, ++ netdev_features_t features) ++{ ++ return features; ++} ++EXPORT_SYMBOL(passthru_features_check); ++ ++static netdev_features_t dflt_features_check(const struct sk_buff *skb, ++ struct net_device *dev, ++ netdev_features_t features) ++{ ++ return vlan_features_check(skb, features); ++} ++ ++netdev_features_t netif_skb_features(struct sk_buff *skb) ++{ ++ struct net_device *dev = skb->dev; ++ netdev_features_t features = dev->features; ++ u16 gso_segs = skb_shinfo(skb)->gso_segs; ++ ++ if (gso_segs > dev->gso_max_segs || gso_segs < dev->gso_min_segs) ++ features &= ~NETIF_F_GSO_MASK; ++ ++ /* If encapsulation offload request, verify we are testing ++ * hardware encapsulation features instead of standard ++ * features for the netdev ++ */ ++ if (skb->encapsulation) ++ features &= dev->hw_enc_features; ++ ++ if (skb_vlan_tagged(skb)) ++ features = netdev_intersect_features(features, ++ dev->vlan_features | ++ NETIF_F_HW_VLAN_CTAG_TX | ++ NETIF_F_HW_VLAN_STAG_TX); ++ ++ if (dev->netdev_ops->ndo_features_check) ++ features &= dev->netdev_ops->ndo_features_check(skb, dev, ++ features); ++ else ++ features &= dflt_features_check(skb, dev, features); ++ ++ return harmonize_features(skb, features); ++} ++EXPORT_SYMBOL(netif_skb_features); ++ ++static int xmit_one(struct sk_buff *skb, struct net_device *dev, ++ struct netdev_queue *txq, bool more) ++{ ++ unsigned int len; ++ int rc; ++ ++ if (!list_empty(&ptype_all) || !list_empty(&dev->ptype_all)) ++ dev_queue_xmit_nit(skb, dev); ++ ++ len = skb->len; ++ trace_net_dev_start_xmit(skb, dev); ++ rc = netdev_start_xmit(skb, dev, txq, more); ++ trace_net_dev_xmit(skb, rc, dev, len); ++ ++ return rc; ++} ++ ++struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev, ++ struct netdev_queue *txq, int *ret) ++{ ++ struct sk_buff *skb = first; ++ int rc = NETDEV_TX_OK; ++ ++ while (skb) { ++ struct sk_buff *next = skb->next; ++ ++ skb->next = NULL; ++ rc = xmit_one(skb, dev, txq, next != NULL); ++ if (unlikely(!dev_xmit_complete(rc))) { ++ skb->next = next; ++ goto out; ++ } ++ ++ skb = next; ++ if (netif_xmit_stopped(txq) && skb) { ++ rc = NETDEV_TX_BUSY; ++ break; ++ } ++ } ++ ++out: ++ *ret = rc; ++ return skb; ++} ++ ++static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb, ++ netdev_features_t features) ++{ ++ if (skb_vlan_tag_present(skb) && ++ !vlan_hw_offload_capable(features, skb->vlan_proto)) ++ skb = __vlan_hwaccel_push_inside(skb); ++ return skb; ++} ++ ++static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev) ++{ ++ netdev_features_t features; ++ ++ if (skb->next) ++ return skb; ++ ++ features = netif_skb_features(skb); ++ skb = validate_xmit_vlan(skb, features); ++ if (unlikely(!skb)) ++ goto out_null; ++ ++ if (netif_needs_gso(skb, features)) { ++ struct sk_buff *segs; ++ ++ segs = skb_gso_segment(skb, features); ++ if (IS_ERR(segs)) { ++ goto out_kfree_skb; ++ } else if (segs) { ++ consume_skb(skb); ++ skb = segs; ++ } ++ } else { ++ if (skb_needs_linearize(skb, features) && ++ __skb_linearize(skb)) ++ goto out_kfree_skb; ++ ++ /* If packet is not checksummed and device does not ++ * support checksumming for this protocol, complete ++ * checksumming here. ++ */ ++ if (skb->ip_summed == CHECKSUM_PARTIAL) { ++ if (skb->encapsulation) ++ skb_set_inner_transport_header(skb, ++ skb_checksum_start_offset(skb)); ++ else ++ skb_set_transport_header(skb, ++ skb_checksum_start_offset(skb)); ++ if (!(features & NETIF_F_ALL_CSUM) && ++ skb_checksum_help(skb)) ++ goto out_kfree_skb; ++ } ++ } ++ ++ return skb; ++ ++out_kfree_skb: ++ kfree_skb(skb); ++out_null: ++ return NULL; ++} ++ ++struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev) ++{ ++ struct sk_buff *next, *head = NULL, *tail; ++ ++ for (; skb != NULL; skb = next) { ++ next = skb->next; ++ skb->next = NULL; ++ ++ /* in case skb wont be segmented, point to itself */ ++ skb->prev = skb; ++ ++ skb = validate_xmit_skb(skb, dev); ++ if (!skb) ++ continue; ++ ++ if (!head) ++ head = skb; ++ else ++ tail->next = skb; ++ /* If skb was segmented, skb->prev points to ++ * the last segment. If not, it still contains skb. ++ */ ++ tail = skb->prev; ++ } ++ return head; ++} ++ ++static void qdisc_pkt_len_init(struct sk_buff *skb) ++{ ++ const struct skb_shared_info *shinfo = skb_shinfo(skb); ++ ++ qdisc_skb_cb(skb)->pkt_len = skb->len; ++ ++ /* To get more precise estimation of bytes sent on wire, ++ * we add to pkt_len the headers size of all segments ++ */ ++ if (shinfo->gso_size) { ++ unsigned int hdr_len; ++ u16 gso_segs = shinfo->gso_segs; ++ ++ /* mac layer + network layer */ ++ hdr_len = skb_transport_header(skb) - skb_mac_header(skb); ++ ++ /* + transport layer */ ++ if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) ++ hdr_len += tcp_hdrlen(skb); ++ else ++ hdr_len += sizeof(struct udphdr); ++ ++ if (shinfo->gso_type & SKB_GSO_DODGY) ++ gso_segs = DIV_ROUND_UP(skb->len - hdr_len, ++ shinfo->gso_size); ++ ++ qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len; ++ } ++} ++ ++static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q, ++ struct net_device *dev, ++ struct netdev_queue *txq) ++{ ++ spinlock_t *root_lock = qdisc_lock(q); ++ bool contended; ++ int rc; ++ ++ qdisc_pkt_len_init(skb); ++ qdisc_calculate_pkt_len(skb, q); ++ /* ++ * Heuristic to force contended enqueues to serialize on a ++ * separate lock before trying to get qdisc main lock. ++ * This permits __QDISC___STATE_RUNNING owner to get the lock more ++ * often and dequeue packets faster. ++ */ ++ contended = qdisc_is_running(q); ++ if (unlikely(contended)) ++ spin_lock(&q->busylock); ++ ++ spin_lock(root_lock); ++ if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) { ++ kfree_skb(skb); ++ rc = NET_XMIT_DROP; ++ } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) && ++ qdisc_run_begin(q)) { ++ /* ++ * This is a work-conserving queue; there are no old skbs ++ * waiting to be sent out; and the qdisc is not running - ++ * xmit the skb directly. ++ */ ++ ++ qdisc_bstats_update(q, skb); ++ ++ if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) { ++ if (unlikely(contended)) { ++ spin_unlock(&q->busylock); ++ contended = false; ++ } ++ __qdisc_run(q); ++ } else ++ qdisc_run_end(q); ++ ++ rc = NET_XMIT_SUCCESS; ++ } else { ++ rc = q->enqueue(skb, q) & NET_XMIT_MASK; ++ if (qdisc_run_begin(q)) { ++ if (unlikely(contended)) { ++ spin_unlock(&q->busylock); ++ contended = false; ++ } ++ __qdisc_run(q); ++ } ++ } ++ spin_unlock(root_lock); ++ if (unlikely(contended)) ++ spin_unlock(&q->busylock); ++ return rc; ++} ++ ++#if IS_ENABLED(CONFIG_CGROUP_NET_PRIO) ++static void skb_update_prio(struct sk_buff *skb) ++{ ++ struct netprio_map *map = rcu_dereference_bh(skb->dev->priomap); ++ ++ if (!skb->priority && skb->sk && map) { ++ unsigned int prioidx = skb->sk->sk_cgrp_prioidx; ++ ++ if (prioidx < map->priomap_len) ++ skb->priority = map->priomap[prioidx]; ++ } ++} ++#else ++#define skb_update_prio(skb) ++#endif ++ ++DEFINE_PER_CPU(int, xmit_recursion); ++EXPORT_SYMBOL(xmit_recursion); ++ ++#define RECURSION_LIMIT 10 ++ ++/** ++ * dev_loopback_xmit - loop back @skb ++ * @skb: buffer to transmit ++ */ ++int dev_loopback_xmit(struct sock *sk, struct sk_buff *skb) ++{ ++ skb_reset_mac_header(skb); ++ __skb_pull(skb, skb_network_offset(skb)); ++ skb->pkt_type = PACKET_LOOPBACK; ++ skb->ip_summed = CHECKSUM_UNNECESSARY; ++ WARN_ON(!skb_dst(skb)); ++ skb_dst_force(skb); ++ netif_rx_ni(skb); ++ return 0; ++} ++EXPORT_SYMBOL(dev_loopback_xmit); ++ ++/** ++ * __dev_queue_xmit - transmit a buffer ++ * @skb: buffer to transmit ++ * @accel_priv: private data used for L2 forwarding offload ++ * ++ * Queue a buffer for transmission to a network device. The caller must ++ * have set the device and priority and built the buffer before calling ++ * this function. The function can be called from an interrupt. ++ * ++ * A negative errno code is returned on a failure. A success does not ++ * guarantee the frame will be transmitted as it may be dropped due ++ * to congestion or traffic shaping. ++ * ++ * ----------------------------------------------------------------------------------- ++ * I notice this method can also return errors from the queue disciplines, ++ * including NET_XMIT_DROP, which is a positive value. So, errors can also ++ * be positive. ++ * ++ * Regardless of the return value, the skb is consumed, so it is currently ++ * difficult to retry a send to this method. (You can bump the ref count ++ * before sending to hold a reference for retry if you are careful.) ++ * ++ * When calling this method, interrupts MUST be enabled. This is because ++ * the BH enable code must have IRQs enabled so that it will not deadlock. ++ * --BLG ++ */ ++static int __dev_queue_xmit(struct sk_buff *skb, void *accel_priv) ++{ ++ struct net_device *dev = skb->dev; ++ struct netdev_queue *txq; ++ struct Qdisc *q; ++ int rc = -ENOMEM; ++ ++ skb_reset_mac_header(skb); ++ ++ if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP)) ++ __skb_tstamp_tx(skb, NULL, skb->sk, SCM_TSTAMP_SCHED); ++ ++ /* Disable soft irqs for various locks below. Also ++ * stops preemption for RCU. ++ */ ++ rcu_read_lock_bh(); ++ ++ skb_update_prio(skb); ++ ++ /* If device/qdisc don't need skb->dst, release it right now while ++ * its hot in this cpu cache. ++ */ ++ if (dev->priv_flags & IFF_XMIT_DST_RELEASE) ++ skb_dst_drop(skb); ++ else ++ skb_dst_force(skb); ++ ++ txq = netdev_pick_tx(dev, skb, accel_priv); ++ q = rcu_dereference_bh(txq->qdisc); ++ ++#ifdef CONFIG_NET_CLS_ACT ++ skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_EGRESS); ++#endif ++ trace_net_dev_queue(skb); ++ if (q->enqueue) { ++ rc = __dev_xmit_skb(skb, q, dev, txq); ++ goto out; ++ } ++ ++ /* The device has no queue. Common case for software devices: ++ loopback, all the sorts of tunnels... ++ ++ Really, it is unlikely that netif_tx_lock protection is necessary ++ here. (f.e. loopback and IP tunnels are clean ignoring statistics ++ counters.) ++ However, it is possible, that they rely on protection ++ made by us here. ++ ++ Check this and shot the lock. It is not prone from deadlocks. ++ Either shot noqueue qdisc, it is even simpler 8) ++ */ ++ if (dev->flags & IFF_UP) { ++ int cpu = smp_processor_id(); /* ok because BHs are off */ ++ ++ if (txq->xmit_lock_owner != cpu) { ++ ++ if (__this_cpu_read(xmit_recursion) > RECURSION_LIMIT) ++ goto recursion_alert; ++ ++ skb = validate_xmit_skb(skb, dev); ++ if (!skb) ++ goto drop; ++ ++ HARD_TX_LOCK(dev, txq, cpu); ++ ++ if (!netif_xmit_stopped(txq)) { ++ __this_cpu_inc(xmit_recursion); ++ skb = dev_hard_start_xmit(skb, dev, txq, &rc); ++ __this_cpu_dec(xmit_recursion); ++ if (dev_xmit_complete(rc)) { ++ HARD_TX_UNLOCK(dev, txq); ++ goto out; ++ } ++ } ++ HARD_TX_UNLOCK(dev, txq); ++ net_crit_ratelimited("Virtual device %s asks to queue packet!\n", ++ dev->name); ++ } else { ++ /* Recursion is detected! It is possible, ++ * unfortunately ++ */ ++recursion_alert: ++ net_crit_ratelimited("Dead loop on virtual device %s, fix it urgently!\n", ++ dev->name); ++ } ++ } ++ ++ rc = -ENETDOWN; ++drop: ++ rcu_read_unlock_bh(); ++ ++ atomic_long_inc(&dev->tx_dropped); ++ kfree_skb_list(skb); ++ return rc; ++out: ++ rcu_read_unlock_bh(); ++ return rc; ++} ++ ++int dev_queue_xmit_sk(struct sock *sk, struct sk_buff *skb) ++{ ++ return __dev_queue_xmit(skb, NULL); ++} ++EXPORT_SYMBOL(dev_queue_xmit_sk); ++ ++int dev_queue_xmit_accel(struct sk_buff *skb, void *accel_priv) ++{ ++ return __dev_queue_xmit(skb, accel_priv); ++} ++EXPORT_SYMBOL(dev_queue_xmit_accel); ++ ++ ++/*======================================================================= ++ Receiver routines ++ =======================================================================*/ ++ ++int netdev_max_backlog __read_mostly = 1000; ++EXPORT_SYMBOL(netdev_max_backlog); ++ ++int netdev_tstamp_prequeue __read_mostly = 1; ++int netdev_budget __read_mostly = 300; ++int weight_p __read_mostly = 64; /* old backlog weight */ ++ ++/* Called with irq disabled */ ++static inline void ____napi_schedule(struct softnet_data *sd, ++ struct napi_struct *napi) ++{ ++ list_add_tail(&napi->poll_list, &sd->poll_list); ++ __raise_softirq_irqoff(NET_RX_SOFTIRQ); ++} ++ ++#ifdef CONFIG_RPS ++ ++/* One global table that all flow-based protocols share. */ ++struct rps_sock_flow_table __rcu *rps_sock_flow_table __read_mostly; ++EXPORT_SYMBOL(rps_sock_flow_table); ++u32 rps_cpu_mask __read_mostly; ++EXPORT_SYMBOL(rps_cpu_mask); ++ ++struct static_key rps_needed __read_mostly; ++ ++static struct rps_dev_flow * ++set_rps_cpu(struct net_device *dev, struct sk_buff *skb, ++ struct rps_dev_flow *rflow, u16 next_cpu) ++{ ++ if (next_cpu < nr_cpu_ids) { ++#ifdef CONFIG_RFS_ACCEL ++ struct netdev_rx_queue *rxqueue; ++ struct rps_dev_flow_table *flow_table; ++ struct rps_dev_flow *old_rflow; ++ u32 flow_id; ++ u16 rxq_index; ++ int rc; ++ ++ /* Should we steer this flow to a different hardware queue? */ ++ if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap || ++ !(dev->features & NETIF_F_NTUPLE)) ++ goto out; ++ rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu); ++ if (rxq_index == skb_get_rx_queue(skb)) ++ goto out; ++ ++ rxqueue = dev->_rx + rxq_index; ++ flow_table = rcu_dereference(rxqueue->rps_flow_table); ++ if (!flow_table) ++ goto out; ++ flow_id = skb_get_hash(skb) & flow_table->mask; ++ rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb, ++ rxq_index, flow_id); ++ if (rc < 0) ++ goto out; ++ old_rflow = rflow; ++ rflow = &flow_table->flows[flow_id]; ++ rflow->filter = rc; ++ if (old_rflow->filter == rflow->filter) ++ old_rflow->filter = RPS_NO_FILTER; ++ out: ++#endif ++ rflow->last_qtail = ++ per_cpu(softnet_data, next_cpu).input_queue_head; ++ } ++ ++ rflow->cpu = next_cpu; ++ return rflow; ++} ++ ++/* ++ * get_rps_cpu is called from netif_receive_skb and returns the target ++ * CPU from the RPS map of the receiving queue for a given skb. ++ * rcu_read_lock must be held on entry. ++ */ ++static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb, ++ struct rps_dev_flow **rflowp) ++{ ++ const struct rps_sock_flow_table *sock_flow_table; ++ struct netdev_rx_queue *rxqueue = dev->_rx; ++ struct rps_dev_flow_table *flow_table; ++ struct rps_map *map; ++ int cpu = -1; ++ u32 tcpu; ++ u32 hash; ++ ++ if (skb_rx_queue_recorded(skb)) { ++ u16 index = skb_get_rx_queue(skb); ++ ++ if (unlikely(index >= dev->real_num_rx_queues)) { ++ WARN_ONCE(dev->real_num_rx_queues > 1, ++ "%s received packet on queue %u, but number " ++ "of RX queues is %u\n", ++ dev->name, index, dev->real_num_rx_queues); ++ goto done; ++ } ++ rxqueue += index; ++ } ++ ++ /* Avoid computing hash if RFS/RPS is not active for this rxqueue */ ++ ++ flow_table = rcu_dereference(rxqueue->rps_flow_table); ++ map = rcu_dereference(rxqueue->rps_map); ++ if (!flow_table && !map) ++ goto done; ++ ++ skb_reset_network_header(skb); ++ hash = skb_get_hash(skb); ++ if (!hash) ++ goto done; ++ ++ sock_flow_table = rcu_dereference(rps_sock_flow_table); ++ if (flow_table && sock_flow_table) { ++ struct rps_dev_flow *rflow; ++ u32 next_cpu; ++ u32 ident; ++ ++ /* First check into global flow table if there is a match */ ++ ident = sock_flow_table->ents[hash & sock_flow_table->mask]; ++ if ((ident ^ hash) & ~rps_cpu_mask) ++ goto try_rps; ++ ++ next_cpu = ident & rps_cpu_mask; ++ ++ /* OK, now we know there is a match, ++ * we can look at the local (per receive queue) flow table ++ */ ++ rflow = &flow_table->flows[hash & flow_table->mask]; ++ tcpu = rflow->cpu; ++ ++ /* ++ * If the desired CPU (where last recvmsg was done) is ++ * different from current CPU (one in the rx-queue flow ++ * table entry), switch if one of the following holds: ++ * - Current CPU is unset (>= nr_cpu_ids). ++ * - Current CPU is offline. ++ * - The current CPU's queue tail has advanced beyond the ++ * last packet that was enqueued using this table entry. ++ * This guarantees that all previous packets for the flow ++ * have been dequeued, thus preserving in order delivery. ++ */ ++ if (unlikely(tcpu != next_cpu) && ++ (tcpu >= nr_cpu_ids || !cpu_online(tcpu) || ++ ((int)(per_cpu(softnet_data, tcpu).input_queue_head - ++ rflow->last_qtail)) >= 0)) { ++ tcpu = next_cpu; ++ rflow = set_rps_cpu(dev, skb, rflow, next_cpu); ++ } ++ ++ if (tcpu < nr_cpu_ids && cpu_online(tcpu)) { ++ *rflowp = rflow; ++ cpu = tcpu; ++ goto done; ++ } ++ } ++ ++try_rps: ++ ++ if (map) { ++ tcpu = map->cpus[reciprocal_scale(hash, map->len)]; ++ if (cpu_online(tcpu)) { ++ cpu = tcpu; ++ goto done; ++ } ++ } ++ ++done: ++ return cpu; ++} ++ ++#ifdef CONFIG_RFS_ACCEL ++ ++/** ++ * rps_may_expire_flow - check whether an RFS hardware filter may be removed ++ * @dev: Device on which the filter was set ++ * @rxq_index: RX queue index ++ * @flow_id: Flow ID passed to ndo_rx_flow_steer() ++ * @filter_id: Filter ID returned by ndo_rx_flow_steer() ++ * ++ * Drivers that implement ndo_rx_flow_steer() should periodically call ++ * this function for each installed filter and remove the filters for ++ * which it returns %true. ++ */ ++bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index, ++ u32 flow_id, u16 filter_id) ++{ ++ struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index; ++ struct rps_dev_flow_table *flow_table; ++ struct rps_dev_flow *rflow; ++ bool expire = true; ++ unsigned int cpu; ++ ++ rcu_read_lock(); ++ flow_table = rcu_dereference(rxqueue->rps_flow_table); ++ if (flow_table && flow_id <= flow_table->mask) { ++ rflow = &flow_table->flows[flow_id]; ++ cpu = ACCESS_ONCE(rflow->cpu); ++ if (rflow->filter == filter_id && cpu < nr_cpu_ids && ++ ((int)(per_cpu(softnet_data, cpu).input_queue_head - ++ rflow->last_qtail) < ++ (int)(10 * flow_table->mask))) ++ expire = false; ++ } ++ rcu_read_unlock(); ++ return expire; ++} ++EXPORT_SYMBOL(rps_may_expire_flow); ++ ++#endif /* CONFIG_RFS_ACCEL */ ++ ++/* Called from hardirq (IPI) context */ ++static void rps_trigger_softirq(void *data) ++{ ++ struct softnet_data *sd = data; ++ ++ ____napi_schedule(sd, &sd->backlog); ++ sd->received_rps++; ++} ++ ++#endif /* CONFIG_RPS */ ++ ++/* ++ * Check if this softnet_data structure is another cpu one ++ * If yes, queue it to our IPI list and return 1 ++ * If no, return 0 ++ */ ++static int rps_ipi_queued(struct softnet_data *sd) ++{ ++#ifdef CONFIG_RPS ++ struct softnet_data *mysd = this_cpu_ptr(&softnet_data); ++ ++ if (sd != mysd) { ++ sd->rps_ipi_next = mysd->rps_ipi_list; ++ mysd->rps_ipi_list = sd; ++ ++ __raise_softirq_irqoff(NET_RX_SOFTIRQ); ++ return 1; ++ } ++#endif /* CONFIG_RPS */ ++ return 0; ++} ++ ++#ifdef CONFIG_NET_FLOW_LIMIT ++int netdev_flow_limit_table_len __read_mostly = (1 << 12); ++#endif ++ ++static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen) ++{ ++#ifdef CONFIG_NET_FLOW_LIMIT ++ struct sd_flow_limit *fl; ++ struct softnet_data *sd; ++ unsigned int old_flow, new_flow; ++ ++ if (qlen < (netdev_max_backlog >> 1)) ++ return false; ++ ++ sd = this_cpu_ptr(&softnet_data); ++ ++ rcu_read_lock(); ++ fl = rcu_dereference(sd->flow_limit); ++ if (fl) { ++ new_flow = skb_get_hash(skb) & (fl->num_buckets - 1); ++ old_flow = fl->history[fl->history_head]; ++ fl->history[fl->history_head] = new_flow; ++ ++ fl->history_head++; ++ fl->history_head &= FLOW_LIMIT_HISTORY - 1; ++ ++ if (likely(fl->buckets[old_flow])) ++ fl->buckets[old_flow]--; ++ ++ if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) { ++ fl->count++; ++ rcu_read_unlock(); ++ return true; ++ } ++ } ++ rcu_read_unlock(); ++#endif ++ return false; ++} ++ ++/* ++ * enqueue_to_backlog is called to queue an skb to a per CPU backlog ++ * queue (may be a remote CPU queue). ++ */ ++static int enqueue_to_backlog(struct sk_buff *skb, int cpu, ++ unsigned int *qtail) ++{ ++ struct softnet_data *sd; ++ unsigned long flags; ++ unsigned int qlen; ++ ++ sd = &per_cpu(softnet_data, cpu); ++ ++ local_irq_save(flags); ++ ++ rps_lock(sd); ++ if (!netif_running(skb->dev)) ++ goto drop; ++ qlen = skb_queue_len(&sd->input_pkt_queue); ++ if (qlen <= netdev_max_backlog && !skb_flow_limit(skb, qlen)) { ++ if (qlen) { ++enqueue: ++ __skb_queue_tail(&sd->input_pkt_queue, skb); ++ input_queue_tail_incr_save(sd, qtail); ++ rps_unlock(sd); ++ local_irq_restore(flags); ++ return NET_RX_SUCCESS; ++ } ++ ++ /* Schedule NAPI for backlog device ++ * We can use non atomic operation since we own the queue lock ++ */ ++ if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) { ++ if (!rps_ipi_queued(sd)) ++ ____napi_schedule(sd, &sd->backlog); ++ } ++ goto enqueue; ++ } ++ ++drop: ++ sd->dropped++; ++ rps_unlock(sd); ++ ++ local_irq_restore(flags); ++ ++ atomic_long_inc(&skb->dev->rx_dropped); ++ kfree_skb(skb); ++ return NET_RX_DROP; ++} ++ ++static int netif_rx_internal(struct sk_buff *skb) ++{ ++ int ret; ++ ++ net_timestamp_check(netdev_tstamp_prequeue, skb); ++ ++ trace_netif_rx(skb); ++#ifdef CONFIG_RPS ++ if (static_key_false(&rps_needed)) { ++ struct rps_dev_flow voidflow, *rflow = &voidflow; ++ int cpu; ++ ++ preempt_disable(); ++ rcu_read_lock(); ++ ++ cpu = get_rps_cpu(skb->dev, skb, &rflow); ++ if (cpu < 0) ++ cpu = smp_processor_id(); ++ ++ ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail); ++ ++ rcu_read_unlock(); ++ preempt_enable(); ++ } else ++#endif ++ { ++ unsigned int qtail; ++ ret = enqueue_to_backlog(skb, get_cpu(), &qtail); ++ put_cpu(); ++ } ++ return ret; ++} ++ ++/** ++ * netif_rx - post buffer to the network code ++ * @skb: buffer to post ++ * ++ * This function receives a packet from a device driver and queues it for ++ * the upper (protocol) levels to process. It always succeeds. The buffer ++ * may be dropped during processing for congestion control or by the ++ * protocol layers. ++ * ++ * return values: ++ * NET_RX_SUCCESS (no congestion) ++ * NET_RX_DROP (packet was dropped) ++ * ++ */ ++ ++int netif_rx(struct sk_buff *skb) ++{ ++ trace_netif_rx_entry(skb); ++ ++ return netif_rx_internal(skb); ++} ++EXPORT_SYMBOL(netif_rx); ++ ++int netif_rx_ni(struct sk_buff *skb) ++{ ++ int err; ++ ++ trace_netif_rx_ni_entry(skb); ++ ++ preempt_disable(); ++ err = netif_rx_internal(skb); ++ if (local_softirq_pending()) ++ do_softirq(); ++ preempt_enable(); ++ ++ return err; ++} ++EXPORT_SYMBOL(netif_rx_ni); ++ ++static void net_tx_action(struct softirq_action *h) ++{ ++ struct softnet_data *sd = this_cpu_ptr(&softnet_data); ++ ++ if (sd->completion_queue) { ++ struct sk_buff *clist; ++ ++ local_irq_disable(); ++ clist = sd->completion_queue; ++ sd->completion_queue = NULL; ++ local_irq_enable(); ++ ++ while (clist) { ++ struct sk_buff *skb = clist; ++ clist = clist->next; ++ ++ WARN_ON(atomic_read(&skb->users)); ++ if (likely(get_kfree_skb_cb(skb)->reason == SKB_REASON_CONSUMED)) ++ trace_consume_skb(skb); ++ else ++ trace_kfree_skb(skb, net_tx_action); ++ __kfree_skb(skb); ++ } ++ } ++ ++ if (sd->output_queue) { ++ struct Qdisc *head; ++ ++ local_irq_disable(); ++ head = sd->output_queue; ++ sd->output_queue = NULL; ++ sd->output_queue_tailp = &sd->output_queue; ++ local_irq_enable(); ++ ++ while (head) { ++ struct Qdisc *q = head; ++ spinlock_t *root_lock; ++ ++ head = head->next_sched; ++ ++ root_lock = qdisc_lock(q); ++ if (spin_trylock(root_lock)) { ++ smp_mb__before_atomic(); ++ clear_bit(__QDISC_STATE_SCHED, ++ &q->state); ++ qdisc_run(q); ++ spin_unlock(root_lock); ++ } else { ++ if (!test_bit(__QDISC_STATE_DEACTIVATED, ++ &q->state)) { ++ __netif_reschedule(q); ++ } else { ++ smp_mb__before_atomic(); ++ clear_bit(__QDISC_STATE_SCHED, ++ &q->state); ++ } ++ } ++ } ++ } ++} ++ ++#if (defined(CONFIG_BRIDGE) || defined(CONFIG_BRIDGE_MODULE)) && \ ++ (defined(CONFIG_ATM_LANE) || defined(CONFIG_ATM_LANE_MODULE)) ++/* This hook is defined here for ATM LANE */ ++int (*br_fdb_test_addr_hook)(struct net_device *dev, ++ unsigned char *addr) __read_mostly; ++EXPORT_SYMBOL_GPL(br_fdb_test_addr_hook); ++#endif ++ ++#ifdef CONFIG_NET_CLS_ACT ++/* TODO: Maybe we should just force sch_ingress to be compiled in ++ * when CONFIG_NET_CLS_ACT is? otherwise some useless instructions ++ * a compare and 2 stores extra right now if we dont have it on ++ * but have CONFIG_NET_CLS_ACT ++ * NOTE: This doesn't stop any functionality; if you dont have ++ * the ingress scheduler, you just can't add policies on ingress. ++ * ++ */ ++static int ing_filter(struct sk_buff *skb, struct netdev_queue *rxq) ++{ ++ struct net_device *dev = skb->dev; ++ u32 ttl = G_TC_RTTL(skb->tc_verd); ++ int result = TC_ACT_OK; ++ struct Qdisc *q; ++ ++ if (unlikely(MAX_RED_LOOP < ttl++)) { ++ net_warn_ratelimited("Redir loop detected Dropping packet (%d->%d)\n", ++ skb->skb_iif, dev->ifindex); ++ return TC_ACT_SHOT; ++ } ++ ++ skb->tc_verd = SET_TC_RTTL(skb->tc_verd, ttl); ++ skb->tc_verd = SET_TC_AT(skb->tc_verd, AT_INGRESS); ++ ++ q = rcu_dereference(rxq->qdisc); ++ if (q != &noop_qdisc) { ++ spin_lock(qdisc_lock(q)); ++ if (likely(!test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) ++ result = qdisc_enqueue_root(skb, q); ++ spin_unlock(qdisc_lock(q)); ++ } ++ ++ return result; ++} ++ ++static inline struct sk_buff *handle_ing(struct sk_buff *skb, ++ struct packet_type **pt_prev, ++ int *ret, struct net_device *orig_dev) ++{ ++ struct netdev_queue *rxq = rcu_dereference(skb->dev->ingress_queue); ++ ++ if (!rxq || rcu_access_pointer(rxq->qdisc) == &noop_qdisc) ++ return skb; ++ ++ if (*pt_prev) { ++ *ret = deliver_skb(skb, *pt_prev, orig_dev); ++ *pt_prev = NULL; ++ } ++ ++ switch (ing_filter(skb, rxq)) { ++ case TC_ACT_SHOT: ++ case TC_ACT_STOLEN: ++ kfree_skb(skb); ++ return NULL; ++ } ++ ++ return skb; ++} ++#endif ++ ++/** ++ * netdev_rx_handler_register - register receive handler ++ * @dev: device to register a handler for ++ * @rx_handler: receive handler to register ++ * @rx_handler_data: data pointer that is used by rx handler ++ * ++ * Register a receive handler for a device. This handler will then be ++ * called from __netif_receive_skb. A negative errno code is returned ++ * on a failure. ++ * ++ * The caller must hold the rtnl_mutex. ++ * ++ * For a general description of rx_handler, see enum rx_handler_result. ++ */ ++int netdev_rx_handler_register(struct net_device *dev, ++ rx_handler_func_t *rx_handler, ++ void *rx_handler_data) ++{ ++ ASSERT_RTNL(); ++ ++ if (dev->rx_handler) ++ return -EBUSY; ++ ++ /* Note: rx_handler_data must be set before rx_handler */ ++ rcu_assign_pointer(dev->rx_handler_data, rx_handler_data); ++ rcu_assign_pointer(dev->rx_handler, rx_handler); ++ ++ return 0; ++} ++EXPORT_SYMBOL_GPL(netdev_rx_handler_register); ++ ++/** ++ * netdev_rx_handler_unregister - unregister receive handler ++ * @dev: device to unregister a handler from ++ * ++ * Unregister a receive handler from a device. ++ * ++ * The caller must hold the rtnl_mutex. ++ */ ++void netdev_rx_handler_unregister(struct net_device *dev) ++{ ++ ++ ASSERT_RTNL(); ++ RCU_INIT_POINTER(dev->rx_handler, NULL); ++ /* a reader seeing a non NULL rx_handler in a rcu_read_lock() ++ * section has a guarantee to see a non NULL rx_handler_data ++ * as well. ++ */ ++ synchronize_net(); ++ RCU_INIT_POINTER(dev->rx_handler_data, NULL); ++} ++EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister); ++ ++/* ++ * Limit the use of PFMEMALLOC reserves to those protocols that implement ++ * the special handling of PFMEMALLOC skbs. ++ */ ++static bool skb_pfmemalloc_protocol(struct sk_buff *skb) ++{ ++ switch (skb->protocol) { ++ case htons(ETH_P_ARP): ++ case htons(ETH_P_IP): ++ case htons(ETH_P_IPV6): ++ case htons(ETH_P_8021Q): ++ case htons(ETH_P_8021AD): ++ return true; ++ default: ++ return false; ++ } ++} ++ ++static int __netif_receive_skb_core(struct sk_buff *skb, bool pfmemalloc) ++{ ++ struct packet_type *ptype, *pt_prev; ++ rx_handler_func_t *rx_handler; ++ struct net_device *orig_dev; ++ bool deliver_exact = false; ++ int ret = NET_RX_DROP; ++ __be16 type; ++ ++ net_timestamp_check(!netdev_tstamp_prequeue, skb); ++ ++ trace_netif_receive_skb(skb); ++ ++ orig_dev = skb->dev; ++ ++ skb_reset_network_header(skb); ++ if (!skb_transport_header_was_set(skb)) ++ skb_reset_transport_header(skb); ++ skb_reset_mac_len(skb); ++ ++ pt_prev = NULL; ++ ++another_round: ++ skb->skb_iif = skb->dev->ifindex; ++ ++ __this_cpu_inc(softnet_data.processed); ++ ++ if (skb->protocol == cpu_to_be16(ETH_P_8021Q) || ++ skb->protocol == cpu_to_be16(ETH_P_8021AD)) { ++ skb = skb_vlan_untag(skb); ++ if (unlikely(!skb)) ++ goto out; ++ } ++ ++#ifdef CONFIG_NET_CLS_ACT ++ if (skb->tc_verd & TC_NCLS) { ++ skb->tc_verd = CLR_TC_NCLS(skb->tc_verd); ++ goto ncls; ++ } ++#endif ++ ++ if (pfmemalloc) ++ goto skip_taps; ++ ++ list_for_each_entry_rcu(ptype, &ptype_all, list) { ++ if (pt_prev) ++ ret = deliver_skb(skb, pt_prev, orig_dev); ++ pt_prev = ptype; ++ } ++ ++ list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) { ++ if (pt_prev) ++ ret = deliver_skb(skb, pt_prev, orig_dev); ++ pt_prev = ptype; ++ } ++ ++skip_taps: ++#ifdef CONFIG_NET_CLS_ACT ++ if (static_key_false(&ingress_needed)) { ++ skb = handle_ing(skb, &pt_prev, &ret, orig_dev); ++ if (!skb) ++ goto out; ++ } ++ ++ skb->tc_verd = 0; ++ncls: ++#endif ++ if (pfmemalloc && !skb_pfmemalloc_protocol(skb)) ++ goto drop; ++ ++ if (skb_vlan_tag_present(skb)) { ++ if (pt_prev) { ++ ret = deliver_skb(skb, pt_prev, orig_dev); ++ pt_prev = NULL; ++ } ++ if (vlan_do_receive(&skb)) ++ goto another_round; ++ else if (unlikely(!skb)) ++ goto out; ++ } ++ ++ rx_handler = rcu_dereference(skb->dev->rx_handler); ++ if (rx_handler) { ++ if (pt_prev) { ++ ret = deliver_skb(skb, pt_prev, orig_dev); ++ pt_prev = NULL; ++ } ++ switch (rx_handler(&skb)) { ++ case RX_HANDLER_CONSUMED: ++ ret = NET_RX_SUCCESS; ++ goto out; ++ case RX_HANDLER_ANOTHER: ++ goto another_round; ++ case RX_HANDLER_EXACT: ++ deliver_exact = true; ++ case RX_HANDLER_PASS: ++ break; ++ default: ++ BUG(); ++ } ++ } ++ ++ if (unlikely(skb_vlan_tag_present(skb))) { ++ if (skb_vlan_tag_get_id(skb)) ++ skb->pkt_type = PACKET_OTHERHOST; ++ /* Note: we might in the future use prio bits ++ * and set skb->priority like in vlan_do_receive() ++ * For the time being, just ignore Priority Code Point ++ */ ++ skb->vlan_tci = 0; ++ } ++ ++ type = skb->protocol; ++ ++ /* deliver only exact match when indicated */ ++ if (likely(!deliver_exact)) { ++ deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type, ++ &ptype_base[ntohs(type) & ++ PTYPE_HASH_MASK]); ++ } ++ ++ deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type, ++ &orig_dev->ptype_specific); ++ ++ if (unlikely(skb->dev != orig_dev)) { ++ deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type, ++ &skb->dev->ptype_specific); ++ } ++ ++ if (pt_prev) { ++ if (unlikely(skb_orphan_frags(skb, GFP_ATOMIC))) ++ goto drop; ++ else ++ ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev); ++ } else { ++drop: ++ atomic_long_inc(&skb->dev->rx_dropped); ++ kfree_skb(skb); ++ /* Jamal, now you will not able to escape explaining ++ * me how you were going to use this. :-) ++ */ ++ ret = NET_RX_DROP; ++ } ++ ++out: ++ return ret; ++} ++ ++static int __netif_receive_skb(struct sk_buff *skb) ++{ ++ int ret; ++ ++ if (sk_memalloc_socks() && skb_pfmemalloc(skb)) { ++ unsigned long pflags = current->flags; ++ ++ /* ++ * PFMEMALLOC skbs are special, they should ++ * - be delivered to SOCK_MEMALLOC sockets only ++ * - stay away from userspace ++ * - have bounded memory usage ++ * ++ * Use PF_MEMALLOC as this saves us from propagating the allocation ++ * context down to all allocation sites. ++ */ ++ current->flags |= PF_MEMALLOC; ++ ret = __netif_receive_skb_core(skb, true); ++ tsk_restore_flags(current, pflags, PF_MEMALLOC); ++ } else ++ ret = __netif_receive_skb_core(skb, false); ++ ++ return ret; ++} ++ ++static int netif_receive_skb_internal(struct sk_buff *skb) ++{ ++ int ret; ++ ++ net_timestamp_check(netdev_tstamp_prequeue, skb); ++ ++ if (skb_defer_rx_timestamp(skb)) ++ return NET_RX_SUCCESS; ++ ++ rcu_read_lock(); ++ ++#ifdef CONFIG_RPS ++ if (static_key_false(&rps_needed)) { ++ struct rps_dev_flow voidflow, *rflow = &voidflow; ++ int cpu = get_rps_cpu(skb->dev, skb, &rflow); ++ ++ if (cpu >= 0) { ++ ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail); ++ rcu_read_unlock(); ++ return ret; ++ } ++ } ++#endif ++ ret = __netif_receive_skb(skb); ++ rcu_read_unlock(); ++ return ret; ++} ++ ++/** ++ * netif_receive_skb - process receive buffer from network ++ * @skb: buffer to process ++ * ++ * netif_receive_skb() is the main receive data processing function. ++ * It always succeeds. The buffer may be dropped during processing ++ * for congestion control or by the protocol layers. ++ * ++ * This function may only be called from softirq context and interrupts ++ * should be enabled. ++ * ++ * Return values (usually ignored): ++ * NET_RX_SUCCESS: no congestion ++ * NET_RX_DROP: packet was dropped ++ */ ++int netif_receive_skb_sk(struct sock *sk, struct sk_buff *skb) ++{ ++ trace_netif_receive_skb_entry(skb); ++ ++ return netif_receive_skb_internal(skb); ++} ++EXPORT_SYMBOL(netif_receive_skb_sk); ++ ++/* Network device is going away, flush any packets still pending ++ * Called with irqs disabled. ++ */ ++static void flush_backlog(void *arg) ++{ ++ struct net_device *dev = arg; ++ struct softnet_data *sd = this_cpu_ptr(&softnet_data); ++ struct sk_buff *skb, *tmp; ++ ++ rps_lock(sd); ++ skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) { ++ if (skb->dev == dev) { ++ __skb_unlink(skb, &sd->input_pkt_queue); ++ kfree_skb(skb); ++ input_queue_head_incr(sd); ++ } ++ } ++ rps_unlock(sd); ++ ++ skb_queue_walk_safe(&sd->process_queue, skb, tmp) { ++ if (skb->dev == dev) { ++ __skb_unlink(skb, &sd->process_queue); ++ kfree_skb(skb); ++ input_queue_head_incr(sd); ++ } ++ } ++} ++ ++static int napi_gro_complete(struct sk_buff *skb) ++{ ++ struct packet_offload *ptype; ++ __be16 type = skb->protocol; ++ struct list_head *head = &offload_base; ++ int err = -ENOENT; ++ ++ BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb)); ++ ++ if (NAPI_GRO_CB(skb)->count == 1) { ++ skb_shinfo(skb)->gso_size = 0; ++ goto out; ++ } ++ ++ rcu_read_lock(); ++ list_for_each_entry_rcu(ptype, head, list) { ++ if (ptype->type != type || !ptype->callbacks.gro_complete) ++ continue; ++ ++ err = ptype->callbacks.gro_complete(skb, 0); ++ break; ++ } ++ rcu_read_unlock(); ++ ++ if (err) { ++ WARN_ON(&ptype->list == head); ++ kfree_skb(skb); ++ return NET_RX_SUCCESS; ++ } ++ ++out: ++ return netif_receive_skb_internal(skb); ++} ++ ++/* napi->gro_list contains packets ordered by age. ++ * youngest packets at the head of it. ++ * Complete skbs in reverse order to reduce latencies. ++ */ ++void napi_gro_flush(struct napi_struct *napi, bool flush_old) ++{ ++ struct sk_buff *skb, *prev = NULL; ++ ++ /* scan list and build reverse chain */ ++ for (skb = napi->gro_list; skb != NULL; skb = skb->next) { ++ skb->prev = prev; ++ prev = skb; ++ } ++ ++ for (skb = prev; skb; skb = prev) { ++ skb->next = NULL; ++ ++ if (flush_old && NAPI_GRO_CB(skb)->age == jiffies) ++ return; ++ ++ prev = skb->prev; ++ napi_gro_complete(skb); ++ napi->gro_count--; ++ } ++ ++ napi->gro_list = NULL; ++} ++EXPORT_SYMBOL(napi_gro_flush); ++ ++static void gro_list_prepare(struct napi_struct *napi, struct sk_buff *skb) ++{ ++ struct sk_buff *p; ++ unsigned int maclen = skb->dev->hard_header_len; ++ u32 hash = skb_get_hash_raw(skb); ++ ++ for (p = napi->gro_list; p; p = p->next) { ++ unsigned long diffs; ++ ++ NAPI_GRO_CB(p)->flush = 0; ++ ++ if (hash != skb_get_hash_raw(p)) { ++ NAPI_GRO_CB(p)->same_flow = 0; ++ continue; ++ } ++ ++ diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev; ++ diffs |= p->vlan_tci ^ skb->vlan_tci; ++ if (maclen == ETH_HLEN) ++ diffs |= compare_ether_header(skb_mac_header(p), ++ skb_mac_header(skb)); ++ else if (!diffs) ++ diffs = memcmp(skb_mac_header(p), ++ skb_mac_header(skb), ++ maclen); ++ NAPI_GRO_CB(p)->same_flow = !diffs; ++ } ++} ++ ++static void skb_gro_reset_offset(struct sk_buff *skb) ++{ ++ const struct skb_shared_info *pinfo = skb_shinfo(skb); ++ const skb_frag_t *frag0 = &pinfo->frags[0]; ++ ++ NAPI_GRO_CB(skb)->data_offset = 0; ++ NAPI_GRO_CB(skb)->frag0 = NULL; ++ NAPI_GRO_CB(skb)->frag0_len = 0; ++ ++ if (skb_mac_header(skb) == skb_tail_pointer(skb) && ++ pinfo->nr_frags && ++ !PageHighMem(skb_frag_page(frag0))) { ++ NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0); ++ NAPI_GRO_CB(skb)->frag0_len = skb_frag_size(frag0); ++ } ++} ++ ++static void gro_pull_from_frag0(struct sk_buff *skb, int grow) ++{ ++ struct skb_shared_info *pinfo = skb_shinfo(skb); ++ ++ BUG_ON(skb->end - skb->tail < grow); ++ ++ memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow); ++ ++ skb->data_len -= grow; ++ skb->tail += grow; ++ ++ pinfo->frags[0].page_offset += grow; ++ skb_frag_size_sub(&pinfo->frags[0], grow); ++ ++ if (unlikely(!skb_frag_size(&pinfo->frags[0]))) { ++ skb_frag_unref(skb, 0); ++ memmove(pinfo->frags, pinfo->frags + 1, ++ --pinfo->nr_frags * sizeof(pinfo->frags[0])); ++ } ++} ++ ++static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb) ++{ ++ struct sk_buff **pp = NULL; ++ struct packet_offload *ptype; ++ __be16 type = skb->protocol; ++ struct list_head *head = &offload_base; ++ int same_flow; ++ enum gro_result ret; ++ int grow; ++ ++ if (!(skb->dev->features & NETIF_F_GRO)) ++ goto normal; ++ ++ if (skb_is_gso(skb) || skb_has_frag_list(skb) || skb->csum_bad) ++ goto normal; ++ ++ gro_list_prepare(napi, skb); ++ ++ rcu_read_lock(); ++ list_for_each_entry_rcu(ptype, head, list) { ++ if (ptype->type != type || !ptype->callbacks.gro_receive) ++ continue; ++ ++ skb_set_network_header(skb, skb_gro_offset(skb)); ++ skb_reset_mac_len(skb); ++ NAPI_GRO_CB(skb)->same_flow = 0; ++ NAPI_GRO_CB(skb)->flush = 0; ++ NAPI_GRO_CB(skb)->free = 0; ++ NAPI_GRO_CB(skb)->udp_mark = 0; ++ NAPI_GRO_CB(skb)->gro_remcsum_start = 0; ++ ++ /* Setup for GRO checksum validation */ ++ switch (skb->ip_summed) { ++ case CHECKSUM_COMPLETE: ++ NAPI_GRO_CB(skb)->csum = skb->csum; ++ NAPI_GRO_CB(skb)->csum_valid = 1; ++ NAPI_GRO_CB(skb)->csum_cnt = 0; ++ break; ++ case CHECKSUM_UNNECESSARY: ++ NAPI_GRO_CB(skb)->csum_cnt = skb->csum_level + 1; ++ NAPI_GRO_CB(skb)->csum_valid = 0; ++ break; ++ default: ++ NAPI_GRO_CB(skb)->csum_cnt = 0; ++ NAPI_GRO_CB(skb)->csum_valid = 0; ++ } ++ ++ pp = ptype->callbacks.gro_receive(&napi->gro_list, skb); ++ break; ++ } ++ rcu_read_unlock(); ++ ++ if (&ptype->list == head) ++ goto normal; ++ ++ same_flow = NAPI_GRO_CB(skb)->same_flow; ++ ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED; ++ ++ if (pp) { ++ struct sk_buff *nskb = *pp; ++ ++ *pp = nskb->next; ++ nskb->next = NULL; ++ napi_gro_complete(nskb); ++ napi->gro_count--; ++ } ++ ++ if (same_flow) ++ goto ok; ++ ++ if (NAPI_GRO_CB(skb)->flush) ++ goto normal; ++ ++ if (unlikely(napi->gro_count >= MAX_GRO_SKBS)) { ++ struct sk_buff *nskb = napi->gro_list; ++ ++ /* locate the end of the list to select the 'oldest' flow */ ++ while (nskb->next) { ++ pp = &nskb->next; ++ nskb = *pp; ++ } ++ *pp = NULL; ++ nskb->next = NULL; ++ napi_gro_complete(nskb); ++ } else { ++ napi->gro_count++; ++ } ++ NAPI_GRO_CB(skb)->count = 1; ++ NAPI_GRO_CB(skb)->age = jiffies; ++ NAPI_GRO_CB(skb)->last = skb; ++ skb_shinfo(skb)->gso_size = skb_gro_len(skb); ++ skb->next = napi->gro_list; ++ napi->gro_list = skb; ++ ret = GRO_HELD; ++ ++pull: ++ grow = skb_gro_offset(skb) - skb_headlen(skb); ++ if (grow > 0) ++ gro_pull_from_frag0(skb, grow); ++ok: ++ return ret; ++ ++normal: ++ ret = GRO_NORMAL; ++ goto pull; ++} ++ ++struct packet_offload *gro_find_receive_by_type(__be16 type) ++{ ++ struct list_head *offload_head = &offload_base; ++ struct packet_offload *ptype; ++ ++ list_for_each_entry_rcu(ptype, offload_head, list) { ++ if (ptype->type != type || !ptype->callbacks.gro_receive) ++ continue; ++ return ptype; ++ } ++ return NULL; ++} ++EXPORT_SYMBOL(gro_find_receive_by_type); ++ ++struct packet_offload *gro_find_complete_by_type(__be16 type) ++{ ++ struct list_head *offload_head = &offload_base; ++ struct packet_offload *ptype; ++ ++ list_for_each_entry_rcu(ptype, offload_head, list) { ++ if (ptype->type != type || !ptype->callbacks.gro_complete) ++ continue; ++ return ptype; ++ } ++ return NULL; ++} ++EXPORT_SYMBOL(gro_find_complete_by_type); ++ ++static gro_result_t napi_skb_finish(gro_result_t ret, struct sk_buff *skb) ++{ ++ switch (ret) { ++ case GRO_NORMAL: ++ if (netif_receive_skb_internal(skb)) ++ ret = GRO_DROP; ++ break; ++ ++ case GRO_DROP: ++ kfree_skb(skb); ++ break; ++ ++ case GRO_MERGED_FREE: ++ if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD) ++ kmem_cache_free(skbuff_head_cache, skb); ++ else ++ __kfree_skb(skb); ++ break; ++ ++ case GRO_HELD: ++ case GRO_MERGED: ++ break; ++ } ++ ++ return ret; ++} ++ ++gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb) ++{ ++ trace_napi_gro_receive_entry(skb); ++ ++ skb_gro_reset_offset(skb); ++ ++ return napi_skb_finish(dev_gro_receive(napi, skb), skb); ++} ++EXPORT_SYMBOL(napi_gro_receive); ++ ++static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb) ++{ ++ if (unlikely(skb->pfmemalloc)) { ++ consume_skb(skb); ++ return; ++ } ++ __skb_pull(skb, skb_headlen(skb)); ++ /* restore the reserve we had after netdev_alloc_skb_ip_align() */ ++ skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb)); ++ skb->vlan_tci = 0; ++ skb->dev = napi->dev; ++ skb->skb_iif = 0; ++ skb->encapsulation = 0; ++ skb_shinfo(skb)->gso_type = 0; ++ skb->truesize = SKB_TRUESIZE(skb_end_offset(skb)); ++ ++ napi->skb = skb; ++} ++ ++struct sk_buff *napi_get_frags(struct napi_struct *napi) ++{ ++ struct sk_buff *skb = napi->skb; ++ ++ if (!skb) { ++ skb = napi_alloc_skb(napi, GRO_MAX_HEAD); ++ napi->skb = skb; ++ } ++ return skb; ++} ++EXPORT_SYMBOL(napi_get_frags); ++ ++static gro_result_t napi_frags_finish(struct napi_struct *napi, ++ struct sk_buff *skb, ++ gro_result_t ret) ++{ ++ switch (ret) { ++ case GRO_NORMAL: ++ case GRO_HELD: ++ __skb_push(skb, ETH_HLEN); ++ skb->protocol = eth_type_trans(skb, skb->dev); ++ if (ret == GRO_NORMAL && netif_receive_skb_internal(skb)) ++ ret = GRO_DROP; ++ break; ++ ++ case GRO_DROP: ++ case GRO_MERGED_FREE: ++ napi_reuse_skb(napi, skb); ++ break; ++ ++ case GRO_MERGED: ++ break; ++ } ++ ++ return ret; ++} ++ ++/* Upper GRO stack assumes network header starts at gro_offset=0 ++ * Drivers could call both napi_gro_frags() and napi_gro_receive() ++ * We copy ethernet header into skb->data to have a common layout. ++ */ ++static struct sk_buff *napi_frags_skb(struct napi_struct *napi) ++{ ++ struct sk_buff *skb = napi->skb; ++ const struct ethhdr *eth; ++ unsigned int hlen = sizeof(*eth); ++ ++ napi->skb = NULL; ++ ++ skb_reset_mac_header(skb); ++ skb_gro_reset_offset(skb); ++ ++ eth = skb_gro_header_fast(skb, 0); ++ if (unlikely(skb_gro_header_hard(skb, hlen))) { ++ eth = skb_gro_header_slow(skb, hlen, 0); ++ if (unlikely(!eth)) { ++ napi_reuse_skb(napi, skb); ++ return NULL; ++ } ++ } else { ++ gro_pull_from_frag0(skb, hlen); ++ NAPI_GRO_CB(skb)->frag0 += hlen; ++ NAPI_GRO_CB(skb)->frag0_len -= hlen; ++ } ++ __skb_pull(skb, hlen); ++ ++ /* ++ * This works because the only protocols we care about don't require ++ * special handling. ++ * We'll fix it up properly in napi_frags_finish() ++ */ ++ skb->protocol = eth->h_proto; ++ ++ return skb; ++} ++ ++gro_result_t napi_gro_frags(struct napi_struct *napi) ++{ ++ struct sk_buff *skb = napi_frags_skb(napi); ++ ++ if (!skb) ++ return GRO_DROP; ++ ++ trace_napi_gro_frags_entry(skb); ++ ++ return napi_frags_finish(napi, skb, dev_gro_receive(napi, skb)); ++} ++EXPORT_SYMBOL(napi_gro_frags); ++ ++/* Compute the checksum from gro_offset and return the folded value ++ * after adding in any pseudo checksum. ++ */ ++__sum16 __skb_gro_checksum_complete(struct sk_buff *skb) ++{ ++ __wsum wsum; ++ __sum16 sum; ++ ++ wsum = skb_checksum(skb, skb_gro_offset(skb), skb_gro_len(skb), 0); ++ ++ /* NAPI_GRO_CB(skb)->csum holds pseudo checksum */ ++ sum = csum_fold(csum_add(NAPI_GRO_CB(skb)->csum, wsum)); ++ if (likely(!sum)) { ++ if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) && ++ !skb->csum_complete_sw) ++ netdev_rx_csum_fault(skb->dev); ++ } ++ ++ NAPI_GRO_CB(skb)->csum = wsum; ++ NAPI_GRO_CB(skb)->csum_valid = 1; ++ ++ return sum; ++} ++EXPORT_SYMBOL(__skb_gro_checksum_complete); ++ ++/* ++ * net_rps_action_and_irq_enable sends any pending IPI's for rps. ++ * Note: called with local irq disabled, but exits with local irq enabled. ++ */ ++static void net_rps_action_and_irq_enable(struct softnet_data *sd) ++{ ++#ifdef CONFIG_RPS ++ struct softnet_data *remsd = sd->rps_ipi_list; ++ ++ if (remsd) { ++ sd->rps_ipi_list = NULL; ++ ++ local_irq_enable(); ++ ++ /* Send pending IPI's to kick RPS processing on remote cpus. */ ++ while (remsd) { ++ struct softnet_data *next = remsd->rps_ipi_next; ++ ++ if (cpu_online(remsd->cpu)) ++ smp_call_function_single_async(remsd->cpu, ++ &remsd->csd); ++ remsd = next; ++ } ++ } else ++#endif ++ local_irq_enable(); ++} ++ ++static bool sd_has_rps_ipi_waiting(struct softnet_data *sd) ++{ ++#ifdef CONFIG_RPS ++ return sd->rps_ipi_list != NULL; ++#else ++ return false; ++#endif ++} ++ ++static int process_backlog(struct napi_struct *napi, int quota) ++{ ++ int work = 0; ++ struct softnet_data *sd = container_of(napi, struct softnet_data, backlog); ++ ++ /* Check if we have pending ipi, its better to send them now, ++ * not waiting net_rx_action() end. ++ */ ++ if (sd_has_rps_ipi_waiting(sd)) { ++ local_irq_disable(); ++ net_rps_action_and_irq_enable(sd); ++ } ++ ++ napi->weight = weight_p; ++ local_irq_disable(); ++ while (1) { ++ struct sk_buff *skb; ++ ++ while ((skb = __skb_dequeue(&sd->process_queue))) { ++ rcu_read_lock(); ++ local_irq_enable(); ++ __netif_receive_skb(skb); ++ rcu_read_unlock(); ++ local_irq_disable(); ++ input_queue_head_incr(sd); ++ if (++work >= quota) { ++ local_irq_enable(); ++ return work; ++ } ++ } ++ ++ rps_lock(sd); ++ if (skb_queue_empty(&sd->input_pkt_queue)) { ++ /* ++ * Inline a custom version of __napi_complete(). ++ * only current cpu owns and manipulates this napi, ++ * and NAPI_STATE_SCHED is the only possible flag set ++ * on backlog. ++ * We can use a plain write instead of clear_bit(), ++ * and we dont need an smp_mb() memory barrier. ++ */ ++ napi->state = 0; ++ rps_unlock(sd); ++ ++ break; ++ } ++ ++ skb_queue_splice_tail_init(&sd->input_pkt_queue, ++ &sd->process_queue); ++ rps_unlock(sd); ++ } ++ local_irq_enable(); ++ ++ return work; ++} ++ ++/** ++ * __napi_schedule - schedule for receive ++ * @n: entry to schedule ++ * ++ * The entry's receive function will be scheduled to run. ++ * Consider using __napi_schedule_irqoff() if hard irqs are masked. ++ */ ++void __napi_schedule(struct napi_struct *n) ++{ ++ unsigned long flags; ++ ++ local_irq_save(flags); ++ ____napi_schedule(this_cpu_ptr(&softnet_data), n); ++ local_irq_restore(flags); ++} ++EXPORT_SYMBOL(__napi_schedule); ++ ++/** ++ * __napi_schedule_irqoff - schedule for receive ++ * @n: entry to schedule ++ * ++ * Variant of __napi_schedule() assuming hard irqs are masked ++ */ ++void __napi_schedule_irqoff(struct napi_struct *n) ++{ ++ ____napi_schedule(this_cpu_ptr(&softnet_data), n); ++} ++EXPORT_SYMBOL(__napi_schedule_irqoff); ++ ++void __napi_complete(struct napi_struct *n) ++{ ++ BUG_ON(!test_bit(NAPI_STATE_SCHED, &n->state)); ++ ++ list_del_init(&n->poll_list); ++ smp_mb__before_atomic(); ++ clear_bit(NAPI_STATE_SCHED, &n->state); ++} ++EXPORT_SYMBOL(__napi_complete); ++ ++void napi_complete_done(struct napi_struct *n, int work_done) ++{ ++ unsigned long flags; ++ ++ /* ++ * don't let napi dequeue from the cpu poll list ++ * just in case its running on a different cpu ++ */ ++ if (unlikely(test_bit(NAPI_STATE_NPSVC, &n->state))) ++ return; ++ ++ if (n->gro_list) { ++ unsigned long timeout = 0; ++ ++ if (work_done) ++ timeout = n->dev->gro_flush_timeout; ++ ++ if (timeout) ++ hrtimer_start(&n->timer, ns_to_ktime(timeout), ++ HRTIMER_MODE_REL_PINNED); ++ else ++ napi_gro_flush(n, false); ++ } ++ if (likely(list_empty(&n->poll_list))) { ++ WARN_ON_ONCE(!test_and_clear_bit(NAPI_STATE_SCHED, &n->state)); ++ } else { ++ /* If n->poll_list is not empty, we need to mask irqs */ ++ local_irq_save(flags); ++ __napi_complete(n); ++ local_irq_restore(flags); ++ } ++} ++EXPORT_SYMBOL(napi_complete_done); ++ ++/* must be called under rcu_read_lock(), as we dont take a reference */ ++struct napi_struct *napi_by_id(unsigned int napi_id) ++{ ++ unsigned int hash = napi_id % HASH_SIZE(napi_hash); ++ struct napi_struct *napi; ++ ++ hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node) ++ if (napi->napi_id == napi_id) ++ return napi; ++ ++ return NULL; ++} ++EXPORT_SYMBOL_GPL(napi_by_id); ++ ++void napi_hash_add(struct napi_struct *napi) ++{ ++ if (!test_and_set_bit(NAPI_STATE_HASHED, &napi->state)) { ++ ++ spin_lock(&napi_hash_lock); ++ ++ /* 0 is not a valid id, we also skip an id that is taken ++ * we expect both events to be extremely rare ++ */ ++ napi->napi_id = 0; ++ while (!napi->napi_id) { ++ napi->napi_id = ++napi_gen_id; ++ if (napi_by_id(napi->napi_id)) ++ napi->napi_id = 0; ++ } ++ ++ hlist_add_head_rcu(&napi->napi_hash_node, ++ &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]); ++ ++ spin_unlock(&napi_hash_lock); ++ } ++} ++EXPORT_SYMBOL_GPL(napi_hash_add); ++ ++/* Warning : caller is responsible to make sure rcu grace period ++ * is respected before freeing memory containing @napi ++ */ ++void napi_hash_del(struct napi_struct *napi) ++{ ++ spin_lock(&napi_hash_lock); ++ ++ if (test_and_clear_bit(NAPI_STATE_HASHED, &napi->state)) ++ hlist_del_rcu(&napi->napi_hash_node); ++ ++ spin_unlock(&napi_hash_lock); ++} ++EXPORT_SYMBOL_GPL(napi_hash_del); ++ ++static enum hrtimer_restart napi_watchdog(struct hrtimer *timer) ++{ ++ struct napi_struct *napi; ++ ++ napi = container_of(timer, struct napi_struct, timer); ++ if (napi->gro_list) ++ napi_schedule(napi); ++ ++ return HRTIMER_NORESTART; ++} ++ ++void netif_napi_add(struct net_device *dev, struct napi_struct *napi, ++ int (*poll)(struct napi_struct *, int), int weight) ++{ ++ INIT_LIST_HEAD(&napi->poll_list); ++ hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED); ++ napi->timer.function = napi_watchdog; ++ napi->gro_count = 0; ++ napi->gro_list = NULL; ++ napi->skb = NULL; ++ napi->poll = poll; ++ if (weight > NAPI_POLL_WEIGHT) ++ pr_err_once("netif_napi_add() called with weight %d on device %s\n", ++ weight, dev->name); ++ napi->weight = weight; ++ list_add(&napi->dev_list, &dev->napi_list); ++ napi->dev = dev; ++#ifdef CONFIG_NETPOLL ++ spin_lock_init(&napi->poll_lock); ++ napi->poll_owner = -1; ++#endif ++ set_bit(NAPI_STATE_SCHED, &napi->state); ++} ++EXPORT_SYMBOL(netif_napi_add); ++ ++void napi_disable(struct napi_struct *n) ++{ ++ might_sleep(); ++ set_bit(NAPI_STATE_DISABLE, &n->state); ++ ++ while (test_and_set_bit(NAPI_STATE_SCHED, &n->state)) ++ msleep(1); ++ ++ hrtimer_cancel(&n->timer); ++ ++ clear_bit(NAPI_STATE_DISABLE, &n->state); ++} ++EXPORT_SYMBOL(napi_disable); ++ ++void netif_napi_del(struct napi_struct *napi) ++{ ++ list_del_init(&napi->dev_list); ++ napi_free_frags(napi); ++ ++ kfree_skb_list(napi->gro_list); ++ napi->gro_list = NULL; ++ napi->gro_count = 0; ++} ++EXPORT_SYMBOL(netif_napi_del); ++ ++static int napi_poll(struct napi_struct *n, struct list_head *repoll) ++{ ++ void *have; ++ int work, weight; ++ ++ list_del_init(&n->poll_list); ++ ++ have = netpoll_poll_lock(n); ++ ++ weight = n->weight; ++ ++ /* This NAPI_STATE_SCHED test is for avoiding a race ++ * with netpoll's poll_napi(). Only the entity which ++ * obtains the lock and sees NAPI_STATE_SCHED set will ++ * actually make the ->poll() call. Therefore we avoid ++ * accidentally calling ->poll() when NAPI is not scheduled. ++ */ ++ work = 0; ++ if (test_bit(NAPI_STATE_SCHED, &n->state)) { ++ work = n->poll(n, weight); ++ trace_napi_poll(n); ++ } ++ ++ WARN_ON_ONCE(work > weight); ++ ++ if (likely(work < weight)) ++ goto out_unlock; ++ ++ /* Drivers must not modify the NAPI state if they ++ * consume the entire weight. In such cases this code ++ * still "owns" the NAPI instance and therefore can ++ * move the instance around on the list at-will. ++ */ ++ if (unlikely(napi_disable_pending(n))) { ++ napi_complete(n); ++ goto out_unlock; ++ } ++ ++ if (n->gro_list) { ++ /* flush too old packets ++ * If HZ < 1000, flush all packets. ++ */ ++ napi_gro_flush(n, HZ >= 1000); ++ } ++ ++ /* Some drivers may have called napi_schedule ++ * prior to exhausting their budget. ++ */ ++ if (unlikely(!list_empty(&n->poll_list))) { ++ pr_warn_once("%s: Budget exhausted after napi rescheduled\n", ++ n->dev ? n->dev->name : "backlog"); ++ goto out_unlock; ++ } ++ ++ list_add_tail(&n->poll_list, repoll); ++ ++out_unlock: ++ netpoll_poll_unlock(have); ++ ++ return work; ++} ++ ++static void net_rx_action(struct softirq_action *h) ++{ ++ struct softnet_data *sd = this_cpu_ptr(&softnet_data); ++ unsigned long time_limit = jiffies + 2; ++ int budget = netdev_budget; ++ LIST_HEAD(list); ++ LIST_HEAD(repoll); ++ ++ local_irq_disable(); ++ list_splice_init(&sd->poll_list, &list); ++ local_irq_enable(); ++ ++ for (;;) { ++ struct napi_struct *n; ++ ++ if (list_empty(&list)) { ++ if (!sd_has_rps_ipi_waiting(sd) && list_empty(&repoll)) ++ return; ++ break; ++ } ++ ++ n = list_first_entry(&list, struct napi_struct, poll_list); ++ budget -= napi_poll(n, &repoll); ++ ++ /* If softirq window is exhausted then punt. ++ * Allow this to run for 2 jiffies since which will allow ++ * an average latency of 1.5/HZ. ++ */ ++ if (unlikely(budget <= 0 || ++ time_after_eq(jiffies, time_limit))) { ++ sd->time_squeeze++; ++ break; ++ } ++ } ++ ++ local_irq_disable(); ++ ++ list_splice_tail_init(&sd->poll_list, &list); ++ list_splice_tail(&repoll, &list); ++ list_splice(&list, &sd->poll_list); ++ if (!list_empty(&sd->poll_list)) ++ __raise_softirq_irqoff(NET_RX_SOFTIRQ); ++ ++ net_rps_action_and_irq_enable(sd); ++} ++ ++struct netdev_adjacent { ++ struct net_device *dev; ++ ++ /* upper master flag, there can only be one master device per list */ ++ bool master; ++ ++ /* counter for the number of times this device was added to us */ ++ u16 ref_nr; ++ ++ /* private field for the users */ ++ void *private; ++ ++ struct list_head list; ++ struct rcu_head rcu; ++}; ++ ++static struct netdev_adjacent *__netdev_find_adj(struct net_device *dev, ++ struct net_device *adj_dev, ++ struct list_head *adj_list) ++{ ++ struct netdev_adjacent *adj; ++ ++ list_for_each_entry(adj, adj_list, list) { ++ if (adj->dev == adj_dev) ++ return adj; ++ } ++ return NULL; ++} ++ ++/** ++ * netdev_has_upper_dev - Check if device is linked to an upper device ++ * @dev: device ++ * @upper_dev: upper device to check ++ * ++ * Find out if a device is linked to specified upper device and return true ++ * in case it is. Note that this checks only immediate upper device, ++ * not through a complete stack of devices. The caller must hold the RTNL lock. ++ */ ++bool netdev_has_upper_dev(struct net_device *dev, ++ struct net_device *upper_dev) ++{ ++ ASSERT_RTNL(); ++ ++ return __netdev_find_adj(dev, upper_dev, &dev->all_adj_list.upper); ++} ++EXPORT_SYMBOL(netdev_has_upper_dev); ++ ++/** ++ * netdev_has_any_upper_dev - Check if device is linked to some device ++ * @dev: device ++ * ++ * Find out if a device is linked to an upper device and return true in case ++ * it is. The caller must hold the RTNL lock. ++ */ ++static bool netdev_has_any_upper_dev(struct net_device *dev) ++{ ++ ASSERT_RTNL(); ++ ++ return !list_empty(&dev->all_adj_list.upper); ++} ++ ++/** ++ * netdev_master_upper_dev_get - Get master upper device ++ * @dev: device ++ * ++ * Find a master upper device and return pointer to it or NULL in case ++ * it's not there. The caller must hold the RTNL lock. ++ */ ++struct net_device *netdev_master_upper_dev_get(struct net_device *dev) ++{ ++ struct netdev_adjacent *upper; ++ ++ ASSERT_RTNL(); ++ ++ if (list_empty(&dev->adj_list.upper)) ++ return NULL; ++ ++ upper = list_first_entry(&dev->adj_list.upper, ++ struct netdev_adjacent, list); ++ if (likely(upper->master)) ++ return upper->dev; ++ return NULL; ++} ++EXPORT_SYMBOL(netdev_master_upper_dev_get); ++ ++void *netdev_adjacent_get_private(struct list_head *adj_list) ++{ ++ struct netdev_adjacent *adj; ++ ++ adj = list_entry(adj_list, struct netdev_adjacent, list); ++ ++ return adj->private; ++} ++EXPORT_SYMBOL(netdev_adjacent_get_private); ++ ++/** ++ * netdev_upper_get_next_dev_rcu - Get the next dev from upper list ++ * @dev: device ++ * @iter: list_head ** of the current position ++ * ++ * Gets the next device from the dev's upper list, starting from iter ++ * position. The caller must hold RCU read lock. ++ */ ++struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev, ++ struct list_head **iter) ++{ ++ struct netdev_adjacent *upper; ++ ++ WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held()); ++ ++ upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); ++ ++ if (&upper->list == &dev->adj_list.upper) ++ return NULL; ++ ++ *iter = &upper->list; ++ ++ return upper->dev; ++} ++EXPORT_SYMBOL(netdev_upper_get_next_dev_rcu); ++ ++/** ++ * netdev_all_upper_get_next_dev_rcu - Get the next dev from upper list ++ * @dev: device ++ * @iter: list_head ** of the current position ++ * ++ * Gets the next device from the dev's upper list, starting from iter ++ * position. The caller must hold RCU read lock. ++ */ ++struct net_device *netdev_all_upper_get_next_dev_rcu(struct net_device *dev, ++ struct list_head **iter) ++{ ++ struct netdev_adjacent *upper; ++ ++ WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held()); ++ ++ upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); ++ ++ if (&upper->list == &dev->all_adj_list.upper) ++ return NULL; ++ ++ *iter = &upper->list; ++ ++ return upper->dev; ++} ++EXPORT_SYMBOL(netdev_all_upper_get_next_dev_rcu); ++ ++/** ++ * netdev_lower_get_next_private - Get the next ->private from the ++ * lower neighbour list ++ * @dev: device ++ * @iter: list_head ** of the current position ++ * ++ * Gets the next netdev_adjacent->private from the dev's lower neighbour ++ * list, starting from iter position. The caller must hold either hold the ++ * RTNL lock or its own locking that guarantees that the neighbour lower ++ * list will remain unchainged. ++ */ ++void *netdev_lower_get_next_private(struct net_device *dev, ++ struct list_head **iter) ++{ ++ struct netdev_adjacent *lower; ++ ++ lower = list_entry(*iter, struct netdev_adjacent, list); ++ ++ if (&lower->list == &dev->adj_list.lower) ++ return NULL; ++ ++ *iter = lower->list.next; ++ ++ return lower->private; ++} ++EXPORT_SYMBOL(netdev_lower_get_next_private); ++ ++/** ++ * netdev_lower_get_next_private_rcu - Get the next ->private from the ++ * lower neighbour list, RCU ++ * variant ++ * @dev: device ++ * @iter: list_head ** of the current position ++ * ++ * Gets the next netdev_adjacent->private from the dev's lower neighbour ++ * list, starting from iter position. The caller must hold RCU read lock. ++ */ ++void *netdev_lower_get_next_private_rcu(struct net_device *dev, ++ struct list_head **iter) ++{ ++ struct netdev_adjacent *lower; ++ ++ WARN_ON_ONCE(!rcu_read_lock_held()); ++ ++ lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); ++ ++ if (&lower->list == &dev->adj_list.lower) ++ return NULL; ++ ++ *iter = &lower->list; ++ ++ return lower->private; ++} ++EXPORT_SYMBOL(netdev_lower_get_next_private_rcu); ++ ++/** ++ * netdev_lower_get_next - Get the next device from the lower neighbour ++ * list ++ * @dev: device ++ * @iter: list_head ** of the current position ++ * ++ * Gets the next netdev_adjacent from the dev's lower neighbour ++ * list, starting from iter position. The caller must hold RTNL lock or ++ * its own locking that guarantees that the neighbour lower ++ * list will remain unchainged. ++ */ ++void *netdev_lower_get_next(struct net_device *dev, struct list_head **iter) ++{ ++ struct netdev_adjacent *lower; ++ ++ lower = list_entry((*iter)->next, struct netdev_adjacent, list); ++ ++ if (&lower->list == &dev->adj_list.lower) ++ return NULL; ++ ++ *iter = &lower->list; ++ ++ return lower->dev; ++} ++EXPORT_SYMBOL(netdev_lower_get_next); ++ ++/** ++ * netdev_lower_get_first_private_rcu - Get the first ->private from the ++ * lower neighbour list, RCU ++ * variant ++ * @dev: device ++ * ++ * Gets the first netdev_adjacent->private from the dev's lower neighbour ++ * list. The caller must hold RCU read lock. ++ */ ++void *netdev_lower_get_first_private_rcu(struct net_device *dev) ++{ ++ struct netdev_adjacent *lower; ++ ++ lower = list_first_or_null_rcu(&dev->adj_list.lower, ++ struct netdev_adjacent, list); ++ if (lower) ++ return lower->private; ++ return NULL; ++} ++EXPORT_SYMBOL(netdev_lower_get_first_private_rcu); ++ ++/** ++ * netdev_master_upper_dev_get_rcu - Get master upper device ++ * @dev: device ++ * ++ * Find a master upper device and return pointer to it or NULL in case ++ * it's not there. The caller must hold the RCU read lock. ++ */ ++struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev) ++{ ++ struct netdev_adjacent *upper; ++ ++ upper = list_first_or_null_rcu(&dev->adj_list.upper, ++ struct netdev_adjacent, list); ++ if (upper && likely(upper->master)) ++ return upper->dev; ++ return NULL; ++} ++EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu); ++ ++static int netdev_adjacent_sysfs_add(struct net_device *dev, ++ struct net_device *adj_dev, ++ struct list_head *dev_list) ++{ ++ char linkname[IFNAMSIZ+7]; ++ sprintf(linkname, dev_list == &dev->adj_list.upper ? ++ "upper_%s" : "lower_%s", adj_dev->name); ++ return sysfs_create_link(&(dev->dev.kobj), &(adj_dev->dev.kobj), ++ linkname); ++} ++static void netdev_adjacent_sysfs_del(struct net_device *dev, ++ char *name, ++ struct list_head *dev_list) ++{ ++ char linkname[IFNAMSIZ+7]; ++ sprintf(linkname, dev_list == &dev->adj_list.upper ? ++ "upper_%s" : "lower_%s", name); ++ sysfs_remove_link(&(dev->dev.kobj), linkname); ++} ++ ++static inline bool netdev_adjacent_is_neigh_list(struct net_device *dev, ++ struct net_device *adj_dev, ++ struct list_head *dev_list) ++{ ++ return (dev_list == &dev->adj_list.upper || ++ dev_list == &dev->adj_list.lower) && ++ net_eq(dev_net(dev), dev_net(adj_dev)); ++} ++ ++static int __netdev_adjacent_dev_insert(struct net_device *dev, ++ struct net_device *adj_dev, ++ struct list_head *dev_list, ++ void *private, bool master) ++{ ++ struct netdev_adjacent *adj; ++ int ret; ++ ++ adj = __netdev_find_adj(dev, adj_dev, dev_list); ++ ++ if (adj) { ++ adj->ref_nr++; ++ return 0; ++ } ++ ++ adj = kmalloc(sizeof(*adj), GFP_KERNEL); ++ if (!adj) ++ return -ENOMEM; ++ ++ adj->dev = adj_dev; ++ adj->master = master; ++ adj->ref_nr = 1; ++ adj->private = private; ++ dev_hold(adj_dev); ++ ++ pr_debug("dev_hold for %s, because of link added from %s to %s\n", ++ adj_dev->name, dev->name, adj_dev->name); ++ ++ if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) { ++ ret = netdev_adjacent_sysfs_add(dev, adj_dev, dev_list); ++ if (ret) ++ goto free_adj; ++ } ++ ++ /* Ensure that master link is always the first item in list. */ ++ if (master) { ++ ret = sysfs_create_link(&(dev->dev.kobj), ++ &(adj_dev->dev.kobj), "master"); ++ if (ret) ++ goto remove_symlinks; ++ ++ list_add_rcu(&adj->list, dev_list); ++ } else { ++ list_add_tail_rcu(&adj->list, dev_list); ++ } ++ ++ return 0; ++ ++remove_symlinks: ++ if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) ++ netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list); ++free_adj: ++ kfree(adj); ++ dev_put(adj_dev); ++ ++ return ret; ++} ++ ++static void __netdev_adjacent_dev_remove(struct net_device *dev, ++ struct net_device *adj_dev, ++ struct list_head *dev_list) ++{ ++ struct netdev_adjacent *adj; ++ ++ adj = __netdev_find_adj(dev, adj_dev, dev_list); ++ ++ if (!adj) { ++ pr_err("tried to remove device %s from %s\n", ++ dev->name, adj_dev->name); ++ BUG(); ++ } ++ ++ if (adj->ref_nr > 1) { ++ pr_debug("%s to %s ref_nr-- = %d\n", dev->name, adj_dev->name, ++ adj->ref_nr-1); ++ adj->ref_nr--; ++ return; ++ } ++ ++ if (adj->master) ++ sysfs_remove_link(&(dev->dev.kobj), "master"); ++ ++ if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) ++ netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list); ++ ++ list_del_rcu(&adj->list); ++ pr_debug("dev_put for %s, because link removed from %s to %s\n", ++ adj_dev->name, dev->name, adj_dev->name); ++ dev_put(adj_dev); ++ kfree_rcu(adj, rcu); ++} ++ ++static int __netdev_adjacent_dev_link_lists(struct net_device *dev, ++ struct net_device *upper_dev, ++ struct list_head *up_list, ++ struct list_head *down_list, ++ void *private, bool master) ++{ ++ int ret; ++ ++ ret = __netdev_adjacent_dev_insert(dev, upper_dev, up_list, private, ++ master); ++ if (ret) ++ return ret; ++ ++ ret = __netdev_adjacent_dev_insert(upper_dev, dev, down_list, private, ++ false); ++ if (ret) { ++ __netdev_adjacent_dev_remove(dev, upper_dev, up_list); ++ return ret; ++ } ++ ++ return 0; ++} ++ ++static int __netdev_adjacent_dev_link(struct net_device *dev, ++ struct net_device *upper_dev) ++{ ++ return __netdev_adjacent_dev_link_lists(dev, upper_dev, ++ &dev->all_adj_list.upper, ++ &upper_dev->all_adj_list.lower, ++ NULL, false); ++} ++ ++static void __netdev_adjacent_dev_unlink_lists(struct net_device *dev, ++ struct net_device *upper_dev, ++ struct list_head *up_list, ++ struct list_head *down_list) ++{ ++ __netdev_adjacent_dev_remove(dev, upper_dev, up_list); ++ __netdev_adjacent_dev_remove(upper_dev, dev, down_list); ++} ++ ++static void __netdev_adjacent_dev_unlink(struct net_device *dev, ++ struct net_device *upper_dev) ++{ ++ __netdev_adjacent_dev_unlink_lists(dev, upper_dev, ++ &dev->all_adj_list.upper, ++ &upper_dev->all_adj_list.lower); ++} ++ ++static int __netdev_adjacent_dev_link_neighbour(struct net_device *dev, ++ struct net_device *upper_dev, ++ void *private, bool master) ++{ ++ int ret = __netdev_adjacent_dev_link(dev, upper_dev); ++ ++ if (ret) ++ return ret; ++ ++ ret = __netdev_adjacent_dev_link_lists(dev, upper_dev, ++ &dev->adj_list.upper, ++ &upper_dev->adj_list.lower, ++ private, master); ++ if (ret) { ++ __netdev_adjacent_dev_unlink(dev, upper_dev); ++ return ret; ++ } ++ ++ return 0; ++} ++ ++static void __netdev_adjacent_dev_unlink_neighbour(struct net_device *dev, ++ struct net_device *upper_dev) ++{ ++ __netdev_adjacent_dev_unlink(dev, upper_dev); ++ __netdev_adjacent_dev_unlink_lists(dev, upper_dev, ++ &dev->adj_list.upper, ++ &upper_dev->adj_list.lower); ++} ++ ++static int __netdev_upper_dev_link(struct net_device *dev, ++ struct net_device *upper_dev, bool master, ++ void *private) ++{ ++ struct netdev_adjacent *i, *j, *to_i, *to_j; ++ int ret = 0; ++ ++ ASSERT_RTNL(); ++ ++ if (dev == upper_dev) ++ return -EBUSY; ++ ++ /* To prevent loops, check if dev is not upper device to upper_dev. */ ++ if (__netdev_find_adj(upper_dev, dev, &upper_dev->all_adj_list.upper)) ++ return -EBUSY; ++ ++ if (__netdev_find_adj(dev, upper_dev, &dev->adj_list.upper)) ++ return -EEXIST; ++ ++ if (master && netdev_master_upper_dev_get(dev)) ++ return -EBUSY; ++ ++ ret = __netdev_adjacent_dev_link_neighbour(dev, upper_dev, private, ++ master); ++ if (ret) ++ return ret; ++ ++ /* Now that we linked these devs, make all the upper_dev's ++ * all_adj_list.upper visible to every dev's all_adj_list.lower an ++ * versa, and don't forget the devices itself. All of these ++ * links are non-neighbours. ++ */ ++ list_for_each_entry(i, &dev->all_adj_list.lower, list) { ++ list_for_each_entry(j, &upper_dev->all_adj_list.upper, list) { ++ pr_debug("Interlinking %s with %s, non-neighbour\n", ++ i->dev->name, j->dev->name); ++ ret = __netdev_adjacent_dev_link(i->dev, j->dev); ++ if (ret) ++ goto rollback_mesh; ++ } ++ } ++ ++ /* add dev to every upper_dev's upper device */ ++ list_for_each_entry(i, &upper_dev->all_adj_list.upper, list) { ++ pr_debug("linking %s's upper device %s with %s\n", ++ upper_dev->name, i->dev->name, dev->name); ++ ret = __netdev_adjacent_dev_link(dev, i->dev); ++ if (ret) ++ goto rollback_upper_mesh; ++ } ++ ++ /* add upper_dev to every dev's lower device */ ++ list_for_each_entry(i, &dev->all_adj_list.lower, list) { ++ pr_debug("linking %s's lower device %s with %s\n", dev->name, ++ i->dev->name, upper_dev->name); ++ ret = __netdev_adjacent_dev_link(i->dev, upper_dev); ++ if (ret) ++ goto rollback_lower_mesh; ++ } ++ ++ call_netdevice_notifiers(NETDEV_CHANGEUPPER, dev); ++ return 0; ++ ++rollback_lower_mesh: ++ to_i = i; ++ list_for_each_entry(i, &dev->all_adj_list.lower, list) { ++ if (i == to_i) ++ break; ++ __netdev_adjacent_dev_unlink(i->dev, upper_dev); ++ } ++ ++ i = NULL; ++ ++rollback_upper_mesh: ++ to_i = i; ++ list_for_each_entry(i, &upper_dev->all_adj_list.upper, list) { ++ if (i == to_i) ++ break; ++ __netdev_adjacent_dev_unlink(dev, i->dev); ++ } ++ ++ i = j = NULL; ++ ++rollback_mesh: ++ to_i = i; ++ to_j = j; ++ list_for_each_entry(i, &dev->all_adj_list.lower, list) { ++ list_for_each_entry(j, &upper_dev->all_adj_list.upper, list) { ++ if (i == to_i && j == to_j) ++ break; ++ __netdev_adjacent_dev_unlink(i->dev, j->dev); ++ } ++ if (i == to_i) ++ break; ++ } ++ ++ __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev); ++ ++ return ret; ++} ++ ++/** ++ * netdev_upper_dev_link - Add a link to the upper device ++ * @dev: device ++ * @upper_dev: new upper device ++ * ++ * Adds a link to device which is upper to this one. The caller must hold ++ * the RTNL lock. On a failure a negative errno code is returned. ++ * On success the reference counts are adjusted and the function ++ * returns zero. ++ */ ++int netdev_upper_dev_link(struct net_device *dev, ++ struct net_device *upper_dev) ++{ ++ return __netdev_upper_dev_link(dev, upper_dev, false, NULL); ++} ++EXPORT_SYMBOL(netdev_upper_dev_link); ++ ++/** ++ * netdev_master_upper_dev_link - Add a master link to the upper device ++ * @dev: device ++ * @upper_dev: new upper device ++ * ++ * Adds a link to device which is upper to this one. In this case, only ++ * one master upper device can be linked, although other non-master devices ++ * might be linked as well. The caller must hold the RTNL lock. ++ * On a failure a negative errno code is returned. On success the reference ++ * counts are adjusted and the function returns zero. ++ */ ++int netdev_master_upper_dev_link(struct net_device *dev, ++ struct net_device *upper_dev) ++{ ++ return __netdev_upper_dev_link(dev, upper_dev, true, NULL); ++} ++EXPORT_SYMBOL(netdev_master_upper_dev_link); ++ ++int netdev_master_upper_dev_link_private(struct net_device *dev, ++ struct net_device *upper_dev, ++ void *private) ++{ ++ return __netdev_upper_dev_link(dev, upper_dev, true, private); ++} ++EXPORT_SYMBOL(netdev_master_upper_dev_link_private); ++ ++/** ++ * netdev_upper_dev_unlink - Removes a link to upper device ++ * @dev: device ++ * @upper_dev: new upper device ++ * ++ * Removes a link to device which is upper to this one. The caller must hold ++ * the RTNL lock. ++ */ ++void netdev_upper_dev_unlink(struct net_device *dev, ++ struct net_device *upper_dev) ++{ ++ struct netdev_adjacent *i, *j; ++ ASSERT_RTNL(); ++ ++ __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev); ++ ++ /* Here is the tricky part. We must remove all dev's lower ++ * devices from all upper_dev's upper devices and vice ++ * versa, to maintain the graph relationship. ++ */ ++ list_for_each_entry(i, &dev->all_adj_list.lower, list) ++ list_for_each_entry(j, &upper_dev->all_adj_list.upper, list) ++ __netdev_adjacent_dev_unlink(i->dev, j->dev); ++ ++ /* remove also the devices itself from lower/upper device ++ * list ++ */ ++ list_for_each_entry(i, &dev->all_adj_list.lower, list) ++ __netdev_adjacent_dev_unlink(i->dev, upper_dev); ++ ++ list_for_each_entry(i, &upper_dev->all_adj_list.upper, list) ++ __netdev_adjacent_dev_unlink(dev, i->dev); ++ ++ call_netdevice_notifiers(NETDEV_CHANGEUPPER, dev); ++} ++EXPORT_SYMBOL(netdev_upper_dev_unlink); ++ ++/** ++ * netdev_bonding_info_change - Dispatch event about slave change ++ * @dev: device ++ * @bonding_info: info to dispatch ++ * ++ * Send NETDEV_BONDING_INFO to netdev notifiers with info. ++ * The caller must hold the RTNL lock. ++ */ ++void netdev_bonding_info_change(struct net_device *dev, ++ struct netdev_bonding_info *bonding_info) ++{ ++ struct netdev_notifier_bonding_info info; ++ ++ memcpy(&info.bonding_info, bonding_info, ++ sizeof(struct netdev_bonding_info)); ++ call_netdevice_notifiers_info(NETDEV_BONDING_INFO, dev, ++ &info.info); ++} ++EXPORT_SYMBOL(netdev_bonding_info_change); ++ ++static void netdev_adjacent_add_links(struct net_device *dev) ++{ ++ struct netdev_adjacent *iter; ++ ++ struct net *net = dev_net(dev); ++ ++ list_for_each_entry(iter, &dev->adj_list.upper, list) { ++ if (!net_eq(net,dev_net(iter->dev))) ++ continue; ++ netdev_adjacent_sysfs_add(iter->dev, dev, ++ &iter->dev->adj_list.lower); ++ netdev_adjacent_sysfs_add(dev, iter->dev, ++ &dev->adj_list.upper); ++ } ++ ++ list_for_each_entry(iter, &dev->adj_list.lower, list) { ++ if (!net_eq(net,dev_net(iter->dev))) ++ continue; ++ netdev_adjacent_sysfs_add(iter->dev, dev, ++ &iter->dev->adj_list.upper); ++ netdev_adjacent_sysfs_add(dev, iter->dev, ++ &dev->adj_list.lower); ++ } ++} ++ ++static void netdev_adjacent_del_links(struct net_device *dev) ++{ ++ struct netdev_adjacent *iter; ++ ++ struct net *net = dev_net(dev); ++ ++ list_for_each_entry(iter, &dev->adj_list.upper, list) { ++ if (!net_eq(net,dev_net(iter->dev))) ++ continue; ++ netdev_adjacent_sysfs_del(iter->dev, dev->name, ++ &iter->dev->adj_list.lower); ++ netdev_adjacent_sysfs_del(dev, iter->dev->name, ++ &dev->adj_list.upper); ++ } ++ ++ list_for_each_entry(iter, &dev->adj_list.lower, list) { ++ if (!net_eq(net,dev_net(iter->dev))) ++ continue; ++ netdev_adjacent_sysfs_del(iter->dev, dev->name, ++ &iter->dev->adj_list.upper); ++ netdev_adjacent_sysfs_del(dev, iter->dev->name, ++ &dev->adj_list.lower); ++ } ++} ++ ++void netdev_adjacent_rename_links(struct net_device *dev, char *oldname) ++{ ++ struct netdev_adjacent *iter; ++ ++ struct net *net = dev_net(dev); ++ ++ list_for_each_entry(iter, &dev->adj_list.upper, list) { ++ if (!net_eq(net,dev_net(iter->dev))) ++ continue; ++ netdev_adjacent_sysfs_del(iter->dev, oldname, ++ &iter->dev->adj_list.lower); ++ netdev_adjacent_sysfs_add(iter->dev, dev, ++ &iter->dev->adj_list.lower); ++ } ++ ++ list_for_each_entry(iter, &dev->adj_list.lower, list) { ++ if (!net_eq(net,dev_net(iter->dev))) ++ continue; ++ netdev_adjacent_sysfs_del(iter->dev, oldname, ++ &iter->dev->adj_list.upper); ++ netdev_adjacent_sysfs_add(iter->dev, dev, ++ &iter->dev->adj_list.upper); ++ } ++} ++ ++void *netdev_lower_dev_get_private(struct net_device *dev, ++ struct net_device *lower_dev) ++{ ++ struct netdev_adjacent *lower; ++ ++ if (!lower_dev) ++ return NULL; ++ lower = __netdev_find_adj(dev, lower_dev, &dev->adj_list.lower); ++ if (!lower) ++ return NULL; ++ ++ return lower->private; ++} ++EXPORT_SYMBOL(netdev_lower_dev_get_private); ++ ++ ++int dev_get_nest_level(struct net_device *dev, ++ bool (*type_check)(struct net_device *dev)) ++{ ++ struct net_device *lower = NULL; ++ struct list_head *iter; ++ int max_nest = -1; ++ int nest; ++ ++ ASSERT_RTNL(); ++ ++ netdev_for_each_lower_dev(dev, lower, iter) { ++ nest = dev_get_nest_level(lower, type_check); ++ if (max_nest < nest) ++ max_nest = nest; ++ } ++ ++ if (type_check(dev)) ++ max_nest++; ++ ++ return max_nest; ++} ++EXPORT_SYMBOL(dev_get_nest_level); ++ ++static void dev_change_rx_flags(struct net_device *dev, int flags) ++{ ++ const struct net_device_ops *ops = dev->netdev_ops; ++ ++ if (ops->ndo_change_rx_flags) ++ ops->ndo_change_rx_flags(dev, flags); ++} ++ ++static int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify) ++{ ++ unsigned int old_flags = dev->flags; ++ kuid_t uid; ++ kgid_t gid; ++ ++ ASSERT_RTNL(); ++ ++ dev->flags |= IFF_PROMISC; ++ dev->promiscuity += inc; ++ if (dev->promiscuity == 0) { ++ /* ++ * Avoid overflow. ++ * If inc causes overflow, untouch promisc and return error. ++ */ ++ if (inc < 0) ++ dev->flags &= ~IFF_PROMISC; ++ else { ++ dev->promiscuity -= inc; ++ pr_warn("%s: promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n", ++ dev->name); ++ return -EOVERFLOW; ++ } ++ } ++ if (dev->flags != old_flags) { ++ pr_info("device %s %s promiscuous mode\n", ++ dev->name, ++ dev->flags & IFF_PROMISC ? "entered" : "left"); ++ if (audit_enabled) { ++ current_uid_gid(&uid, &gid); ++ audit_log(current->audit_context, GFP_ATOMIC, ++ AUDIT_ANOM_PROMISCUOUS, ++ "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u", ++ dev->name, (dev->flags & IFF_PROMISC), ++ (old_flags & IFF_PROMISC), ++ from_kuid(&init_user_ns, audit_get_loginuid(current)), ++ from_kuid(&init_user_ns, uid), ++ from_kgid(&init_user_ns, gid), ++ audit_get_sessionid(current)); ++ } ++ ++ dev_change_rx_flags(dev, IFF_PROMISC); ++ } ++ if (notify) ++ __dev_notify_flags(dev, old_flags, IFF_PROMISC); ++ return 0; ++} ++ ++/** ++ * dev_set_promiscuity - update promiscuity count on a device ++ * @dev: device ++ * @inc: modifier ++ * ++ * Add or remove promiscuity from a device. While the count in the device ++ * remains above zero the interface remains promiscuous. Once it hits zero ++ * the device reverts back to normal filtering operation. A negative inc ++ * value is used to drop promiscuity on the device. ++ * Return 0 if successful or a negative errno code on error. ++ */ ++int dev_set_promiscuity(struct net_device *dev, int inc) ++{ ++ unsigned int old_flags = dev->flags; ++ int err; ++ ++ err = __dev_set_promiscuity(dev, inc, true); ++ if (err < 0) ++ return err; ++ if (dev->flags != old_flags) ++ dev_set_rx_mode(dev); ++ return err; ++} ++EXPORT_SYMBOL(dev_set_promiscuity); ++ ++static int __dev_set_allmulti(struct net_device *dev, int inc, bool notify) ++{ ++ unsigned int old_flags = dev->flags, old_gflags = dev->gflags; ++ ++ ASSERT_RTNL(); ++ ++ dev->flags |= IFF_ALLMULTI; ++ dev->allmulti += inc; ++ if (dev->allmulti == 0) { ++ /* ++ * Avoid overflow. ++ * If inc causes overflow, untouch allmulti and return error. ++ */ ++ if (inc < 0) ++ dev->flags &= ~IFF_ALLMULTI; ++ else { ++ dev->allmulti -= inc; ++ pr_warn("%s: allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n", ++ dev->name); ++ return -EOVERFLOW; ++ } ++ } ++ if (dev->flags ^ old_flags) { ++ dev_change_rx_flags(dev, IFF_ALLMULTI); ++ dev_set_rx_mode(dev); ++ if (notify) ++ __dev_notify_flags(dev, old_flags, ++ dev->gflags ^ old_gflags); ++ } ++ return 0; ++} ++ ++/** ++ * dev_set_allmulti - update allmulti count on a device ++ * @dev: device ++ * @inc: modifier ++ * ++ * Add or remove reception of all multicast frames to a device. While the ++ * count in the device remains above zero the interface remains listening ++ * to all interfaces. Once it hits zero the device reverts back to normal ++ * filtering operation. A negative @inc value is used to drop the counter ++ * when releasing a resource needing all multicasts. ++ * Return 0 if successful or a negative errno code on error. ++ */ ++ ++int dev_set_allmulti(struct net_device *dev, int inc) ++{ ++ return __dev_set_allmulti(dev, inc, true); ++} ++EXPORT_SYMBOL(dev_set_allmulti); ++ ++/* ++ * Upload unicast and multicast address lists to device and ++ * configure RX filtering. When the device doesn't support unicast ++ * filtering it is put in promiscuous mode while unicast addresses ++ * are present. ++ */ ++void __dev_set_rx_mode(struct net_device *dev) ++{ ++ const struct net_device_ops *ops = dev->netdev_ops; ++ ++ /* dev_open will call this function so the list will stay sane. */ ++ if (!(dev->flags&IFF_UP)) ++ return; ++ ++ if (!netif_device_present(dev)) ++ return; ++ ++ if (!(dev->priv_flags & IFF_UNICAST_FLT)) { ++ /* Unicast addresses changes may only happen under the rtnl, ++ * therefore calling __dev_set_promiscuity here is safe. ++ */ ++ if (!netdev_uc_empty(dev) && !dev->uc_promisc) { ++ __dev_set_promiscuity(dev, 1, false); ++ dev->uc_promisc = true; ++ } else if (netdev_uc_empty(dev) && dev->uc_promisc) { ++ __dev_set_promiscuity(dev, -1, false); ++ dev->uc_promisc = false; ++ } ++ } ++ ++ if (ops->ndo_set_rx_mode) ++ ops->ndo_set_rx_mode(dev); ++} ++ ++void dev_set_rx_mode(struct net_device *dev) ++{ ++ netif_addr_lock_bh(dev); ++ __dev_set_rx_mode(dev); ++ netif_addr_unlock_bh(dev); ++} ++ ++/** ++ * dev_get_flags - get flags reported to userspace ++ * @dev: device ++ * ++ * Get the combination of flag bits exported through APIs to userspace. ++ */ ++unsigned int dev_get_flags(const struct net_device *dev) ++{ ++ unsigned int flags; ++ ++ flags = (dev->flags & ~(IFF_PROMISC | ++ IFF_ALLMULTI | ++ IFF_RUNNING | ++ IFF_LOWER_UP | ++ IFF_DORMANT)) | ++ (dev->gflags & (IFF_PROMISC | ++ IFF_ALLMULTI)); ++ ++ if (netif_running(dev)) { ++ if (netif_oper_up(dev)) ++ flags |= IFF_RUNNING; ++ if (netif_carrier_ok(dev)) ++ flags |= IFF_LOWER_UP; ++ if (netif_dormant(dev)) ++ flags |= IFF_DORMANT; ++ } ++ ++ return flags; ++} ++EXPORT_SYMBOL(dev_get_flags); ++ ++int __dev_change_flags(struct net_device *dev, unsigned int flags) ++{ ++ unsigned int old_flags = dev->flags; ++ int ret; ++ ++ ASSERT_RTNL(); ++ ++ /* ++ * Set the flags on our device. ++ */ ++ ++ dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP | ++ IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL | ++ IFF_AUTOMEDIA)) | ++ (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC | ++ IFF_ALLMULTI)); ++ ++ /* ++ * Load in the correct multicast list now the flags have changed. ++ */ ++ ++ if ((old_flags ^ flags) & IFF_MULTICAST) ++ dev_change_rx_flags(dev, IFF_MULTICAST); ++ ++ dev_set_rx_mode(dev); ++ ++ /* ++ * Have we downed the interface. We handle IFF_UP ourselves ++ * according to user attempts to set it, rather than blindly ++ * setting it. ++ */ ++ ++ ret = 0; ++ if ((old_flags ^ flags) & IFF_UP) ++ ret = ((old_flags & IFF_UP) ? __dev_close : __dev_open)(dev); ++ ++ if ((flags ^ dev->gflags) & IFF_PROMISC) { ++ int inc = (flags & IFF_PROMISC) ? 1 : -1; ++ unsigned int old_flags = dev->flags; ++ ++ dev->gflags ^= IFF_PROMISC; ++ ++ if (__dev_set_promiscuity(dev, inc, false) >= 0) ++ if (dev->flags != old_flags) ++ dev_set_rx_mode(dev); ++ } ++ ++ /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI ++ is important. Some (broken) drivers set IFF_PROMISC, when ++ IFF_ALLMULTI is requested not asking us and not reporting. ++ */ ++ if ((flags ^ dev->gflags) & IFF_ALLMULTI) { ++ int inc = (flags & IFF_ALLMULTI) ? 1 : -1; ++ ++ dev->gflags ^= IFF_ALLMULTI; ++ __dev_set_allmulti(dev, inc, false); ++ } ++ ++ return ret; ++} ++ ++void __dev_notify_flags(struct net_device *dev, unsigned int old_flags, ++ unsigned int gchanges) ++{ ++ unsigned int changes = dev->flags ^ old_flags; ++ ++ if (gchanges) ++ rtmsg_ifinfo(RTM_NEWLINK, dev, gchanges, GFP_ATOMIC); ++ ++ if (changes & IFF_UP) { ++ if (dev->flags & IFF_UP) ++ call_netdevice_notifiers(NETDEV_UP, dev); ++ else ++ call_netdevice_notifiers(NETDEV_DOWN, dev); ++ } ++ ++ if (dev->flags & IFF_UP && ++ (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE))) { ++ struct netdev_notifier_change_info change_info; ++ ++ change_info.flags_changed = changes; ++ call_netdevice_notifiers_info(NETDEV_CHANGE, dev, ++ &change_info.info); ++ } ++} ++ ++/** ++ * dev_change_flags - change device settings ++ * @dev: device ++ * @flags: device state flags ++ * ++ * Change settings on device based state flags. The flags are ++ * in the userspace exported format. ++ */ ++int dev_change_flags(struct net_device *dev, unsigned int flags) ++{ ++ int ret; ++ unsigned int changes, old_flags = dev->flags, old_gflags = dev->gflags; ++ ++ ret = __dev_change_flags(dev, flags); ++ if (ret < 0) ++ return ret; ++ ++ changes = (old_flags ^ dev->flags) | (old_gflags ^ dev->gflags); ++ __dev_notify_flags(dev, old_flags, changes); ++ return ret; ++} ++EXPORT_SYMBOL(dev_change_flags); ++ ++static int __dev_set_mtu(struct net_device *dev, int new_mtu) ++{ ++ const struct net_device_ops *ops = dev->netdev_ops; ++ ++ if (ops->ndo_change_mtu) ++ return ops->ndo_change_mtu(dev, new_mtu); ++ ++ dev->mtu = new_mtu; ++ return 0; ++} ++ ++/** ++ * dev_set_mtu - Change maximum transfer unit ++ * @dev: device ++ * @new_mtu: new transfer unit ++ * ++ * Change the maximum transfer size of the network device. ++ */ ++int dev_set_mtu(struct net_device *dev, int new_mtu) ++{ ++ int err, orig_mtu; ++ ++ if (new_mtu == dev->mtu) ++ return 0; ++ ++ /* MTU must be positive. */ ++ if (new_mtu < 0) ++ return -EINVAL; ++ ++ if (!netif_device_present(dev)) ++ return -ENODEV; ++ ++ err = call_netdevice_notifiers(NETDEV_PRECHANGEMTU, dev); ++ err = notifier_to_errno(err); ++ if (err) ++ return err; ++ ++ orig_mtu = dev->mtu; ++ err = __dev_set_mtu(dev, new_mtu); ++ ++ if (!err) { ++ err = call_netdevice_notifiers(NETDEV_CHANGEMTU, dev); ++ err = notifier_to_errno(err); ++ if (err) { ++ /* setting mtu back and notifying everyone again, ++ * so that they have a chance to revert changes. ++ */ ++ __dev_set_mtu(dev, orig_mtu); ++ call_netdevice_notifiers(NETDEV_CHANGEMTU, dev); ++ } ++ } ++ return err; ++} ++EXPORT_SYMBOL(dev_set_mtu); ++ ++/** ++ * dev_set_group - Change group this device belongs to ++ * @dev: device ++ * @new_group: group this device should belong to ++ */ ++void dev_set_group(struct net_device *dev, int new_group) ++{ ++ dev->group = new_group; ++} ++EXPORT_SYMBOL(dev_set_group); ++ ++/** ++ * dev_set_mac_address - Change Media Access Control Address ++ * @dev: device ++ * @sa: new address ++ * ++ * Change the hardware (MAC) address of the device ++ */ ++int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa) ++{ ++ const struct net_device_ops *ops = dev->netdev_ops; ++ int err; ++ ++ if (!ops->ndo_set_mac_address) ++ return -EOPNOTSUPP; ++ if (sa->sa_family != dev->type) ++ return -EINVAL; ++ if (!netif_device_present(dev)) ++ return -ENODEV; ++ err = ops->ndo_set_mac_address(dev, sa); ++ if (err) ++ return err; ++ dev->addr_assign_type = NET_ADDR_SET; ++ call_netdevice_notifiers(NETDEV_CHANGEADDR, dev); ++ add_device_randomness(dev->dev_addr, dev->addr_len); ++ return 0; ++} ++EXPORT_SYMBOL(dev_set_mac_address); ++ ++/** ++ * dev_change_carrier - Change device carrier ++ * @dev: device ++ * @new_carrier: new value ++ * ++ * Change device carrier ++ */ ++int dev_change_carrier(struct net_device *dev, bool new_carrier) ++{ ++ const struct net_device_ops *ops = dev->netdev_ops; ++ ++ if (!ops->ndo_change_carrier) ++ return -EOPNOTSUPP; ++ if (!netif_device_present(dev)) ++ return -ENODEV; ++ return ops->ndo_change_carrier(dev, new_carrier); ++} ++EXPORT_SYMBOL(dev_change_carrier); ++ ++/** ++ * dev_get_phys_port_id - Get device physical port ID ++ * @dev: device ++ * @ppid: port ID ++ * ++ * Get device physical port ID ++ */ ++int dev_get_phys_port_id(struct net_device *dev, ++ struct netdev_phys_item_id *ppid) ++{ ++ const struct net_device_ops *ops = dev->netdev_ops; ++ ++ if (!ops->ndo_get_phys_port_id) ++ return -EOPNOTSUPP; ++ return ops->ndo_get_phys_port_id(dev, ppid); ++} ++EXPORT_SYMBOL(dev_get_phys_port_id); ++ ++/** ++ * dev_get_phys_port_name - Get device physical port name ++ * @dev: device ++ * @name: port name ++ * ++ * Get device physical port name ++ */ ++int dev_get_phys_port_name(struct net_device *dev, ++ char *name, size_t len) ++{ ++ const struct net_device_ops *ops = dev->netdev_ops; ++ ++ if (!ops->ndo_get_phys_port_name) ++ return -EOPNOTSUPP; ++ return ops->ndo_get_phys_port_name(dev, name, len); ++} ++EXPORT_SYMBOL(dev_get_phys_port_name); ++ ++/** ++ * dev_new_index - allocate an ifindex ++ * @net: the applicable net namespace ++ * ++ * Returns a suitable unique value for a new device interface ++ * number. The caller must hold the rtnl semaphore or the ++ * dev_base_lock to be sure it remains unique. ++ */ ++static int dev_new_index(struct net *net) ++{ ++ int ifindex = net->ifindex; ++ for (;;) { ++ if (++ifindex <= 0) ++ ifindex = 1; ++ if (!__dev_get_by_index(net, ifindex)) ++ return net->ifindex = ifindex; ++ } ++} ++ ++/* Delayed registration/unregisteration */ ++static LIST_HEAD(net_todo_list); ++DECLARE_WAIT_QUEUE_HEAD(netdev_unregistering_wq); ++ ++static void net_set_todo(struct net_device *dev) ++{ ++ list_add_tail(&dev->todo_list, &net_todo_list); ++ dev_net(dev)->dev_unreg_count++; ++} ++ ++static void rollback_registered_many(struct list_head *head) ++{ ++ struct net_device *dev, *tmp; ++ LIST_HEAD(close_head); ++ ++ BUG_ON(dev_boot_phase); ++ ASSERT_RTNL(); ++ ++ list_for_each_entry_safe(dev, tmp, head, unreg_list) { ++ /* Some devices call without registering ++ * for initialization unwind. Remove those ++ * devices and proceed with the remaining. ++ */ ++ if (dev->reg_state == NETREG_UNINITIALIZED) { ++ pr_debug("unregister_netdevice: device %s/%p never was registered\n", ++ dev->name, dev); ++ ++ WARN_ON(1); ++ list_del(&dev->unreg_list); ++ continue; ++ } ++ dev->dismantle = true; ++ BUG_ON(dev->reg_state != NETREG_REGISTERED); ++ } ++ ++ /* If device is running, close it first. */ ++ list_for_each_entry(dev, head, unreg_list) ++ list_add_tail(&dev->close_list, &close_head); ++ dev_close_many(&close_head, true); ++ ++ list_for_each_entry(dev, head, unreg_list) { ++ /* And unlink it from device chain. */ ++ unlist_netdevice(dev); ++ ++ dev->reg_state = NETREG_UNREGISTERING; ++ on_each_cpu(flush_backlog, dev, 1); ++ } ++ ++ synchronize_net(); ++ ++ list_for_each_entry(dev, head, unreg_list) { ++ struct sk_buff *skb = NULL; ++ ++ /* Shutdown queueing discipline. */ ++ dev_shutdown(dev); ++ ++ ++ /* Notify protocols, that we are about to destroy ++ this device. They should clean all the things. ++ */ ++ call_netdevice_notifiers(NETDEV_UNREGISTER, dev); ++ ++ if (!dev->rtnl_link_ops || ++ dev->rtnl_link_state == RTNL_LINK_INITIALIZED) ++ skb = rtmsg_ifinfo_build_skb(RTM_DELLINK, dev, ~0U, ++ GFP_KERNEL); ++ ++ /* ++ * Flush the unicast and multicast chains ++ */ ++ dev_uc_flush(dev); ++ dev_mc_flush(dev); ++ ++ if (dev->netdev_ops->ndo_uninit) ++ dev->netdev_ops->ndo_uninit(dev); ++ ++ if (skb) ++ rtmsg_ifinfo_send(skb, dev, GFP_KERNEL); ++ ++ /* Notifier chain MUST detach us all upper devices. */ ++ WARN_ON(netdev_has_any_upper_dev(dev)); ++ ++ /* Remove entries from kobject tree */ ++ netdev_unregister_kobject(dev); ++#ifdef CONFIG_XPS ++ /* Remove XPS queueing entries */ ++ netif_reset_xps_queues_gt(dev, 0); ++#endif ++ } ++ ++ synchronize_net(); ++ ++ list_for_each_entry(dev, head, unreg_list) ++ dev_put(dev); ++} ++ ++static void rollback_registered(struct net_device *dev) ++{ ++ LIST_HEAD(single); ++ ++ list_add(&dev->unreg_list, &single); ++ rollback_registered_many(&single); ++ list_del(&single); ++} ++ ++static netdev_features_t netdev_fix_features(struct net_device *dev, ++ netdev_features_t features) ++{ ++ /* Fix illegal checksum combinations */ ++ if ((features & NETIF_F_HW_CSUM) && ++ (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) { ++ netdev_warn(dev, "mixed HW and IP checksum settings.\n"); ++ features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM); ++ } ++ ++ /* TSO requires that SG is present as well. */ ++ if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) { ++ netdev_dbg(dev, "Dropping TSO features since no SG feature.\n"); ++ features &= ~NETIF_F_ALL_TSO; ++ } ++ ++ if ((features & NETIF_F_TSO) && !(features & NETIF_F_HW_CSUM) && ++ !(features & NETIF_F_IP_CSUM)) { ++ netdev_dbg(dev, "Dropping TSO features since no CSUM feature.\n"); ++ features &= ~NETIF_F_TSO; ++ features &= ~NETIF_F_TSO_ECN; ++ } ++ ++ if ((features & NETIF_F_TSO6) && !(features & NETIF_F_HW_CSUM) && ++ !(features & NETIF_F_IPV6_CSUM)) { ++ netdev_dbg(dev, "Dropping TSO6 features since no CSUM feature.\n"); ++ features &= ~NETIF_F_TSO6; ++ } ++ ++ /* TSO ECN requires that TSO is present as well. */ ++ if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN) ++ features &= ~NETIF_F_TSO_ECN; ++ ++ /* Software GSO depends on SG. */ ++ if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) { ++ netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n"); ++ features &= ~NETIF_F_GSO; ++ } ++ ++ /* UFO needs SG and checksumming */ ++ if (features & NETIF_F_UFO) { ++ /* maybe split UFO into V4 and V6? */ ++ if (!((features & NETIF_F_GEN_CSUM) || ++ (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM)) ++ == (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) { ++ netdev_dbg(dev, ++ "Dropping NETIF_F_UFO since no checksum offload features.\n"); ++ features &= ~NETIF_F_UFO; ++ } ++ ++ if (!(features & NETIF_F_SG)) { ++ netdev_dbg(dev, ++ "Dropping NETIF_F_UFO since no NETIF_F_SG feature.\n"); ++ features &= ~NETIF_F_UFO; ++ } ++ } ++ ++#ifdef CONFIG_NET_RX_BUSY_POLL ++ if (dev->netdev_ops->ndo_busy_poll) ++ features |= NETIF_F_BUSY_POLL; ++ else ++#endif ++ features &= ~NETIF_F_BUSY_POLL; ++ ++ return features; ++} ++ ++int __netdev_update_features(struct net_device *dev) ++{ ++ netdev_features_t features; ++ int err = 0; ++ ++ ASSERT_RTNL(); ++ ++ features = netdev_get_wanted_features(dev); ++ ++ if (dev->netdev_ops->ndo_fix_features) ++ features = dev->netdev_ops->ndo_fix_features(dev, features); ++ ++ /* driver might be less strict about feature dependencies */ ++ features = netdev_fix_features(dev, features); ++ ++ if (dev->features == features) ++ return 0; ++ ++ netdev_dbg(dev, "Features changed: %pNF -> %pNF\n", ++ &dev->features, &features); ++ ++ if (dev->netdev_ops->ndo_set_features) ++ err = dev->netdev_ops->ndo_set_features(dev, features); ++ ++ if (unlikely(err < 0)) { ++ netdev_err(dev, ++ "set_features() failed (%d); wanted %pNF, left %pNF\n", ++ err, &features, &dev->features); ++ return -1; ++ } ++ ++ if (!err) ++ dev->features = features; ++ ++ return 1; ++} ++ ++/** ++ * netdev_update_features - recalculate device features ++ * @dev: the device to check ++ * ++ * Recalculate dev->features set and send notifications if it ++ * has changed. Should be called after driver or hardware dependent ++ * conditions might have changed that influence the features. ++ */ ++void netdev_update_features(struct net_device *dev) ++{ ++ if (__netdev_update_features(dev)) ++ netdev_features_change(dev); ++} ++EXPORT_SYMBOL(netdev_update_features); ++ ++/** ++ * netdev_change_features - recalculate device features ++ * @dev: the device to check ++ * ++ * Recalculate dev->features set and send notifications even ++ * if they have not changed. Should be called instead of ++ * netdev_update_features() if also dev->vlan_features might ++ * have changed to allow the changes to be propagated to stacked ++ * VLAN devices. ++ */ ++void netdev_change_features(struct net_device *dev) ++{ ++ __netdev_update_features(dev); ++ netdev_features_change(dev); ++} ++EXPORT_SYMBOL(netdev_change_features); ++ ++/** ++ * netif_stacked_transfer_operstate - transfer operstate ++ * @rootdev: the root or lower level device to transfer state from ++ * @dev: the device to transfer operstate to ++ * ++ * Transfer operational state from root to device. This is normally ++ * called when a stacking relationship exists between the root ++ * device and the device(a leaf device). ++ */ ++void netif_stacked_transfer_operstate(const struct net_device *rootdev, ++ struct net_device *dev) ++{ ++ if (rootdev->operstate == IF_OPER_DORMANT) ++ netif_dormant_on(dev); ++ else ++ netif_dormant_off(dev); ++ ++ if (netif_carrier_ok(rootdev)) { ++ if (!netif_carrier_ok(dev)) ++ netif_carrier_on(dev); ++ } else { ++ if (netif_carrier_ok(dev)) ++ netif_carrier_off(dev); ++ } ++} ++EXPORT_SYMBOL(netif_stacked_transfer_operstate); ++ ++#ifdef CONFIG_SYSFS ++static int netif_alloc_rx_queues(struct net_device *dev) ++{ ++ unsigned int i, count = dev->num_rx_queues; ++ struct netdev_rx_queue *rx; ++ size_t sz = count * sizeof(*rx); ++ ++ BUG_ON(count < 1); ++ ++ rx = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT); ++ if (!rx) { ++ rx = vzalloc(sz); ++ if (!rx) ++ return -ENOMEM; ++ } ++ dev->_rx = rx; ++ ++ for (i = 0; i < count; i++) ++ rx[i].dev = dev; ++ return 0; ++} ++#endif ++ ++static void netdev_init_one_queue(struct net_device *dev, ++ struct netdev_queue *queue, void *_unused) ++{ ++ /* Initialize queue lock */ ++ spin_lock_init(&queue->_xmit_lock); ++ netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type); ++ queue->xmit_lock_owner = -1; ++ netdev_queue_numa_node_write(queue, NUMA_NO_NODE); ++ queue->dev = dev; ++#ifdef CONFIG_BQL ++ dql_init(&queue->dql, HZ); ++#endif ++} ++ ++static void netif_free_tx_queues(struct net_device *dev) ++{ ++ kvfree(dev->_tx); ++} ++ ++static int netif_alloc_netdev_queues(struct net_device *dev) ++{ ++ unsigned int count = dev->num_tx_queues; ++ struct netdev_queue *tx; ++ size_t sz = count * sizeof(*tx); ++ ++ if (count < 1 || count > 0xffff) ++ return -EINVAL; ++ ++ tx = kzalloc(sz, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT); ++ if (!tx) { ++ tx = vzalloc(sz); ++ if (!tx) ++ return -ENOMEM; ++ } ++ dev->_tx = tx; ++ ++ netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL); ++ spin_lock_init(&dev->tx_global_lock); ++ ++ return 0; ++} ++ ++/** ++ * register_netdevice - register a network device ++ * @dev: device to register ++ * ++ * Take a completed network device structure and add it to the kernel ++ * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier ++ * chain. 0 is returned on success. A negative errno code is returned ++ * on a failure to set up the device, or if the name is a duplicate. ++ * ++ * Callers must hold the rtnl semaphore. You may want ++ * register_netdev() instead of this. ++ * ++ * BUGS: ++ * The locking appears insufficient to guarantee two parallel registers ++ * will not get the same name. ++ */ ++ ++int register_netdevice(struct net_device *dev) ++{ ++ int ret; ++ struct net *net = dev_net(dev); ++ ++ BUG_ON(dev_boot_phase); ++ ASSERT_RTNL(); ++ ++ might_sleep(); ++ ++ /* When net_device's are persistent, this will be fatal. */ ++ BUG_ON(dev->reg_state != NETREG_UNINITIALIZED); ++ BUG_ON(!net); ++ ++ spin_lock_init(&dev->addr_list_lock); ++ netdev_set_addr_lockdep_class(dev); ++ ++ ret = dev_get_valid_name(net, dev, dev->name); ++ if (ret < 0) ++ goto out; ++ ++ /* Init, if this function is available */ ++ if (dev->netdev_ops->ndo_init) { ++ ret = dev->netdev_ops->ndo_init(dev); ++ if (ret) { ++ if (ret > 0) ++ ret = -EIO; ++ goto out; ++ } ++ } ++ ++ if (((dev->hw_features | dev->features) & ++ NETIF_F_HW_VLAN_CTAG_FILTER) && ++ (!dev->netdev_ops->ndo_vlan_rx_add_vid || ++ !dev->netdev_ops->ndo_vlan_rx_kill_vid)) { ++ netdev_WARN(dev, "Buggy VLAN acceleration in driver!\n"); ++ ret = -EINVAL; ++ goto err_uninit; ++ } ++ ++ ret = -EBUSY; ++ if (!dev->ifindex) ++ dev->ifindex = dev_new_index(net); ++ else if (__dev_get_by_index(net, dev->ifindex)) ++ goto err_uninit; ++ ++ /* Transfer changeable features to wanted_features and enable ++ * software offloads (GSO and GRO). ++ */ ++ dev->hw_features |= NETIF_F_SOFT_FEATURES; ++ dev->features |= NETIF_F_SOFT_FEATURES; ++ dev->wanted_features = dev->features & dev->hw_features; ++ ++ if (!(dev->flags & IFF_LOOPBACK)) { ++ dev->hw_features |= NETIF_F_NOCACHE_COPY; ++ } ++ ++ /* Make NETIF_F_HIGHDMA inheritable to VLAN devices. ++ */ ++ dev->vlan_features |= NETIF_F_HIGHDMA; ++ ++ /* Make NETIF_F_SG inheritable to tunnel devices. ++ */ ++ dev->hw_enc_features |= NETIF_F_SG; ++ ++ /* Make NETIF_F_SG inheritable to MPLS. ++ */ ++ dev->mpls_features |= NETIF_F_SG; ++ ++ ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev); ++ ret = notifier_to_errno(ret); ++ if (ret) ++ goto err_uninit; ++ ++ ret = netdev_register_kobject(dev); ++ if (ret) ++ goto err_uninit; ++ dev->reg_state = NETREG_REGISTERED; ++ ++ __netdev_update_features(dev); ++ ++ /* ++ * Default initial state at registry is that the ++ * device is present. ++ */ ++ ++ set_bit(__LINK_STATE_PRESENT, &dev->state); ++ ++ linkwatch_init_dev(dev); ++ ++ dev_init_scheduler(dev); ++ dev_hold(dev); ++ list_netdevice(dev); ++ add_device_randomness(dev->dev_addr, dev->addr_len); ++ ++ /* If the device has permanent device address, driver should ++ * set dev_addr and also addr_assign_type should be set to ++ * NET_ADDR_PERM (default value). ++ */ ++ if (dev->addr_assign_type == NET_ADDR_PERM) ++ memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len); ++ ++ /* Notify protocols, that a new device appeared. */ ++ ret = call_netdevice_notifiers(NETDEV_REGISTER, dev); ++ ret = notifier_to_errno(ret); ++ if (ret) { ++ rollback_registered(dev); ++ dev->reg_state = NETREG_UNREGISTERED; ++ } ++ /* ++ * Prevent userspace races by waiting until the network ++ * device is fully setup before sending notifications. ++ */ ++ if (!dev->rtnl_link_ops || ++ dev->rtnl_link_state == RTNL_LINK_INITIALIZED) ++ rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL); ++ ++out: ++ return ret; ++ ++err_uninit: ++ if (dev->netdev_ops->ndo_uninit) ++ dev->netdev_ops->ndo_uninit(dev); ++ goto out; ++} ++EXPORT_SYMBOL(register_netdevice); ++ ++/** ++ * init_dummy_netdev - init a dummy network device for NAPI ++ * @dev: device to init ++ * ++ * This takes a network device structure and initialize the minimum ++ * amount of fields so it can be used to schedule NAPI polls without ++ * registering a full blown interface. This is to be used by drivers ++ * that need to tie several hardware interfaces to a single NAPI ++ * poll scheduler due to HW limitations. ++ */ ++int init_dummy_netdev(struct net_device *dev) ++{ ++ /* Clear everything. Note we don't initialize spinlocks ++ * are they aren't supposed to be taken by any of the ++ * NAPI code and this dummy netdev is supposed to be ++ * only ever used for NAPI polls ++ */ ++ memset(dev, 0, sizeof(struct net_device)); ++ ++ /* make sure we BUG if trying to hit standard ++ * register/unregister code path ++ */ ++ dev->reg_state = NETREG_DUMMY; ++ ++ /* NAPI wants this */ ++ INIT_LIST_HEAD(&dev->napi_list); ++ ++ /* a dummy interface is started by default */ ++ set_bit(__LINK_STATE_PRESENT, &dev->state); ++ set_bit(__LINK_STATE_START, &dev->state); + -+outunlock: -+ __raw_write_seqcount_end(&devnet_rename_seq); -+ mutex_unlock(&devnet_rename_mutex); ++ /* Note : We dont allocate pcpu_refcnt for dummy devices, ++ * because users of this 'device' dont need to change ++ * its refcount. ++ */ ++ ++ return 0; ++} ++EXPORT_SYMBOL_GPL(init_dummy_netdev); ++ ++ ++/** ++ * register_netdev - register a network device ++ * @dev: device to register ++ * ++ * Take a completed network device structure and add it to the kernel ++ * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier ++ * chain. 0 is returned on success. A negative errno code is returned ++ * on a failure to set up the device, or if the name is a duplicate. ++ * ++ * This is a wrapper around register_netdevice that takes the rtnl semaphore ++ * and expands the device name if you passed a format string to ++ * alloc_netdev. ++ */ ++int register_netdev(struct net_device *dev) ++{ ++ int err; ++ ++ rtnl_lock(); ++ err = register_netdevice(dev); ++ rtnl_unlock(); + return err; - } - - /** -@@ -2218,6 +2224,7 @@ - sd->output_queue_tailp = &q->next_sched; - raise_softirq_irqoff(NET_TX_SOFTIRQ); - local_irq_restore(flags); -+ preempt_check_resched_rt(); - } - - void __netif_schedule(struct Qdisc *q) -@@ -2299,6 +2306,7 @@ - __this_cpu_write(softnet_data.completion_queue, skb); - raise_softirq_irqoff(NET_TX_SOFTIRQ); - local_irq_restore(flags); -+ preempt_check_resched_rt(); - } - EXPORT_SYMBOL(__dev_kfree_skb_irq); - -@@ -3366,6 +3374,7 @@ - rps_unlock(sd); - - local_irq_restore(flags); -+ preempt_check_resched_rt(); - - atomic_long_inc(&skb->dev->rx_dropped); - kfree_skb(skb); -@@ -3384,7 +3393,7 @@ - struct rps_dev_flow voidflow, *rflow = &voidflow; - int cpu; - -- preempt_disable(); -+ migrate_disable(); - rcu_read_lock(); - - cpu = get_rps_cpu(skb->dev, skb, &rflow); -@@ -3394,13 +3403,13 @@ - ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail); - - rcu_read_unlock(); -- preempt_enable(); -+ migrate_enable(); - } else - #endif - { - unsigned int qtail; -- ret = enqueue_to_backlog(skb, get_cpu(), &qtail); -- put_cpu(); -+ ret = enqueue_to_backlog(skb, get_cpu_light(), &qtail); -+ put_cpu_light(); - } - return ret; - } -@@ -3434,16 +3443,44 @@ - - trace_netif_rx_ni_entry(skb); - -- preempt_disable(); -+ local_bh_disable(); - err = netif_rx_internal(skb); -- if (local_softirq_pending()) -- do_softirq(); -- preempt_enable(); -+ local_bh_enable(); - - return err; - } - EXPORT_SYMBOL(netif_rx_ni); - -+#ifdef CONFIG_PREEMPT_RT_FULL -+/* -+ * RT runs ksoftirqd as a real time thread and the root_lock is a -+ * "sleeping spinlock". If the trylock fails then we can go into an -+ * infinite loop when ksoftirqd preempted the task which actually -+ * holds the lock, because we requeue q and raise NET_TX softirq -+ * causing ksoftirqd to loop forever. ++} ++EXPORT_SYMBOL(register_netdev); ++ ++int netdev_refcnt_read(const struct net_device *dev) ++{ ++ int i, refcnt = 0; ++ ++ for_each_possible_cpu(i) ++ refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i); ++ return refcnt; ++} ++EXPORT_SYMBOL(netdev_refcnt_read); ++ ++/** ++ * netdev_wait_allrefs - wait until all references are gone. ++ * @dev: target net_device + * -+ * It's safe to use spin_lock on RT here as softirqs run in thread -+ * context and cannot deadlock against the thread which is holding -+ * root_lock. ++ * This is called when unregistering network devices. + * -+ * On !RT the trylock might fail, but there we bail out from the -+ * softirq loop after 10 attempts which we can't do on RT. And the -+ * task holding root_lock cannot be preempted, so the only downside of -+ * that trylock is that we need 10 loops to decide that we should have -+ * given up in the first one :) ++ * Any protocol or device that holds a reference should register ++ * for netdevice notification, and cleanup and put back the ++ * reference if they receive an UNREGISTER event. ++ * We can get stuck here if buggy protocols don't correctly ++ * call dev_put. + */ -+static inline int take_root_lock(spinlock_t *lock) ++static void netdev_wait_allrefs(struct net_device *dev) +{ -+ spin_lock(lock); -+ return 1; ++ unsigned long rebroadcast_time, warning_time; ++ int refcnt; ++ ++ linkwatch_forget_dev(dev); ++ ++ rebroadcast_time = warning_time = jiffies; ++ refcnt = netdev_refcnt_read(dev); ++ ++ while (refcnt != 0) { ++ if (time_after(jiffies, rebroadcast_time + 1 * HZ)) { ++ rtnl_lock(); ++ ++ /* Rebroadcast unregister notification */ ++ call_netdevice_notifiers(NETDEV_UNREGISTER, dev); ++ ++ __rtnl_unlock(); ++ rcu_barrier(); ++ rtnl_lock(); ++ ++ call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev); ++ if (test_bit(__LINK_STATE_LINKWATCH_PENDING, ++ &dev->state)) { ++ /* We must not have linkwatch events ++ * pending on unregister. If this ++ * happens, we simply run the queue ++ * unscheduled, resulting in a noop ++ * for this device. ++ */ ++ linkwatch_run_queue(); ++ } ++ ++ __rtnl_unlock(); ++ ++ rebroadcast_time = jiffies; ++ } ++ ++ msleep(250); ++ ++ refcnt = netdev_refcnt_read(dev); ++ ++ if (time_after(jiffies, warning_time + 10 * HZ)) { ++ pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n", ++ dev->name, refcnt); ++ warning_time = jiffies; ++ } ++ } ++} ++ ++/* The sequence is: ++ * ++ * rtnl_lock(); ++ * ... ++ * register_netdevice(x1); ++ * register_netdevice(x2); ++ * ... ++ * unregister_netdevice(y1); ++ * unregister_netdevice(y2); ++ * ... ++ * rtnl_unlock(); ++ * free_netdev(y1); ++ * free_netdev(y2); ++ * ++ * We are invoked by rtnl_unlock(). ++ * This allows us to deal with problems: ++ * 1) We can delete sysfs objects which invoke hotplug ++ * without deadlocking with linkwatch via keventd. ++ * 2) Since we run with the RTNL semaphore not held, we can sleep ++ * safely in order to wait for the netdev refcnt to drop to zero. ++ * ++ * We must not return until all unregister events added during ++ * the interval the lock was held have been completed. ++ */ ++void netdev_run_todo(void) ++{ ++ struct list_head list; ++ ++ /* Snapshot list, allow later requests */ ++ list_replace_init(&net_todo_list, &list); ++ ++ __rtnl_unlock(); ++ ++ ++ /* Wait for rcu callbacks to finish before next phase */ ++ if (!list_empty(&list)) ++ rcu_barrier(); ++ ++ while (!list_empty(&list)) { ++ struct net_device *dev ++ = list_first_entry(&list, struct net_device, todo_list); ++ list_del(&dev->todo_list); ++ ++ rtnl_lock(); ++ call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev); ++ __rtnl_unlock(); ++ ++ if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) { ++ pr_err("network todo '%s' but state %d\n", ++ dev->name, dev->reg_state); ++ dump_stack(); ++ continue; ++ } ++ ++ dev->reg_state = NETREG_UNREGISTERED; ++ ++ netdev_wait_allrefs(dev); ++ ++ /* paranoia */ ++ BUG_ON(netdev_refcnt_read(dev)); ++ BUG_ON(!list_empty(&dev->ptype_all)); ++ BUG_ON(!list_empty(&dev->ptype_specific)); ++ WARN_ON(rcu_access_pointer(dev->ip_ptr)); ++ WARN_ON(rcu_access_pointer(dev->ip6_ptr)); ++ WARN_ON(dev->dn_ptr); ++ ++ if (dev->destructor) ++ dev->destructor(dev); ++ ++ /* Report a network device has been unregistered */ ++ rtnl_lock(); ++ dev_net(dev)->dev_unreg_count--; ++ __rtnl_unlock(); ++ wake_up(&netdev_unregistering_wq); ++ ++ /* Free network device */ ++ kobject_put(&dev->dev.kobj); ++ } +} ++ ++/* Convert net_device_stats to rtnl_link_stats64. They have the same ++ * fields in the same order, with only the type differing. ++ */ ++void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64, ++ const struct net_device_stats *netdev_stats) ++{ ++#if BITS_PER_LONG == 64 ++ BUILD_BUG_ON(sizeof(*stats64) != sizeof(*netdev_stats)); ++ memcpy(stats64, netdev_stats, sizeof(*stats64)); +#else -+static inline int take_root_lock(spinlock_t *lock) ++ size_t i, n = sizeof(*stats64) / sizeof(u64); ++ const unsigned long *src = (const unsigned long *)netdev_stats; ++ u64 *dst = (u64 *)stats64; ++ ++ BUILD_BUG_ON(sizeof(*netdev_stats) / sizeof(unsigned long) != ++ sizeof(*stats64) / sizeof(u64)); ++ for (i = 0; i < n; i++) ++ dst[i] = src[i]; ++#endif ++} ++EXPORT_SYMBOL(netdev_stats_to_stats64); ++ ++/** ++ * dev_get_stats - get network device statistics ++ * @dev: device to get statistics from ++ * @storage: place to store stats ++ * ++ * Get network statistics from device. Return @storage. ++ * The device driver may provide its own method by setting ++ * dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats; ++ * otherwise the internal statistics structure is used. ++ */ ++struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev, ++ struct rtnl_link_stats64 *storage) +{ -+ return spin_trylock(lock); ++ const struct net_device_ops *ops = dev->netdev_ops; ++ ++ if (ops->ndo_get_stats64) { ++ memset(storage, 0, sizeof(*storage)); ++ ops->ndo_get_stats64(dev, storage); ++ } else if (ops->ndo_get_stats) { ++ netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev)); ++ } else { ++ netdev_stats_to_stats64(storage, &dev->stats); ++ } ++ storage->rx_dropped += atomic_long_read(&dev->rx_dropped); ++ storage->tx_dropped += atomic_long_read(&dev->tx_dropped); ++ return storage; +} ++EXPORT_SYMBOL(dev_get_stats); ++ ++struct netdev_queue *dev_ingress_queue_create(struct net_device *dev) ++{ ++ struct netdev_queue *queue = dev_ingress_queue(dev); ++ ++#ifdef CONFIG_NET_CLS_ACT ++ if (queue) ++ return queue; ++ queue = kzalloc(sizeof(*queue), GFP_KERNEL); ++ if (!queue) ++ return NULL; ++ netdev_init_one_queue(dev, queue, NULL); ++ RCU_INIT_POINTER(queue->qdisc, &noop_qdisc); ++ queue->qdisc_sleeping = &noop_qdisc; ++ rcu_assign_pointer(dev->ingress_queue, queue); +#endif ++ return queue; ++} + - static void net_tx_action(struct softirq_action *h) - { - struct softnet_data *sd = this_cpu_ptr(&softnet_data); -@@ -3485,7 +3522,7 @@ - head = head->next_sched; - - root_lock = qdisc_lock(q); -- if (spin_trylock(root_lock)) { -+ if (take_root_lock(root_lock)) { - smp_mb__before_atomic(); - clear_bit(__QDISC_STATE_SCHED, - &q->state); -@@ -3884,7 +3921,7 @@ - skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) { - if (skb->dev == dev) { - __skb_unlink(skb, &sd->input_pkt_queue); -- kfree_skb(skb); -+ __skb_queue_tail(&sd->tofree_queue, skb); - input_queue_head_incr(sd); - } - } -@@ -3893,10 +3930,13 @@ - skb_queue_walk_safe(&sd->process_queue, skb, tmp) { - if (skb->dev == dev) { - __skb_unlink(skb, &sd->process_queue); -- kfree_skb(skb); -+ __skb_queue_tail(&sd->tofree_queue, skb); - input_queue_head_incr(sd); - } - } ++static const struct ethtool_ops default_ethtool_ops; + -+ if (!skb_queue_empty(&sd->tofree_queue)) -+ raise_softirq_irqoff(NET_RX_SOFTIRQ); - } - - static int napi_gro_complete(struct sk_buff *skb) -@@ -4347,6 +4387,7 @@ - sd->rps_ipi_list = NULL; - - local_irq_enable(); -+ preempt_check_resched_rt(); - - /* Send pending IPI's to kick RPS processing on remote cpus. */ - while (remsd) { -@@ -4360,6 +4401,7 @@ - } else - #endif - local_irq_enable(); -+ preempt_check_resched_rt(); - } - - static bool sd_has_rps_ipi_waiting(struct softnet_data *sd) -@@ -4439,6 +4481,7 @@ - local_irq_save(flags); - ____napi_schedule(this_cpu_ptr(&softnet_data), n); - local_irq_restore(flags); -+ preempt_check_resched_rt(); - } - EXPORT_SYMBOL(__napi_schedule); - -@@ -7168,6 +7211,7 @@ - - raise_softirq_irqoff(NET_TX_SOFTIRQ); - local_irq_enable(); -+ preempt_check_resched_rt(); - - /* Process offline CPU's input_pkt_queue */ - while ((skb = __skb_dequeue(&oldsd->process_queue))) { -@@ -7178,6 +7222,9 @@ - netif_rx_ni(skb); - input_queue_head_incr(oldsd); - } -+ while ((skb = __skb_dequeue(&oldsd->tofree_queue))) { -+ kfree_skb(skb); ++void netdev_set_default_ethtool_ops(struct net_device *dev, ++ const struct ethtool_ops *ops) ++{ ++ if (dev->ethtool_ops == &default_ethtool_ops) ++ dev->ethtool_ops = ops; ++} ++EXPORT_SYMBOL_GPL(netdev_set_default_ethtool_ops); ++ ++void netdev_freemem(struct net_device *dev) ++{ ++ char *addr = (char *)dev - dev->padded; ++ ++ kvfree(addr); ++} ++ ++/** ++ * alloc_netdev_mqs - allocate network device ++ * @sizeof_priv: size of private data to allocate space for ++ * @name: device name format string ++ * @name_assign_type: origin of device name ++ * @setup: callback to initialize device ++ * @txqs: the number of TX subqueues to allocate ++ * @rxqs: the number of RX subqueues to allocate ++ * ++ * Allocates a struct net_device with private data area for driver use ++ * and performs basic initialization. Also allocates subqueue structs ++ * for each queue on the device. ++ */ ++struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name, ++ unsigned char name_assign_type, ++ void (*setup)(struct net_device *), ++ unsigned int txqs, unsigned int rxqs) ++{ ++ struct net_device *dev; ++ size_t alloc_size; ++ struct net_device *p; ++ ++ BUG_ON(strlen(name) >= sizeof(dev->name)); ++ ++ if (txqs < 1) { ++ pr_err("alloc_netdev: Unable to allocate device with zero queues\n"); ++ return NULL; + } - - return NOTIFY_OK; - } -@@ -7479,8 +7526,9 @@ - for_each_possible_cpu(i) { - struct softnet_data *sd = &per_cpu(softnet_data, i); - -- skb_queue_head_init(&sd->input_pkt_queue); -- skb_queue_head_init(&sd->process_queue); -+ skb_queue_head_init_raw(&sd->input_pkt_queue); -+ skb_queue_head_init_raw(&sd->process_queue); -+ skb_queue_head_init_raw(&sd->tofree_queue); - INIT_LIST_HEAD(&sd->poll_list); - sd->output_queue_tailp = &sd->output_queue; - #ifdef CONFIG_RPS -diff -Nur linux-4.1.6.orig/net/core/skbuff.c linux-4.1.6/net/core/skbuff.c ---- linux-4.1.6.orig/net/core/skbuff.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/net/core/skbuff.c 2015-09-08 23:49:08.445829687 +0200 ++ ++#ifdef CONFIG_SYSFS ++ if (rxqs < 1) { ++ pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n"); ++ return NULL; ++ } ++#endif ++ ++ alloc_size = sizeof(struct net_device); ++ if (sizeof_priv) { ++ /* ensure 32-byte alignment of private area */ ++ alloc_size = ALIGN(alloc_size, NETDEV_ALIGN); ++ alloc_size += sizeof_priv; ++ } ++ /* ensure 32-byte alignment of whole construct */ ++ alloc_size += NETDEV_ALIGN - 1; ++ ++ p = kzalloc(alloc_size, GFP_KERNEL | __GFP_NOWARN | __GFP_REPEAT); ++ if (!p) ++ p = vzalloc(alloc_size); ++ if (!p) ++ return NULL; ++ ++ dev = PTR_ALIGN(p, NETDEV_ALIGN); ++ dev->padded = (char *)dev - (char *)p; ++ ++ dev->pcpu_refcnt = alloc_percpu(int); ++ if (!dev->pcpu_refcnt) ++ goto free_dev; ++ ++ if (dev_addr_init(dev)) ++ goto free_pcpu; ++ ++ dev_mc_init(dev); ++ dev_uc_init(dev); ++ ++ dev_net_set(dev, &init_net); ++ ++ dev->gso_max_size = GSO_MAX_SIZE; ++ dev->gso_max_segs = GSO_MAX_SEGS; ++ dev->gso_min_segs = 0; ++ ++ INIT_LIST_HEAD(&dev->napi_list); ++ INIT_LIST_HEAD(&dev->unreg_list); ++ INIT_LIST_HEAD(&dev->close_list); ++ INIT_LIST_HEAD(&dev->link_watch_list); ++ INIT_LIST_HEAD(&dev->adj_list.upper); ++ INIT_LIST_HEAD(&dev->adj_list.lower); ++ INIT_LIST_HEAD(&dev->all_adj_list.upper); ++ INIT_LIST_HEAD(&dev->all_adj_list.lower); ++ INIT_LIST_HEAD(&dev->ptype_all); ++ INIT_LIST_HEAD(&dev->ptype_specific); ++ dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM; ++ setup(dev); ++ ++ dev->num_tx_queues = txqs; ++ dev->real_num_tx_queues = txqs; ++ if (netif_alloc_netdev_queues(dev)) ++ goto free_all; ++ ++#ifdef CONFIG_SYSFS ++ dev->num_rx_queues = rxqs; ++ dev->real_num_rx_queues = rxqs; ++ if (netif_alloc_rx_queues(dev)) ++ goto free_all; ++#endif ++ ++ strcpy(dev->name, name); ++ dev->name_assign_type = name_assign_type; ++ dev->group = INIT_NETDEV_GROUP; ++ if (!dev->ethtool_ops) ++ dev->ethtool_ops = &default_ethtool_ops; ++ return dev; ++ ++free_all: ++ free_netdev(dev); ++ return NULL; ++ ++free_pcpu: ++ free_percpu(dev->pcpu_refcnt); ++free_dev: ++ netdev_freemem(dev); ++ return NULL; ++} ++EXPORT_SYMBOL(alloc_netdev_mqs); ++ ++/** ++ * free_netdev - free network device ++ * @dev: device ++ * ++ * This function does the last stage of destroying an allocated device ++ * interface. The reference to the device object is released. ++ * If this is the last reference then it will be freed. ++ */ ++void free_netdev(struct net_device *dev) ++{ ++ struct napi_struct *p, *n; ++ ++ netif_free_tx_queues(dev); ++#ifdef CONFIG_SYSFS ++ kvfree(dev->_rx); ++#endif ++ ++ kfree(rcu_dereference_protected(dev->ingress_queue, 1)); ++ ++ /* Flush device addresses */ ++ dev_addr_flush(dev); ++ ++ list_for_each_entry_safe(p, n, &dev->napi_list, dev_list) ++ netif_napi_del(p); ++ ++ free_percpu(dev->pcpu_refcnt); ++ dev->pcpu_refcnt = NULL; ++ ++ /* Compatibility with error handling in drivers */ ++ if (dev->reg_state == NETREG_UNINITIALIZED) { ++ netdev_freemem(dev); ++ return; ++ } ++ ++ BUG_ON(dev->reg_state != NETREG_UNREGISTERED); ++ dev->reg_state = NETREG_RELEASED; ++ ++ /* will free via device release */ ++ put_device(&dev->dev); ++} ++EXPORT_SYMBOL(free_netdev); ++ ++/** ++ * synchronize_net - Synchronize with packet receive processing ++ * ++ * Wait for packets currently being received to be done. ++ * Does not block later packets from starting. ++ */ ++void synchronize_net(void) ++{ ++ might_sleep(); ++ if (rtnl_is_locked()) ++ synchronize_rcu_expedited(); ++ else ++ synchronize_rcu(); ++} ++EXPORT_SYMBOL(synchronize_net); ++ ++/** ++ * unregister_netdevice_queue - remove device from the kernel ++ * @dev: device ++ * @head: list ++ * ++ * This function shuts down a device interface and removes it ++ * from the kernel tables. ++ * If head not NULL, device is queued to be unregistered later. ++ * ++ * Callers must hold the rtnl semaphore. You may want ++ * unregister_netdev() instead of this. ++ */ ++ ++void unregister_netdevice_queue(struct net_device *dev, struct list_head *head) ++{ ++ ASSERT_RTNL(); ++ ++ if (head) { ++ list_move_tail(&dev->unreg_list, head); ++ } else { ++ rollback_registered(dev); ++ /* Finish processing unregister after unlock */ ++ net_set_todo(dev); ++ } ++} ++EXPORT_SYMBOL(unregister_netdevice_queue); ++ ++/** ++ * unregister_netdevice_many - unregister many devices ++ * @head: list of devices ++ * ++ * Note: As most callers use a stack allocated list_head, ++ * we force a list_del() to make sure stack wont be corrupted later. ++ */ ++void unregister_netdevice_many(struct list_head *head) ++{ ++ struct net_device *dev; ++ ++ if (!list_empty(head)) { ++ rollback_registered_many(head); ++ list_for_each_entry(dev, head, unreg_list) ++ net_set_todo(dev); ++ list_del(head); ++ } ++} ++EXPORT_SYMBOL(unregister_netdevice_many); ++ ++/** ++ * unregister_netdev - remove device from the kernel ++ * @dev: device ++ * ++ * This function shuts down a device interface and removes it ++ * from the kernel tables. ++ * ++ * This is just a wrapper for unregister_netdevice that takes ++ * the rtnl semaphore. In general you want to use this and not ++ * unregister_netdevice. ++ */ ++void unregister_netdev(struct net_device *dev) ++{ ++ rtnl_lock(); ++ unregister_netdevice(dev); ++ rtnl_unlock(); ++} ++EXPORT_SYMBOL(unregister_netdev); ++ ++/** ++ * dev_change_net_namespace - move device to different nethost namespace ++ * @dev: device ++ * @net: network namespace ++ * @pat: If not NULL name pattern to try if the current device name ++ * is already taken in the destination network namespace. ++ * ++ * This function shuts down a device interface and moves it ++ * to a new network namespace. On success 0 is returned, on ++ * a failure a netagive errno code is returned. ++ * ++ * Callers must hold the rtnl semaphore. ++ */ ++ ++int dev_change_net_namespace(struct net_device *dev, struct net *net, const char *pat) ++{ ++ int err; ++ ++ ASSERT_RTNL(); ++ ++ /* Don't allow namespace local devices to be moved. */ ++ err = -EINVAL; ++ if (dev->features & NETIF_F_NETNS_LOCAL) ++ goto out; ++ ++ /* Ensure the device has been registrered */ ++ if (dev->reg_state != NETREG_REGISTERED) ++ goto out; ++ ++ /* Get out if there is nothing todo */ ++ err = 0; ++ if (net_eq(dev_net(dev), net)) ++ goto out; ++ ++ /* Pick the destination device name, and ensure ++ * we can use it in the destination network namespace. ++ */ ++ err = -EEXIST; ++ if (__dev_get_by_name(net, dev->name)) { ++ /* We get here if we can't use the current device name */ ++ if (!pat) ++ goto out; ++ if (dev_get_valid_name(net, dev, pat) < 0) ++ goto out; ++ } ++ ++ /* ++ * And now a mini version of register_netdevice unregister_netdevice. ++ */ ++ ++ /* If device is running close it first. */ ++ dev_close(dev); ++ ++ /* And unlink it from device chain */ ++ err = -ENODEV; ++ unlist_netdevice(dev); ++ ++ synchronize_net(); ++ ++ /* Shutdown queueing discipline. */ ++ dev_shutdown(dev); ++ ++ /* Notify protocols, that we are about to destroy ++ this device. They should clean all the things. ++ ++ Note that dev->reg_state stays at NETREG_REGISTERED. ++ This is wanted because this way 8021q and macvlan know ++ the device is just moving and can keep their slaves up. ++ */ ++ call_netdevice_notifiers(NETDEV_UNREGISTER, dev); ++ rcu_barrier(); ++ call_netdevice_notifiers(NETDEV_UNREGISTER_FINAL, dev); ++ rtmsg_ifinfo(RTM_DELLINK, dev, ~0U, GFP_KERNEL); ++ ++ /* ++ * Flush the unicast and multicast chains ++ */ ++ dev_uc_flush(dev); ++ dev_mc_flush(dev); ++ ++ /* Send a netdev-removed uevent to the old namespace */ ++ kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE); ++ netdev_adjacent_del_links(dev); ++ ++ /* Actually switch the network namespace */ ++ dev_net_set(dev, net); ++ ++ /* If there is an ifindex conflict assign a new one */ ++ if (__dev_get_by_index(net, dev->ifindex)) ++ dev->ifindex = dev_new_index(net); ++ ++ /* Send a netdev-add uevent to the new namespace */ ++ kobject_uevent(&dev->dev.kobj, KOBJ_ADD); ++ netdev_adjacent_add_links(dev); ++ ++ /* Fixup kobjects */ ++ err = device_rename(&dev->dev, dev->name); ++ WARN_ON(err); ++ ++ /* Add the device back in the hashes */ ++ list_netdevice(dev); ++ ++ /* Notify protocols, that a new device appeared. */ ++ call_netdevice_notifiers(NETDEV_REGISTER, dev); ++ ++ /* ++ * Prevent userspace races by waiting until the network ++ * device is fully setup before sending notifications. ++ */ ++ rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL); ++ ++ synchronize_net(); ++ err = 0; ++out: ++ return err; ++} ++EXPORT_SYMBOL_GPL(dev_change_net_namespace); ++ ++static int dev_cpu_callback(struct notifier_block *nfb, ++ unsigned long action, ++ void *ocpu) ++{ ++ struct sk_buff **list_skb; ++ struct sk_buff *skb; ++ unsigned int cpu, oldcpu = (unsigned long)ocpu; ++ struct softnet_data *sd, *oldsd; ++ ++ if (action != CPU_DEAD && action != CPU_DEAD_FROZEN) ++ return NOTIFY_OK; ++ ++ local_irq_disable(); ++ cpu = smp_processor_id(); ++ sd = &per_cpu(softnet_data, cpu); ++ oldsd = &per_cpu(softnet_data, oldcpu); ++ ++ /* Find end of our completion_queue. */ ++ list_skb = &sd->completion_queue; ++ while (*list_skb) ++ list_skb = &(*list_skb)->next; ++ /* Append completion queue from offline CPU. */ ++ *list_skb = oldsd->completion_queue; ++ oldsd->completion_queue = NULL; ++ ++ /* Append output queue from offline CPU. */ ++ if (oldsd->output_queue) { ++ *sd->output_queue_tailp = oldsd->output_queue; ++ sd->output_queue_tailp = oldsd->output_queue_tailp; ++ oldsd->output_queue = NULL; ++ oldsd->output_queue_tailp = &oldsd->output_queue; ++ } ++ /* Append NAPI poll list from offline CPU, with one exception : ++ * process_backlog() must be called by cpu owning percpu backlog. ++ * We properly handle process_queue & input_pkt_queue later. ++ */ ++ while (!list_empty(&oldsd->poll_list)) { ++ struct napi_struct *napi = list_first_entry(&oldsd->poll_list, ++ struct napi_struct, ++ poll_list); ++ ++ list_del_init(&napi->poll_list); ++ if (napi->poll == process_backlog) ++ napi->state = 0; ++ else ++ ____napi_schedule(sd, napi); ++ } ++ ++ raise_softirq_irqoff(NET_TX_SOFTIRQ); ++ local_irq_enable(); ++ ++ /* Process offline CPU's input_pkt_queue */ ++ while ((skb = __skb_dequeue(&oldsd->process_queue))) { ++ netif_rx_ni(skb); ++ input_queue_head_incr(oldsd); ++ } ++ while ((skb = skb_dequeue(&oldsd->input_pkt_queue))) { ++ netif_rx_ni(skb); ++ input_queue_head_incr(oldsd); ++ } ++ ++ return NOTIFY_OK; ++} ++ ++ ++/** ++ * netdev_increment_features - increment feature set by one ++ * @all: current feature set ++ * @one: new feature set ++ * @mask: mask feature set ++ * ++ * Computes a new feature set after adding a device with feature set ++ * @one to the master device with current feature set @all. Will not ++ * enable anything that is off in @mask. Returns the new feature set. ++ */ ++netdev_features_t netdev_increment_features(netdev_features_t all, ++ netdev_features_t one, netdev_features_t mask) ++{ ++ if (mask & NETIF_F_GEN_CSUM) ++ mask |= NETIF_F_ALL_CSUM; ++ mask |= NETIF_F_VLAN_CHALLENGED; ++ ++ all |= one & (NETIF_F_ONE_FOR_ALL|NETIF_F_ALL_CSUM) & mask; ++ all &= one | ~NETIF_F_ALL_FOR_ALL; ++ ++ /* If one device supports hw checksumming, set for all. */ ++ if (all & NETIF_F_GEN_CSUM) ++ all &= ~(NETIF_F_ALL_CSUM & ~NETIF_F_GEN_CSUM); ++ ++ return all; ++} ++EXPORT_SYMBOL(netdev_increment_features); ++ ++static struct hlist_head * __net_init netdev_create_hash(void) ++{ ++ int i; ++ struct hlist_head *hash; ++ ++ hash = kmalloc(sizeof(*hash) * NETDEV_HASHENTRIES, GFP_KERNEL); ++ if (hash != NULL) ++ for (i = 0; i < NETDEV_HASHENTRIES; i++) ++ INIT_HLIST_HEAD(&hash[i]); ++ ++ return hash; ++} ++ ++/* Initialize per network namespace state */ ++static int __net_init netdev_init(struct net *net) ++{ ++ if (net != &init_net) ++ INIT_LIST_HEAD(&net->dev_base_head); ++ ++ net->dev_name_head = netdev_create_hash(); ++ if (net->dev_name_head == NULL) ++ goto err_name; ++ ++ net->dev_index_head = netdev_create_hash(); ++ if (net->dev_index_head == NULL) ++ goto err_idx; ++ ++ return 0; ++ ++err_idx: ++ kfree(net->dev_name_head); ++err_name: ++ return -ENOMEM; ++} ++ ++/** ++ * netdev_drivername - network driver for the device ++ * @dev: network device ++ * ++ * Determine network driver for device. ++ */ ++const char *netdev_drivername(const struct net_device *dev) ++{ ++ const struct device_driver *driver; ++ const struct device *parent; ++ const char *empty = ""; ++ ++ parent = dev->dev.parent; ++ if (!parent) ++ return empty; ++ ++ driver = parent->driver; ++ if (driver && driver->name) ++ return driver->name; ++ return empty; ++} ++ ++static void __netdev_printk(const char *level, const struct net_device *dev, ++ struct va_format *vaf) ++{ ++ if (dev && dev->dev.parent) { ++ dev_printk_emit(level[1] - '0', ++ dev->dev.parent, ++ "%s %s %s%s: %pV", ++ dev_driver_string(dev->dev.parent), ++ dev_name(dev->dev.parent), ++ netdev_name(dev), netdev_reg_state(dev), ++ vaf); ++ } else if (dev) { ++ printk("%s%s%s: %pV", ++ level, netdev_name(dev), netdev_reg_state(dev), vaf); ++ } else { ++ printk("%s(NULL net_device): %pV", level, vaf); ++ } ++} ++ ++void netdev_printk(const char *level, const struct net_device *dev, ++ const char *format, ...) ++{ ++ struct va_format vaf; ++ va_list args; ++ ++ va_start(args, format); ++ ++ vaf.fmt = format; ++ vaf.va = &args; ++ ++ __netdev_printk(level, dev, &vaf); ++ ++ va_end(args); ++} ++EXPORT_SYMBOL(netdev_printk); ++ ++#define define_netdev_printk_level(func, level) \ ++void func(const struct net_device *dev, const char *fmt, ...) \ ++{ \ ++ struct va_format vaf; \ ++ va_list args; \ ++ \ ++ va_start(args, fmt); \ ++ \ ++ vaf.fmt = fmt; \ ++ vaf.va = &args; \ ++ \ ++ __netdev_printk(level, dev, &vaf); \ ++ \ ++ va_end(args); \ ++} \ ++EXPORT_SYMBOL(func); ++ ++define_netdev_printk_level(netdev_emerg, KERN_EMERG); ++define_netdev_printk_level(netdev_alert, KERN_ALERT); ++define_netdev_printk_level(netdev_crit, KERN_CRIT); ++define_netdev_printk_level(netdev_err, KERN_ERR); ++define_netdev_printk_level(netdev_warn, KERN_WARNING); ++define_netdev_printk_level(netdev_notice, KERN_NOTICE); ++define_netdev_printk_level(netdev_info, KERN_INFO); ++ ++static void __net_exit netdev_exit(struct net *net) ++{ ++ kfree(net->dev_name_head); ++ kfree(net->dev_index_head); ++} ++ ++static struct pernet_operations __net_initdata netdev_net_ops = { ++ .init = netdev_init, ++ .exit = netdev_exit, ++}; ++ ++static void __net_exit default_device_exit(struct net *net) ++{ ++ struct net_device *dev, *aux; ++ /* ++ * Push all migratable network devices back to the ++ * initial network namespace ++ */ ++ rtnl_lock(); ++ for_each_netdev_safe(net, dev, aux) { ++ int err; ++ char fb_name[IFNAMSIZ]; ++ ++ /* Ignore unmoveable devices (i.e. loopback) */ ++ if (dev->features & NETIF_F_NETNS_LOCAL) ++ continue; ++ ++ /* Leave virtual devices for the generic cleanup */ ++ if (dev->rtnl_link_ops) ++ continue; ++ ++ /* Push remaining network devices to init_net */ ++ snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex); ++ err = dev_change_net_namespace(dev, &init_net, fb_name); ++ if (err) { ++ pr_emerg("%s: failed to move %s to init_net: %d\n", ++ __func__, dev->name, err); ++ BUG(); ++ } ++ } ++ rtnl_unlock(); ++} ++ ++static void __net_exit rtnl_lock_unregistering(struct list_head *net_list) ++{ ++ /* Return with the rtnl_lock held when there are no network ++ * devices unregistering in any network namespace in net_list. ++ */ ++ struct net *net; ++ bool unregistering; ++ DEFINE_WAIT_FUNC(wait, woken_wake_function); ++ ++ add_wait_queue(&netdev_unregistering_wq, &wait); ++ for (;;) { ++ unregistering = false; ++ rtnl_lock(); ++ list_for_each_entry(net, net_list, exit_list) { ++ if (net->dev_unreg_count > 0) { ++ unregistering = true; ++ break; ++ } ++ } ++ if (!unregistering) ++ break; ++ __rtnl_unlock(); ++ ++ wait_woken(&wait, TASK_UNINTERRUPTIBLE, MAX_SCHEDULE_TIMEOUT); ++ } ++ remove_wait_queue(&netdev_unregistering_wq, &wait); ++} ++ ++static void __net_exit default_device_exit_batch(struct list_head *net_list) ++{ ++ /* At exit all network devices most be removed from a network ++ * namespace. Do this in the reverse order of registration. ++ * Do this across as many network namespaces as possible to ++ * improve batching efficiency. ++ */ ++ struct net_device *dev; ++ struct net *net; ++ LIST_HEAD(dev_kill_list); ++ ++ /* To prevent network device cleanup code from dereferencing ++ * loopback devices or network devices that have been freed ++ * wait here for all pending unregistrations to complete, ++ * before unregistring the loopback device and allowing the ++ * network namespace be freed. ++ * ++ * The netdev todo list containing all network devices ++ * unregistrations that happen in default_device_exit_batch ++ * will run in the rtnl_unlock() at the end of ++ * default_device_exit_batch. ++ */ ++ rtnl_lock_unregistering(net_list); ++ list_for_each_entry(net, net_list, exit_list) { ++ for_each_netdev_reverse(net, dev) { ++ if (dev->rtnl_link_ops && dev->rtnl_link_ops->dellink) ++ dev->rtnl_link_ops->dellink(dev, &dev_kill_list); ++ else ++ unregister_netdevice_queue(dev, &dev_kill_list); ++ } ++ } ++ unregister_netdevice_many(&dev_kill_list); ++ rtnl_unlock(); ++} ++ ++static struct pernet_operations __net_initdata default_device_ops = { ++ .exit = default_device_exit, ++ .exit_batch = default_device_exit_batch, ++}; ++ ++/* ++ * Initialize the DEV module. At boot time this walks the device list and ++ * unhooks any devices that fail to initialise (normally hardware not ++ * present) and leaves us with a valid list of present and active devices. ++ * ++ */ ++ ++/* ++ * This is called single threaded during boot, so no need ++ * to take the rtnl semaphore. ++ */ ++static int __init net_dev_init(void) ++{ ++ int i, rc = -ENOMEM; ++ ++ BUG_ON(!dev_boot_phase); ++ ++ if (dev_proc_init()) ++ goto out; ++ ++ if (netdev_kobject_init()) ++ goto out; ++ ++ INIT_LIST_HEAD(&ptype_all); ++ for (i = 0; i < PTYPE_HASH_SIZE; i++) ++ INIT_LIST_HEAD(&ptype_base[i]); ++ ++ INIT_LIST_HEAD(&offload_base); ++ ++ if (register_pernet_subsys(&netdev_net_ops)) ++ goto out; ++ ++ /* ++ * Initialise the packet receive queues. ++ */ ++ ++ for_each_possible_cpu(i) { ++ struct softnet_data *sd = &per_cpu(softnet_data, i); ++ ++ skb_queue_head_init(&sd->input_pkt_queue); ++ skb_queue_head_init(&sd->process_queue); ++ INIT_LIST_HEAD(&sd->poll_list); ++ sd->output_queue_tailp = &sd->output_queue; ++#ifdef CONFIG_RPS ++ sd->csd.func = rps_trigger_softirq; ++ sd->csd.info = sd; ++ sd->cpu = i; ++#endif ++ ++ sd->backlog.poll = process_backlog; ++ sd->backlog.weight = weight_p; ++ } ++ ++ dev_boot_phase = 0; ++ ++ /* The loopback device is special if any other network devices ++ * is present in a network namespace the loopback device must ++ * be present. Since we now dynamically allocate and free the ++ * loopback device ensure this invariant is maintained by ++ * keeping the loopback device as the first device on the ++ * list of network devices. Ensuring the loopback devices ++ * is the first device that appears and the last network device ++ * that disappears. ++ */ ++ if (register_pernet_device(&loopback_net_ops)) ++ goto out; ++ ++ if (register_pernet_device(&default_device_ops)) ++ goto out; ++ ++ open_softirq(NET_TX_SOFTIRQ, net_tx_action); ++ open_softirq(NET_RX_SOFTIRQ, net_rx_action); ++ ++ hotcpu_notifier(dev_cpu_callback, 0); ++ dst_init(); ++ rc = 0; ++out: ++ return rc; ++} ++ ++subsys_initcall(net_dev_init); +diff -Nur linux-4.1.10.orig/net/core/skbuff.c linux-4.1.10/net/core/skbuff.c +--- linux-4.1.10.orig/net/core/skbuff.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/net/core/skbuff.c 2015-10-07 18:00:08.000000000 +0200 @@ -63,6 +63,7 @@ #include #include @@ -26299,9 +70088,9 @@ diff -Nur linux-4.1.6.orig/net/core/skbuff.c linux-4.1.6/net/core/skbuff.c return data; } -diff -Nur linux-4.1.6.orig/net/core/sock.c linux-4.1.6/net/core/sock.c ---- linux-4.1.6.orig/net/core/sock.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/net/core/sock.c 2015-09-08 23:49:08.445829687 +0200 +diff -Nur linux-4.1.10.orig/net/core/sock.c linux-4.1.10/net/core/sock.c +--- linux-4.1.10.orig/net/core/sock.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/net/core/sock.c 2015-10-07 18:00:08.000000000 +0200 @@ -2370,12 +2370,11 @@ if (sk->sk_lock.owned) __lock_sock(sk); @@ -26316,9 +70105,9 @@ diff -Nur linux-4.1.6.orig/net/core/sock.c linux-4.1.6/net/core/sock.c } EXPORT_SYMBOL(lock_sock_nested); -diff -Nur linux-4.1.6.orig/net/ipv4/icmp.c linux-4.1.6/net/ipv4/icmp.c ---- linux-4.1.6.orig/net/ipv4/icmp.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/net/ipv4/icmp.c 2015-09-08 23:49:08.461827913 +0200 +diff -Nur linux-4.1.10.orig/net/ipv4/icmp.c linux-4.1.10/net/ipv4/icmp.c +--- linux-4.1.10.orig/net/ipv4/icmp.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/net/ipv4/icmp.c 2015-10-07 18:00:08.000000000 +0200 @@ -69,6 +69,7 @@ #include #include @@ -26370,9 +70159,9 @@ diff -Nur linux-4.1.6.orig/net/ipv4/icmp.c linux-4.1.6/net/ipv4/icmp.c } /* should there be an ICMP stat for ignored echos? */ return true; -diff -Nur linux-4.1.6.orig/net/ipv4/sysctl_net_ipv4.c linux-4.1.6/net/ipv4/sysctl_net_ipv4.c ---- linux-4.1.6.orig/net/ipv4/sysctl_net_ipv4.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/net/ipv4/sysctl_net_ipv4.c 2015-09-08 23:49:08.461827913 +0200 +diff -Nur linux-4.1.10.orig/net/ipv4/sysctl_net_ipv4.c linux-4.1.10/net/ipv4/sysctl_net_ipv4.c +--- linux-4.1.10.orig/net/ipv4/sysctl_net_ipv4.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/net/ipv4/sysctl_net_ipv4.c 2015-10-07 18:00:08.000000000 +0200 @@ -779,6 +779,13 @@ .proc_handler = proc_dointvec }, @@ -26387,9 +70176,9 @@ diff -Nur linux-4.1.6.orig/net/ipv4/sysctl_net_ipv4.c linux-4.1.6/net/ipv4/sysct .procname = "icmp_ignore_bogus_error_responses", .data = &init_net.ipv4.sysctl_icmp_ignore_bogus_error_responses, .maxlen = sizeof(int), -diff -Nur linux-4.1.6.orig/net/mac80211/rx.c linux-4.1.6/net/mac80211/rx.c ---- linux-4.1.6.orig/net/mac80211/rx.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/net/mac80211/rx.c 2015-09-08 23:49:08.461827913 +0200 +diff -Nur linux-4.1.10.orig/net/mac80211/rx.c linux-4.1.10/net/mac80211/rx.c +--- linux-4.1.10.orig/net/mac80211/rx.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/net/mac80211/rx.c 2015-10-07 18:00:08.000000000 +0200 @@ -3554,7 +3554,7 @@ struct ieee80211_supported_band *sband; struct ieee80211_rx_status *status = IEEE80211_SKB_RXCB(skb); @@ -26399,9 +70188,9 @@ diff -Nur linux-4.1.6.orig/net/mac80211/rx.c linux-4.1.6/net/mac80211/rx.c if (WARN_ON(status->band >= IEEE80211_NUM_BANDS)) goto drop; -diff -Nur linux-4.1.6.orig/net/netfilter/core.c linux-4.1.6/net/netfilter/core.c ---- linux-4.1.6.orig/net/netfilter/core.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/net/netfilter/core.c 2015-09-08 23:49:08.461827913 +0200 +diff -Nur linux-4.1.10.orig/net/netfilter/core.c linux-4.1.10/net/netfilter/core.c +--- linux-4.1.10.orig/net/netfilter/core.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/net/netfilter/core.c 2015-10-07 18:00:08.000000000 +0200 @@ -22,11 +22,17 @@ #include #include @@ -26420,9 +70209,9 @@ diff -Nur linux-4.1.6.orig/net/netfilter/core.c linux-4.1.6/net/netfilter/core.c static DEFINE_MUTEX(afinfo_mutex); const struct nf_afinfo __rcu *nf_afinfo[NFPROTO_NUMPROTO] __read_mostly; -diff -Nur linux-4.1.6.orig/net/packet/af_packet.c linux-4.1.6/net/packet/af_packet.c ---- linux-4.1.6.orig/net/packet/af_packet.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/net/packet/af_packet.c 2015-09-08 23:49:08.465827470 +0200 +diff -Nur linux-4.1.10.orig/net/packet/af_packet.c linux-4.1.10/net/packet/af_packet.c +--- linux-4.1.10.orig/net/packet/af_packet.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/net/packet/af_packet.c 2015-10-07 18:00:08.000000000 +0200 @@ -63,6 +63,7 @@ #include #include @@ -26449,9 +70238,9 @@ diff -Nur linux-4.1.6.orig/net/packet/af_packet.c linux-4.1.6/net/packet/af_pack } } prb_close_block(pkc, pbd, po, status); -diff -Nur linux-4.1.6.orig/net/rds/ib_rdma.c linux-4.1.6/net/rds/ib_rdma.c ---- linux-4.1.6.orig/net/rds/ib_rdma.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/net/rds/ib_rdma.c 2015-09-08 23:49:08.465827470 +0200 +diff -Nur linux-4.1.10.orig/net/rds/ib_rdma.c linux-4.1.10/net/rds/ib_rdma.c +--- linux-4.1.10.orig/net/rds/ib_rdma.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/net/rds/ib_rdma.c 2015-10-07 18:00:08.000000000 +0200 @@ -34,6 +34,7 @@ #include #include @@ -26469,9 +70258,9 @@ diff -Nur linux-4.1.6.orig/net/rds/ib_rdma.c linux-4.1.6/net/rds/ib_rdma.c } } -diff -Nur linux-4.1.6.orig/net/sched/sch_generic.c linux-4.1.6/net/sched/sch_generic.c ---- linux-4.1.6.orig/net/sched/sch_generic.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/net/sched/sch_generic.c 2015-09-08 23:49:08.465827470 +0200 +diff -Nur linux-4.1.10.orig/net/sched/sch_generic.c linux-4.1.10/net/sched/sch_generic.c +--- linux-4.1.10.orig/net/sched/sch_generic.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/net/sched/sch_generic.c 2015-10-07 18:00:08.000000000 +0200 @@ -894,7 +894,7 @@ /* Wait for outstanding qdisc_run calls. */ list_for_each_entry(dev, head, close_list) @@ -26481,9 +70270,9 @@ diff -Nur linux-4.1.6.orig/net/sched/sch_generic.c linux-4.1.6/net/sched/sch_gen } void dev_deactivate(struct net_device *dev) -diff -Nur linux-4.1.6.orig/net/sunrpc/svc_xprt.c linux-4.1.6/net/sunrpc/svc_xprt.c ---- linux-4.1.6.orig/net/sunrpc/svc_xprt.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/net/sunrpc/svc_xprt.c 2015-09-08 23:49:08.465827470 +0200 +diff -Nur linux-4.1.10.orig/net/sunrpc/svc_xprt.c linux-4.1.10/net/sunrpc/svc_xprt.c +--- linux-4.1.10.orig/net/sunrpc/svc_xprt.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/net/sunrpc/svc_xprt.c 2015-10-07 18:00:08.000000000 +0200 @@ -341,7 +341,7 @@ goto out; } @@ -26511,9 +70300,9 @@ diff -Nur linux-4.1.6.orig/net/sunrpc/svc_xprt.c linux-4.1.6/net/sunrpc/svc_xprt out: trace_svc_xprt_do_enqueue(xprt, rqstp); } -diff -Nur linux-4.1.6.orig/scripts/mkcompile_h linux-4.1.6/scripts/mkcompile_h ---- linux-4.1.6.orig/scripts/mkcompile_h 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/scripts/mkcompile_h 2015-09-08 23:49:08.465827470 +0200 +diff -Nur linux-4.1.10.orig/scripts/mkcompile_h linux-4.1.10/scripts/mkcompile_h +--- linux-4.1.10.orig/scripts/mkcompile_h 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/scripts/mkcompile_h 2015-10-07 18:00:08.000000000 +0200 @@ -4,7 +4,8 @@ ARCH=$2 SMP=$3 @@ -26532,9 +70321,9 @@ diff -Nur linux-4.1.6.orig/scripts/mkcompile_h linux-4.1.6/scripts/mkcompile_h UTS_VERSION="$UTS_VERSION $CONFIG_FLAGS $TIMESTAMP" # Truncate to maximum length -diff -Nur linux-4.1.6.orig/sound/core/pcm_native.c linux-4.1.6/sound/core/pcm_native.c ---- linux-4.1.6.orig/sound/core/pcm_native.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/sound/core/pcm_native.c 2015-09-08 23:49:08.465827470 +0200 +diff -Nur linux-4.1.10.orig/sound/core/pcm_native.c linux-4.1.10/sound/core/pcm_native.c +--- linux-4.1.10.orig/sound/core/pcm_native.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/sound/core/pcm_native.c 2015-10-07 18:00:08.000000000 +0200 @@ -123,7 +123,7 @@ void snd_pcm_stream_lock_irq(struct snd_pcm_substream *substream) { @@ -26571,9 +70360,9 @@ diff -Nur linux-4.1.6.orig/sound/core/pcm_native.c linux-4.1.6/sound/core/pcm_na } EXPORT_SYMBOL_GPL(snd_pcm_stream_unlock_irqrestore); -diff -Nur linux-4.1.6.orig/sound/soc/intel/atom/sst/sst.c linux-4.1.6/sound/soc/intel/atom/sst/sst.c ---- linux-4.1.6.orig/sound/soc/intel/atom/sst/sst.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/sound/soc/intel/atom/sst/sst.c 2015-09-08 23:49:08.465827470 +0200 +diff -Nur linux-4.1.10.orig/sound/soc/intel/atom/sst/sst.c linux-4.1.10/sound/soc/intel/atom/sst/sst.c +--- linux-4.1.10.orig/sound/soc/intel/atom/sst/sst.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/sound/soc/intel/atom/sst/sst.c 2015-10-07 18:00:08.000000000 +0200 @@ -368,8 +368,8 @@ * initialize by FW or driver when firmware is loaded */ @@ -26585,9 +70374,9 @@ diff -Nur linux-4.1.6.orig/sound/soc/intel/atom/sst/sst.c linux-4.1.6/sound/soc/ spin_unlock_irqrestore(&ctx->ipc_spin_lock, irq_flags); } -diff -Nur linux-4.1.6.orig/virt/kvm/async_pf.c linux-4.1.6/virt/kvm/async_pf.c ---- linux-4.1.6.orig/virt/kvm/async_pf.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/virt/kvm/async_pf.c 2015-09-08 23:49:08.465827470 +0200 +diff -Nur linux-4.1.10.orig/virt/kvm/async_pf.c linux-4.1.10/virt/kvm/async_pf.c +--- linux-4.1.10.orig/virt/kvm/async_pf.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/virt/kvm/async_pf.c 2015-10-07 18:00:08.000000000 +0200 @@ -94,8 +94,8 @@ trace_kvm_async_pf_completed(addr, gva); @@ -26599,9 +70388,9 @@ diff -Nur linux-4.1.6.orig/virt/kvm/async_pf.c linux-4.1.6/virt/kvm/async_pf.c mmput(mm); kvm_put_kvm(vcpu->kvm); -diff -Nur linux-4.1.6.orig/virt/kvm/kvm_main.c linux-4.1.6/virt/kvm/kvm_main.c ---- linux-4.1.6.orig/virt/kvm/kvm_main.c 2015-08-17 05:52:51.000000000 +0200 -+++ linux-4.1.6/virt/kvm/kvm_main.c 2015-09-08 23:49:08.465827470 +0200 +diff -Nur linux-4.1.10.orig/virt/kvm/kvm_main.c linux-4.1.10/virt/kvm/kvm_main.c +--- linux-4.1.10.orig/virt/kvm/kvm_main.c 2015-10-03 13:49:38.000000000 +0200 ++++ linux-4.1.10/virt/kvm/kvm_main.c 2015-10-07 18:00:08.000000000 +0200 @@ -218,7 +218,7 @@ vcpu->kvm = kvm; vcpu->vcpu_id = id; -- cgit v1.2.3